Closes #3061 (#3145)

* Good start

* Misc

* Make ApiAuthenticationMiddleware use new json

* Remove first newtonsoft dependency

* Pull latest ASFB json enhancements

* Start reimplementing newtonsoft!

* One thing at a time

* Keep doing all kind of breaking changes which need to be tested later

* Add back ShouldSerialize() support

* Misc

* Eradicate remaining parts of newtonsoft

* WIP

* Workaround STJ stupidity in regards to derived types

STJ can't serialize derived type properties by default, so we'll use another approach in our serializable file function

* Make CI happy

* Bunch of further fixes

* Fix AddFreeLicense() after rewrite

* Add full support for JsonDisallowNullAttribute

* Optimize our json utilities even further

* Misc

* Add support for fields in disallow null

* Misc optimization

* Fix deserialization of GlobalCache in STD

* Fix non-public [JsonExtensionData]

* Fix IM missing method exception, correct db storage helpers

* Fix saving into generic databases

Thanks STJ

* Make Save() function abstract to force inheritors to implement it properly

* Correct ShouldSerializeAdditionalProperties to be a method

* Misc cleanup

* Code review

* Allow JSON comments in configs, among other

* Allow trailing commas in configs

Users very often add them accidentally, no reason to throw on them

* Fix confirmation ID

Probably needs further fixes, will need to check later

* Correct confirmations deserialization

* Use JsonNumberHandling

* Misc

* Misc

* [JsonDisallowNull] corrections

* Forbid [JsonDisallowNull] on non-nullable structs

* Not really but okay

* Add and use ToJson() helpers

* Misc

* Misc

ArchiSteamFarm/Helpers/Json/JsonDisallowNullAttribute.cs

@@ -0,0 +1,30 @@
+//     _                _      _  ____   _                           _____
+//    / \    _ __  ___ | |__  (_)/ ___| | |_  ___   __ _  _ __ ___  |  ___|__ _  _ __  _ __ ___
+//   / _ \  | '__|/ __|| '_ \ | |\___ \ | __|/ _ \ / _` || '_ ` _ \ | |_  / _` || '__|| '_ ` _ \
+//  / ___ \ | |  | (__ | | | || | ___) || |_|  __/| (_| || | | | | ||  _|| (_| || |   | | | | | |
+// /_/   \_\|_|   \___||_| |_||_||____/  \__|\___| \__,_||_| |_| |_||_|   \__,_||_|   |_| |_| |_|
+// |
+// Copyright 2015-2024 Łukasz "JustArchi" Domeradzki
+// Contact: JustArchi@JustArchi.net
+// |
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// |
+// http://www.apache.org/licenses/LICENSE-2.0
+// |
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System;
+using System.Text.Json.Serialization;
+using JetBrains.Annotations;
+
+namespace ArchiSteamFarm.Helpers.Json;
+
+[AttributeUsage(AttributeTargets.Field | AttributeTargets.Property)]
+[PublicAPI]
+public sealed class JsonDisallowNullAttribute : JsonAttribute;

ArchiSteamFarm/Helpers/Json/JsonUtilities.cs

@@ -0,0 +1,186 @@
+//     _                _      _  ____   _                           _____
+//    / \    _ __  ___ | |__  (_)/ ___| | |_  ___   __ _  _ __ ___  |  ___|__ _  _ __  _ __ ___
+//   / _ \  | '__|/ __|| '_ \ | |\___ \ | __|/ _ \ / _` || '_ ` _ \ | |_  / _` || '__|| '_ ` _ \
+//  / ___ \ | |  | (__ | | | || | ___) || |_|  __/| (_| || | | | | ||  _|| (_| || |   | | | | | |
+// /_/   \_\|_|   \___||_| |_||_||____/  \__|\___| \__,_||_| |_| |_||_|   \__,_||_|   |_| |_| |_|
+// |
+// Copyright 2015-2024 Łukasz "JustArchi" Domeradzki
+// Contact: JustArchi@JustArchi.net
+// |
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// |
+// http://www.apache.org/licenses/LICENSE-2.0
+// |
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System;
+using System.Diagnostics.CodeAnalysis;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+using System.Text.Json;
+using System.Text.Json.Serialization.Metadata;
+using System.Threading;
+using System.Threading.Tasks;
+using ArchiSteamFarm.Localization;
+using JetBrains.Annotations;
+
+namespace ArchiSteamFarm.Helpers.Json;
+
+public static class JsonUtilities {
+	[PublicAPI]
+	public static readonly JsonSerializerOptions DefaultJsonSerialierOptions = CreateDefaultJsonSerializerOptions();
+
+	[PublicAPI]
+	public static readonly JsonSerializerOptions IndentedJsonSerialierOptions = CreateDefaultJsonSerializerOptions(true);
+
+	[PublicAPI]
+	public static JsonElement ToJsonElement<T>(this T obj, bool writeIndented = false) where T : notnull {
+		ArgumentNullException.ThrowIfNull(obj);
+
+		return JsonSerializer.SerializeToElement(obj, writeIndented ? IndentedJsonSerialierOptions : DefaultJsonSerialierOptions);
+	}
+
+	[PublicAPI]
+	public static T? ToJsonObject<T>(this JsonElement jsonElement, CancellationToken cancellationToken = default) => jsonElement.Deserialize<T>(DefaultJsonSerialierOptions);
+
+	[PublicAPI]
+	public static async ValueTask<T?> ToJsonObject<T>(this Stream stream, CancellationToken cancellationToken = default) {
+		ArgumentNullException.ThrowIfNull(stream);
+
+		return await JsonSerializer.DeserializeAsync<T>(stream, DefaultJsonSerialierOptions, cancellationToken).ConfigureAwait(false);
+	}
+
+	[PublicAPI]
+	public static T? ToJsonObject<T>([StringSyntax(StringSyntaxAttribute.Json)] this string json) {
+		ArgumentException.ThrowIfNullOrEmpty(json);
+
+		return JsonSerializer.Deserialize<T>(json, DefaultJsonSerialierOptions);
+	}
+
+	[PublicAPI]
+	public static string ToJsonText<T>(this T obj, bool writeIndented = false) => JsonSerializer.Serialize(obj, writeIndented ? IndentedJsonSerialierOptions : DefaultJsonSerialierOptions);
+
+	private static void ApplyCustomModifiers(JsonTypeInfo jsonTypeInfo) {
+		ArgumentNullException.ThrowIfNull(jsonTypeInfo);
+
+		bool potentialDisallowedNullsPossible = false;
+
+		foreach (JsonPropertyInfo property in jsonTypeInfo.Properties) {
+			// All our modifications require a valid Get method on a property
+			if (property.Get == null) {
+				continue;
+			}
+
+			// The object should be validated against potential nulls if at least one property has [JsonDisallowNull] declared, avoid performance penalty otherwise
+			if (property.AttributeProvider?.IsDefined(typeof(JsonDisallowNullAttribute), false) == true) {
+				if (property.PropertyType.IsValueType && (Nullable.GetUnderlyingType(property.PropertyType) == null)) {
+					// We should have no [JsonDisallowNull] declared on non-nullable types, this requires developer correction
+					throw new InvalidOperationException(string.Format(CultureInfo.CurrentCulture, Strings.WarningUnknownValuePleaseReport, nameof(JsonDisallowNullAttribute), $"{property.Name} ({jsonTypeInfo.Type})"));
+				}
+
+				potentialDisallowedNullsPossible = true;
+			}
+
+			// The property should be checked against ShouldSerialize if there is a valid method to invoke, avoid performance penalty otherwise
+			MethodInfo? shouldSerializeMethod = GetShouldSerializeMethod(jsonTypeInfo.Type, property);
+
+			if (shouldSerializeMethod != null) {
+				property.ShouldSerialize = (parent, _) => ShouldSerialize(shouldSerializeMethod, parent);
+			}
+		}
+
+		if (potentialDisallowedNullsPossible) {
+			jsonTypeInfo.OnDeserialized = OnPotentialDisallowedNullsDeserialized;
+		}
+	}
+
+	private static JsonSerializerOptions CreateDefaultJsonSerializerOptions(bool writeIndented = false) =>
+		new() {
+			AllowTrailingCommas = true,
+			PropertyNamingPolicy = null,
+			ReadCommentHandling = JsonCommentHandling.Skip,
+			TypeInfoResolver = new DefaultJsonTypeInfoResolver { Modifiers = { ApplyCustomModifiers } },
+			WriteIndented = writeIndented
+		};
+
+	[UnconditionalSuppressMessage("AssemblyLoadTrimming", "IL2070", Justification = "We don't care about trimmed methods, it's not like we can make it work differently anyway")]
+	[UnconditionalSuppressMessage("AssemblyLoadTrimming", "IL2075", Justification = "We don't care about trimmed properties, it's not like we can make it work differently anyway")]
+	private static MethodInfo? GetShouldSerializeMethod([SuppressMessage("ReSharper", "SuggestBaseTypeForParameter")] Type parent, JsonPropertyInfo property) {
+		ArgumentNullException.ThrowIfNull(parent);
+		ArgumentNullException.ThrowIfNull(property);
+
+		// Handle most common case where ShouldSerializeXYZ() matches property name
+		MethodInfo? result = parent.GetMethod($"ShouldSerialize{property.Name}", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Static, null, Type.EmptyTypes, null);
+
+		if (result?.ReturnType == typeof(bool)) {
+			// Method exists and returns a boolean, that's what we'd like to hear
+			return result;
+		}
+
+		// Handle less common case where ShouldSerializeXYZ() matches original member name
+		PropertyInfo? memberNameProperty = property.GetType().GetProperty("MemberName", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public);
+
+		if (memberNameProperty == null) {
+			// Should never happen, investigate if it does
+			throw new InvalidOperationException(nameof(memberNameProperty));
+		}
+
+		object? memberNameResult = memberNameProperty.GetValue(property);
+
+		if (memberNameResult is not string memberName) {
+			// Should never happen, investigate if it does
+			throw new InvalidOperationException(nameof(memberName));
+		}
+
+		if (string.IsNullOrEmpty(memberName) || (memberName == property.Name)) {
+			// We don't have anything to work with further, there is no ShouldSerialize() method
+			return null;
+		}
+
+		result = parent.GetMethod($"ShouldSerialize{memberName}", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Static, null, Type.EmptyTypes, null);
+
+		// Use alternative method if it exists and returns a boolean
+		return result?.ReturnType == typeof(bool) ? result : null;
+	}
+
+	[UnconditionalSuppressMessage("AssemblyLoadTrimming", "IL2075", Justification = "We don't care about trimmed properties, it's not like we can make it work differently anyway")]
+	private static void OnPotentialDisallowedNullsDeserialized(object obj) {
+		ArgumentNullException.ThrowIfNull(obj);
+
+		Type type = obj.GetType();
+
+		foreach (FieldInfo field in type.GetFields(BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Static).Where(field => field.IsDefined(typeof(JsonDisallowNullAttribute), false) && (field.GetValue(obj) == null))) {
+			throw new JsonException($"Required field {field.Name} expects a non-null value.");
+		}
+
+		foreach (PropertyInfo property in type.GetProperties(BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Static).Where(property => (property.GetMethod != null) && property.IsDefined(typeof(JsonDisallowNullAttribute), false) && (property.GetValue(obj) == null))) {
+			throw new JsonException($"Required property {property.Name} expects a non-null value.");
+		}
+	}
+
+	private static bool ShouldSerialize(MethodInfo shouldSerializeMethod, object parent) {
+		ArgumentNullException.ThrowIfNull(shouldSerializeMethod);
+		ArgumentNullException.ThrowIfNull(parent);
+
+		if (shouldSerializeMethod.ReturnType != typeof(bool)) {
+			throw new InvalidOperationException(nameof(shouldSerializeMethod));
+		}
+
+		object? shouldSerialize = shouldSerializeMethod.Invoke(parent, null);
+
+		if (shouldSerialize is not bool result) {
+			// Should not happen, we've already determined we have a method that returns a boolean
+			throw new InvalidOperationException(nameof(shouldSerialize));
+		}
+
+		return result;
+	}
+}

ArchiSteamFarm/Helpers/SerializableFile.cs

@@ -24,7 +24,8 @@ using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
 using ArchiSteamFarm.Core;
-using Newtonsoft.Json;
+using ArchiSteamFarm.Helpers.Json;
+using JetBrains.Annotations;
 
 namespace ArchiSteamFarm.Helpers;
 
@@ -49,47 +50,60 @@ public abstract class SerializableFile : IDisposable {
 		}
 	}
 
-	protected async Task Save() {
-		if (string.IsNullOrEmpty(FilePath)) {
+	/// <summary>
+	///     Implementing this method in your target class is crucial for providing supported functionality.
+	///     In order to do so, it's enough to call static <see cref="Save" /> function from the parent class, providing <code>this</code> as input parameter.
+	///     Afterwards, simply call your <see cref="Save" /> function whenever you need to save changes.
+	///     This approach will allow JSON serializer used in the <see cref="SerializableFile" /> to properly discover all of the properties used in your class.
+	///     Unfortunately, due to STJ's limitations, called by some "security", it's not possible for base class to resolve your properties automatically otherwise.
+	/// </summary>
+	/// <example>protected override Task Save() => Save(this);</example>
+	[UsedImplicitly]
+	protected abstract Task Save();
+
+	protected static async Task Save<T>(T serializableFile) where T : SerializableFile {
+		ArgumentNullException.ThrowIfNull(serializableFile);
+
+		if (string.IsNullOrEmpty(serializableFile.FilePath)) {
 			throw new InvalidOperationException(nameof(FilePath));
 		}
 
-		if (ReadOnly) {
+		if (serializableFile.ReadOnly) {
 			return;
 		}
 
 		// ReSharper disable once SuspiciousLockOverSynchronizationPrimitive - this is not a mistake, we need extra synchronization, and we can re-use the semaphore object for that
-		lock (FileSemaphore) {
-			if (SavingScheduled) {
+		lock (serializableFile.FileSemaphore) {
+			if (serializableFile.SavingScheduled) {
 				return;
 			}
 
-			SavingScheduled = true;
+			serializableFile.SavingScheduled = true;
 		}
 
-		await FileSemaphore.WaitAsync().ConfigureAwait(false);
+		await serializableFile.FileSemaphore.WaitAsync().ConfigureAwait(false);
 
 		try {
 			// ReSharper disable once SuspiciousLockOverSynchronizationPrimitive - this is not a mistake, we need extra synchronization, and we can re-use the semaphore object for that
-			lock (FileSemaphore) {
-				SavingScheduled = false;
+			lock (serializableFile.FileSemaphore) {
+				serializableFile.SavingScheduled = false;
 			}
 
-			if (ReadOnly) {
+			if (serializableFile.ReadOnly) {
 				return;
 			}
 
-			string json = JsonConvert.SerializeObject(this, Debugging.IsUserDebugging ? Formatting.Indented : Formatting.None);
+			string json = serializableFile.ToJsonText(Debugging.IsUserDebugging);
 
 			if (string.IsNullOrEmpty(json)) {
 				throw new InvalidOperationException(nameof(json));
 			}
 
 			// We always want to write entire content to temporary file first, in order to never load corrupted data, also when target file doesn't exist
-			string newFilePath = $"{FilePath}.new";
+			string newFilePath = $"{serializableFile.FilePath}.new";
 
-			if (File.Exists(FilePath)) {
-				string currentJson = await File.ReadAllTextAsync(FilePath).ConfigureAwait(false);
+			if (File.Exists(serializableFile.FilePath)) {
+				string currentJson = await File.ReadAllTextAsync(serializableFile.FilePath).ConfigureAwait(false);
 
 				if (json == currentJson) {
 					return;
@@ -97,16 +111,16 @@ public abstract class SerializableFile : IDisposable {
 
 				await File.WriteAllTextAsync(newFilePath, json).ConfigureAwait(false);
 
-				File.Replace(newFilePath, FilePath, null);
+				File.Replace(newFilePath, serializableFile.FilePath, null);
 			} else {
 				await File.WriteAllTextAsync(newFilePath, json).ConfigureAwait(false);
 
-				File.Move(newFilePath, FilePath);
+				File.Move(newFilePath, serializableFile.FilePath);
 			}
 		} catch (Exception e) {
 			ASF.ArchiLogger.LogGenericException(e);
 		} finally {
-			FileSemaphore.Release();
+			serializableFile.FileSemaphore.Release();
 		}
 	}