Implement cache validity for PackagesData

For unknown to me reason, this breaks for many people with Steam reporting invalid data and ASF caching it until new change number, which may never arrive. Add our own 7-days validity on top, to ensure that user never needs to delete ASF.db manually.

ArchiSteamFarm.OfficialPlugins.SteamTokenDumper/SteamTokenDumperPlugin.cs

@@ -22,7 +22,6 @@
 using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
-using System.Collections.Immutable;
 using System.ComponentModel;
 using System.Composition;
 using System.Globalization;
@@ -35,6 +34,7 @@ using ArchiSteamFarm.OfficialPlugins.SteamTokenDumper.Localization;
 using ArchiSteamFarm.Plugins;
 using ArchiSteamFarm.Plugins.Interfaces;
 using ArchiSteamFarm.Steam;
+using ArchiSteamFarm.Storage;
 using ArchiSteamFarm.Web;
 using ArchiSteamFarm.Web.Responses;
 using Newtonsoft.Json;
@@ -341,7 +341,7 @@ internal sealed class SteamTokenDumperPlugin : OfficialPlugin, IASF, IBot, IBotC
 			HashSet<uint> appIDsToRefresh = new();
 
 			foreach (uint packageID in packageIDs.Where(static packageID => !Config.SecretPackageIDs.Contains(packageID))) {
-				if (!ASF.GlobalDatabase.PackagesDataReadOnly.TryGetValue(packageID, out (uint ChangeNumber, ImmutableHashSet<uint>? AppIDs) packageData) || (packageData.AppIDs == null)) {
+				if (!ASF.GlobalDatabase.PackagesDataReadOnly.TryGetValue(packageID, out PackageData? packageData) || (packageData.AppIDs == null)) {
 					// ASF might not have the package info for us at the moment, we'll retry later
 					continue;
 				}

ArchiSteamFarm/Steam/Bot.cs

@@ -1170,7 +1170,7 @@ public sealed class Bot : IAsyncDisposable, IDisposable {
 
 	internal Task<HashSet<uint>?> GetMarketableAppIDs() => ArchiWebHandler.GetAppList();
 
-	internal async Task<Dictionary<uint, (uint ChangeNumber, ImmutableHashSet<uint>? AppIDs)>?> GetPackagesData(IReadOnlyCollection<uint> packageIDs) {
+	internal async Task<Dictionary<uint, PackageData>?> GetPackagesData(IReadOnlyCollection<uint> packageIDs) {
 		if ((packageIDs == null) || (packageIDs.Count == 0)) {
 			throw new ArgumentNullException(nameof(packageIDs));
 		}
@@ -1190,7 +1190,7 @@ public sealed class Bot : IAsyncDisposable, IDisposable {
 		}
 
 		if (packageRequests.Count == 0) {
-			return new Dictionary<uint, (uint ChangeNumber, ImmutableHashSet<uint>? AppIDs)>(0);
+			return new Dictionary<uint, PackageData>(0);
 		}
 
 		AsyncJobMultiple<SteamApps.PICSProductInfoCallback>.ResultSet? productInfoResultSet = null;
@@ -1207,7 +1207,9 @@ public sealed class Bot : IAsyncDisposable, IDisposable {
 			return null;
 		}
 
-		Dictionary<uint, (uint ChangeNumber, ImmutableHashSet<uint>? AppIDs)> result = new();
+		DateTime validUntil = DateTime.UtcNow.AddDays(7);
+
+		Dictionary<uint, PackageData> result = new();
 
 		foreach (SteamApps.PICSProductInfoCallback.PICSProductInfo productInfo in productInfoResultSet.Results.SelectMany(static productInfoResult => productInfoResult.Packages).Where(static productInfoPackages => productInfoPackages.Key != 0).Select(static productInfoPackages => productInfoPackages.Value)) {
 			if (productInfo.KeyValues == KeyValue.Invalid) {
@@ -1238,7 +1240,7 @@ public sealed class Bot : IAsyncDisposable, IDisposable {
 					appIDs.Add(appID);
 				}
 			} finally {
-				result[productInfo.ID] = (changeNumber, appIDs?.ToImmutableHashSet());
+				result[productInfo.ID] = new PackageData(changeNumber, validUntil, appIDs?.ToImmutableHashSet());
 			}
 		}
 
@@ -2665,7 +2667,7 @@ public sealed class Bot : IAsyncDisposable, IDisposable {
 
 				// Package is always due to refresh with access token change
 				packagesToRefresh[license.PackageID] = (uint) license.LastChangeNumber;
-			} else if (!ASF.GlobalDatabase.PackagesDataReadOnly.TryGetValue(license.PackageID, out (uint ChangeNumber, ImmutableHashSet<uint>? AppIDs) packageData) || (packageData.ChangeNumber < license.LastChangeNumber)) {
+			} else if (!ASF.GlobalDatabase.PackagesDataReadOnly.TryGetValue(license.PackageID, out PackageData? packageData) || (packageData.ChangeNumber < license.LastChangeNumber)) {
 				packagesToRefresh[license.PackageID] = (uint) license.LastChangeNumber;
 			}
 		}

ArchiSteamFarm/Storage/GlobalDatabase.cs

@@ -22,7 +22,6 @@
 using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
-using System.Collections.Immutable;
 using System.Globalization;
 using System.IO;
 using System.Linq;
@@ -47,7 +46,7 @@ public sealed class GlobalDatabase : SerializableFile {
 
 	[JsonIgnore]
 	[PublicAPI]
-	public IReadOnlyDictionary<uint, (uint ChangeNumber, ImmutableHashSet<uint>? AppIDs)> PackagesDataReadOnly => PackagesData;
+	public IReadOnlyDictionary<uint, PackageData> PackagesDataReadOnly => PackagesData;
 
 	[JsonProperty(Required = Required.DisallowNull)]
 	internal readonly ObservableConcurrentDictionary<uint, byte> CardCountsPerGame = new();
@@ -62,7 +61,7 @@ public sealed class GlobalDatabase : SerializableFile {
 	private readonly ConcurrentDictionary<uint, ulong> PackagesAccessTokens = new();
 
 	[JsonProperty(Required = Required.DisallowNull)]
-	private readonly ConcurrentDictionary<uint, (uint ChangeNumber, ImmutableHashSet<uint>? AppIDs)> PackagesData = new();
+	private readonly ConcurrentDictionary<uint, PackageData> PackagesData = new();
 
 	private readonly SemaphoreSlim PackagesRefreshSemaphore = new(1, 1);
 
@@ -247,7 +246,7 @@ public sealed class GlobalDatabase : SerializableFile {
 		HashSet<uint> result = new();
 
 		foreach (uint packageID in packageIDs.Where(static packageID => packageID != 0)) {
-			if (!PackagesData.TryGetValue(packageID, out (uint ChangeNumber, ImmutableHashSet<uint>? AppIDs) packagesData) || (packagesData.AppIDs?.Contains(appID) != true)) {
+			if (!PackagesData.TryGetValue(packageID, out PackageData? packageEntry) || (packageEntry.AppIDs?.Contains(appID) != true)) {
 				continue;
 			}
 
@@ -316,13 +315,15 @@ public sealed class GlobalDatabase : SerializableFile {
 		await PackagesRefreshSemaphore.WaitAsync().ConfigureAwait(false);
 
 		try {
-			HashSet<uint> packageIDs = packages.Where(package => (package.Key != 0) && (!PackagesData.TryGetValue(package.Key, out (uint ChangeNumber, ImmutableHashSet<uint>? AppIDs) previousData) || (previousData.ChangeNumber < package.Value))).Select(static package => package.Key).ToHashSet();
+			DateTime now = DateTime.UtcNow;
+
+			HashSet<uint> packageIDs = packages.Where(package => (package.Key != 0) && (!PackagesData.TryGetValue(package.Key, out PackageData? previousData) || (previousData.ChangeNumber < package.Value) || (previousData.ValidUntil < now))).Select(static package => package.Key).ToHashSet();
 
 			if (packageIDs.Count == 0) {
 				return;
 			}
 
-			Dictionary<uint, (uint ChangeNumber, ImmutableHashSet<uint>? AppIDs)>? packagesData = await bot.GetPackagesData(packageIDs).ConfigureAwait(false);
+			Dictionary<uint, PackageData>? packagesData = await bot.GetPackagesData(packageIDs).ConfigureAwait(false);
 
 			if (packagesData == null) {
 				bot.ArchiLogger.LogGenericWarning(Strings.WarningFailed);
@@ -330,20 +331,11 @@ public sealed class GlobalDatabase : SerializableFile {
 				return;
 			}
 
-			bool save = false;
+			foreach ((uint packageID, PackageData packageData) in packagesData) {
-
-			foreach ((uint packageID, (uint ChangeNumber, ImmutableHashSet<uint>? AppIDs) packageData) in packagesData) {
-				if (PackagesData.TryGetValue(packageID, out (uint ChangeNumber, ImmutableHashSet<uint>? AppIDs) previousData) && (packageData.ChangeNumber <= previousData.ChangeNumber)) {
-					continue;
-				}
-
 				PackagesData[packageID] = packageData;
-				save = true;
 			}
 
-			if (save) {
+			Utilities.InBackground(Save);
-				Utilities.InBackground(Save);
-			}
 		} finally {
 			PackagesRefreshSemaphore.Release();
 		}

ArchiSteamFarm/Storage/PackageData.cs

@@ -0,0 +1,70 @@
+//     _                _      _  ____   _                           _____
+//    / \    _ __  ___ | |__  (_)/ ___| | |_  ___   __ _  _ __ ___  |  ___|__ _  _ __  _ __ ___
+//   / _ \  | '__|/ __|| '_ \ | |\___ \ | __|/ _ \ / _` || '_ ` _ \ | |_  / _` || '__|| '_ ` _ \
+//  / ___ \ | |  | (__ | | | || | ___) || |_|  __/| (_| || | | | | ||  _|| (_| || |   | | | | | |
+// /_/   \_\|_|   \___||_| |_||_||____/  \__|\___| \__,_||_| |_| |_||_|   \__,_||_|   |_| |_| |_|
+// |
+// Copyright 2015-2022 Łukasz "JustArchi" Domeradzki
+// Contact: JustArchi@JustArchi.net
+// |
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// |
+// http://www.apache.org/licenses/LICENSE-2.0
+// |
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System;
+using System.Collections.Immutable;
+using JetBrains.Annotations;
+using Newtonsoft.Json;
+
+namespace ArchiSteamFarm.Storage;
+
+public sealed class PackageData {
+	[JsonProperty]
+	public ImmutableHashSet<uint>? AppIDs { get; private set; }
+
+	[JsonProperty]
+	public uint ChangeNumber { get; private set; }
+
+	[JsonProperty]
+	public DateTime ValidUntil { get; private set; }
+
+	[JsonProperty("Item2")]
+	[Obsolete("TODO: Delete me")]
+	private ImmutableHashSet<uint> AppIDsOld {
+		set => AppIDs = value;
+	}
+
+	[JsonProperty("Item1")]
+	[Obsolete("TODO: Delete me and make ChangeNumber and ValidUntil - Required.Always")]
+	private uint ChangeNumberOld {
+		set => ChangeNumber = value;
+	}
+
+	internal PackageData(uint changeNumber, DateTime validUntil, ImmutableHashSet<uint>? appIDs = null) {
+		if (changeNumber == 0) {
+			throw new ArgumentOutOfRangeException(nameof(changeNumber));
+		}
+
+		if (validUntil <= DateTime.UnixEpoch) {
+			throw new ArgumentOutOfRangeException(nameof(validUntil));
+		}
+
+		ChangeNumber = changeNumber;
+		ValidUntil = validUntil;
+		AppIDs = appIDs;
+	}
+
+	[JsonConstructor]
+	private PackageData() { }
+
+	[UsedImplicitly]
+	public bool ShouldSerializeAppIDs() => AppIDs is { IsEmpty: false };
+}