From d67366cd12798d342b156e91d62ebc2b885ff438 Mon Sep 17 00:00:00 2001 From: JustArchi Date: Thu, 11 Aug 2022 22:38:19 +0200 Subject: [PATCH] Implement cache validity for PackagesData For unknown to me reason, this breaks for many people with Steam reporting invalid data and ASF caching it until new change number, which may never arrive. Add our own 7-days validity on top, to ensure that user never needs to delete ASF.db manually. --- .../SteamTokenDumperPlugin.cs | 4 +- ArchiSteamFarm/Steam/Bot.cs | 12 ++-- ArchiSteamFarm/Storage/GlobalDatabase.cs | 26 +++---- ArchiSteamFarm/Storage/PackageData.cs | 70 +++++++++++++++++++ 4 files changed, 88 insertions(+), 24 deletions(-) create mode 100644 ArchiSteamFarm/Storage/PackageData.cs diff --git a/ArchiSteamFarm.OfficialPlugins.SteamTokenDumper/SteamTokenDumperPlugin.cs b/ArchiSteamFarm.OfficialPlugins.SteamTokenDumper/SteamTokenDumperPlugin.cs index 6ec7f28c9..e0c9d25ff 100644 --- a/ArchiSteamFarm.OfficialPlugins.SteamTokenDumper/SteamTokenDumperPlugin.cs +++ b/ArchiSteamFarm.OfficialPlugins.SteamTokenDumper/SteamTokenDumperPlugin.cs @@ -22,7 +22,6 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; -using System.Collections.Immutable; using System.ComponentModel; using System.Composition; using System.Globalization; @@ -35,6 +34,7 @@ using ArchiSteamFarm.OfficialPlugins.SteamTokenDumper.Localization; using ArchiSteamFarm.Plugins; using ArchiSteamFarm.Plugins.Interfaces; using ArchiSteamFarm.Steam; +using ArchiSteamFarm.Storage; using ArchiSteamFarm.Web; using ArchiSteamFarm.Web.Responses; using Newtonsoft.Json; @@ -341,7 +341,7 @@ internal sealed class SteamTokenDumperPlugin : OfficialPlugin, IASF, IBot, IBotC HashSet appIDsToRefresh = new(); foreach (uint packageID in packageIDs.Where(static packageID => !Config.SecretPackageIDs.Contains(packageID))) { - if (!ASF.GlobalDatabase.PackagesDataReadOnly.TryGetValue(packageID, out (uint ChangeNumber, ImmutableHashSet? AppIDs) packageData) || (packageData.AppIDs == null)) { + if (!ASF.GlobalDatabase.PackagesDataReadOnly.TryGetValue(packageID, out PackageData? packageData) || (packageData.AppIDs == null)) { // ASF might not have the package info for us at the moment, we'll retry later continue; } diff --git a/ArchiSteamFarm/Steam/Bot.cs b/ArchiSteamFarm/Steam/Bot.cs index 8f522b977..84a6e30f5 100644 --- a/ArchiSteamFarm/Steam/Bot.cs +++ b/ArchiSteamFarm/Steam/Bot.cs @@ -1170,7 +1170,7 @@ public sealed class Bot : IAsyncDisposable, IDisposable { internal Task?> GetMarketableAppIDs() => ArchiWebHandler.GetAppList(); - internal async Task? AppIDs)>?> GetPackagesData(IReadOnlyCollection packageIDs) { + internal async Task?> GetPackagesData(IReadOnlyCollection packageIDs) { if ((packageIDs == null) || (packageIDs.Count == 0)) { throw new ArgumentNullException(nameof(packageIDs)); } @@ -1190,7 +1190,7 @@ public sealed class Bot : IAsyncDisposable, IDisposable { } if (packageRequests.Count == 0) { - return new Dictionary? AppIDs)>(0); + return new Dictionary(0); } AsyncJobMultiple.ResultSet? productInfoResultSet = null; @@ -1207,7 +1207,9 @@ public sealed class Bot : IAsyncDisposable, IDisposable { return null; } - Dictionary? AppIDs)> result = new(); + DateTime validUntil = DateTime.UtcNow.AddDays(7); + + Dictionary result = new(); foreach (SteamApps.PICSProductInfoCallback.PICSProductInfo productInfo in productInfoResultSet.Results.SelectMany(static productInfoResult => productInfoResult.Packages).Where(static productInfoPackages => productInfoPackages.Key != 0).Select(static productInfoPackages => productInfoPackages.Value)) { if (productInfo.KeyValues == KeyValue.Invalid) { @@ -1238,7 +1240,7 @@ public sealed class Bot : IAsyncDisposable, IDisposable { appIDs.Add(appID); } } finally { - result[productInfo.ID] = (changeNumber, appIDs?.ToImmutableHashSet()); + result[productInfo.ID] = new PackageData(changeNumber, validUntil, appIDs?.ToImmutableHashSet()); } } @@ -2665,7 +2667,7 @@ public sealed class Bot : IAsyncDisposable, IDisposable { // Package is always due to refresh with access token change packagesToRefresh[license.PackageID] = (uint) license.LastChangeNumber; - } else if (!ASF.GlobalDatabase.PackagesDataReadOnly.TryGetValue(license.PackageID, out (uint ChangeNumber, ImmutableHashSet? AppIDs) packageData) || (packageData.ChangeNumber < license.LastChangeNumber)) { + } else if (!ASF.GlobalDatabase.PackagesDataReadOnly.TryGetValue(license.PackageID, out PackageData? packageData) || (packageData.ChangeNumber < license.LastChangeNumber)) { packagesToRefresh[license.PackageID] = (uint) license.LastChangeNumber; } } diff --git a/ArchiSteamFarm/Storage/GlobalDatabase.cs b/ArchiSteamFarm/Storage/GlobalDatabase.cs index b09d93126..ae6455091 100644 --- a/ArchiSteamFarm/Storage/GlobalDatabase.cs +++ b/ArchiSteamFarm/Storage/GlobalDatabase.cs @@ -22,7 +22,6 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; -using System.Collections.Immutable; using System.Globalization; using System.IO; using System.Linq; @@ -47,7 +46,7 @@ public sealed class GlobalDatabase : SerializableFile { [JsonIgnore] [PublicAPI] - public IReadOnlyDictionary? AppIDs)> PackagesDataReadOnly => PackagesData; + public IReadOnlyDictionary PackagesDataReadOnly => PackagesData; [JsonProperty(Required = Required.DisallowNull)] internal readonly ObservableConcurrentDictionary CardCountsPerGame = new(); @@ -62,7 +61,7 @@ public sealed class GlobalDatabase : SerializableFile { private readonly ConcurrentDictionary PackagesAccessTokens = new(); [JsonProperty(Required = Required.DisallowNull)] - private readonly ConcurrentDictionary? AppIDs)> PackagesData = new(); + private readonly ConcurrentDictionary PackagesData = new(); private readonly SemaphoreSlim PackagesRefreshSemaphore = new(1, 1); @@ -247,7 +246,7 @@ public sealed class GlobalDatabase : SerializableFile { HashSet result = new(); foreach (uint packageID in packageIDs.Where(static packageID => packageID != 0)) { - if (!PackagesData.TryGetValue(packageID, out (uint ChangeNumber, ImmutableHashSet? AppIDs) packagesData) || (packagesData.AppIDs?.Contains(appID) != true)) { + if (!PackagesData.TryGetValue(packageID, out PackageData? packageEntry) || (packageEntry.AppIDs?.Contains(appID) != true)) { continue; } @@ -316,13 +315,15 @@ public sealed class GlobalDatabase : SerializableFile { await PackagesRefreshSemaphore.WaitAsync().ConfigureAwait(false); try { - HashSet packageIDs = packages.Where(package => (package.Key != 0) && (!PackagesData.TryGetValue(package.Key, out (uint ChangeNumber, ImmutableHashSet? AppIDs) previousData) || (previousData.ChangeNumber < package.Value))).Select(static package => package.Key).ToHashSet(); + DateTime now = DateTime.UtcNow; + + HashSet packageIDs = packages.Where(package => (package.Key != 0) && (!PackagesData.TryGetValue(package.Key, out PackageData? previousData) || (previousData.ChangeNumber < package.Value) || (previousData.ValidUntil < now))).Select(static package => package.Key).ToHashSet(); if (packageIDs.Count == 0) { return; } - Dictionary? AppIDs)>? packagesData = await bot.GetPackagesData(packageIDs).ConfigureAwait(false); + Dictionary? packagesData = await bot.GetPackagesData(packageIDs).ConfigureAwait(false); if (packagesData == null) { bot.ArchiLogger.LogGenericWarning(Strings.WarningFailed); @@ -330,20 +331,11 @@ public sealed class GlobalDatabase : SerializableFile { return; } - bool save = false; - - foreach ((uint packageID, (uint ChangeNumber, ImmutableHashSet? AppIDs) packageData) in packagesData) { - if (PackagesData.TryGetValue(packageID, out (uint ChangeNumber, ImmutableHashSet? AppIDs) previousData) && (packageData.ChangeNumber <= previousData.ChangeNumber)) { - continue; - } - + foreach ((uint packageID, PackageData packageData) in packagesData) { PackagesData[packageID] = packageData; - save = true; } - if (save) { - Utilities.InBackground(Save); - } + Utilities.InBackground(Save); } finally { PackagesRefreshSemaphore.Release(); } diff --git a/ArchiSteamFarm/Storage/PackageData.cs b/ArchiSteamFarm/Storage/PackageData.cs new file mode 100644 index 000000000..78d83f84e --- /dev/null +++ b/ArchiSteamFarm/Storage/PackageData.cs @@ -0,0 +1,70 @@ +// _ _ _ ____ _ _____ +// / \ _ __ ___ | |__ (_)/ ___| | |_ ___ __ _ _ __ ___ | ___|__ _ _ __ _ __ ___ +// / _ \ | '__|/ __|| '_ \ | |\___ \ | __|/ _ \ / _` || '_ ` _ \ | |_ / _` || '__|| '_ ` _ \ +// / ___ \ | | | (__ | | | || | ___) || |_| __/| (_| || | | | | || _|| (_| || | | | | | | | +// /_/ \_\|_| \___||_| |_||_||____/ \__|\___| \__,_||_| |_| |_||_| \__,_||_| |_| |_| |_| +// | +// Copyright 2015-2022 Łukasz "JustArchi" Domeradzki +// Contact: JustArchi@JustArchi.net +// | +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// | +// http://www.apache.org/licenses/LICENSE-2.0 +// | +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Immutable; +using JetBrains.Annotations; +using Newtonsoft.Json; + +namespace ArchiSteamFarm.Storage; + +public sealed class PackageData { + [JsonProperty] + public ImmutableHashSet? AppIDs { get; private set; } + + [JsonProperty] + public uint ChangeNumber { get; private set; } + + [JsonProperty] + public DateTime ValidUntil { get; private set; } + + [JsonProperty("Item2")] + [Obsolete("TODO: Delete me")] + private ImmutableHashSet AppIDsOld { + set => AppIDs = value; + } + + [JsonProperty("Item1")] + [Obsolete("TODO: Delete me and make ChangeNumber and ValidUntil - Required.Always")] + private uint ChangeNumberOld { + set => ChangeNumber = value; + } + + internal PackageData(uint changeNumber, DateTime validUntil, ImmutableHashSet? appIDs = null) { + if (changeNumber == 0) { + throw new ArgumentOutOfRangeException(nameof(changeNumber)); + } + + if (validUntil <= DateTime.UnixEpoch) { + throw new ArgumentOutOfRangeException(nameof(validUntil)); + } + + ChangeNumber = changeNumber; + ValidUntil = validUntil; + AppIDs = appIDs; + } + + [JsonConstructor] + private PackageData() { } + + [UsedImplicitly] + public bool ShouldSerializeAppIDs() => AppIDs is { IsEmpty: false }; +}