From 273a6b0fbe2f8c3f8da1f5ede0605f0570d79d58 Mon Sep 17 00:00:00 2001 From: JustArchi Date: Sun, 3 Jan 2021 21:37:16 +0100 Subject: [PATCH] Invalidate cache upon pics changes PICS restart indicates that we don't know what changes exactly we've missed. If package doesn't emit any change number (and it doesn't have to), we might miss the update that happened in the meantime without being aware of that happening. --- .../SteamTokenDumperPlugin.cs | 3 +- ArchiSteamFarm/Bot.cs | 23 ++++++------- ArchiSteamFarm/GlobalDatabase.cs | 33 +++++++++++++++---- ArchiSteamFarm/SteamPICSChanges.cs | 10 +++++- 4 files changed, 49 insertions(+), 20 deletions(-) diff --git a/ArchiSteamFarm.OfficialPlugins.SteamTokenDumper/SteamTokenDumperPlugin.cs b/ArchiSteamFarm.OfficialPlugins.SteamTokenDumper/SteamTokenDumperPlugin.cs index 8df266078..744ec6160 100644 --- a/ArchiSteamFarm.OfficialPlugins.SteamTokenDumper/SteamTokenDumperPlugin.cs +++ b/ArchiSteamFarm.OfficialPlugins.SteamTokenDumper/SteamTokenDumperPlugin.cs @@ -22,6 +22,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Collections.Immutable; using System.Composition; using System.Globalization; using System.Linq; @@ -258,7 +259,7 @@ namespace ArchiSteamFarm.OfficialPlugins.SteamTokenDumper { HashSet appIDsToRefresh = new(); foreach (uint packageID in packageIDs) { - if (!ASF.GlobalDatabase.PackagesDataReadOnly.TryGetValue(packageID, out (uint ChangeNumber, HashSet? AppIDs) packageData) || (packageData.AppIDs == null)) { + if (!ASF.GlobalDatabase.PackagesDataReadOnly.TryGetValue(packageID, out (uint ChangeNumber, ImmutableHashSet? AppIDs) packageData) || (packageData.AppIDs == null)) { // ASF might not have the package info for us at the moment, we'll retry later continue; } diff --git a/ArchiSteamFarm/Bot.cs b/ArchiSteamFarm/Bot.cs index 262ca5445..158d57850 100755 --- a/ArchiSteamFarm/Bot.cs +++ b/ArchiSteamFarm/Bot.cs @@ -1219,7 +1219,7 @@ namespace ArchiSteamFarm { internal async Task?> GetMarketableAppIDs() => await ArchiWebHandler.GetAppList().ConfigureAwait(false); - internal async Task? AppIDs)>?> GetPackagesData(IReadOnlyCollection packageIDs) { + internal async Task? AppIDs)>?> GetPackagesData(IReadOnlyCollection packageIDs) { if ((packageIDs == null) || (packageIDs.Count == 0)) { throw new ArgumentNullException(nameof(packageIDs)); } @@ -1239,7 +1239,7 @@ namespace ArchiSteamFarm { } if (packageRequests.Count == 0) { - return new Dictionary? AppIDs)>(0); + return new Dictionary? AppIDs)>(0); } AsyncJobMultiple.ResultSet? productInfoResultSet = null; @@ -1256,7 +1256,7 @@ namespace ArchiSteamFarm { return null; } - Dictionary? AppIDs)> result = new(); + Dictionary? AppIDs)> result = new(); foreach (SteamApps.PICSProductInfoCallback.PICSProductInfo productInfo in productInfoResultSet.Results.SelectMany(productInfoResult => productInfoResult.Packages).Where(productInfoPackages => productInfoPackages.Key != 0).Select(productInfoPackages => productInfoPackages.Value)) { if (productInfo.KeyValues == KeyValue.Invalid) { @@ -1265,28 +1265,29 @@ namespace ArchiSteamFarm { return null; } - (uint ChangeNumber, HashSet? AppIDs) value = (productInfo.ChangeNumber, null); + uint changeNumber = productInfo.ChangeNumber; + HashSet? appIDs = null; try { - KeyValue appIDs = productInfo.KeyValues["appids"]; + KeyValue appIDsKv = productInfo.KeyValues["appids"]; - if (appIDs == KeyValue.Invalid) { + if (appIDsKv == KeyValue.Invalid) { continue; } - value.AppIDs = new HashSet(appIDs.Children.Count); + appIDs = new HashSet(appIDsKv.Children.Count); - foreach (string? appIDText in appIDs.Children.Select(app => app.Value)) { + foreach (string? appIDText in appIDsKv.Children.Select(app => app.Value)) { if (!uint.TryParse(appIDText, out uint appID) || (appID == 0)) { ArchiLogger.LogNullError(nameof(appID)); return null; } - value.AppIDs.Add(appID); + appIDs.Add(appID); } } finally { - result[productInfo.ID] = value; + result[productInfo.ID] = (changeNumber, appIDs?.ToImmutableHashSet()); } } @@ -2620,7 +2621,7 @@ namespace ArchiSteamFarm { // Package is always due to refresh with access token change packagesToRefresh[license.PackageID] = (uint) license.LastChangeNumber; - } else if (!ASF.GlobalDatabase.PackagesDataReadOnly.TryGetValue(license.PackageID, out (uint ChangeNumber, HashSet? AppIDs) packageData) || (packageData.ChangeNumber < license.LastChangeNumber)) { + } else if (!ASF.GlobalDatabase.PackagesDataReadOnly.TryGetValue(license.PackageID, out (uint ChangeNumber, ImmutableHashSet? AppIDs) packageData) || (packageData.ChangeNumber < license.LastChangeNumber)) { packagesToRefresh[license.PackageID] = (uint) license.LastChangeNumber; } } diff --git a/ArchiSteamFarm/GlobalDatabase.cs b/ArchiSteamFarm/GlobalDatabase.cs index d1cf1323e..e28c8ff35 100644 --- a/ArchiSteamFarm/GlobalDatabase.cs +++ b/ArchiSteamFarm/GlobalDatabase.cs @@ -22,6 +22,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Collections.Immutable; using System.Globalization; using System.IO; using System.Linq; @@ -41,7 +42,7 @@ namespace ArchiSteamFarm { [JsonIgnore] [PublicAPI] - public IReadOnlyDictionary? AppIDs)> PackagesDataReadOnly => PackagesData; + public IReadOnlyDictionary? AppIDs)> PackagesDataReadOnly => PackagesData; [JsonProperty(Required = Required.DisallowNull)] internal readonly InMemoryServerListProvider ServerListProvider = new(); @@ -50,7 +51,7 @@ namespace ArchiSteamFarm { private readonly ConcurrentDictionary PackagesAccessTokens = new(); [JsonProperty(Required = Required.DisallowNull)] - private readonly ConcurrentDictionary? AppIDs)> PackagesData = new(); + private readonly ConcurrentDictionary? AppIDs)> PackagesData = new(); private readonly SemaphoreSlim PackagesRefreshSemaphore = new(1, 1); @@ -151,7 +152,7 @@ namespace ArchiSteamFarm { HashSet result = new(); foreach (uint packageID in packageIDs.Where(packageID => packageID != 0)) { - if (!PackagesData.TryGetValue(packageID, out (uint ChangeNumber, HashSet? AppIDs) packagesData) || (packagesData.AppIDs?.Contains(appID) != true)) { + if (!PackagesData.TryGetValue(packageID, out (uint ChangeNumber, ImmutableHashSet? AppIDs) packagesData) || (packagesData.AppIDs?.Contains(appID) != true)) { continue; } @@ -161,6 +162,24 @@ namespace ArchiSteamFarm { return result; } + internal void OnPICSChangesRestart() { + bool save = false; + + if (!PackagesData.IsEmpty) { + PackagesData.Clear(); + save = true; + } + + if (!PackagesAccessTokens.IsEmpty) { + PackagesAccessTokens.Clear(); + save = true; + } + + if (save) { + Utilities.InBackground(Save); + } + } + internal void RefreshPackageAccessTokens(IReadOnlyDictionary packageAccessTokens) { if ((packageAccessTokens == null) || (packageAccessTokens.Count == 0)) { throw new ArgumentNullException(nameof(packageAccessTokens)); @@ -192,13 +211,13 @@ namespace ArchiSteamFarm { await PackagesRefreshSemaphore.WaitAsync().ConfigureAwait(false); try { - HashSet packageIDs = packages.Where(package => (package.Key != 0) && (!PackagesData.TryGetValue(package.Key, out (uint ChangeNumber, HashSet? AppIDs) packageData) || (packageData.ChangeNumber < package.Value))).Select(package => package.Key).ToHashSet(); + HashSet packageIDs = packages.Where(package => (package.Key != 0) && (!PackagesData.TryGetValue(package.Key, out (uint ChangeNumber, ImmutableHashSet? AppIDs) packageData) || (packageData.ChangeNumber < package.Value))).Select(package => package.Key).ToHashSet(); if (packageIDs.Count == 0) { return; } - Dictionary? AppIDs)>? packagesData = await bot.GetPackagesData(packageIDs).ConfigureAwait(false); + Dictionary? AppIDs)>? packagesData = await bot.GetPackagesData(packageIDs).ConfigureAwait(false); if (packagesData == null) { bot.ArchiLogger.LogGenericWarning(Strings.WarningFailed); @@ -208,8 +227,8 @@ namespace ArchiSteamFarm { bool save = false; - foreach ((uint packageID, (uint ChangeNumber, HashSet? AppIDs) packageData) in packagesData) { - if (PackagesData.TryGetValue(packageID, out (uint ChangeNumber, HashSet? AppIDs) previousData) && (packageData.ChangeNumber < previousData.ChangeNumber)) { + foreach ((uint packageID, (uint ChangeNumber, ImmutableHashSet? AppIDs) packageData) in packagesData) { + if (PackagesData.TryGetValue(packageID, out (uint ChangeNumber, ImmutableHashSet? AppIDs) previousData) && (packageData.ChangeNumber < previousData.ChangeNumber)) { continue; } diff --git a/ArchiSteamFarm/SteamPICSChanges.cs b/ArchiSteamFarm/SteamPICSChanges.cs index f3c86b5eb..267b5b739 100644 --- a/ArchiSteamFarm/SteamPICSChanges.cs +++ b/ArchiSteamFarm/SteamPICSChanges.cs @@ -63,7 +63,7 @@ namespace ArchiSteamFarm { SteamApps.PICSChangesCallback? picsChanges = null; for (byte i = 0; (i < WebBrowser.MaxTries) && (picsChanges == null); i++) { - refreshBot = Bot.Bots?.Values.FirstOrDefault(bot => bot.IsConnectedAndLoggedOn); + refreshBot = Bot.Bots?.Values.Where(bot => bot.IsConnectedAndLoggedOn).OrderByDescending(bot => bot.OwnedPackageIDs.Count).FirstOrDefault(); if (refreshBot == null) { return; @@ -89,6 +89,14 @@ namespace ArchiSteamFarm { LastChangeNumber = picsChanges.CurrentChangeNumber; if (picsChanges.RequiresFullAppUpdate || picsChanges.RequiresFullPackageUpdate || ((picsChanges.AppChanges.Count == 0) && (picsChanges.PackageChanges.Count == 0))) { + if (ASF.GlobalDatabase != null) { + ASF.GlobalDatabase.OnPICSChangesRestart(); + + if (refreshBot.OwnedPackageIDs.Count > 0) { + await ASF.GlobalDatabase.RefreshPackages(refreshBot, refreshBot.OwnedPackageIDs.Keys.ToDictionary(packageID => packageID, _ => uint.MinValue)).ConfigureAwait(false); + } + } + await PluginsCore.OnPICSChangesRestart(picsChanges.CurrentChangeNumber).ConfigureAwait(false); return;