From 0fa4d5e2a68b41442be44c2914c0097144797bb7 Mon Sep 17 00:00:00 2001 From: JustArchi Date: Tue, 9 Jun 2020 22:56:04 +0200 Subject: [PATCH] Refactor package info and their access tokens, add ISteamPICSChanges plugin interface --- ArchiSteamFarm/ASF.cs | 4 + ArchiSteamFarm/Bot.cs | 33 ++++-- ArchiSteamFarm/CardsFarmer.cs | 2 +- ArchiSteamFarm/GlobalDatabase.cs | 36 ++++++- ArchiSteamFarm/Plugins/ISteamPICSChanges.cs | 50 +++++++++ ArchiSteamFarm/Plugins/PluginsCore.cs | 60 +++++++++++ ArchiSteamFarm/SteamPICSChanges.cs | 108 ++++++++++++++++++++ 7 files changed, 282 insertions(+), 11 deletions(-) create mode 100644 ArchiSteamFarm/Plugins/ISteamPICSChanges.cs create mode 100644 ArchiSteamFarm/SteamPICSChanges.cs diff --git a/ArchiSteamFarm/ASF.cs b/ArchiSteamFarm/ASF.cs index 550d71635..4df9a4a4e 100644 --- a/ArchiSteamFarm/ASF.cs +++ b/ArchiSteamFarm/ASF.cs @@ -125,6 +125,10 @@ namespace ArchiSteamFarm { await ArchiKestrel.Start().ConfigureAwait(false); } + uint changeNumberToStartFrom = await PluginsCore.GetChangeNumberToStartFrom().ConfigureAwait(false); + + SteamPICSChanges.Init(changeNumberToStartFrom); + await RegisterBots().ConfigureAwait(false); InitEvents(); diff --git a/ArchiSteamFarm/Bot.cs b/ArchiSteamFarm/Bot.cs index 93b349ee1..0077b9eab 100755 --- a/ArchiSteamFarm/Bot.cs +++ b/ArchiSteamFarm/Bot.cs @@ -117,7 +117,7 @@ namespace ArchiSteamFarm { internal readonly ArchiHandler ArchiHandler; internal readonly BotDatabase BotDatabase; - internal readonly ConcurrentDictionary OwnedPackageIDs = new ConcurrentDictionary(); + internal readonly ConcurrentDictionary OwnedPackageIDs = new ConcurrentDictionary(); internal bool CanReceiveSteamCards => !IsAccountLimited && !IsAccountLocked; internal bool IsAccountLimited => AccountFlags.HasFlag(EAccountFlags.LimitedUser) || AccountFlags.HasFlag(EAccountFlags.LimitedUserForce); @@ -648,7 +648,7 @@ namespace ArchiSteamFarm { DateTime mostRecent = DateTime.MinValue; foreach (uint packageID in packageIDs) { - if (!OwnedPackageIDs.TryGetValue(packageID, out (EPaymentMethod PaymentMethod, DateTime TimeCreated, ulong AccessToken) packageData)) { + if (!OwnedPackageIDs.TryGetValue(packageID, out (EPaymentMethod PaymentMethod, DateTime TimeCreated) packageData)) { continue; } @@ -822,11 +822,21 @@ namespace ArchiSteamFarm { return null; } + HashSet packageRequests = new HashSet(); + + foreach (uint packageID in packageIDs) { + if (!ASF.GlobalDatabase.PackageAccessTokensReadOnly.TryGetValue(packageID, out ulong packageAccessToken)) { + continue; + } + + packageRequests.Add(new SteamApps.PICSRequest(packageID, packageAccessToken, false)); + } + AsyncJobMultiple.ResultSet productInfoResultSet = null; for (byte i = 0; (i < WebBrowser.MaxTries) && (productInfoResultSet == null) && IsConnectedAndLoggedOn; i++) { try { - productInfoResultSet = await SteamApps.PICSGetProductInfo(Enumerable.Empty(), packageIDs.Select(packageID => new SteamApps.PICSRequest(packageID, OwnedPackageIDs.TryGetValue(packageID, out (EPaymentMethod PaymentMethod, DateTime TimeCreated, ulong AccessToken) value) ? value.AccessToken : 0, false))); + productInfoResultSet = await SteamApps.PICSGetProductInfo(Enumerable.Empty(), packageRequests); } catch (Exception e) { ArchiLogger.LogGenericWarningException(e); } @@ -2354,16 +2364,25 @@ namespace ArchiSteamFarm { Commands.OnNewLicenseList(); OwnedPackageIDs.Clear(); + Dictionary packageAccessTokens = new Dictionary(); Dictionary packagesToRefresh = new Dictionary(); - foreach (SteamApps.LicenseListCallback.License license in callback.LicenseList.Where(license => license.PackageID != 0)) { - OwnedPackageIDs[license.PackageID] = (license.PaymentMethod, license.TimeCreated, license.AccessToken); + foreach (SteamApps.LicenseListCallback.License license in callback.LicenseList) { + OwnedPackageIDs[license.PackageID] = (license.PaymentMethod, license.TimeCreated); - if (!ASF.GlobalDatabase.PackagesDataReadOnly.TryGetValue(license.PackageID, out (uint ChangeNumber, HashSet AppIDs) packageData) || (packageData.ChangeNumber < license.LastChangeNumber) || (packageData.AppIDs == null)) { + if (!ASF.GlobalDatabase.PackageAccessTokensReadOnly.TryGetValue(license.PackageID, out ulong packageAccessToken) || (packageAccessToken != license.AccessToken)) { + packageAccessTokens[license.PackageID] = license.AccessToken; + } + + if (!ASF.GlobalDatabase.PackagesDataReadOnly.TryGetValue(license.PackageID, out (uint ChangeNumber, HashSet AppIDs) packageData) || (packageData.ChangeNumber < license.LastChangeNumber)) { packagesToRefresh[license.PackageID] = (uint) license.LastChangeNumber; } } + if (packageAccessTokens.Count > 0) { + ASF.GlobalDatabase.RefreshPackageAccessTokens(packageAccessTokens); + } + if (packagesToRefresh.Count > 0) { ArchiLogger.LogGenericTrace(Strings.BotRefreshingPackagesData); await ASF.GlobalDatabase.RefreshPackages(this, packagesToRefresh).ConfigureAwait(false); @@ -2578,6 +2597,8 @@ namespace ArchiSteamFarm { ); } + SteamPICSChanges.OnBotLoggedOn(); + await PluginsCore.OnBotLoggedOn(this).ConfigureAwait(false); break; diff --git a/ArchiSteamFarm/CardsFarmer.cs b/ArchiSteamFarm/CardsFarmer.cs index 6d531c087..1ee892f59 100755 --- a/ArchiSteamFarm/CardsFarmer.cs +++ b/ArchiSteamFarm/CardsFarmer.cs @@ -1181,7 +1181,7 @@ namespace ArchiSteamFarm { if (packageIDs != null) { foreach (uint packageID in packageIDs) { - if (!Bot.OwnedPackageIDs.TryGetValue(packageID, out (EPaymentMethod PaymentMethod, DateTime TimeCreated, ulong AccessToken) packageData)) { + if (!Bot.OwnedPackageIDs.TryGetValue(packageID, out (EPaymentMethod PaymentMethod, DateTime TimeCreated) packageData)) { Bot.ArchiLogger.LogNullError(nameof(packageData)); return; diff --git a/ArchiSteamFarm/GlobalDatabase.cs b/ArchiSteamFarm/GlobalDatabase.cs index 25c76a966..32dee6233 100644 --- a/ArchiSteamFarm/GlobalDatabase.cs +++ b/ArchiSteamFarm/GlobalDatabase.cs @@ -38,6 +38,10 @@ namespace ArchiSteamFarm { [PublicAPI] public readonly Guid Guid = Guid.NewGuid(); + [JsonIgnore] + [PublicAPI] + public IReadOnlyDictionary PackageAccessTokensReadOnly => PackagesAccessTokens; + [JsonIgnore] [PublicAPI] public IReadOnlyDictionary AppIDs)> PackagesDataReadOnly => PackagesData; @@ -45,6 +49,9 @@ namespace ArchiSteamFarm { [JsonProperty(Required = Required.DisallowNull)] internal readonly InMemoryServerListProvider ServerListProvider = new InMemoryServerListProvider(); + [JsonProperty(Required = Required.DisallowNull)] + private readonly ConcurrentDictionary PackagesAccessTokens = new ConcurrentDictionary(); + [JsonProperty(Required = Required.DisallowNull)] private readonly ConcurrentDictionary AppIDs)> PackagesData = new ConcurrentDictionary AppIDs)>(); @@ -139,7 +146,7 @@ namespace ArchiSteamFarm { HashSet result = new HashSet(); foreach (uint packageID in packageIDs.Where(packageID => packageID != 0)) { - if (!PackagesData.TryGetValue(packageID, out (uint _, HashSet AppIDs) packagesData) || (packagesData.AppIDs?.Contains(appID) != true)) { + if (!PackagesData.TryGetValue(packageID, out (uint ChangeNumber, HashSet AppIDs) packagesData) || (packagesData.AppIDs?.Contains(appID) != true)) { continue; } @@ -149,6 +156,27 @@ namespace ArchiSteamFarm { return result; } + internal void RefreshPackageAccessTokens(IReadOnlyDictionary packageAccessTokens) { + if ((packageAccessTokens == null) || (packageAccessTokens.Count == 0)) { + ASF.ArchiLogger.LogNullError(nameof(packageAccessTokens)); + + return; + } + + bool save = false; + + foreach ((uint packageID, ulong currentAccessToken) in packageAccessTokens) { + if (!PackagesAccessTokens.TryGetValue(packageID, out ulong previousAccessToken) || (previousAccessToken != currentAccessToken)) { + PackagesAccessTokens[packageID] = currentAccessToken; + save = true; + } + } + + if (save) { + Utilities.InBackground(Save); + } + } + internal async Task RefreshPackages(Bot bot, IReadOnlyDictionary packages) { if ((bot == null) || (packages == null) || (packages.Count == 0)) { ASF.ArchiLogger.LogNullError(nameof(bot) + " || " + nameof(packages)); @@ -159,7 +187,7 @@ namespace ArchiSteamFarm { await PackagesRefreshSemaphore.WaitAsync().ConfigureAwait(false); try { - HashSet packageIDs = packages.Where(package => (package.Key != 0) && (!PackagesData.TryGetValue(package.Key, out (uint ChangeNumber, HashSet AppIDs) packageData) || (packageData.ChangeNumber < package.Value) || (packageData.AppIDs == null))).Select(package => package.Key).ToHashSet(); + HashSet packageIDs = packages.Where(package => (package.Key != 0) && (!PackagesData.TryGetValue(package.Key, out (uint ChangeNumber, HashSet AppIDs) packageData) || (packageData.ChangeNumber < package.Value))).Select(package => package.Key).ToHashSet(); if (packageIDs.Count == 0) { return; @@ -173,8 +201,8 @@ namespace ArchiSteamFarm { return; } - foreach ((uint packageID, (uint ChangeNumber, HashSet AppIDs) package) in packagesData) { - PackagesData[packageID] = package; + foreach ((uint packageID, (uint ChangeNumber, HashSet AppIDs) packageData) in packagesData) { + PackagesData[packageID] = packageData; } Utilities.InBackground(Save); diff --git a/ArchiSteamFarm/Plugins/ISteamPICSChanges.cs b/ArchiSteamFarm/Plugins/ISteamPICSChanges.cs new file mode 100644 index 000000000..28adc3f3f --- /dev/null +++ b/ArchiSteamFarm/Plugins/ISteamPICSChanges.cs @@ -0,0 +1,50 @@ +// _ _ _ ____ _ _____ +// / \ _ __ ___ | |__ (_)/ ___| | |_ ___ __ _ _ __ ___ | ___|__ _ _ __ _ __ ___ +// / _ \ | '__|/ __|| '_ \ | |\___ \ | __|/ _ \ / _` || '_ ` _ \ | |_ / _` || '__|| '_ ` _ \ +// / ___ \ | | | (__ | | | || | ___) || |_| __/| (_| || | | | | || _|| (_| || | | | | | | | +// /_/ \_\|_| \___||_| |_||_||____/ \__|\___| \__,_||_| |_| |_||_| \__,_||_| |_| |_| |_| +// | +// Copyright 2015-2020 Łukasz "JustArchi" Domeradzki +// Contact: JustArchi@JustArchi.net +// | +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// | +// http://www.apache.org/licenses/LICENSE-2.0 +// | +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Collections.Generic; +using System.Threading.Tasks; +using JetBrains.Annotations; +using SteamKit2; + +namespace ArchiSteamFarm.Plugins { + [PublicAPI] + public interface ISteamPICSChanges : IPlugin { + /// + /// ASF uses this method for determining the point in time from which it should keep history going upon a restart. The actual point in time that will be used is calculated as the lowest change number from all loaded plugins, to guarantee that no plugin will miss any changes, while allowing possible duplicates for those plugins that were already synchronized with newer changes. If you don't care about persistent state and just want to receive the ongoing history, you should return 0 (which is equal to "I'm fine with any"). If there won't be any plugin asking for a specific point in time, ASF will start returning entries since the start of the program. + /// + /// The most recent change number from which you're fine to receive + Task GetPreferredChangeNumberToStartFrom(); + + /// + /// ASF will call this method upon receiving any app/package PICS changes. The history is guaranteed to be precise and continuous starting from until is called. It's possible for this method to have duplicated calls across different runs, in particular when some other plugin asks for lower , therefore you should keep that in mind (and refer to change number of standalone apps/packages). + /// + /// The change number of current callback. + /// App changes that happened since the previous call of this method. Can be empty. + /// Package changes that happened since the previous call of this method. Can be empty. + void OnPICSChanges(uint currentChangeNumber, [NotNull] IReadOnlyDictionary appChanges, [NotNull] IReadOnlyDictionary packageChanges); + + /// + /// ASF will call this method when it'll be necessary to restart the history of PICS changes. This can happen due to Steam limitation in which we're unable to keep history going if we're too far behind (approx 5k changeNumbers). If you're relying on continuous history of app/package PICS changes sent by , ASF can no longer guarantee that upon calling this method, therefore you should start clean. + /// + /// The change number from which we're restarting the PICS history. + void OnPICSChangesRestart(uint currentChangeNumber); + } +} diff --git a/ArchiSteamFarm/Plugins/PluginsCore.cs b/ArchiSteamFarm/Plugins/PluginsCore.cs index dccf7bc1e..3d4510842 100644 --- a/ArchiSteamFarm/Plugins/PluginsCore.cs +++ b/ArchiSteamFarm/Plugins/PluginsCore.cs @@ -63,6 +63,30 @@ namespace ArchiSteamFarm.Plugins { return result ?? StringComparer.Ordinal; } + internal static async Task GetChangeNumberToStartFrom() { + if (!HasActivePluginsLoaded) { + return 0; + } + + IList results; + + try { + results = await Utilities.InParallel(ActivePlugins.OfType().Select(plugin => plugin.GetPreferredChangeNumberToStartFrom())).ConfigureAwait(false); + } catch (Exception e) { + ASF.ArchiLogger.LogGenericException(e); + + return 0; + } + + uint changeNumberToStartFrom = uint.MaxValue; + + foreach (uint result in results.Where(result => (result > 0) && (result < changeNumberToStartFrom))) { + changeNumberToStartFrom = result; + } + + return changeNumberToStartFrom == uint.MaxValue ? 0 : changeNumberToStartFrom; + } + internal static bool InitPlugins() { if (HasActivePluginsLoaded) { return false; @@ -493,6 +517,42 @@ namespace ArchiSteamFarm.Plugins { } } + internal static async Task OnPICSChanges(uint currentChangeNumber, IReadOnlyDictionary appChanges, IReadOnlyDictionary packageChanges) { + if ((currentChangeNumber == 0) || (appChanges == null) || (packageChanges == null)) { + ASF.ArchiLogger.LogNullError(nameof(currentChangeNumber) + " || " + nameof(appChanges) + " || " + nameof(packageChanges)); + + return; + } + + if (!HasActivePluginsLoaded) { + return; + } + + try { + await Utilities.InParallel(ActivePlugins.OfType().Select(plugin => Task.Run(() => plugin.OnPICSChanges(currentChangeNumber, appChanges, packageChanges)))).ConfigureAwait(false); + } catch (Exception e) { + ASF.ArchiLogger.LogGenericException(e); + } + } + + internal static async Task OnPICSChangesRestart(uint currentChangeNumber) { + if (currentChangeNumber == 0) { + ASF.ArchiLogger.LogNullError(nameof(currentChangeNumber)); + + return; + } + + if (!HasActivePluginsLoaded) { + return; + } + + try { + await Utilities.InParallel(ActivePlugins.OfType().Select(plugin => Task.Run(() => plugin.OnPICSChangesRestart(currentChangeNumber)))).ConfigureAwait(false); + } catch (Exception e) { + ASF.ArchiLogger.LogGenericException(e); + } + } + private static HashSet LoadAssembliesFrom(string path) { if (string.IsNullOrEmpty(path)) { ASF.ArchiLogger.LogNullError(nameof(path)); diff --git a/ArchiSteamFarm/SteamPICSChanges.cs b/ArchiSteamFarm/SteamPICSChanges.cs new file mode 100644 index 000000000..7720733ae --- /dev/null +++ b/ArchiSteamFarm/SteamPICSChanges.cs @@ -0,0 +1,108 @@ +// _ _ _ ____ _ _____ +// / \ _ __ ___ | |__ (_)/ ___| | |_ ___ __ _ _ __ ___ | ___|__ _ _ __ _ __ ___ +// / _ \ | '__|/ __|| '_ \ | |\___ \ | __|/ _ \ / _` || '_ ` _ \ | |_ / _` || '__|| '_ ` _ \ +// / ___ \ | | | (__ | | | || | ___) || |_| __/| (_| || | | | | || _|| (_| || | | | | | | | +// /_/ \_\|_| \___||_| |_||_||____/ \__|\___| \__,_||_| |_| |_||_| \__,_||_| |_| |_| |_| +// | +// Copyright 2015-2020 Łukasz "JustArchi" Domeradzki +// Contact: JustArchi@JustArchi.net +// | +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// | +// http://www.apache.org/licenses/LICENSE-2.0 +// | +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using ArchiSteamFarm.Localization; +using ArchiSteamFarm.Plugins; +using SteamKit2; + +namespace ArchiSteamFarm { + internal static class SteamPICSChanges { + private const byte RefreshTimerInMinutes = 5; + + private static readonly SemaphoreSlim RefreshSemaphore = new SemaphoreSlim(1, 1); + private static readonly Timer RefreshTimer = new Timer(async e => await RefreshChanges().ConfigureAwait(false)); + + private static uint LastChangeNumber; + private static bool TimerAlreadySet; + + internal static void Init(uint changeNumberToStartFrom) => LastChangeNumber = changeNumberToStartFrom; + + internal static void OnBotLoggedOn() { + if (TimerAlreadySet) { + return; + } + + lock (RefreshTimer) { + if (TimerAlreadySet) { + return; + } + + TimerAlreadySet = true; + RefreshTimer.Change(TimeSpan.Zero, TimeSpan.FromMinutes(RefreshTimerInMinutes)); + } + } + + private static async Task RefreshChanges() { + if (!await RefreshSemaphore.WaitAsync(0).ConfigureAwait(false)) { + return; + } + + try { + Bot refreshBot = null; + SteamApps.PICSChangesCallback picsChanges = null; + + for (byte i = 0; (i < WebBrowser.MaxTries) && (picsChanges == null); i++) { + refreshBot = Bot.Bots.Values.FirstOrDefault(bot => bot.IsConnectedAndLoggedOn); + + if (refreshBot == null) { + return; + } + + try { + picsChanges = await refreshBot.SteamApps.PICSGetChangesSince(LastChangeNumber, true, true); + } catch (Exception e) { + refreshBot.ArchiLogger.LogGenericWarningException(e); + } + } + + if ((refreshBot == null) || (picsChanges == null)) { + ASF.ArchiLogger.LogGenericWarning(Strings.WarningFailed); + + return; + } + + if (picsChanges.CurrentChangeNumber == picsChanges.LastChangeNumber) { + return; + } + + LastChangeNumber = picsChanges.CurrentChangeNumber; + + if ((picsChanges.AppChanges.Count == 0) && (picsChanges.PackageChanges.Count == 0)) { + await PluginsCore.OnPICSChangesRestart(picsChanges.CurrentChangeNumber).ConfigureAwait(false); + + return; + } + + if (picsChanges.PackageChanges.Count > 0) { + await ASF.GlobalDatabase.RefreshPackages(refreshBot, picsChanges.PackageChanges.ToDictionary(package => package.Key, package => package.Value.ChangeNumber)).ConfigureAwait(false); + } + + await PluginsCore.OnPICSChanges(picsChanges.CurrentChangeNumber, picsChanges.AppChanges, picsChanges.PackageChanges).ConfigureAwait(false); + } finally { + RefreshSemaphore.Release(); + } + } + } +}