From 8c0a100ae80737aa724b4efe52f04205f4fb18ec Mon Sep 17 00:00:00 2001 From: JustArchi Date: Sat, 8 Apr 2017 04:47:38 +0200 Subject: [PATCH] Enhance InventoryLimiterDelay Gifts/Login limiter is actually working decent, because response is nearly instant and fast enough to not worry about it in long-run. With inventory things are entirely different, as inventory fetching might take even a very long time, and while fetching one inventory, we might already run out of our delay and start fetching another one. This is not a big functionality-wise, as it's nothing new for ASF to parse multiple inventories concurrently, but Steam Community actually counts number of requests, and our inventory function might ask for multiple pages during execution, which could quickly lead to a situation of 10+ ongoing inventory requests being sent concurrently for too many accounts at once, as we can't predict not only how long the request will be handled, but also how many sub-requests we will do across one. This means that for optimal performance in terms of rate-limiting, we must limit ASF to one inventory request at a time, with mandatory InventoryLimiterDelay before asking for another one. This can degrade performance of previously fast !loot requests on multiple accounts at once (especially with bigger inventories), but it will also decrease significantly a chance of getting rate-limited and requests failing. --- ArchiSteamFarm/ArchiWebHandler.cs | 193 +++++++++++++++++------------- ArchiSteamFarm/Bot.cs | 3 - ArchiSteamFarm/Statistics.cs | 1 - ArchiSteamFarm/Trading.cs | 12 -- 4 files changed, 108 insertions(+), 101 deletions(-) diff --git a/ArchiSteamFarm/ArchiWebHandler.cs b/ArchiSteamFarm/ArchiWebHandler.cs index 33f1810a4..4f28acac8 100644 --- a/ArchiSteamFarm/ArchiWebHandler.cs +++ b/ArchiSteamFarm/ArchiWebHandler.cs @@ -24,6 +24,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Net; using System.Text; @@ -55,6 +56,8 @@ namespace ArchiSteamFarm { private const string SteamStoreHost = "store.steampowered.com"; private const string SteamStoreURL = "http://" + SteamStoreHost; + private static readonly SemaphoreSlim InventorySemaphore = new SemaphoreSlim(1); + private static int Timeout = GlobalConfig.DefaultConnectionTimeout * 1000; // This must be int type private readonly Bot Bot; @@ -477,6 +480,7 @@ namespace ArchiSteamFarm { return result; } + [SuppressMessage("ReSharper", "FunctionComplexityOverflow")] internal async Task> GetMySteamInventory(bool tradable, HashSet wantedTypes, HashSet wantedRealAppIDs = null) { if ((wantedTypes == null) || (wantedTypes.Count == 0)) { Bot.ArchiLogger.LogNullError(nameof(wantedTypes)); @@ -492,110 +496,119 @@ namespace ArchiSteamFarm { string request = SteamCommunityURL + "/my/inventory/json/" + Steam.Item.SteamAppID + "/" + Steam.Item.SteamCommunityContextID + "?l=english&trading=" + (tradable ? "1" : "0") + "&start="; uint currentPage = 0; - while (true) { - JObject jObject = await WebBrowser.UrlGetToJObjectRetry(request + currentPage).ConfigureAwait(false); + await InventorySemaphore.WaitAsync().ConfigureAwait(false); - IEnumerable descriptions = jObject?.SelectTokens("$.rgDescriptions.*"); - if (descriptions == null) { - return null; // OK, empty inventory - } + try { + while (true) { + JObject jObject = await WebBrowser.UrlGetToJObjectRetry(request + currentPage).ConfigureAwait(false); - Dictionary> descriptionMap = new Dictionary>(); - foreach (JToken description in descriptions.Where(description => description != null)) { - string classIDString = description["classid"]?.ToString(); - if (string.IsNullOrEmpty(classIDString)) { - Bot.ArchiLogger.LogNullError(nameof(classIDString)); - continue; + IEnumerable descriptions = jObject?.SelectTokens("$.rgDescriptions.*"); + if (descriptions == null) { + return null; // OK, empty inventory } - if (!ulong.TryParse(classIDString, out ulong classID) || (classID == 0)) { - Bot.ArchiLogger.LogNullError(nameof(classID)); - continue; - } - - if (descriptionMap.ContainsKey(classID)) { - continue; - } - - uint appID = 0; - - string hashName = description["market_hash_name"]?.ToString(); - if (!string.IsNullOrEmpty(hashName)) { - appID = GetAppIDFromMarketHashName(hashName); - } - - if (appID == 0) { - string appIDString = description["appid"]?.ToString(); - if (string.IsNullOrEmpty(appIDString)) { - Bot.ArchiLogger.LogNullError(nameof(appIDString)); + Dictionary> descriptionMap = new Dictionary>(); + foreach (JToken description in descriptions.Where(description => description != null)) { + string classIDString = description["classid"]?.ToString(); + if (string.IsNullOrEmpty(classIDString)) { + Bot.ArchiLogger.LogNullError(nameof(classIDString)); continue; } - if (!uint.TryParse(appIDString, out appID) || (appID == 0)) { - Bot.ArchiLogger.LogNullError(nameof(appID)); + if (!ulong.TryParse(classIDString, out ulong classID) || (classID == 0)) { + Bot.ArchiLogger.LogNullError(nameof(classID)); continue; } + + if (descriptionMap.ContainsKey(classID)) { + continue; + } + + uint appID = 0; + + string hashName = description["market_hash_name"]?.ToString(); + if (!string.IsNullOrEmpty(hashName)) { + appID = GetAppIDFromMarketHashName(hashName); + } + + if (appID == 0) { + string appIDString = description["appid"]?.ToString(); + if (string.IsNullOrEmpty(appIDString)) { + Bot.ArchiLogger.LogNullError(nameof(appIDString)); + continue; + } + + if (!uint.TryParse(appIDString, out appID) || (appID == 0)) { + Bot.ArchiLogger.LogNullError(nameof(appID)); + continue; + } + } + + Steam.Item.EType type = Steam.Item.EType.Unknown; + + string descriptionType = description["type"]?.ToString(); + if (!string.IsNullOrEmpty(descriptionType)) { + type = GetItemType(descriptionType); + } + + descriptionMap[classID] = new Tuple(appID, type); } - Steam.Item.EType type = Steam.Item.EType.Unknown; - - string descriptionType = description["type"]?.ToString(); - if (!string.IsNullOrEmpty(descriptionType)) { - type = GetItemType(descriptionType); - } - - descriptionMap[classID] = new Tuple(appID, type); - } - - IEnumerable items = jObject.SelectTokens("$.rgInventory.*"); - if (items == null) { - Bot.ArchiLogger.LogNullError(nameof(items)); - return null; - } - - foreach (JToken item in items.Where(item => item != null)) { - Steam.Item steamItem; - - try { - steamItem = item.ToObject(); - } catch (JsonException e) { - Bot.ArchiLogger.LogGenericException(e); + IEnumerable items = jObject.SelectTokens("$.rgInventory.*"); + if (items == null) { + Bot.ArchiLogger.LogNullError(nameof(items)); return null; } - if (steamItem == null) { - Bot.ArchiLogger.LogNullError(nameof(steamItem)); + foreach (JToken item in items.Where(item => item != null)) { + Steam.Item steamItem; + + try { + steamItem = item.ToObject(); + } catch (JsonException e) { + Bot.ArchiLogger.LogGenericException(e); + return null; + } + + if (steamItem == null) { + Bot.ArchiLogger.LogNullError(nameof(steamItem)); + return null; + } + + steamItem.AppID = Steam.Item.SteamAppID; + steamItem.ContextID = Steam.Item.SteamCommunityContextID; + + if (descriptionMap.TryGetValue(steamItem.ClassID, out Tuple description)) { + steamItem.RealAppID = description.Item1; + steamItem.Type = description.Item2; + } + + if (!wantedTypes.Contains(steamItem.Type) || (wantedRealAppIDs?.Contains(steamItem.RealAppID) == false)) { + continue; + } + + result.Add(steamItem); + } + + if (!bool.TryParse(jObject["more"]?.ToString(), out bool more) || !more) { + break; // OK, last page + } + + if (!uint.TryParse(jObject["more_start"]?.ToString(), out uint nextPage) || (nextPage <= currentPage)) { + Bot.ArchiLogger.LogNullError(nameof(nextPage)); return null; } - steamItem.AppID = Steam.Item.SteamAppID; - steamItem.ContextID = Steam.Item.SteamCommunityContextID; - - if (descriptionMap.TryGetValue(steamItem.ClassID, out Tuple description)) { - steamItem.RealAppID = description.Item1; - steamItem.Type = description.Item2; - } - - if (!wantedTypes.Contains(steamItem.Type) || (wantedRealAppIDs?.Contains(steamItem.RealAppID) == false)) { - continue; - } - - result.Add(steamItem); + currentPage = nextPage; } - if (!bool.TryParse(jObject["more"]?.ToString(), out bool more) || !more) { - break; // OK, last page - } - - if (!uint.TryParse(jObject["more_start"]?.ToString(), out uint nextPage) || (nextPage <= currentPage)) { - Bot.ArchiLogger.LogNullError(nameof(nextPage)); - return null; - } - - currentPage = nextPage; + return result; + } finally { + Task.Run(async () => { + await Task.Delay(Program.GlobalConfig.InventoryLimiterDelay * 1000).ConfigureAwait(false); + InventorySemaphore.Release(); + }).Forget(); } - - return result; } internal async Task> GetOwnedGames(ulong steamID) { @@ -926,7 +939,17 @@ namespace ArchiSteamFarm { } const string request = SteamCommunityURL + "/my/inventory"; - return await WebBrowser.UrlHeadRetry(request).ConfigureAwait(false); + + await InventorySemaphore.WaitAsync().ConfigureAwait(false); + + try { + return await WebBrowser.UrlHeadRetry(request).ConfigureAwait(false); + } finally { + Task.Run(async () => { + await Task.Delay(Program.GlobalConfig.InventoryLimiterDelay * 1000).ConfigureAwait(false); + InventorySemaphore.Release(); + }).Forget(); + } } internal async Task MarkSentTrades() { diff --git a/ArchiSteamFarm/Bot.cs b/ArchiSteamFarm/Bot.cs index 99525591a..04a36057a 100755 --- a/ArchiSteamFarm/Bot.cs +++ b/ArchiSteamFarm/Bot.cs @@ -1214,7 +1214,6 @@ namespace ArchiSteamFarm { return; } - await Trading.LimitInventoryRequestsAsync().ConfigureAwait(false); await ArchiWebHandler.MarkInventory().ConfigureAwait(false); } @@ -2449,8 +2448,6 @@ namespace ArchiSteamFarm { return FormatBotResponse(Strings.BotLootingYourself); } - await Trading.LimitInventoryRequestsAsync().ConfigureAwait(false); - HashSet inventory = await ArchiWebHandler.GetMySteamInventory(true, BotConfig.LootableTypes).ConfigureAwait(false); if ((inventory == null) || (inventory.Count == 0)) { return FormatBotResponse(string.Format(Strings.ErrorIsEmpty, nameof(inventory))); diff --git a/ArchiSteamFarm/Statistics.cs b/ArchiSteamFarm/Statistics.cs index ecf71af06..2d79e578b 100644 --- a/ArchiSteamFarm/Statistics.cs +++ b/ArchiSteamFarm/Statistics.cs @@ -123,7 +123,6 @@ namespace ArchiSteamFarm { return; } - await Trading.LimitInventoryRequestsAsync().ConfigureAwait(false); HashSet inventory = await Bot.ArchiWebHandler.GetMySteamInventory(true, new HashSet { Steam.Item.EType.TradingCard }).ConfigureAwait(false); // This is actually inventory failure, so we'll stop sending heartbeats but not record it as valid check diff --git a/ArchiSteamFarm/Trading.cs b/ArchiSteamFarm/Trading.cs index 94eeeee80..d0cf1d50b 100644 --- a/ArchiSteamFarm/Trading.cs +++ b/ArchiSteamFarm/Trading.cs @@ -35,8 +35,6 @@ namespace ArchiSteamFarm { internal const byte MaxItemsPerTrade = 150; // This is due to limit on POST size in WebBrowser internal const byte MaxTradesPerAccount = 5; // This is limit introduced by Valve - private static readonly SemaphoreSlim InventorySemaphore = new SemaphoreSlim(1); - private readonly Bot Bot; private readonly ConcurrentHashSet IgnoredTrades = new ConcurrentHashSet(); private readonly SemaphoreSlim TradesSemaphore = new SemaphoreSlim(1); @@ -71,14 +69,6 @@ namespace ArchiSteamFarm { } } - internal static async Task LimitInventoryRequestsAsync() { - await InventorySemaphore.WaitAsync().ConfigureAwait(false); - Task.Run(async () => { - await Task.Delay(Program.GlobalConfig.InventoryLimiterDelay * 1000).ConfigureAwait(false); - InventorySemaphore.Release(); - }).Forget(); - } - internal void OnDisconnected() => IgnoredTrades.ClearAndTrim(); private async Task ParseActiveTrades() { @@ -249,8 +239,6 @@ namespace ArchiSteamFarm { HashSet appIDs = new HashSet(tradeOffer.ItemsToGive.Select(item => item.RealAppID)); // Now check if it's worth for us to do the trade - await LimitInventoryRequestsAsync().ConfigureAwait(false); - HashSet inventory = await Bot.ArchiWebHandler.GetMySteamInventory(false, new HashSet { Steam.Item.EType.TradingCard }, appIDs).ConfigureAwait(false); if ((inventory == null) || (inventory.Count == 0)) { // If we can't check our inventory when not using MatchEverything, this is a temporary failure