From 91e495605b41c39fd218484dab18439ae353350e Mon Sep 17 00:00:00 2001 From: JustArchi Date: Fri, 5 Apr 2019 16:24:02 +0200 Subject: [PATCH] Port ArchiBot's WebBrowser client errors handling First important change is for all requests sent by ASF. Across those 4 years of development I do not remember a single situation where retrying on 4xx status code brought any improvement, bad request has to be handled by us, access denied and not found won't disappear after retry, all other ones are rather unused. Therefore, it makes sense to skip remaining tries on 4xx errors and do not flood the service with requests that are very unlikely to change anything. Second change is smaller one and it allows the consumer of WebBrowser to declare that he's interested in handling client error codes himself. This way he can add extra logic and appropriately react to them - ASF uses it in statistics module, where the listing can signal refusal to list due to e.g. outdated ASF version through 403. Previously this was treated on the same level as timeout, which wasn't optimal. --- ArchiSteamFarm/Statistics.cs | 32 +++-- ArchiSteamFarm/Utilities.cs | 3 + ArchiSteamFarm/WebBrowser.cs | 220 +++++++++++++++++++++++++++-------- 3 files changed, 193 insertions(+), 62 deletions(-) diff --git a/ArchiSteamFarm/Statistics.cs b/ArchiSteamFarm/Statistics.cs index 80a3990eb..d6d3a87bc 100644 --- a/ArchiSteamFarm/Statistics.cs +++ b/ArchiSteamFarm/Statistics.cs @@ -33,7 +33,6 @@ using Newtonsoft.Json; namespace ArchiSteamFarm { internal sealed class Statistics : IDisposable { - private const byte MaxHeartBeatTTL = MinHeartBeatTTL + WebBrowser.MaxTries; // Maximum amount of minutes until we give up on sending further HeartBeats private const byte MaxMatchedBotsHard = 40; private const byte MaxMatchedBotsSoft = 20; private const byte MaxMatchingRounds = 10; @@ -102,12 +101,15 @@ namespace ArchiSteamFarm { { "SteamID", Bot.SteamID.ToString() } }; - // Listing is free to deny our announce request, hence we don't retry - if (await Bot.ArchiWebHandler.WebBrowser.UrlPost(request, data, maxTries: 1).ConfigureAwait(false) == null) { - if (DateTime.UtcNow > LastHeartBeat.AddMinutes(MaxHeartBeatTTL)) { - ShouldSendHeartBeats = false; - Bot.RequestPersonaStateUpdate(); - } + WebBrowser.BasicResponse response = await Bot.ArchiWebHandler.WebBrowser.UrlPost(request, data, requestOptions: WebBrowser.ERequestOptions.ReturnClientErrors).ConfigureAwait(false); + + if (response == null) { + return; + } + + if (response.StatusCode.IsClientErrorCode()) { + LastHeartBeat = DateTime.MinValue; + ShouldSendHeartBeats = false; return; } @@ -200,11 +202,19 @@ namespace ArchiSteamFarm { { "TradeToken", tradeToken } }; - // Listing is free to deny our announce request, hence we don't retry - bool announced = await Bot.ArchiWebHandler.WebBrowser.UrlPost(request, data, maxTries: 1).ConfigureAwait(false) != null; + WebBrowser.BasicResponse response = await Bot.ArchiWebHandler.WebBrowser.UrlPost(request, data, requestOptions: WebBrowser.ERequestOptions.ReturnClientErrors).ConfigureAwait(false); - LastHeartBeat = announced ? DateTime.UtcNow : DateTime.MinValue; - ShouldSendHeartBeats = announced; + if (response == null) { + return; + } + + if (response.StatusCode.IsClientErrorCode()) { + LastHeartBeat = DateTime.MinValue; + ShouldSendHeartBeats = false; + } + + LastHeartBeat = DateTime.UtcNow; + ShouldSendHeartBeats = true; } finally { RequestsSemaphore.Release(); } diff --git a/ArchiSteamFarm/Utilities.cs b/ArchiSteamFarm/Utilities.cs index 692795993..27194ca59 100644 --- a/ArchiSteamFarm/Utilities.cs +++ b/ArchiSteamFarm/Utilities.cs @@ -149,6 +149,9 @@ namespace ArchiSteamFarm { } } + [PublicAPI] + public static bool IsClientErrorCode(this HttpStatusCode statusCode) => (statusCode >= HttpStatusCode.BadRequest) && (statusCode < HttpStatusCode.InternalServerError); + [PublicAPI] public static bool IsValidCdKey(string key) { if (string.IsNullOrEmpty(key)) { diff --git a/ArchiSteamFarm/WebBrowser.cs b/ArchiSteamFarm/WebBrowser.cs index 5e3964fbc..5e79879e8 100644 --- a/ArchiSteamFarm/WebBrowser.cs +++ b/ArchiSteamFarm/WebBrowser.cs @@ -90,31 +90,46 @@ namespace ArchiSteamFarm { [ItemCanBeNull] [PublicAPI] - public async Task UrlGetToHtmlDocument(string request, string referer = null, byte maxTries = MaxTries) { + public async Task UrlGetToHtmlDocument(string request, string referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries) { if (string.IsNullOrEmpty(request) || (maxTries == 0)) { ArchiLogger.LogNullError(nameof(request) + " || " + nameof(maxTries)); return null; } - StringResponse response = await UrlGetToString(request, referer, maxTries).ConfigureAwait(false); + StringResponse response = await UrlGetToString(request, referer, requestOptions, maxTries).ConfigureAwait(false); return response != null ? new HtmlDocumentResponse(response) : null; } [ItemCanBeNull] [PublicAPI] - public async Task> UrlGetToJsonObject(string request, string referer = null, byte maxTries = MaxTries) where T : class { + public async Task> UrlGetToJsonObject(string request, string referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries) where T : class { if (string.IsNullOrEmpty(request) || (maxTries == 0)) { ArchiLogger.LogNullError(nameof(request) + " || " + nameof(maxTries)); return null; } - for (byte i = 0; i < maxTries; i++) { - StringResponse response = await UrlGetToString(request, referer, 1).ConfigureAwait(false); + ObjectResponse result = null; - if (string.IsNullOrEmpty(response?.Content)) { + for (byte i = 0; i < maxTries; i++) { + StringResponse response = await UrlGetToString(request, referer, requestOptions | ERequestOptions.ReturnClientErrors, 1).ConfigureAwait(false); + + // ReSharper disable once UseNullPropagationWhenPossible - false check + if (response == null) { + continue; + } + + if (response.StatusCode.IsClientErrorCode()) { + if (requestOptions.HasFlag(ERequestOptions.ReturnClientErrors)) { + result = new ObjectResponse(response); + } + + break; + } + + if (string.IsNullOrEmpty(response.Content)) { continue; } @@ -140,22 +155,37 @@ namespace ArchiSteamFarm { ArchiLogger.LogGenericDebug(string.Format(Strings.ErrorFailingRequest, request)); } - return null; + return result; } [ItemCanBeNull] [PublicAPI] - public async Task UrlGetToXmlDocument(string request, string referer = null, byte maxTries = MaxTries) { + public async Task UrlGetToXmlDocument(string request, string referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries) { if (string.IsNullOrEmpty(request) || (maxTries == 0)) { ArchiLogger.LogNullError(nameof(request) + " || " + nameof(maxTries)); return null; } - for (byte i = 0; i < maxTries; i++) { - StringResponse response = await UrlGetToString(request, referer, 1).ConfigureAwait(false); + XmlDocumentResponse result = null; - if (string.IsNullOrEmpty(response?.Content)) { + for (byte i = 0; i < maxTries; i++) { + StringResponse response = await UrlGetToString(request, referer, requestOptions | ERequestOptions.ReturnClientErrors, 1).ConfigureAwait(false); + + // ReSharper disable once UseNullPropagationWhenPossible - false check + if (response == null) { + continue; + } + + if (response.StatusCode.IsClientErrorCode()) { + if (requestOptions.HasFlag(ERequestOptions.ReturnClientErrors)) { + result = new XmlDocumentResponse(response); + } + + break; + } + + if (string.IsNullOrEmpty(response.Content)) { continue; } @@ -177,49 +207,32 @@ namespace ArchiSteamFarm { ArchiLogger.LogGenericDebug(string.Format(Strings.ErrorFailingRequest, request)); } - return null; + return result; } [ItemCanBeNull] [PublicAPI] - public async Task UrlHead(string request, string referer = null, byte maxTries = MaxTries) { + public async Task UrlHead(string request, string referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries) { if (string.IsNullOrEmpty(request) || (maxTries == 0)) { ArchiLogger.LogNullError(nameof(request) + " || " + nameof(maxTries)); return null; } + BasicResponse result = null; + for (byte i = 0; i < maxTries; i++) { using (HttpResponseMessage response = await InternalHead(request, referer).ConfigureAwait(false)) { if (response == null) { continue; } - return new BasicResponse(response); - } - } + if (response.StatusCode.IsClientErrorCode()) { + if (requestOptions.HasFlag(ERequestOptions.ReturnClientErrors)) { + result = new BasicResponse(response); + } - if (maxTries > 1) { - ArchiLogger.LogGenericWarning(string.Format(Strings.ErrorRequestFailedTooManyTimes, maxTries)); - ArchiLogger.LogGenericDebug(string.Format(Strings.ErrorFailingRequest, request)); - } - - return null; - } - - [ItemCanBeNull] - [PublicAPI] - public async Task UrlPost(string request, IReadOnlyCollection> data = null, string referer = null, byte maxTries = MaxTries) { - if (string.IsNullOrEmpty(request) || (maxTries == 0)) { - ArchiLogger.LogNullError(nameof(request) + " || " + nameof(maxTries)); - - return null; - } - - for (byte i = 0; i < maxTries; i++) { - using (HttpResponseMessage response = await InternalPost(request, data, referer).ConfigureAwait(false)) { - if (response == null) { - continue; + break; } return new BasicResponse(response); @@ -231,36 +244,87 @@ namespace ArchiSteamFarm { ArchiLogger.LogGenericDebug(string.Format(Strings.ErrorFailingRequest, request)); } - return null; + return result; } [ItemCanBeNull] [PublicAPI] - public async Task UrlPostToHtmlDocument(string request, IReadOnlyCollection> data = null, string referer = null, byte maxTries = MaxTries) { + public async Task UrlPost(string request, IReadOnlyCollection> data = null, string referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries) { if (string.IsNullOrEmpty(request) || (maxTries == 0)) { ArchiLogger.LogNullError(nameof(request) + " || " + nameof(maxTries)); return null; } - StringResponse response = await UrlPostToString(request, data, referer, maxTries).ConfigureAwait(false); + BasicResponse result = null; + + for (byte i = 0; i < maxTries; i++) { + using (HttpResponseMessage response = await InternalPost(request, data, referer).ConfigureAwait(false)) { + if (response == null) { + continue; + } + + if (response.StatusCode.IsClientErrorCode()) { + if (requestOptions.HasFlag(ERequestOptions.ReturnClientErrors)) { + result = new BasicResponse(response); + } + + break; + } + + return new BasicResponse(response); + } + } + + if (maxTries > 1) { + ArchiLogger.LogGenericWarning(string.Format(Strings.ErrorRequestFailedTooManyTimes, maxTries)); + ArchiLogger.LogGenericDebug(string.Format(Strings.ErrorFailingRequest, request)); + } + + return result; + } + + [ItemCanBeNull] + [PublicAPI] + public async Task UrlPostToHtmlDocument(string request, IReadOnlyCollection> data = null, string referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries) { + if (string.IsNullOrEmpty(request) || (maxTries == 0)) { + ArchiLogger.LogNullError(nameof(request) + " || " + nameof(maxTries)); + + return null; + } + + StringResponse response = await UrlPostToString(request, data, referer, requestOptions, maxTries).ConfigureAwait(false); return response != null ? new HtmlDocumentResponse(response) : null; } [ItemCanBeNull] [PublicAPI] - public async Task> UrlPostToJsonObject(string request, IReadOnlyCollection> data = null, string referer = null, byte maxTries = MaxTries) where T : class { + public async Task> UrlPostToJsonObject(string request, IReadOnlyCollection> data = null, string referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries) where T : class { if (string.IsNullOrEmpty(request) || (maxTries == 0)) { ArchiLogger.LogNullError(nameof(request) + " || " + nameof(maxTries)); return null; } - for (byte i = 0; i < maxTries; i++) { - StringResponse response = await UrlPostToString(request, data, referer, maxTries).ConfigureAwait(false); + ObjectResponse result = null; - if (string.IsNullOrEmpty(response?.Content)) { + for (byte i = 0; i < maxTries; i++) { + StringResponse response = await UrlPostToString(request, data, referer, requestOptions | ERequestOptions.ReturnClientErrors, maxTries).ConfigureAwait(false); + + if (response == null) { + return null; + } + + if (response.StatusCode.IsClientErrorCode()) { + if (requestOptions.HasFlag(ERequestOptions.ReturnClientErrors)) { + result = new ObjectResponse(response); + } + + break; + } + + if (string.IsNullOrEmpty(response.Content)) { continue; } @@ -286,7 +350,7 @@ namespace ArchiSteamFarm { ArchiLogger.LogGenericDebug(string.Format(Strings.ErrorFailingRequest, request)); } - return null; + return result; } internal static void Init() { @@ -319,13 +383,15 @@ namespace ArchiSteamFarm { } [ItemCanBeNull] - internal async Task UrlGetToBinaryWithProgress(string request, string referer = null, byte maxTries = MaxTries) { + internal async Task UrlGetToBinaryWithProgress(string request, string referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries) { if (string.IsNullOrEmpty(request) || (maxTries == 0)) { ArchiLogger.LogNullError(nameof(request) + " || " + nameof(maxTries)); return null; } + BinaryResponse result = null; + for (byte i = 0; i < maxTries; i++) { const byte printPercentage = 10; const byte maxBatches = 99 / printPercentage; @@ -335,6 +401,14 @@ namespace ArchiSteamFarm { continue; } + if (response.StatusCode.IsClientErrorCode()) { + if (requestOptions.HasFlag(ERequestOptions.ReturnClientErrors)) { + result = new BinaryResponse(response); + } + + break; + } + ArchiLogger.LogGenericDebug("0%..."); uint contentLength = (uint) response.Content.Headers.ContentLength.GetValueOrDefault(); @@ -387,23 +461,33 @@ namespace ArchiSteamFarm { ArchiLogger.LogGenericDebug(string.Format(Strings.ErrorFailingRequest, request)); } - return null; + return result; } [ItemCanBeNull] - internal async Task UrlGetToString(string request, string referer = null, byte maxTries = MaxTries) { + internal async Task UrlGetToString(string request, string referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries) { if (string.IsNullOrEmpty(request) || (maxTries == 0)) { ArchiLogger.LogNullError(nameof(request) + " || " + nameof(maxTries)); return null; } + StringResponse result = null; + for (byte i = 0; i < maxTries; i++) { using (HttpResponseMessage response = await InternalGet(request, referer).ConfigureAwait(false)) { if (response == null) { continue; } + if (response.StatusCode.IsClientErrorCode()) { + if (requestOptions.HasFlag(ERequestOptions.ReturnClientErrors)) { + result = new StringResponse(response); + } + + break; + } + return new StringResponse(response, await response.Content.ReadAsStringAsync().ConfigureAwait(false)); } } @@ -413,7 +497,7 @@ namespace ArchiSteamFarm { ArchiLogger.LogGenericDebug(string.Format(Strings.ErrorFailingRequest, request)); } - return null; + return result; } private async Task InternalGet(string request, string referer = null, HttpCompletionOption httpCompletionOptions = HttpCompletionOption.ResponseContentRead) { @@ -548,6 +632,11 @@ namespace ArchiSteamFarm { return await InternalRequest(redirectUri, httpMethod, data, referer, httpCompletionOption, --maxRedirections).ConfigureAwait(false); } + if ((response.StatusCode >= HttpStatusCode.BadRequest) && (response.StatusCode < HttpStatusCode.InternalServerError)) { + // Do not retry on client errors + return response; + } + using (response) { if (Debugging.IsUserDebugging) { ArchiLogger.LogGenericDebug(string.Format(Strings.Content, await response.Content.ReadAsStringAsync().ConfigureAwait(false))); @@ -558,19 +647,29 @@ namespace ArchiSteamFarm { } [ItemCanBeNull] - private async Task UrlPostToString(string request, IReadOnlyCollection> data = null, string referer = null, byte maxTries = MaxTries) { + private async Task UrlPostToString(string request, IReadOnlyCollection> data = null, string referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries) { if (string.IsNullOrEmpty(request) || (maxTries == 0)) { ArchiLogger.LogNullError(nameof(request) + " || " + nameof(maxTries)); return null; } + StringResponse result = null; + for (byte i = 0; i < maxTries; i++) { using (HttpResponseMessage response = await InternalPost(request, data, referer).ConfigureAwait(false)) { if (response == null) { continue; } + if (response.StatusCode.IsClientErrorCode()) { + if (requestOptions.HasFlag(ERequestOptions.ReturnClientErrors)) { + result = new StringResponse(response); + } + + break; + } + return new StringResponse(response, await response.Content.ReadAsStringAsync().ConfigureAwait(false)); } } @@ -580,10 +679,13 @@ namespace ArchiSteamFarm { ArchiLogger.LogGenericDebug(string.Format(Strings.ErrorFailingRequest, request)); } - return null; + return result; } public class BasicResponse { + [PublicAPI] + public readonly HttpStatusCode StatusCode; + internal readonly Uri FinalUri; internal BasicResponse([NotNull] HttpResponseMessage httpResponseMessage) { @@ -592,6 +694,7 @@ namespace ArchiSteamFarm { } FinalUri = httpResponseMessage.Headers.Location ?? httpResponseMessage.RequestMessage.RequestUri; + StatusCode = httpResponseMessage.StatusCode; } internal BasicResponse([NotNull] BasicResponse basicResponse) { @@ -600,6 +703,7 @@ namespace ArchiSteamFarm { } FinalUri = basicResponse.FinalUri; + StatusCode = basicResponse.StatusCode; } } @@ -627,6 +731,8 @@ namespace ArchiSteamFarm { Content = content; } + + internal ObjectResponse([NotNull] BasicResponse basicResponse) : base(basicResponse) { } } public sealed class XmlDocumentResponse : BasicResponse { @@ -640,6 +746,14 @@ namespace ArchiSteamFarm { Content = content; } + + internal XmlDocumentResponse([NotNull] BasicResponse basicResponse) : base(basicResponse) { } + } + + [Flags] + public enum ERequestOptions : byte { + None = 0, + ReturnClientErrors = 1 } internal sealed class BinaryResponse : BasicResponse { @@ -652,6 +766,8 @@ namespace ArchiSteamFarm { Content = content; } + + internal BinaryResponse([NotNull] HttpResponseMessage httpResponseMessage) : base(httpResponseMessage) { } } internal sealed class StringResponse : BasicResponse { @@ -664,6 +780,8 @@ namespace ArchiSteamFarm { Content = content; } + + internal StringResponse([NotNull] HttpResponseMessage httpResponseMessage) : base(httpResponseMessage) { } } } }