Implement background announcements for ASF STM

This will be used exclusively by users with extraordinary large inventories, or if ASF backend will just be slower than usual.
This commit is contained in:
Archi
2023-12-02 19:36:34 +01:00
parent b940af6e83
commit eff60bf307
3 changed files with 312 additions and 84 deletions

View File

@@ -41,7 +41,7 @@ using SteamKit2;
namespace ArchiSteamFarm.OfficialPlugins.ItemsMatcher;
internal static class Backend {
internal static async Task<BasicResponse?> AnnounceDiffForListing(WebBrowser webBrowser, ulong steamID, ICollection<AssetForListing> inventory, string inventoryChecksum, IReadOnlyCollection<Asset.EType> acceptedMatchableTypes, uint totalInventoryCount, bool matchEverything, string tradeToken, ICollection<AssetForListing> inventoryRemoved, string? previousInventoryChecksum, string? nickname = null, string? avatarHash = null) {
internal static async Task<ObjectResponse<GenericResponse<BackgroundTaskResponse>>?> AnnounceDiffForListing(WebBrowser webBrowser, ulong steamID, ICollection<AssetForListing> inventory, string inventoryChecksum, IReadOnlyCollection<Asset.EType> acceptedMatchableTypes, uint totalInventoryCount, bool matchEverything, string tradeToken, ICollection<AssetForListing> inventoryRemoved, string? previousInventoryChecksum, string? nickname = null, string? avatarHash = null) {
ArgumentNullException.ThrowIfNull(webBrowser);
if ((steamID == 0) || !new SteamID(steamID).IsIndividualAccount) {
@@ -69,14 +69,14 @@ internal static class Backend {
return null;
}
Uri request = new(ArchiNet.URL, "/Api/Listing/AnnounceDiff");
Uri request = new(ArchiNet.URL, "/Api/Listing/AnnounceDiff/v2");
AnnouncementDiffRequest data = new(ASF.GlobalDatabase?.Identifier ?? Guid.NewGuid(), steamID, inventory, inventoryChecksum, acceptedMatchableTypes, totalInventoryCount, matchEverything, ASF.GlobalConfig?.MaxTradeHoldDuration ?? GlobalConfig.DefaultMaxTradeHoldDuration, tradeToken, inventoryRemoved, previousInventoryChecksum, nickname, avatarHash);
return await webBrowser.UrlPost(request, data: data, requestOptions: WebBrowser.ERequestOptions.ReturnRedirections | WebBrowser.ERequestOptions.ReturnClientErrors | WebBrowser.ERequestOptions.CompressRequest).ConfigureAwait(false);
return await webBrowser.UrlPostToJsonObject<GenericResponse<BackgroundTaskResponse>, AnnouncementDiffRequest>(request, data: data, requestOptions: WebBrowser.ERequestOptions.ReturnRedirections | WebBrowser.ERequestOptions.ReturnClientErrors | WebBrowser.ERequestOptions.AllowInvalidBodyOnErrors | WebBrowser.ERequestOptions.CompressRequest).ConfigureAwait(false);
}
internal static async Task<BasicResponse?> AnnounceForListing(WebBrowser webBrowser, ulong steamID, ICollection<AssetForListing> inventory, string inventoryChecksum, IReadOnlyCollection<Asset.EType> acceptedMatchableTypes, uint totalInventoryCount, bool matchEverything, string tradeToken, string? nickname = null, string? avatarHash = null) {
internal static async Task<ObjectResponse<GenericResponse<BackgroundTaskResponse>>?> AnnounceForListing(WebBrowser webBrowser, ulong steamID, ICollection<AssetForListing> inventory, string inventoryChecksum, IReadOnlyCollection<Asset.EType> acceptedMatchableTypes, uint totalInventoryCount, bool matchEverything, string tradeToken, string? nickname = null, string? avatarHash = null) {
ArgumentNullException.ThrowIfNull(webBrowser);
if ((steamID == 0) || !new SteamID(steamID).IsIndividualAccount) {
@@ -104,11 +104,11 @@ internal static class Backend {
return null;
}
Uri request = new(ArchiNet.URL, "/Api/Listing/Announce/v4");
Uri request = new(ArchiNet.URL, "/Api/Listing/Announce/v5");
AnnouncementRequest data = new(ASF.GlobalDatabase?.Identifier ?? Guid.NewGuid(), steamID, inventory, inventoryChecksum, acceptedMatchableTypes, totalInventoryCount, matchEverything, ASF.GlobalConfig?.MaxTradeHoldDuration ?? GlobalConfig.DefaultMaxTradeHoldDuration, tradeToken, nickname, avatarHash);
return await webBrowser.UrlPost(request, data: data, requestOptions: WebBrowser.ERequestOptions.ReturnRedirections | WebBrowser.ERequestOptions.ReturnClientErrors | WebBrowser.ERequestOptions.CompressRequest).ConfigureAwait(false);
return await webBrowser.UrlPostToJsonObject<GenericResponse<BackgroundTaskResponse>, AnnouncementRequest>(request, data: data, requestOptions: WebBrowser.ERequestOptions.ReturnRedirections | WebBrowser.ERequestOptions.ReturnClientErrors | WebBrowser.ERequestOptions.AllowInvalidBodyOnErrors | WebBrowser.ERequestOptions.CompressRequest).ConfigureAwait(false);
}
internal static string GenerateChecksumFor(IList<AssetForListing> assetsForListings) {
@@ -196,4 +196,22 @@ internal static class Backend {
return await webBrowser.UrlPost(request, data: data, requestOptions: WebBrowser.ERequestOptions.ReturnRedirections | WebBrowser.ERequestOptions.ReturnClientErrors | WebBrowser.ERequestOptions.CompressRequest).ConfigureAwait(false);
}
internal static async Task<ObjectResponse<GenericResponse<BackgroundTaskResponse>>?> PollResult(WebBrowser webBrowser, ulong steamID, Guid requestID) {
ArgumentNullException.ThrowIfNull(webBrowser);
if ((steamID == 0) || !new SteamID(steamID).IsIndividualAccount) {
throw new ArgumentOutOfRangeException(nameof(steamID));
}
ArgumentOutOfRangeException.ThrowIfEqual(requestID, Guid.Empty);
if (SharedInfo.BuildInfo.IsCustomBuild) {
return null;
}
Uri request = new(ArchiNet.URL, $"/Api/Listing/PollResult/{steamID}/{requestID:N}");
return await webBrowser.UrlGetToJsonObject<GenericResponse<BackgroundTaskResponse>>(request, requestOptions: WebBrowser.ERequestOptions.ReturnRedirections | WebBrowser.ERequestOptions.ReturnClientErrors | WebBrowser.ERequestOptions.AllowInvalidBodyOnErrors).ConfigureAwait(false);
}
}

View File

@@ -0,0 +1,42 @@
// _ _ _ ____ _ _____
// / \ _ __ ___ | |__ (_)/ ___| | |_ ___ __ _ _ __ ___ | ___|__ _ _ __ _ __ ___
// / _ \ | '__|/ __|| '_ \ | |\___ \ | __|/ _ \ / _` || '_ ` _ \ | |_ / _` || '__|| '_ ` _ \
// / ___ \ | | | (__ | | | || | ___) || |_| __/| (_| || | | | | || _|| (_| || | | | | | | |
// /_/ \_\|_| \___||_| |_||_||____/ \__|\___| \__,_||_| |_| |_||_| \__,_||_| |_| |_| |_|
// |
// Copyright 2015-2023 Łukasz "JustArchi" Domeradzki
// Contact: JustArchi@JustArchi.net
// |
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// |
// http://www.apache.org/licenses/LICENSE-2.0
// |
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
using System;
using Newtonsoft.Json;
namespace ArchiSteamFarm.OfficialPlugins.ItemsMatcher.Data;
#pragma warning disable CA1812 // False positive, the class is used during json deserialization
internal sealed class BackgroundTaskResponse {
#pragma warning disable CS0649 // False positive, the field is used during json deserialization
[JsonProperty(Required = Required.Always)]
internal readonly bool Finished;
#pragma warning restore CS0649 // False positive, the field is used during json deserialization
#pragma warning disable CS0649 // False positive, the field is used during json deserialization
[JsonProperty(Required = Required.Always)]
internal readonly Guid RequestID;
#pragma warning restore CS0649 // False positive, the field is used during json deserialization
[JsonConstructor]
private BackgroundTaskResponse() { }
}
#pragma warning restore CA1812 // False positive, the class is used during json deserialization

View File

@@ -390,7 +390,58 @@ internal sealed class RemoteCommunication : IAsyncDisposable, IDisposable {
ObjectResponse<GenericResponse<ImmutableHashSet<SetPart>>>? setPartsResponse = await Backend.GetSetParts(WebBrowser, Bot.SteamID, acceptedMatchableTypes, realAppIDs).ConfigureAwait(false);
if (!HandleAnnounceResponse(BotCache, tradeToken, response: setPartsResponse) || (setPartsResponse?.Content?.Result == null)) {
if (setPartsResponse == null) {
// This is actually a network failure, so we'll stop sending heartbeats but not record it as valid check
ShouldSendHeartBeats = false;
Bot.ArchiLogger.LogGenericWarning(string.Format(CultureInfo.CurrentCulture, Strings.ErrorObjectIsNull, nameof(setPartsResponse)));
return;
}
if (setPartsResponse.StatusCode.IsRedirectionCode()) {
ShouldSendHeartBeats = false;
Bot.ArchiLogger.LogGenericWarning(string.Format(CultureInfo.CurrentCulture, Strings.WarningFailedWithError, setPartsResponse.StatusCode));
if (setPartsResponse.FinalUri.Host != ArchiWebHandler.SteamCommunityURL.Host) {
ASF.ArchiLogger.LogGenericError(string.Format(CultureInfo.CurrentCulture, Strings.WarningUnknownValuePleaseReport, nameof(setPartsResponse.FinalUri), setPartsResponse.FinalUri));
return;
}
// We've expected the result, not the redirection to the sign in, we need to authenticate again
SignedInWithSteam = false;
return;
}
if (!setPartsResponse.StatusCode.IsSuccessCode()) {
// ArchiNet told us that we've sent a bad request, so the process should restart from the beginning at later time
ShouldSendHeartBeats = false;
Bot.ArchiLogger.LogGenericWarning(string.Format(CultureInfo.CurrentCulture, Strings.WarningFailedWithError, setPartsResponse.StatusCode));
switch (setPartsResponse.StatusCode) {
case HttpStatusCode.Forbidden:
// ArchiNet told us to stop submitting data for now
LastAnnouncement = DateTime.UtcNow.AddYears(1);
return;
case HttpStatusCode.TooManyRequests:
// ArchiNet told us to try again later
LastAnnouncement = DateTime.UtcNow.AddDays(1);
return;
default:
// There is something wrong with our payload or the server, we shouldn't retry for at least several hours
LastAnnouncement = DateTime.UtcNow.AddHours(6);
return;
}
}
if (setPartsResponse.Content?.Result == null) {
// This should never happen if we got the correct response
Bot.ArchiLogger.LogGenericError(string.Format(CultureInfo.CurrentCulture, Strings.WarningUnknownValuePleaseReport, nameof(setPartsResponse), setPartsResponse.Content?.Result));
return;
}
@@ -511,26 +562,214 @@ internal sealed class RemoteCommunication : IAsyncDisposable, IDisposable {
Bot.ArchiLogger.LogGenericInfo(string.Format(CultureInfo.CurrentCulture, Localization.Strings.ListingAnnouncing, Bot.SteamID, nickname ?? Bot.SteamID.ToString(CultureInfo.InvariantCulture), assetsForListing.Count));
BasicResponse? diffResponse = await Backend.AnnounceDiffForListing(WebBrowser, Bot.SteamID, inventoryAddedChanged, checksum, acceptedMatchableTypes, (uint) inventory.Count, matchEverything, tradeToken, previousInventoryState.Values, previousChecksum, nickname, avatarHash).ConfigureAwait(false);
ObjectResponse<GenericResponse<BackgroundTaskResponse>>? diffResponse = null;
Guid diffRequestID = Guid.Empty;
if (HandleAnnounceResponse(BotCache, tradeToken, inventoryChecksumBeforeDeduplication, assetsForListing, previousChecksum, diffResponse)) {
for (byte i = 0; i < WebBrowser.MaxTries; i++) {
if (diffRequestID != Guid.Empty) {
diffResponse = await Backend.PollResult(WebBrowser, Bot.SteamID, diffRequestID).ConfigureAwait(false);
} else {
diffResponse = await Backend.AnnounceDiffForListing(WebBrowser, Bot.SteamID, inventoryAddedChanged, checksum, acceptedMatchableTypes, (uint) inventory.Count, matchEverything, tradeToken, previousInventoryState.Values, previousChecksum, nickname, avatarHash).ConfigureAwait(false);
}
if (diffResponse == null) {
// This is actually a network failure, so we'll stop sending heartbeats but not record it as valid check
ShouldSendHeartBeats = false;
Bot.ArchiLogger.LogGenericWarning(string.Format(CultureInfo.CurrentCulture, Strings.ErrorObjectIsNull, nameof(diffResponse)));
return;
}
if (diffResponse.StatusCode.IsRedirectionCode()) {
ShouldSendHeartBeats = false;
Bot.ArchiLogger.LogGenericWarning(string.Format(CultureInfo.CurrentCulture, Strings.WarningFailedWithError, diffResponse.StatusCode));
if (diffResponse.FinalUri.Host != ArchiWebHandler.SteamCommunityURL.Host) {
ASF.ArchiLogger.LogGenericError(string.Format(CultureInfo.CurrentCulture, Strings.WarningUnknownValuePleaseReport, nameof(diffResponse.FinalUri), diffResponse.FinalUri));
return;
}
// We've expected the result, not the redirection to the sign in, we need to authenticate again
SignedInWithSteam = false;
return;
}
if (!diffResponse.StatusCode.IsSuccessCode()) {
// ArchiNet told us that we've sent a bad request, so the process should restart from the beginning at later time
ShouldSendHeartBeats = false;
Bot.ArchiLogger.LogGenericWarning(string.Format(CultureInfo.CurrentCulture, Strings.WarningFailedWithError, diffResponse.StatusCode));
switch (diffResponse.StatusCode) {
case HttpStatusCode.Conflict:
// ArchiNet told us to do full announcement instead, the only non-OK response we accept
break;
case HttpStatusCode.Forbidden:
// ArchiNet told us to stop submitting data for now
LastAnnouncement = DateTime.UtcNow.AddYears(1);
return;
case HttpStatusCode.TooManyRequests:
// ArchiNet told us to try again later
LastAnnouncement = DateTime.UtcNow.AddDays(1);
return;
default:
// There is something wrong with our payload or the server, we shouldn't retry for at least several hours
LastAnnouncement = DateTime.UtcNow.AddHours(6);
return;
}
break;
}
// Great, do we need to wait?
if (diffResponse.Content?.Result == null) {
// This should never happen if we got the correct response
Bot.ArchiLogger.LogGenericError(string.Format(CultureInfo.CurrentCulture, Strings.WarningUnknownValuePleaseReport, nameof(diffResponse), diffResponse.Content?.Result));
return;
}
if (diffResponse.Content.Result.Finished) {
break;
}
diffRequestID = diffResponse.Content.Result.RequestID;
diffResponse = null;
}
if (diffResponse == null) {
// We've waited long enough, something is definitely wrong with us or the backend
Bot.ArchiLogger.LogGenericWarning(string.Format(CultureInfo.CurrentCulture, Strings.WarningFailedWithError, nameof(diffResponse)));
return;
}
if (diffResponse.StatusCode.IsSuccessCode() && diffResponse.Content is { Success: true, Result.Finished: true }) {
// Our diff announce has succeeded, we have nothing to do further
Bot.ArchiLogger.LogGenericInfo(Strings.Success);
LastAnnouncement = LastHeartBeat = DateTime.UtcNow;
ShouldSendAnnouncementEarlier = false;
ShouldSendHeartBeats = true;
BotCache.LastAnnouncedAssetsForListing.ReplaceWith(assetsForListing);
BotCache.LastAnnouncedTradeToken = tradeToken;
BotCache.LastInventoryChecksumBeforeDeduplication = inventoryChecksumBeforeDeduplication;
return;
}
}
Bot.ArchiLogger.LogGenericInfo(string.Format(CultureInfo.CurrentCulture, Localization.Strings.ListingAnnouncing, Bot.SteamID, nickname ?? Bot.SteamID.ToString(CultureInfo.InvariantCulture), assetsForListing.Count));
BasicResponse? response = await Backend.AnnounceForListing(WebBrowser, Bot.SteamID, assetsForListing, checksum, acceptedMatchableTypes, (uint) inventory.Count, matchEverything, tradeToken, nickname, avatarHash).ConfigureAwait(false);
ObjectResponse<GenericResponse<BackgroundTaskResponse>>? announceResponse = null;
Guid announceRequestID = Guid.Empty;
HandleAnnounceResponse(BotCache, tradeToken, inventoryChecksumBeforeDeduplication, assetsForListing, response: response);
for (byte i = 0; i < WebBrowser.MaxTries; i++) {
if (announceRequestID != Guid.Empty) {
announceResponse = await Backend.PollResult(WebBrowser, Bot.SteamID, announceRequestID).ConfigureAwait(false);
} else {
announceResponse = await Backend.AnnounceForListing(WebBrowser, Bot.SteamID, assetsForListing, checksum, acceptedMatchableTypes, (uint) inventory.Count, matchEverything, tradeToken, nickname, avatarHash).ConfigureAwait(false);
}
if (announceResponse == null) {
// This is actually a network failure, so we'll stop sending heartbeats but not record it as valid check
ShouldSendHeartBeats = false;
Bot.ArchiLogger.LogGenericWarning(string.Format(CultureInfo.CurrentCulture, Strings.ErrorObjectIsNull, nameof(announceResponse)));
return;
}
if (announceResponse.StatusCode.IsRedirectionCode()) {
ShouldSendHeartBeats = false;
Bot.ArchiLogger.LogGenericWarning(string.Format(CultureInfo.CurrentCulture, Strings.WarningFailedWithError, announceResponse.StatusCode));
if (announceResponse.FinalUri.Host != ArchiWebHandler.SteamCommunityURL.Host) {
ASF.ArchiLogger.LogGenericError(string.Format(CultureInfo.CurrentCulture, Strings.WarningUnknownValuePleaseReport, nameof(announceResponse.FinalUri), announceResponse.FinalUri));
return;
}
// We've expected the result, not the redirection to the sign in, we need to authenticate again
SignedInWithSteam = false;
return;
}
if (!announceResponse.StatusCode.IsSuccessCode()) {
// ArchiNet told us that we've sent a bad request, so the process should restart from the beginning at later time
ShouldSendHeartBeats = false;
Bot.ArchiLogger.LogGenericWarning(string.Format(CultureInfo.CurrentCulture, Strings.WarningFailedWithError, announceResponse.StatusCode));
switch (announceResponse.StatusCode) {
case HttpStatusCode.Conflict:
// ArchiNet told us to that we've applied wrong deduplication logic, we can try again in a second
LastAnnouncement = DateTime.UtcNow.AddMinutes(5);
return;
case HttpStatusCode.Forbidden:
// ArchiNet told us to stop submitting data for now
LastAnnouncement = DateTime.UtcNow.AddYears(1);
return;
case HttpStatusCode.TooManyRequests:
// ArchiNet told us to try again later
LastAnnouncement = DateTime.UtcNow.AddDays(1);
return;
default:
// There is something wrong with our payload or the server, we shouldn't retry for at least several hours
LastAnnouncement = DateTime.UtcNow.AddHours(6);
return;
}
}
// Great, do we need to wait?
if (announceResponse.Content?.Result == null) {
// This should never happen if we got the correct response
Bot.ArchiLogger.LogGenericError(string.Format(CultureInfo.CurrentCulture, Strings.WarningUnknownValuePleaseReport, nameof(announceResponse), announceResponse.Content?.Result));
return;
}
if (announceResponse.Content.Result.Finished) {
break;
}
announceRequestID = announceResponse.Content.Result.RequestID;
announceResponse = null;
}
if (announceResponse == null) {
// We've waited long enough, something is definitely wrong with us or the backend
Bot.ArchiLogger.LogGenericWarning(string.Format(CultureInfo.CurrentCulture, Strings.WarningFailedWithError, nameof(announceResponse)));
return;
}
if (announceResponse.StatusCode.IsSuccessCode() && announceResponse.Content is { Success: true, Result.Finished: true }) {
// Our diff announce has succeeded, we have nothing to do further
Bot.ArchiLogger.LogGenericInfo(Strings.Success);
LastAnnouncement = LastHeartBeat = DateTime.UtcNow;
ShouldSendAnnouncementEarlier = false;
ShouldSendHeartBeats = true;
BotCache.LastAnnouncedAssetsForListing.ReplaceWith(assetsForListing);
BotCache.LastAnnouncedTradeToken = tradeToken;
BotCache.LastInventoryChecksumBeforeDeduplication = inventoryChecksumBeforeDeduplication;
return;
}
// Everything we've tried has failed
Bot.ArchiLogger.LogGenericWarning(Strings.WarningFailed);
} finally {
RequestsSemaphore.Release();
}
Bot.ArchiLogger.LogGenericInfo(Strings.Success);
}
internal void TriggerMatchActivelyEarlier() {
@@ -544,77 +783,6 @@ internal sealed class RemoteCommunication : IAsyncDisposable, IDisposable {
}
}
private bool HandleAnnounceResponse(BotCache botCache, string tradeToken, string? inventoryChecksumBeforeDeduplication = null, ICollection<AssetForListing>? assetsForListing = null, string? previousInventoryChecksum = null, BasicResponse? response = null) {
ArgumentNullException.ThrowIfNull(botCache);
ArgumentException.ThrowIfNullOrEmpty(tradeToken);
if (response == null) {
// This is actually a network failure, so we'll stop sending heartbeats but not record it as valid check
ShouldSendHeartBeats = false;
Bot.ArchiLogger.LogGenericWarning(string.Format(CultureInfo.CurrentCulture, Strings.ErrorObjectIsNull, nameof(response)));
return false;
}
if (response.StatusCode.IsRedirectionCode()) {
ShouldSendHeartBeats = false;
Bot.ArchiLogger.LogGenericWarning(string.Format(CultureInfo.CurrentCulture, Strings.WarningFailedWithError, response.StatusCode));
if (response.FinalUri.Host != ArchiWebHandler.SteamCommunityURL.Host) {
ASF.ArchiLogger.LogGenericError(string.Format(CultureInfo.CurrentCulture, Strings.WarningUnknownValuePleaseReport, nameof(response.FinalUri), response.FinalUri));
return false;
}
// We've expected the result, not the redirection to the sign in, we need to authenticate again
SignedInWithSteam = false;
return false;
}
if (response.StatusCode.IsClientErrorCode()) {
// ArchiNet told us that we've sent a bad request, so the process should restart from the beginning at later time
ShouldSendHeartBeats = false;
Bot.ArchiLogger.LogGenericWarning(string.Format(CultureInfo.CurrentCulture, Strings.WarningFailedWithError, response.StatusCode));
switch (response.StatusCode) {
case HttpStatusCode.Conflict when !string.IsNullOrEmpty(previousInventoryChecksum):
// ArchiNet told us to do full announcement instead
return false;
case HttpStatusCode.Forbidden:
// ArchiNet told us to stop submitting data for now
LastAnnouncement = DateTime.UtcNow.AddYears(1);
return false;
case HttpStatusCode.TooManyRequests:
// ArchiNet told us to try again later
LastAnnouncement = DateTime.UtcNow.AddDays(1);
return false;
default:
// There is something wrong with our payload or the server, we shouldn't retry for at least several hours
LastAnnouncement = DateTime.UtcNow.AddHours(6);
return false;
}
}
if (assetsForListing?.Count > 0) {
LastAnnouncement = LastHeartBeat = DateTime.UtcNow;
ShouldSendAnnouncementEarlier = false;
ShouldSendHeartBeats = true;
botCache.LastAnnouncedAssetsForListing.ReplaceWith(assetsForListing);
botCache.LastAnnouncedTradeToken = tradeToken;
botCache.LastInventoryChecksumBeforeDeduplication = inventoryChecksumBeforeDeduplication;
}
return true;
}
private async Task<bool?> IsEligibleForListing() {
// Bot must be eligible for matching
bool? isEligibleForMatching = await IsEligibleForMatching().ConfigureAwait(false);