Add support for request compression (#2805)

* Add support for brotli request compression

* Refactor and add support for netf

* Use fastest compression

ArchiSteamFarm.OfficialPlugins.ItemsMatcher/Backend.cs

@@ -69,7 +69,7 @@ internal static class Backend {
 
 		AnnouncementRequest data = new(ASF.GlobalDatabase?.Identifier ?? Guid.NewGuid(), steamID, tradeToken, inventory, acceptedMatchableTypes, totalInventoryCount, matchEverything, ASF.GlobalConfig?.MaxTradeHoldDuration ?? GlobalConfig.DefaultMaxTradeHoldDuration, nickname, avatarHash);
 
-		return await webBrowser.UrlPost(request, data: data, requestOptions: WebBrowser.ERequestOptions.ReturnRedirections | WebBrowser.ERequestOptions.ReturnClientErrors).ConfigureAwait(false);
+		return await webBrowser.UrlPost(request, data: data, requestOptions: WebBrowser.ERequestOptions.ReturnRedirections | WebBrowser.ERequestOptions.ReturnClientErrors | WebBrowser.ERequestOptions.CompressRequest).ConfigureAwait(false);
 	}
 
 	internal static async Task<(HttpStatusCode StatusCode, ImmutableHashSet<ListedUser> Users)?> GetListedUsersForMatching(Guid licenseID, Bot bot, WebBrowser webBrowser, IReadOnlyCollection<Asset> inventory, IReadOnlyCollection<Asset.EType> acceptedMatchableTypes) {
@@ -96,7 +96,7 @@ internal static class Backend {
 
 		InventoriesRequest data = new(ASF.GlobalDatabase?.Identifier ?? Guid.NewGuid(), bot.SteamID, inventory, acceptedMatchableTypes);
 
-		ObjectResponse<GenericResponse<ImmutableHashSet<ListedUser>>>? response = await webBrowser.UrlPostToJsonObject<GenericResponse<ImmutableHashSet<ListedUser>>, InventoriesRequest>(request, headers, data, requestOptions: WebBrowser.ERequestOptions.ReturnClientErrors | WebBrowser.ERequestOptions.AllowInvalidBodyOnErrors).ConfigureAwait(false);
+		ObjectResponse<GenericResponse<ImmutableHashSet<ListedUser>>>? response = await webBrowser.UrlPostToJsonObject<GenericResponse<ImmutableHashSet<ListedUser>>, InventoriesRequest>(request, headers, data, requestOptions: WebBrowser.ERequestOptions.ReturnClientErrors | WebBrowser.ERequestOptions.AllowInvalidBodyOnErrors | WebBrowser.ERequestOptions.CompressRequest).ConfigureAwait(false);
 
 		if (response == null) {
 			return null;
@@ -113,6 +113,6 @@ internal static class Backend {
 
 		HeartBeatRequest data = new(ASF.GlobalDatabase?.Identifier ?? Guid.NewGuid(), bot.SteamID);
 
-		return await webBrowser.UrlPost(request, data: data, requestOptions: WebBrowser.ERequestOptions.ReturnRedirections | WebBrowser.ERequestOptions.ReturnClientErrors).ConfigureAwait(false);
+		return await webBrowser.UrlPost(request, data: data, requestOptions: WebBrowser.ERequestOptions.ReturnRedirections | WebBrowser.ERequestOptions.ReturnClientErrors | WebBrowser.ERequestOptions.CompressRequest).ConfigureAwait(false);
 	}
 }

ArchiSteamFarm/Web/CompressedContent.cs

@@ -0,0 +1,70 @@
+//     _                _      _  ____   _                           _____
+//    / \    _ __  ___ | |__  (_)/ ___| | |_  ___   __ _  _ __ ___  |  ___|__ _  _ __  _ __ ___
+//   / _ \  | '__|/ __|| '_ \ | |\___ \ | __|/ _ \ / _` || '_ ` _ \ | |_  / _` || '__|| '_ ` _ \
+//  / ___ \ | |  | (__ | | | || | ___) || |_|  __/| (_| || | | | | ||  _|| (_| || |   | | | | | |
+// /_/   \_\|_|   \___||_| |_||_||____/  \__|\___| \__,_||_| |_| |_||_|   \__,_||_|   |_| |_| |_|
+// |
+// Copyright 2015-2023 Łukasz "JustArchi" Domeradzki
+// Contact: JustArchi@JustArchi.net
+// |
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// |
+// http://www.apache.org/licenses/LICENSE-2.0
+// |
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Compression;
+using System.Net.Http;
+using System.Threading.Tasks;
+
+namespace ArchiSteamFarm.Web;
+
+internal sealed class CompressedContent : StreamContent {
+	private CompressedContent(Stream content) : base(content) => ArgumentNullException.ThrowIfNull(content);
+
+	internal static async Task<CompressedContent> FromHttpContent(HttpContent content) {
+		ArgumentNullException.ThrowIfNull(content);
+
+		// We're going to create compressed stream and copy original content to it
+		MemoryStream compressionOutput = new();
+
+		(Stream compressionInput, string contentEncoding) = GetBestSupportedCompressionMethod(compressionOutput);
+
+		await using (compressionInput.ConfigureAwait(false)) {
+			await content.CopyToAsync(compressionInput).ConfigureAwait(false);
+		}
+
+		// Reset the position back to 0, so HttpClient can read it again
+		compressionOutput.Position = 0;
+
+		CompressedContent result = new(compressionOutput);
+
+		foreach ((string? key, IEnumerable<string>? value) in content.Headers) {
+			result.Headers.Add(key, value);
+		}
+
+		// Inform the server that we're sending compressed data
+		result.Headers.ContentEncoding.Add(contentEncoding);
+
+		return result;
+	}
+
+	private static (Stream CompressionInput, string ContentEncoding) GetBestSupportedCompressionMethod(Stream output) {
+		ArgumentNullException.ThrowIfNull(output);
+
+#if NETFRAMEWORK
+		return (new GZipStream(output, CompressionLevel.Fastest, true), "gzip");
+#else
+		return (new BrotliStream(output, CompressionLevel.Fastest, true), "br");
+#endif
+	}
+}

ArchiSteamFarm/Web/WebBrowser.cs

@@ -825,6 +825,18 @@ public sealed class WebBrowser : IDisposable {
 
 							break;
 					}
+
+					// Compress the request if caller specified it, so he knows that the server supports it, and the content is not compressed yet
+					if (requestOptions.HasFlag(ERequestOptions.CompressRequest) && (requestMessage.Content.Headers.ContentEncoding.Count == 0)) {
+						HttpContent originalContent = requestMessage.Content;
+
+						requestMessage.Content = await CompressedContent.FromHttpContent(originalContent).ConfigureAwait(false);
+
+						if (data is not HttpContent) {
+							// We don't need to keep old HttpContent around anymore, help GC
+							originalContent.Dispose();
+						}
+					}
 				}
 
 				if (referer != null) {
@@ -957,6 +969,7 @@ public sealed class WebBrowser : IDisposable {
 		ReturnServerErrors = 2,
 		ReturnRedirections = 4,
 		AllowInvalidBodyOnSuccess = 8,
-		AllowInvalidBodyOnErrors = 16
+		AllowInvalidBodyOnErrors = 16,
+		CompressRequest = 32
 	}
 }