Akatsuki-Misaki/ArchiSteamFarm
1
0
Fork 0
mirror of https://github.com/JustArchiNET/ArchiSteamFarm.git synced 2026-01-06 17:10:13 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
9bee9425a40b5e96262bc3b0ce316abac782d9f4
ArchiSteamFarm/ArchiSteamFarm/WebBrowser.cs
JustArchi 6bbff6ab4e Happy new year
2018-01-01 02:56:53 +01:00

562 lines
19 KiB
C#
Raw Blame History

// _ _ _ ____ _ _____
// / \ _ __ ___ | |__ (_)/ ___| | |_ ___ __ _ _ __ ___ | ___|__ _ _ __ _ __ ___
// / _ \ | '__|/ __|| '_ \ | |\___ \ | __|/ _ \ / _` || '_ ` _ \ | |_ / _` || '__|| '_ ` _ \
// / ___ \ | | | (__ | | | || | ___) || |_| __/| (_| || | | | | || _|| (_| || | | | | | | |
// /_/ \_\|_| \___||_| |_||_||____/ \__|\___| \__,_||_| |_| |_||_| \__,_||_| |_| |_| |_|
//
// Copyright 2015-2018 Łukasz "JustArchi" Domeradzki
// Contact: JustArchi@JustArchi.net
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
using System;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
using System.Xml;
using ArchiSteamFarm.Localization;
using HtmlAgilityPack;
using Newtonsoft.Json;
namespace ArchiSteamFarm {
internal sealed class WebBrowser : IDisposable {
internal const byte MaxTries = 5; // Defines maximum number of recommended tries for a single request
private const byte ExtendedTimeoutMultiplier = 10; // Defines multiplier of timeout for WebBrowsers dealing with huge data (ASF update)
private const byte MaxConnections = 10; // Defines maximum number of connections per ServicePoint. Be careful, as it also defines maximum number of sockets in CLOSE_WAIT state
private const byte MaxIdleTime = 15; // Defines in seconds, how long socket is allowed to stay in CLOSE_WAIT state after there are no connections to it
internal readonly CookieContainer CookieContainer = new CookieContainer();
internal TimeSpan Timeout => HttpClient.Timeout;
private readonly ArchiLogger ArchiLogger;
private readonly HttpClient HttpClient;
internal WebBrowser(ArchiLogger archiLogger, bool extendedTimeout = false) {
ArchiLogger = archiLogger ?? throw new ArgumentNullException(nameof(archiLogger));
HttpClientHandler httpClientHandler = new HttpClientHandler {
AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip,
CookieContainer = CookieContainer,
MaxConnectionsPerServer = MaxConnections,
UseProxy = false
};
HttpClient = new HttpClient(httpClientHandler) { Timeout = TimeSpan.FromSeconds(extendedTimeout ? ExtendedTimeoutMultiplier * Program.GlobalConfig.ConnectionTimeout : Program.GlobalConfig.ConnectionTimeout) };
// Most web services expect that UserAgent is set, so we declare it globally
HttpClient.DefaultRequestHeaders.UserAgent.ParseAdd(SharedInfo.PublicIdentifier + "/" + SharedInfo.Version);
}
public void Dispose() => HttpClient.Dispose();
internal static void Init() {
// Set max connection limit from default of 2 to desired value
ServicePointManager.DefaultConnectionLimit = MaxConnections;
// Set max idle time from default of 100 seconds (100 * 1000) to desired value
ServicePointManager.MaxServicePointIdleTime = MaxIdleTime * 1000;
// Don't use Expect100Continue, we're sure about our POSTs, save some TCP packets
ServicePointManager.Expect100Continue = false;
// Reuse ports if possible
ServicePointManager.ReusePort = true;
}
internal static HtmlDocument StringToHtmlDocument(string html) {
if (string.IsNullOrEmpty(html)) {
ASF.ArchiLogger.LogNullError(nameof(html));
return null;
}
HtmlDocument htmlDocument = new HtmlDocument();
htmlDocument.LoadHtml(html);
return htmlDocument;
}
internal async Task<byte[]> UrlGetToBytesWithProgressRetry(string request, string referer = null) {
if (string.IsNullOrEmpty(request)) {
ArchiLogger.LogNullError(nameof(request));
return null;
}
byte[] result = null;
for (byte i = 0; (i < MaxTries) && (result == null); i++) {
result = await UrlGetToBytesWithProgress(request, referer).ConfigureAwait(false);
}
if (result != null) {
return result;
}
ArchiLogger.LogGenericWarning(string.Format(Strings.ErrorRequestFailedTooManyTimes, MaxTries));
ArchiLogger.LogGenericDebug(string.Format(Strings.ErrorFailingRequest, request));
return null;
}
internal async Task<HtmlDocument> UrlGetToHtmlDocumentRetry(string request, string referer = null) {
if (string.IsNullOrEmpty(request)) {
ArchiLogger.LogNullError(nameof(request));
return null;
}
HtmlDocument result = null;
for (byte i = 0; (i < MaxTries) && (result == null); i++) {
result = await UrlGetToHtmlDocument(request, referer).ConfigureAwait(false);
}
if (result != null) {
return result;
}
ArchiLogger.LogGenericWarning(string.Format(Strings.ErrorRequestFailedTooManyTimes, MaxTries));
ArchiLogger.LogGenericDebug(string.Format(Strings.ErrorFailingRequest, request));
return null;
}
internal async Task<T> UrlGetToJsonResultRetry<T>(string request, string referer = null) {
if (string.IsNullOrEmpty(request)) {
ArchiLogger.LogNullError(nameof(request));
return default;
}
string json = await UrlGetToContentRetry(request, referer).ConfigureAwait(false);
if (string.IsNullOrEmpty(json)) {
return default;
}
try {
return JsonConvert.DeserializeObject<T>(json);
} catch (JsonException e) {
ArchiLogger.LogGenericException(e);
if (Debugging.IsUserDebugging) {
ArchiLogger.LogGenericDebug(string.Format(Strings.Content, json));
}
return default;
}
}
internal async Task<XmlDocument> UrlGetToXMLRetry(string request, string referer = null) {
if (string.IsNullOrEmpty(request)) {
ArchiLogger.LogNullError(nameof(request));
return null;
}
XmlDocument result = null;
for (byte i = 0; (i < MaxTries) && (result == null); i++) {
result = await UrlGetToXML(request, referer).ConfigureAwait(false);
}
if (result != null) {
return result;
}
ArchiLogger.LogGenericWarning(string.Format(Strings.ErrorRequestFailedTooManyTimes, MaxTries));
ArchiLogger.LogGenericDebug(string.Format(Strings.ErrorFailingRequest, request));
return null;
}
internal async Task<bool> UrlHeadRetry(string request, string referer = null) {
if (string.IsNullOrEmpty(request)) {
ArchiLogger.LogNullError(nameof(request));
return false;
}
bool result = false;
for (byte i = 0; (i < MaxTries) && !result; i++) {
result = await UrlHead(request, referer).ConfigureAwait(false);
}
if (result) {
return true;
}
ArchiLogger.LogGenericWarning(string.Format(Strings.ErrorRequestFailedTooManyTimes, MaxTries));
ArchiLogger.LogGenericDebug(string.Format(Strings.ErrorFailingRequest, request));
return false;
}
internal async Task<Uri> UrlHeadToUriRetry(string request, string referer = null) {
if (string.IsNullOrEmpty(request)) {
ArchiLogger.LogNullError(nameof(request));
return null;
}
Uri result = null;
for (byte i = 0; (i < MaxTries) && (result == null); i++) {
result = await UrlHeadToUri(request, referer).ConfigureAwait(false);
}
if (result != null) {
return result;
}
ArchiLogger.LogGenericWarning(string.Format(Strings.ErrorRequestFailedTooManyTimes, MaxTries));
ArchiLogger.LogGenericDebug(string.Format(Strings.ErrorFailingRequest, request));
return null;
}
internal async Task<bool> UrlPost(string request, IReadOnlyCollection<KeyValuePair<string, string>> data = null, string referer = null) {
if (string.IsNullOrEmpty(request)) {
ArchiLogger.LogNullError(nameof(request));
return false;
}
using (HttpResponseMessage response = await UrlPostToResponse(request, data, referer).ConfigureAwait(false)) {
return response != null;
}
}
internal async Task<bool> UrlPostRetry(string request, IReadOnlyCollection<KeyValuePair<string, string>> data = null, string referer = null) {
if (string.IsNullOrEmpty(request)) {
ArchiLogger.LogNullError(nameof(request));
return false;
}
bool result = false;
for (byte i = 0; (i < MaxTries) && !result; i++) {
result = await UrlPost(request, data, referer).ConfigureAwait(false);
}
if (result) {
return true;
}
ArchiLogger.LogGenericWarning(string.Format(Strings.ErrorRequestFailedTooManyTimes, MaxTries));
ArchiLogger.LogGenericDebug(string.Format(Strings.ErrorFailingRequest, request));
return false;
}
internal async Task<HtmlDocument> UrlPostToHtmlDocumentRetry(string request, IReadOnlyCollection<KeyValuePair<string, string>> data = null, string referer = null) {
if (string.IsNullOrEmpty(request)) {
ArchiLogger.LogNullError(nameof(request));
return null;
}
string content = await UrlPostToContentRetry(request, data, referer).ConfigureAwait(false);
return !string.IsNullOrEmpty(content) ? StringToHtmlDocument(content) : null;
}
internal async Task<T> UrlPostToJsonResultRetry<T>(string request, IReadOnlyCollection<KeyValuePair<string, string>> data = null, string referer = null) {
if (string.IsNullOrEmpty(request)) {
ArchiLogger.LogNullError(nameof(request));
return default;
}
string json = await UrlPostToContentRetry(request, data, referer).ConfigureAwait(false);
if (string.IsNullOrEmpty(json)) {
return default;
}
try {
return JsonConvert.DeserializeObject<T>(json);
} catch (JsonException e) {
ArchiLogger.LogGenericException(e);
if (Debugging.IsUserDebugging) {
ArchiLogger.LogGenericDebug(string.Format(Strings.Content, json));
}
return default;
}
}
private async Task<byte[]> UrlGetToBytesWithProgress(string request, string referer = null) {
if (string.IsNullOrEmpty(request)) {
ArchiLogger.LogNullError(nameof(request));
return null;
}
const byte printPercentage = 10;
const byte maxBatches = 99 / printPercentage;
using (HttpResponseMessage httpResponse = await UrlGetToResponse(request, referer, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false)) {
if (httpResponse == null) {
return null;
}
ArchiLogger.LogGenericDebug("0%...");
uint contentLength = (uint) httpResponse.Content.Headers.ContentLength.GetValueOrDefault();
using (MemoryStream ms = new MemoryStream((int) contentLength)) {
try {
using (Stream contentStream = await httpResponse.Content.ReadAsStreamAsync().ConfigureAwait(false)) {
byte batch = 0;
uint readThisBatch = 0;
byte[] buffer = new byte[8192]; // This is HttpClient's buffer, using more doesn't make sense
while (contentStream.CanRead) {
int read = await contentStream.ReadAsync(buffer, 0, buffer.Length).ConfigureAwait(false);
if (read == 0) {
break;
}
await ms.WriteAsync(buffer, 0, read).ConfigureAwait(false);
if ((contentLength == 0) || (batch >= maxBatches)) {
continue;
}
readThisBatch += (uint) read;
if (readThisBatch < contentLength / printPercentage) {
continue;
}
readThisBatch -= contentLength / printPercentage;
ArchiLogger.LogGenericDebug(++batch * printPercentage + "%...");
}
}
} catch (Exception e) {
ArchiLogger.LogGenericDebuggingException(e);
return null;
}
ArchiLogger.LogGenericDebug("100%");
return ms.ToArray();
}
}
}
private async Task<string> UrlGetToContent(string request, string referer = null) {
if (string.IsNullOrEmpty(request)) {
ArchiLogger.LogNullError(nameof(request));
return null;
}
using (HttpResponseMessage httpResponse = await UrlGetToResponse(request, referer).ConfigureAwait(false)) {
if (httpResponse == null) {
return null;
}
return await httpResponse.Content.ReadAsStringAsync().ConfigureAwait(false);
}
}
private async Task<string> UrlGetToContentRetry(string request, string referer = null) {
if (string.IsNullOrEmpty(request)) {
ArchiLogger.LogNullError(nameof(request));
return null;
}
string result = null;
for (byte i = 0; (i < MaxTries) && string.IsNullOrEmpty(result); i++) {
result = await UrlGetToContent(request, referer).ConfigureAwait(false);
}
if (!string.IsNullOrEmpty(result)) {
return result;
}
ArchiLogger.LogGenericWarning(string.Format(Strings.ErrorRequestFailedTooManyTimes, MaxTries));
ArchiLogger.LogGenericDebug(string.Format(Strings.ErrorFailingRequest, request));
return null;
}
private async Task<HtmlDocument> UrlGetToHtmlDocument(string request, string referer = null) {
if (string.IsNullOrEmpty(request)) {
ArchiLogger.LogNullError(nameof(request));
return null;
}
string content = await UrlGetToContent(request, referer).ConfigureAwait(false);
return !string.IsNullOrEmpty(content) ? StringToHtmlDocument(content) : null;
}
private async Task<HttpResponseMessage> UrlGetToResponse(string request, string referer = null, HttpCompletionOption httpCompletionOptions = HttpCompletionOption.ResponseContentRead) {
if (string.IsNullOrEmpty(request)) {
ArchiLogger.LogNullError(nameof(request));
return null;
}
HttpResponseMessage result = await UrlRequest(new Uri(request), HttpMethod.Get, null, referer, httpCompletionOptions).ConfigureAwait(false);
return result;
}
private async Task<XmlDocument> UrlGetToXML(string request, string referer = null) {
if (string.IsNullOrEmpty(request)) {
ArchiLogger.LogNullError(nameof(request));
return null;
}
string xml = await UrlGetToContent(request, referer).ConfigureAwait(false);
if (string.IsNullOrEmpty(xml)) {
return null;
}
XmlDocument xmlDocument = new XmlDocument();
try {
xmlDocument.LoadXml(xml);
} catch (XmlException e) {
ArchiLogger.LogGenericException(e);
return null;
}
return xmlDocument;
}
private async Task<bool> UrlHead(string request, string referer = null) {
if (string.IsNullOrEmpty(request)) {
ArchiLogger.LogNullError(nameof(request));
return false;
}
using (HttpResponseMessage response = await UrlHeadToResponse(request, referer).ConfigureAwait(false)) {
return response != null;
}
}
private async Task<HttpResponseMessage> UrlHeadToResponse(string request, string referer = null) {
if (!string.IsNullOrEmpty(request)) {
return await UrlRequest(new Uri(request), HttpMethod.Head, null, referer).ConfigureAwait(false);
}
ArchiLogger.LogNullError(nameof(request));
return null;
}
private async Task<Uri> UrlHeadToUri(string request, string referer = null) {
if (string.IsNullOrEmpty(request)) {
ArchiLogger.LogNullError(nameof(request));
return null;
}
using (HttpResponseMessage response = await UrlHeadToResponse(request, referer).ConfigureAwait(false)) {
return response?.RequestMessage.RequestUri;
}
}
private async Task<string> UrlPostToContent(string request, IReadOnlyCollection<KeyValuePair<string, string>> data = null, string referer = null) {
if (string.IsNullOrEmpty(request)) {
ArchiLogger.LogNullError(nameof(request));
return null;
}
using (HttpResponseMessage httpResponse = await UrlPostToResponse(request, data, referer).ConfigureAwait(false)) {
if (httpResponse == null) {
return null;
}
return await httpResponse.Content.ReadAsStringAsync().ConfigureAwait(false);
}
}
private async Task<string> UrlPostToContentRetry(string request, IReadOnlyCollection<KeyValuePair<string, string>> data = null, string referer = null) {
if (string.IsNullOrEmpty(request)) {
ArchiLogger.LogNullError(nameof(request));
return null;
}
string result = null;
for (byte i = 0; (i < MaxTries) && string.IsNullOrEmpty(result); i++) {
result = await UrlPostToContent(request, data, referer).ConfigureAwait(false);
}
if (!string.IsNullOrEmpty(result)) {
return result;
}
ArchiLogger.LogGenericWarning(string.Format(Strings.ErrorRequestFailedTooManyTimes, MaxTries));
ArchiLogger.LogGenericDebug(string.Format(Strings.ErrorFailingRequest, request));
return null;
}
private async Task<HttpResponseMessage> UrlPostToResponse(string request, IReadOnlyCollection<KeyValuePair<string, string>> data = null, string referer = null) {
if (!string.IsNullOrEmpty(request)) {
return await UrlRequest(new Uri(request), HttpMethod.Post, data, referer).ConfigureAwait(false);
}
ArchiLogger.LogNullError(nameof(request));
return null;
}
private async Task<HttpResponseMessage> UrlRequest(Uri requestUri, HttpMethod httpMethod, IReadOnlyCollection<KeyValuePair<string, string>> data = null, string referer = null, HttpCompletionOption httpCompletionOption = HttpCompletionOption.ResponseContentRead, byte maxRedirections = MaxTries) {
if ((requestUri == null) || (httpMethod == null)) {
ArchiLogger.LogNullError(nameof(requestUri) + " || " + nameof(httpMethod));
return null;
}
HttpResponseMessage responseMessage;
using (HttpRequestMessage requestMessage = new HttpRequestMessage(httpMethod, requestUri)) {
if (data != null) {
try {
requestMessage.Content = new FormUrlEncodedContent(data);
} catch (UriFormatException e) {
ArchiLogger.LogGenericException(e);
return null;
}
}
if (!string.IsNullOrEmpty(referer)) {
requestMessage.Headers.Referrer = new Uri(referer);
}
try {
responseMessage = await HttpClient.SendAsync(requestMessage, httpCompletionOption).ConfigureAwait(false);
} catch (Exception e) {
ArchiLogger.LogGenericDebuggingException(e);
return null;
}
}
if (responseMessage == null) {
return null;
}
if (responseMessage.IsSuccessStatusCode) {
return responseMessage;
}
Uri redirectUri;
using (responseMessage) {
ushort status = (ushort) responseMessage.StatusCode;
if ((status >= 300) && (status <= 399) && (maxRedirections > 0)) {
redirectUri = responseMessage.Headers.Location;
if (redirectUri.IsAbsoluteUri) {
switch (redirectUri.Scheme) {
case "http":
case "https":
break;
default:
// Invalid ones such as "steammobile"
return null;
}
} else {
redirectUri = new Uri(requestUri.GetLeftPart(UriPartial.Authority) + redirectUri);
}
} else {
if (!Debugging.IsDebugBuild) {
return null;
}
ArchiLogger.LogGenericDebug(string.Format(Strings.ErrorFailingRequest, requestUri));
ArchiLogger.LogGenericDebug(string.Format(Strings.StatusCode, responseMessage.StatusCode));
ArchiLogger.LogGenericDebug(string.Format(Strings.Content, await responseMessage.Content.ReadAsStringAsync().ConfigureAwait(false)));
return null;
}
}
return await UrlRequest(redirectUri, httpMethod, data, referer, httpCompletionOption, --maxRedirections).ConfigureAwait(false);
}
}
}
Reference in New Issue View Git Blame Copy Permalink