Akatsuki-Misaki/ArchiSteamFarm
1
0
Fork 0
mirror of https://github.com/JustArchiNET/ArchiSteamFarm.git synced 2026-01-01 06:00:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Bring in latest ArchiBoT WebBrowser

Browse Source 
Highlights include: Support for automatic decompression via GZip/Deflate, more aggressive disposal of response messages, rewrite of cookie handling from dictionary to cookiecontainer, debug log of also timed out messages and more.
We sacrifice the performance and scalability of single HttpClient doing the work for being less error prone on eventual steam fuckups (CookieContainer can adapt to always changing structure)
This commit is contained in:
JustArchi
2016-04-12 07:40:02 +02:00
parent 62a6e38e47
commit 65b31e5537
3 changed files with 149 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

93
ArchiSteamFarm/ArchiWebHandler.cs
View File

@@ -44,8 +44,8 @@ namespace ArchiSteamFarm {
private static int Timeout = GlobalConfig.DefaultHttpTimeout * 1000;
private readonly Bot Bot;
private readonly Dictionary<string, string> Cookie = new Dictionary<string, string>(4);
private readonly SemaphoreSlim SessionSemaphore = new SemaphoreSlim(1);
private readonly CookieContainer CookieContainer = new CookieContainer();
private DateTime LastSessionRefreshCheck = DateTime.MinValue;
@@ -112,17 +112,15 @@ namespace ArchiSteamFarm {
return false;
}
Logging.LogGenericInfo("Success!", Bot.BotName);
Logging.LogGenericInfo("Success!");
string steamLogin = authResult["token"].AsString();
string steamLoginSecure = authResult["tokensecure"].AsString();
CookieContainer.Add(new Cookie("sessionid", sessionID, "/", "." + SteamCommunity));
Cookie["sessionid"] = sessionID;
Cookie["steamLogin"] = steamLogin;
Cookie["steamLoginSecure"] = steamLoginSecure;
string steamLogin = authResult["token"].Value;
CookieContainer.Add(new Cookie("steamLogin", steamLogin, "/", "." + SteamCommunity));
// The below is used for display purposes only
Cookie["webTradeEligibility"] = "{\"allowed\":0,\"reason\":0,\"allowed_at_time\":0,\"steamguard_required_days\":0,\"sales_this_year\":0,\"max_sales_per_year\":0,\"forms_requested\":0}";
string steamLoginSecure = authResult["tokensecure"].Value;
CookieContainer.Add(new Cookie("steamLoginSecure", steamLoginSecure, "/", "." + SteamCommunity));
if (!UnlockParentalAccount(parentalPin).Result) {
return false;
@@ -135,7 +133,7 @@ namespace ArchiSteamFarm {
internal async Task<bool?> IsLoggedIn() {
HtmlDocument htmlDocument = null;
for (byte i = 0; i < WebBrowser.MaxRetries && htmlDocument == null; i++) {
htmlDocument = await WebBrowser.UrlGetToHtmlDocument(SteamCommunityURL + "/my/profile", Cookie).ConfigureAwait(false);
htmlDocument = await WebBrowser.UrlGetToHtmlDocument(SteamCommunityURL + "/my/profile", CookieContainer).ConfigureAwait(false);
}
if (htmlDocument == null) {
@@ -186,7 +184,7 @@ namespace ArchiSteamFarm {
XmlDocument response = null;
for (byte i = 0; i < WebBrowser.MaxRetries && response == null; i++) {
response = await WebBrowser.UrlGetToXML(request, Cookie).ConfigureAwait(false);
response = await WebBrowser.UrlGetToXML(request, CookieContainer).ConfigureAwait(false);
}
if (response == null) {
@@ -291,8 +289,9 @@ namespace ArchiSteamFarm {
return false;
}
string sessionID;
if (!Cookie.TryGetValue("sessionid", out sessionID)) {
string sessionID = CookieContainer.GetCookieValue(SteamCommunityURL, "sessionid");
if (string.IsNullOrEmpty(sessionID)) {
Logging.LogNullError("sessionID");
return false;
}
@@ -303,12 +302,12 @@ namespace ArchiSteamFarm {
{"action", "join"}
};
HttpResponseMessage response = null;
for (byte i = 0; i < WebBrowser.MaxRetries && response == null; i++) {
response = await WebBrowser.UrlPost(request, data, Cookie).ConfigureAwait(false);
bool result = false;
for (byte i = 0; i < WebBrowser.MaxRetries && !result; i++) {
result = await WebBrowser.UrlPost(request, data, CookieContainer).ConfigureAwait(false);
}
if (response == null) {
if (!result) {
Logging.LogGenericWTF("Request failed even after " + WebBrowser.MaxRetries + " tries", Bot.BotName);
return false;
}
@@ -325,8 +324,9 @@ namespace ArchiSteamFarm {
return false;
}
string sessionID;
if (!Cookie.TryGetValue("sessionid", out sessionID)) {
string sessionID = CookieContainer.GetCookieValue(SteamCommunityURL, "sessionid");
if (string.IsNullOrEmpty(sessionID)) {
Logging.LogNullError("sessionID");
return false;
}
@@ -339,12 +339,12 @@ namespace ArchiSteamFarm {
{"tradeofferid", tradeID.ToString()}
};
HttpResponseMessage response = null;
for (byte i = 0; i < WebBrowser.MaxRetries && response == null; i++) {
response = await WebBrowser.UrlPost(request, data, Cookie, referer).ConfigureAwait(false);
bool result = false;
for (byte i = 0; i < WebBrowser.MaxRetries && !result; i++) {
result = await WebBrowser.UrlPost(request, data, CookieContainer, referer).ConfigureAwait(false);
}
if (response == null) {
if (!result) {
Logging.LogGenericWTF("Request failed even after " + WebBrowser.MaxRetries + " tries", Bot.BotName);
return false;
}
@@ -359,7 +359,7 @@ namespace ArchiSteamFarm {
JObject jObject = null;
for (byte i = 0; i < WebBrowser.MaxRetries && jObject == null; i++) {
jObject = await WebBrowser.UrlGetToJObject(SteamCommunityURL + "/my/inventory/json/753/6?trading=1", Cookie).ConfigureAwait(false);
jObject = await WebBrowser.UrlGetToJObject(SteamCommunityURL + "/my/inventory/json/753/6?trading=1", CookieContainer).ConfigureAwait(false);
}
if (jObject == null) {
@@ -394,8 +394,9 @@ namespace ArchiSteamFarm {
return false;
}
string sessionID;
if (!Cookie.TryGetValue("sessionid", out sessionID)) {
string sessionID = CookieContainer.GetCookieValue(SteamCommunityURL, "sessionid");
if (string.IsNullOrEmpty(sessionID)) {
Logging.LogNullError("sessionID");
return false;
}
@@ -434,12 +435,12 @@ namespace ArchiSteamFarm {
{"trade_offer_create_params", string.IsNullOrEmpty(token) ? "" : $"{{\"trade_offer_access_token\":\"{token}\"}}"}
};
HttpResponseMessage response = null;
for (byte i = 0; i < WebBrowser.MaxRetries && response == null; i++) {
response = await WebBrowser.UrlPost(request, data, Cookie, referer).ConfigureAwait(false);
bool result = false;
for (byte i = 0; i < WebBrowser.MaxRetries && !result; i++) {
result = await WebBrowser.UrlPost(request, data, CookieContainer, referer).ConfigureAwait(false);
}
if (response == null) {
if (!result) {
Logging.LogGenericWTF("Request failed even after " + WebBrowser.MaxRetries + " tries", Bot.BotName);
return false;
}
@@ -459,7 +460,7 @@ namespace ArchiSteamFarm {
HtmlDocument htmlDocument = null;
for (byte i = 0; i < WebBrowser.MaxRetries && htmlDocument == null; i++) {
htmlDocument = await WebBrowser.UrlGetToHtmlDocument(SteamCommunityURL + "/my/badges?l=english&p=" + page, Cookie).ConfigureAwait(false);
htmlDocument = await WebBrowser.UrlGetToHtmlDocument(SteamCommunityURL + "/my/badges?p=" + page, CookieContainer).ConfigureAwait(false);
}
if (htmlDocument == null) {
@@ -481,7 +482,7 @@ namespace ArchiSteamFarm {
HtmlDocument htmlDocument = null;
for (byte i = 0; i < WebBrowser.MaxRetries && htmlDocument == null; i++) {
htmlDocument = await WebBrowser.UrlGetToHtmlDocument(SteamCommunityURL + "/my/gamecards/" + appID + "?l=english", Cookie).ConfigureAwait(false);
htmlDocument = await WebBrowser.UrlGetToHtmlDocument(SteamCommunityURL + "/my/gamecards/" + appID, CookieContainer).ConfigureAwait(false);
}
if (htmlDocument == null) {
@@ -497,12 +498,12 @@ namespace ArchiSteamFarm {
return false;
}
HttpResponseMessage response = null;
for (byte i = 0; i < WebBrowser.MaxRetries && response == null; i++) {
response = await WebBrowser.UrlGet(SteamCommunityURL + "/my/inventory", Cookie).ConfigureAwait(false);
bool result = false;
for (byte i = 0; i < WebBrowser.MaxRetries && !result; i++) {
result = await WebBrowser.UrlGet(SteamCommunityURL + "/my/inventory", CookieContainer).ConfigureAwait(false);
}
if (response == null) {
if (!result) {
Logging.LogGenericWTF("Request failed even after " + WebBrowser.MaxRetries + " tries", Bot.BotName);
return false;
}
@@ -519,8 +520,9 @@ namespace ArchiSteamFarm {
return false;
}
string sessionID;
if (!Cookie.TryGetValue("sessionid", out sessionID)) {
string sessionID = CookieContainer.GetCookieValue(SteamCommunityURL, "sessionid");
if (string.IsNullOrEmpty(sessionID)) {
Logging.LogNullError("sessionID");
return false;
}
@@ -529,12 +531,12 @@ namespace ArchiSteamFarm {
{ "sessionid", sessionID }
};
HttpResponseMessage response = null;
for (byte i = 0; i < WebBrowser.MaxRetries && response == null; i++) {
response = await WebBrowser.UrlPost(request, data, Cookie).ConfigureAwait(false);
bool result = false;
for (byte i = 0; i < WebBrowser.MaxRetries && !result; i++) {
result = await WebBrowser.UrlPost(request, data, CookieContainer).ConfigureAwait(false);
}
if (response == null) {
if (!result) {
Logging.LogGenericWTF("Request failed even after " + WebBrowser.MaxRetries + " tries", Bot.BotName);
return false;
}
@@ -557,7 +559,7 @@ namespace ArchiSteamFarm {
HttpResponseMessage response = null;
for (byte i = 0; i < WebBrowser.MaxRetries && response == null; i++) {
response = await WebBrowser.UrlPost(request, data, Cookie, referer).ConfigureAwait(false);
response = await WebBrowser.UrlPostToResponse(request, data, CookieContainer, referer).ConfigureAwait(false);
}
if (response == null) {
@@ -567,10 +569,13 @@ namespace ArchiSteamFarm {
IEnumerable<string> setCookieValues;
if (!response.Headers.TryGetValues("Set-Cookie", out setCookieValues)) {
response.Dispose();
Logging.LogNullError("setCookieValues", Bot.BotName);
return false;
}
response.Dispose();
foreach (string setCookieValue in setCookieValues) {
if (!setCookieValue.Contains("steamparental=")) {
continue;
@@ -583,8 +588,8 @@ namespace ArchiSteamFarm {
setCookie = setCookie.Substring(0, index);
}
Cookie["steamparental"] = setCookie;
Logging.LogGenericInfo("Success!", Bot.BotName);
CookieContainer.Add(new Cookie("steamparental", setCookie, "/", "." + SteamCommunity));
return true;
}

22
ArchiSteamFarm/Utilities.cs
View File

@@ -25,6 +25,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
namespace ArchiSteamFarm {
@@ -35,6 +36,27 @@ namespace ArchiSteamFarm {
return Task.WhenAll(sequence.Select(action));
}
internal static string GetCookieValue(this CookieContainer cookieContainer, string URL, string name) {
if (string.IsNullOrEmpty(URL) || string.IsNullOrEmpty(name)) {
return null;
}
CookieCollection cookies = cookieContainer.GetCookies(new Uri(URL));
if (cookies == null || cookies.Count == 0) {
return null;
}
foreach (Cookie cookie in cookies) {
if (!cookie.Name.Equals(name, StringComparison.Ordinal)) {
continue;
}
return cookie.Value;
}
return null;
}
internal static Task SleepAsync(int miliseconds) {
if (miliseconds < 0) {
return Task.FromResult(true);

125
ArchiSteamFarm/WebBrowser.cs
View File

@@ -23,13 +23,13 @@
*/
using HtmlAgilityPack;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;
using System.Xml;
@@ -40,11 +40,16 @@ namespace ArchiSteamFarm {
private const byte MaxConnections = 10; // Defines maximum number of connections per ServicePoint. Be careful, as it also defines maximum number of sockets in CLOSE_WAIT state
private const byte MaxIdleTime = 15; // In seconds, how long socket is allowed to stay in CLOSE_WAIT state after there are no connections to it
internal static readonly CookieContainer CookieContainer = new CookieContainer();
private static readonly string DefaultUserAgent = "ArchiSteamFarm/" + Program.Version;
private static readonly HttpClient HttpClient = new HttpClient(new HttpClientHandler {
UseCookies = false
}) {
Timeout = TimeSpan.FromSeconds(60)
private static readonly HttpClientHandler HttpClientHandler = new HttpClientHandler {
AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip,
CookieContainer = CookieContainer
};
private static readonly HttpClient HttpClient = new HttpClient(HttpClientHandler) {
Timeout = TimeSpan.FromSeconds(GlobalConfig.DefaultHttpTimeout)
};
internal static void Init() {
@@ -63,67 +68,94 @@ namespace ArchiSteamFarm {
// Don't use Expect100Continue, we're sure about our POSTs, save some TCP packets
ServicePointManager.Expect100Continue = false;
// Reuse ports if possible
// TODO: Mono doesn't support that feature yet
#if !__MonoCS__
// Reuse ports if possible (since .NET 4.6+)
//ServicePointManager.ReusePort = true;
#endif
}
internal static async Task<HttpResponseMessage> UrlGet(string request, Dictionary<string, string> cookies = null, string referer = null) {
internal static async Task<bool> UrlGet(string request, CookieContainer cookieContainer = null, string referer = null) {
if (string.IsNullOrEmpty(request)) {
return false;
}
HttpResponseMessage response = await UrlGetToResponse(request, cookieContainer, referer).ConfigureAwait(false);
if (response == null) {
return false;
}
response.Dispose();
return true;
}
internal static async Task<bool> UrlPost(string request, Dictionary<string, string> data = null, CookieContainer cookieContainer = null, string referer = null) {
if (string.IsNullOrEmpty(request)) {
return false;
}
HttpResponseMessage response = await UrlPostToResponse(request, data, cookieContainer, referer).ConfigureAwait(false);
if (response == null) {
return false;
}
response.Dispose();
return true;
}
internal static async Task<HttpResponseMessage> UrlGetToResponse(string request, CookieContainer cookieContainer = null, string referer = null) {
if (string.IsNullOrEmpty(request)) {
return null;
}
return await UrlRequest(request, HttpMethod.Get, null, cookies, referer).ConfigureAwait(false);
return await UrlRequest(request, HttpMethod.Get, null, cookieContainer, referer).ConfigureAwait(false);
}
internal static async Task<HttpResponseMessage> UrlPost(string request, Dictionary<string, string> data = null, Dictionary<string, string> cookies = null, string referer = null) {
internal static async Task<HttpResponseMessage> UrlPostToResponse(string request, Dictionary<string, string> data = null, CookieContainer cookieContainer = null, string referer = null) {
if (string.IsNullOrEmpty(request)) {
return null;
}
return await UrlRequest(request, HttpMethod.Post, data, cookies, referer).ConfigureAwait(false);
return await UrlRequest(request, HttpMethod.Post, data, cookieContainer, referer).ConfigureAwait(false);
}
internal static async Task<string> UrlGetToContent(string request, Dictionary<string, string> cookies = null, string referer = null) {
internal static async Task<string> UrlGetToContent(string request, CookieContainer cookieContainer = null, string referer = null) {
if (string.IsNullOrEmpty(request)) {
return null;
}
HttpResponseMessage httpResponse = await UrlGet(request, cookies, referer).ConfigureAwait(false);
HttpResponseMessage httpResponse = await UrlGetToResponse(request, cookieContainer, referer).ConfigureAwait(false);
if (httpResponse == null) {
return null;
}
if (httpResponse.Content == null) {
return null;
}
string result = await httpResponse.Content.ReadAsStringAsync().ConfigureAwait(false);
httpResponse.Dispose();
return await httpResponse.Content.ReadAsStringAsync().ConfigureAwait(false);
return result;
}
internal static async Task<Stream> UrlGetToStream(string request, Dictionary<string, string> cookies = null, string referer = null) {
internal static async Task<Stream> UrlGetToStream(string request, CookieContainer cookieContainer = null, string referer = null) {
if (string.IsNullOrEmpty(request)) {
return null;
}
HttpResponseMessage httpResponse = await UrlGet(request, cookies, referer).ConfigureAwait(false);
HttpResponseMessage httpResponse = await UrlGetToResponse(request, cookieContainer, referer).ConfigureAwait(false);
if (httpResponse == null) {
return null;
}
if (httpResponse.Content == null) {
return null;
}
Stream result = await httpResponse.Content.ReadAsStreamAsync().ConfigureAwait(false);
httpResponse.Dispose();
return await httpResponse.Content.ReadAsStreamAsync().ConfigureAwait(false);
return result;
}
internal static async Task<HtmlDocument> UrlGetToHtmlDocument(string request, Dictionary<string, string> cookies = null, string referer = null) {
internal static async Task<HtmlDocument> UrlGetToHtmlDocument(string request, CookieContainer cookieContainer = null, string referer = null) {
if (string.IsNullOrEmpty(request)) {
return null;
}
string content = await UrlGetToContent(request, cookies, referer).ConfigureAwait(false);
string content = await UrlGetToContent(request, cookieContainer, referer).ConfigureAwait(false);
if (string.IsNullOrEmpty(content)) {
return null;
}
@@ -135,12 +167,12 @@ namespace ArchiSteamFarm {
return htmlDocument;
}
internal static async Task<JObject> UrlGetToJObject(string request, Dictionary<string, string> cookies = null, string referer = null) {
internal static async Task<JObject> UrlGetToJObject(string request, CookieContainer cookieContainer = null, string referer = null) {
if (string.IsNullOrEmpty(request)) {
return null;
}
string content = await UrlGetToContent(request, cookies, referer).ConfigureAwait(false);
string content = await UrlGetToContent(request, cookieContainer, referer).ConfigureAwait(false);
if (string.IsNullOrEmpty(content)) {
return null;
}
@@ -149,7 +181,7 @@ namespace ArchiSteamFarm {
try {
jObject = JObject.Parse(content);
} catch (Exception e) {
} catch (JsonException e) {
Logging.LogGenericException(e);
return null;
}
@@ -157,12 +189,12 @@ namespace ArchiSteamFarm {
return jObject;
}
internal static async Task<XmlDocument> UrlGetToXML(string request, Dictionary<string, string> cookies = null, string referer = null) {
internal static async Task<XmlDocument> UrlGetToXML(string request, CookieContainer cookieContainer = null, string referer = null) {
if (string.IsNullOrEmpty(request)) {
return null;
}
string content = await UrlGetToContent(request, cookies, referer).ConfigureAwait(false);
string content = await UrlGetToContent(request, cookieContainer, referer).ConfigureAwait(false);
if (string.IsNullOrEmpty(content)) {
return null;
}
@@ -179,7 +211,7 @@ namespace ArchiSteamFarm {
return xmlDocument;
}
private static async Task<HttpResponseMessage> UrlRequest(string request, HttpMethod httpMethod, Dictionary<string, string> data = null, Dictionary<string, string> cookies = null, string referer = null) {
private static async Task<HttpResponseMessage> UrlRequest(string request, HttpMethod httpMethod, Dictionary<string, string> data = null, CookieContainer cookieContainer = null, string referer = null) {
if (string.IsNullOrEmpty(request) || httpMethod == null) {
return null;
}
@@ -199,34 +231,33 @@ namespace ArchiSteamFarm {
}
}
if (cookies != null && cookies.Count > 0) {
StringBuilder cookieHeader = new StringBuilder();
foreach (KeyValuePair<string, string> cookie in cookies) {
cookieHeader.Append(cookie.Key + "=" + cookie.Value + ";");
}
requestMessage.Headers.Add("Cookie", cookieHeader.ToString());
}
if (!string.IsNullOrEmpty(referer)) {
requestMessage.Headers.Referrer = new Uri(referer);
}
try {
responseMessage = await HttpClient.SendAsync(requestMessage).ConfigureAwait(false);
if (cookieContainer == null) {
responseMessage = await HttpClient.SendAsync(requestMessage).ConfigureAwait(false);
} else {
using (HttpClientHandler httpClientHandler = new HttpClientHandler {
AutomaticDecompression = HttpClientHandler.AutomaticDecompression,
CookieContainer = cookieContainer
}) using (HttpClient httpClient = new HttpClient(httpClientHandler)) {
responseMessage = await httpClient.SendAsync(requestMessage).ConfigureAwait(false);
}
}
} catch { // Request failed, we don't need to know the exact reason, swallow exception
return null;
}
}
if (responseMessage == null) {
return null;
}
if (!responseMessage.IsSuccessStatusCode) {
if (responseMessage == null || !responseMessage.IsSuccessStatusCode) {
if (Debugging.IsDebugBuild || Program.GlobalConfig.Debug) {
Logging.LogGenericError("Request: " + request + " failed!");
Logging.LogGenericError("Status code: " + responseMessage.StatusCode);
Logging.LogGenericError("Content: " + Environment.NewLine + await responseMessage.Content.ReadAsStringAsync().ConfigureAwait(false));
if (responseMessage != null) {
Logging.LogGenericError("Status code: " + responseMessage.StatusCode);
Logging.LogGenericError("Content: " + Environment.NewLine + await responseMessage.Content.ReadAsStringAsync().ConfigureAwait(false));
}
}
return null;
}