From fbe5bd12ae5935e4ddec9d247a7048c48fbee286 Mon Sep 17 00:00:00 2001 From: Vitaliy Date: Thu, 2 Apr 2020 18:01:55 +0300 Subject: [PATCH] Use streams for http responses instead of strings where possible, replace HtmlAgilityPack with AngleSharp (#1703) * Replace HAP with AngleSharp, add stream support to WebBrowser * Fix skipped nullable operator * Add extension method * Rename function to be closer to HAP API * Add JSON deserialization from stream, fix variable names, remove obsolete code * Add more extension methods * Fixes after review: Remove excessive dependency Move string value to const Different handling for null and empty cases for confirmations Use more human-friendly names * Add http completion options, make GetToStream private * Cleanup * Add null checks, make StreamResponse disposable * Refactor UrlGetToBinaryWithProgress into using UrlGetToStream --- ArchiSteamFarm/ArchiSteamFarm.csproj | 2 +- ArchiSteamFarm/ArchiWebHandler.cs | 61 ++++----- ArchiSteamFarm/CardsFarmer.cs | 50 ++++---- ArchiSteamFarm/Json/Steam.cs | 12 +- ArchiSteamFarm/MobileAuthenticator.cs | 21 ++-- ArchiSteamFarm/SteamSaleEvent.cs | 8 +- ArchiSteamFarm/Utilities.cs | 31 +++++ ArchiSteamFarm/WebBrowser.cs | 174 ++++++++++++++++++-------- 8 files changed, 232 insertions(+), 127 deletions(-) diff --git a/ArchiSteamFarm/ArchiSteamFarm.csproj b/ArchiSteamFarm/ArchiSteamFarm.csproj index 523f2cad1..90cfa7ffc 100644 --- a/ArchiSteamFarm/ArchiSteamFarm.csproj +++ b/ArchiSteamFarm/ArchiSteamFarm.csproj @@ -51,11 +51,11 @@ + all - diff --git a/ArchiSteamFarm/ArchiWebHandler.cs b/ArchiSteamFarm/ArchiWebHandler.cs index 670908285..19f96fc3d 100644 --- a/ArchiSteamFarm/ArchiWebHandler.cs +++ b/ArchiSteamFarm/ArchiWebHandler.cs @@ -30,10 +30,10 @@ using System.Text; using System.Threading; using System.Threading.Tasks; using System.Xml; +using AngleSharp.Dom; using ArchiSteamFarm.Helpers; using ArchiSteamFarm.Json; using ArchiSteamFarm.Localization; -using HtmlAgilityPack; using JetBrains.Annotations; using Newtonsoft.Json; using Newtonsoft.Json.Linq; @@ -456,7 +456,7 @@ namespace ArchiSteamFarm { } [PublicAPI] - public async Task UrlGetToHtmlDocumentWithSession(string host, string request, bool checkSessionPreemptively = true, byte maxTries = WebBrowser.MaxTries) { + public async Task UrlGetToHtmlDocumentWithSession(string host, string request, bool checkSessionPreemptively = true, byte maxTries = WebBrowser.MaxTries) { if (string.IsNullOrEmpty(host) || string.IsNullOrEmpty(request)) { Bot.ArchiLogger.LogNullError(nameof(host) + " || " + nameof(request)); @@ -756,7 +756,7 @@ namespace ArchiSteamFarm { } [PublicAPI] - public async Task UrlPostToHtmlDocumentWithSession(string host, string request, Dictionary data = null, string referer = null, ESession session = ESession.Lowercase, bool checkSessionPreemptively = true, byte maxTries = WebBrowser.MaxTries) { + public async Task UrlPostToHtmlDocumentWithSession(string host, string request, Dictionary data = null, string referer = null, ESession session = ESession.Lowercase, bool checkSessionPreemptively = true, byte maxTries = WebBrowser.MaxTries) { if (string.IsNullOrEmpty(host) || string.IsNullOrEmpty(request) || !Enum.IsDefined(typeof(ESession), session)) { Bot.ArchiLogger.LogNullError(nameof(host) + " || " + nameof(request) + " || " + nameof(session)); @@ -1306,9 +1306,9 @@ namespace ArchiSteamFarm { { "subid", subID.ToString() } }; - HtmlDocument htmlDocument = await UrlPostToHtmlDocumentWithSession(SteamStoreURL, request, data).ConfigureAwait(false); + IDocument htmlDocument = await UrlPostToHtmlDocumentWithSession(SteamStoreURL, request, data).ConfigureAwait(false); - return htmlDocument?.DocumentNode.SelectSingleNode("//div[@class='add_free_content_success_area']") != null; + return htmlDocument?.SelectSingleNode("//div[@class='add_free_content_success_area']") != null; } internal async Task ChangePrivacySettings(Steam.UserPrivacy userPrivacy) { @@ -1647,7 +1647,7 @@ namespace ArchiSteamFarm { return result; } - internal async Task GetBadgePage(byte page) { + internal async Task GetBadgePage(byte page) { if (page == 0) { Bot.ArchiLogger.LogNullError(nameof(page)); @@ -1692,7 +1692,7 @@ namespace ArchiSteamFarm { return response; } - internal async Task GetConfirmations(string deviceID, string confirmationHash, uint time) { + internal async Task GetConfirmations(string deviceID, string confirmationHash, uint time) { if (string.IsNullOrEmpty(deviceID) || string.IsNullOrEmpty(confirmationHash) || (time == 0)) { Bot.ArchiLogger.LogNullError(nameof(deviceID) + " || " + nameof(confirmationHash) + " || " + nameof(time)); @@ -1719,21 +1719,21 @@ namespace ArchiSteamFarm { [ItemCanBeNull] internal async Task> GetDigitalGiftCards() { const string request = "/gifts"; - HtmlDocument response = await UrlGetToHtmlDocumentWithSession(SteamStoreURL, request).ConfigureAwait(false); + IDocument response = await UrlGetToHtmlDocumentWithSession(SteamStoreURL, request).ConfigureAwait(false); if (response == null) { return null; } - HtmlNodeCollection htmlNodes = response.DocumentNode.SelectNodes("//div[@class='pending_gift']/div[starts-with(@id, 'pending_gift_')][count(div[@class='pending_giftcard_leftcol']) > 0]/@id"); + List htmlNodes = response.SelectNodes("//div[@class='pending_gift']/div[starts-with(@id, 'pending_gift_')][count(div[@class='pending_giftcard_leftcol']) > 0]/@id"); - if (htmlNodes == null) { + if (htmlNodes.Count == 0) { return new HashSet(0); } HashSet results = new HashSet(htmlNodes.Count); - foreach (string giftCardIDText in htmlNodes.Select(node => node.GetAttributeValue("id", null))) { + foreach (string giftCardIDText in htmlNodes.Select(node => node.GetAttributeValue("id"))) { if (string.IsNullOrEmpty(giftCardIDText)) { Bot.ArchiLogger.LogNullError(nameof(giftCardIDText)); @@ -1758,7 +1758,7 @@ namespace ArchiSteamFarm { return results; } - internal async Task GetDiscoveryQueuePage() { + internal async Task GetDiscoveryQueuePage() { const string request = "/explore?l=english"; return await UrlGetToHtmlDocumentWithSession(SteamStoreURL, request).ConfigureAwait(false); @@ -1767,22 +1767,22 @@ namespace ArchiSteamFarm { [ItemCanBeNull] internal async Task> GetFamilySharingSteamIDs() { const string request = "/account/managedevices?l=english"; - HtmlDocument htmlDocument = await UrlGetToHtmlDocumentWithSession(SteamStoreURL, request).ConfigureAwait(false); + IDocument htmlDocument = await UrlGetToHtmlDocumentWithSession(SteamStoreURL, request).ConfigureAwait(false); if (htmlDocument == null) { return null; } - HtmlNodeCollection htmlNodes = htmlDocument.DocumentNode.SelectNodes("(//table[@class='accountTable'])[2]//a/@data-miniprofile"); + List htmlNodes = htmlDocument.SelectNodes("(//table[@class='accountTable'])[2]//a/@data-miniprofile"); - if (htmlNodes == null) { + if (htmlNodes.Count == 0) { // OK, no authorized steamIDs return new HashSet(0); } HashSet result = new HashSet(htmlNodes.Count); - foreach (string miniProfile in htmlNodes.Select(htmlNode => htmlNode.GetAttributeValue("data-miniprofile", null))) { + foreach (string miniProfile in htmlNodes.Select(htmlNode => htmlNode.GetAttributeValue("data-miniprofile"))) { if (string.IsNullOrEmpty(miniProfile)) { Bot.ArchiLogger.LogNullError(nameof(miniProfile)); @@ -1802,7 +1802,7 @@ namespace ArchiSteamFarm { return result; } - internal async Task GetGameCardsPage(ulong appID) { + internal async Task GetGameCardsPage(ulong appID) { if (appID == 0) { Bot.ArchiLogger.LogNullError(nameof(appID)); @@ -1862,16 +1862,16 @@ namespace ArchiSteamFarm { string request = "/tradeoffer/" + tradeID + "?l=english"; - HtmlDocument htmlDocument = await UrlGetToHtmlDocumentWithSession(SteamCommunityURL, request).ConfigureAwait(false); + IDocument htmlDocument = await UrlGetToHtmlDocumentWithSession(SteamCommunityURL, request).ConfigureAwait(false); - HtmlNode htmlNode = htmlDocument?.DocumentNode.SelectSingleNode("//div[@class='pagecontent']/script"); + IElement htmlNode = htmlDocument?.SelectSingleNode("//div[@class='pagecontent']/script"); if (htmlNode == null) { // Trade can be no longer valid return null; } - string text = htmlNode.InnerText; + string text = htmlNode.TextContent; if (string.IsNullOrEmpty(text)) { Bot.ArchiLogger.LogNullError(nameof(text)); @@ -1879,7 +1879,8 @@ namespace ArchiSteamFarm { return null; } - int index = text.IndexOf("g_daysTheirEscrow = ", StringComparison.Ordinal); + const string daysTheirVariableName = "g_daysTheirEscrow = "; + int index = text.IndexOf(daysTheirVariableName, StringComparison.Ordinal); if (index < 0) { Bot.ArchiLogger.LogNullError(nameof(index)); @@ -1887,7 +1888,7 @@ namespace ArchiSteamFarm { return null; } - index += 20; + index += daysTheirVariableName.Length; text = text.Substring(index); index = text.IndexOf(';'); @@ -2312,15 +2313,15 @@ namespace ArchiSteamFarm { private async Task<(ESteamApiKeyState State, string Key)> GetApiKeyState() { const string request = "/dev/apikey?l=english"; - HtmlDocument htmlDocument = await UrlGetToHtmlDocumentWithSession(SteamCommunityURL, request).ConfigureAwait(false); + IDocument htmlDocument = await UrlGetToHtmlDocumentWithSession(SteamCommunityURL, request).ConfigureAwait(false); - HtmlNode titleNode = htmlDocument?.DocumentNode.SelectSingleNode("//div[@id='mainContents']/h2"); + IElement titleNode = htmlDocument?.SelectSingleNode("//div[@id='mainContents']/h2"); if (titleNode == null) { return (ESteamApiKeyState.Timeout, null); } - string title = titleNode.InnerText; + string title = titleNode.TextContent; if (string.IsNullOrEmpty(title)) { Bot.ArchiLogger.LogNullError(nameof(title)); @@ -2332,7 +2333,7 @@ namespace ArchiSteamFarm { return (ESteamApiKeyState.AccessDenied, null); } - HtmlNode htmlNode = htmlDocument.DocumentNode.SelectSingleNode("//div[@id='bodyContents_ex']/p"); + IElement htmlNode = htmlDocument.SelectSingleNode("//div[@id='bodyContents_ex']/p"); if (htmlNode == null) { Bot.ArchiLogger.LogNullError(nameof(htmlNode)); @@ -2340,7 +2341,7 @@ namespace ArchiSteamFarm { return (ESteamApiKeyState.Error, null); } - string text = htmlNode.InnerText; + string text = htmlNode.TextContent; if (string.IsNullOrEmpty(text)) { Bot.ArchiLogger.LogNullError(nameof(text)); @@ -2620,13 +2621,13 @@ namespace ArchiSteamFarm { private async Task<(bool Success, bool Result)> ResolvePublicInventory() { const string request = "/my/edit/settings?l=english"; - HtmlDocument htmlDocument = await UrlGetToHtmlDocumentWithSession(SteamCommunityURL, request, false).ConfigureAwait(false); + IDocument htmlDocument = await UrlGetToHtmlDocumentWithSession(SteamCommunityURL, request, false).ConfigureAwait(false); if (htmlDocument == null) { return (false, false); } - HtmlNode htmlNode = htmlDocument.DocumentNode.SelectSingleNode("//div[@data-component='ProfilePrivacySettings']/@data-privacysettings"); + IElement htmlNode = htmlDocument.SelectSingleNode("//div[@data-component='ProfilePrivacySettings']/@data-privacysettings"); if (htmlNode == null) { Bot.ArchiLogger.LogNullError(nameof(htmlNode)); @@ -2634,7 +2635,7 @@ namespace ArchiSteamFarm { return (false, false); } - string json = htmlNode.GetAttributeValue("data-privacysettings", null); + string json = htmlNode.GetAttributeValue("data-privacysettings"); if (string.IsNullOrEmpty(json)) { Bot.ArchiLogger.LogNullError(nameof(json)); diff --git a/ArchiSteamFarm/CardsFarmer.cs b/ArchiSteamFarm/CardsFarmer.cs index 0779ab93c..d8a07a0fb 100755 --- a/ArchiSteamFarm/CardsFarmer.cs +++ b/ArchiSteamFarm/CardsFarmer.cs @@ -30,10 +30,10 @@ using System.Net; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; +using AngleSharp.Dom; using ArchiSteamFarm.Collections; using ArchiSteamFarm.Localization; using ArchiSteamFarm.Plugins; -using HtmlAgilityPack; using JetBrains.Annotations; using Newtonsoft.Json; using SteamKit2; @@ -372,32 +372,32 @@ namespace ArchiSteamFarm { } [SuppressMessage("ReSharper", "FunctionComplexityOverflow")] - private async Task CheckPage(HtmlDocument htmlDocument, ISet parsedAppIDs) { + private async Task CheckPage(IDocument htmlDocument, ISet parsedAppIDs) { if ((htmlDocument == null) || (parsedAppIDs == null)) { Bot.ArchiLogger.LogNullError(nameof(htmlDocument) + " || " + nameof(parsedAppIDs)); return; } - HtmlNodeCollection htmlNodes = htmlDocument.DocumentNode.SelectNodes("//div[@class='badge_row_inner']"); + List htmlNodes = htmlDocument.SelectNodes("//div[@class='badge_row_inner']"); - if (htmlNodes == null) { + if (htmlNodes.Count == 0) { // No eligible badges whatsoever return; } HashSet backgroundTasks = null; - foreach (HtmlNode htmlNode in htmlNodes) { - HtmlNode statsNode = htmlNode.SelectSingleNode(".//div[@class='badge_title_stats_content']"); - HtmlNode appIDNode = statsNode?.SelectSingleNode(".//div[@class='card_drop_info_dialog']"); + foreach (IElement htmlNode in htmlNodes) { + IElement statsNode = htmlNode.SelectSingleElementNode(".//div[@class='badge_title_stats_content']"); + IElement appIDNode = statsNode?.SelectSingleElementNode(".//div[@class='card_drop_info_dialog']"); if (appIDNode == null) { // It's just a badge, nothing more continue; } - string appIDText = appIDNode.GetAttributeValue("id", null); + string appIDText = appIDNode.GetAttributeValue("id"); if (string.IsNullOrEmpty(appIDText)) { Bot.ArchiLogger.LogNullError(nameof(appIDText)); @@ -454,7 +454,7 @@ namespace ArchiSteamFarm { } // Cards - HtmlNode progressNode = statsNode.SelectSingleNode(".//span[@class='progress_info_bold']"); + IElement progressNode = statsNode.SelectSingleElementNode(".//span[@class='progress_info_bold']"); if (progressNode == null) { Bot.ArchiLogger.LogNullError(nameof(progressNode)); @@ -462,7 +462,7 @@ namespace ArchiSteamFarm { continue; } - string progressText = progressNode.InnerText; + string progressText = progressNode.TextContent; if (string.IsNullOrEmpty(progressText)) { Bot.ArchiLogger.LogNullError(nameof(progressText)); @@ -493,7 +493,7 @@ namespace ArchiSteamFarm { } // To save us on extra work, check cards earned so far first - HtmlNode cardsEarnedNode = statsNode.SelectSingleNode(".//div[@class='card_drop_info_header']"); + IElement cardsEarnedNode = statsNode.SelectSingleElementNode(".//div[@class='card_drop_info_header']"); if (cardsEarnedNode == null) { Bot.ArchiLogger.LogNullError(nameof(cardsEarnedNode)); @@ -501,7 +501,7 @@ namespace ArchiSteamFarm { continue; } - string cardsEarnedText = cardsEarnedNode.InnerText; + string cardsEarnedText = cardsEarnedNode.TextContent; if (string.IsNullOrEmpty(cardsEarnedText)) { Bot.ArchiLogger.LogNullError(nameof(cardsEarnedText)); @@ -538,7 +538,7 @@ namespace ArchiSteamFarm { } // Hours - HtmlNode timeNode = statsNode.SelectSingleNode(".//div[@class='badge_title_stats_playtime']"); + IElement timeNode = statsNode.SelectSingleElementNode(".//div[@class='badge_title_stats_playtime']"); if (timeNode == null) { Bot.ArchiLogger.LogNullError(nameof(timeNode)); @@ -546,7 +546,7 @@ namespace ArchiSteamFarm { continue; } - string hoursText = timeNode.InnerText; + string hoursText = timeNode.TextContent; if (string.IsNullOrEmpty(hoursText)) { Bot.ArchiLogger.LogNullError(nameof(hoursText)); @@ -567,7 +567,7 @@ namespace ArchiSteamFarm { } // Names - HtmlNode nameNode = statsNode.SelectSingleNode("(.//div[@class='card_drop_info_body'])[last()]"); + IElement nameNode = statsNode.SelectSingleElementNode("(.//div[@class='card_drop_info_body'])[last()]"); if (nameNode == null) { Bot.ArchiLogger.LogNullError(nameof(nameNode)); @@ -575,7 +575,7 @@ namespace ArchiSteamFarm { continue; } - string name = nameNode.InnerText; + string name = nameNode.TextContent; if (string.IsNullOrEmpty(name)) { Bot.ArchiLogger.LogNullError(nameof(name)); @@ -619,11 +619,11 @@ namespace ArchiSteamFarm { // Levels byte badgeLevel = 0; - HtmlNode levelNode = htmlNode.SelectSingleNode(".//div[@class='badge_info_description']/div[2]"); + IElement levelNode = htmlNode.SelectSingleElementNode(".//div[@class='badge_info_description']/div[2]"); if (levelNode != null) { // There is no levelNode if we didn't craft that badge yet (level 0) - string levelText = levelNode.InnerText; + string levelText = levelNode.TextContent; if (string.IsNullOrEmpty(levelText)) { Bot.ArchiLogger.LogNullError(nameof(levelText)); @@ -694,7 +694,7 @@ namespace ArchiSteamFarm { return; } - HtmlDocument htmlDocument = await Bot.ArchiWebHandler.GetBadgePage(page).ConfigureAwait(false); + IDocument htmlDocument = await Bot.ArchiWebHandler.GetBadgePage(page).ConfigureAwait(false); if (htmlDocument == null) { return; @@ -942,15 +942,15 @@ namespace ArchiSteamFarm { return 0; } - HtmlDocument htmlDocument = await Bot.ArchiWebHandler.GetGameCardsPage(appID).ConfigureAwait(false); + IDocument htmlDocument = await Bot.ArchiWebHandler.GetGameCardsPage(appID).ConfigureAwait(false); - HtmlNode progressNode = htmlDocument?.DocumentNode.SelectSingleNode("//span[@class='progress_info_bold']"); + IElement progressNode = htmlDocument?.SelectSingleNode("//span[@class='progress_info_bold']"); if (progressNode == null) { return null; } - string progress = progressNode.InnerText; + string progress = progressNode.TextContent; if (string.IsNullOrEmpty(progress)) { Bot.ArchiLogger.LogNullError(nameof(progress)); @@ -976,7 +976,7 @@ namespace ArchiSteamFarm { private async Task IsAnythingToFarm() { // Find the number of badge pages Bot.ArchiLogger.LogGenericInfo(Strings.CheckingFirstBadgePage); - HtmlDocument htmlDocument = await Bot.ArchiWebHandler.GetBadgePage(1).ConfigureAwait(false); + IDocument htmlDocument = await Bot.ArchiWebHandler.GetBadgePage(1).ConfigureAwait(false); if (htmlDocument == null) { Bot.ArchiLogger.LogGenericWarning(Strings.WarningCouldNotCheckBadges); @@ -986,10 +986,10 @@ namespace ArchiSteamFarm { byte maxPages = 1; - HtmlNode htmlNode = htmlDocument.DocumentNode.SelectSingleNode("(//a[@class='pagelink'])[last()]"); + IElement htmlNode = htmlDocument.SelectSingleNode("(//a[@class='pagelink'])[last()]"); if (htmlNode != null) { - string lastPage = htmlNode.InnerText; + string lastPage = htmlNode.TextContent; if (string.IsNullOrEmpty(lastPage)) { Bot.ArchiLogger.LogNullError(nameof(lastPage)); diff --git a/ArchiSteamFarm/Json/Steam.cs b/ArchiSteamFarm/Json/Steam.cs index a6b686120..3ab166ec2 100644 --- a/ArchiSteamFarm/Json/Steam.cs +++ b/ArchiSteamFarm/Json/Steam.cs @@ -24,8 +24,8 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics.CodeAnalysis; using System.Linq; +using AngleSharp.Dom; using ArchiSteamFarm.Localization; -using HtmlAgilityPack; using JetBrains.Annotations; using Newtonsoft.Json; using Newtonsoft.Json.Linq; @@ -300,7 +300,7 @@ namespace ArchiSteamFarm.Json { return; } - HtmlDocument htmlDocument = WebBrowser.StringToHtmlDocument(value); + IDocument htmlDocument = WebBrowser.StringToHtmlDocument(value).Result; if (htmlDocument == null) { ASF.ArchiLogger.LogNullError(nameof(htmlDocument)); @@ -308,10 +308,10 @@ namespace ArchiSteamFarm.Json { return; } - if (htmlDocument.DocumentNode.SelectSingleNode("//div[@class='mobileconf_trade_area']") != null) { + if (htmlDocument.SelectSingleNode("//div[@class='mobileconf_trade_area']") != null) { Type = EType.Trade; - HtmlNode tradeOfferNode = htmlDocument.DocumentNode.SelectSingleNode("//div[@class='tradeoffer']"); + IElement tradeOfferNode = htmlDocument.SelectSingleNode("//div[@class='tradeoffer']"); if (tradeOfferNode == null) { ASF.ArchiLogger.LogNullError(nameof(tradeOfferNode)); @@ -319,7 +319,7 @@ namespace ArchiSteamFarm.Json { return; } - string idText = tradeOfferNode.GetAttributeValue("id", null); + string idText = tradeOfferNode.GetAttributeValue("id"); if (string.IsNullOrEmpty(idText)) { ASF.ArchiLogger.LogNullError(nameof(idText)); @@ -352,7 +352,7 @@ namespace ArchiSteamFarm.Json { } TradeOfferID = tradeOfferID; - } else if (htmlDocument.DocumentNode.SelectSingleNode("//div[@class='mobileconf_listing_prices']") != null) { + } else if (htmlDocument.SelectSingleNode("//div[@class='mobileconf_listing_prices']") != null) { Type = EType.Market; } else { // Normally this should be reported, but under some specific circumstances we might actually receive this one diff --git a/ArchiSteamFarm/MobileAuthenticator.cs b/ArchiSteamFarm/MobileAuthenticator.cs index 31bea0a1d..4be238530 100644 --- a/ArchiSteamFarm/MobileAuthenticator.cs +++ b/ArchiSteamFarm/MobileAuthenticator.cs @@ -26,9 +26,9 @@ using System.Security.Cryptography; using System.Text; using System.Threading; using System.Threading.Tasks; +using AngleSharp.Dom; using ArchiSteamFarm.Json; using ArchiSteamFarm.Localization; -using HtmlAgilityPack; using JetBrains.Annotations; using Newtonsoft.Json; @@ -155,18 +155,21 @@ namespace ArchiSteamFarm { await LimitConfirmationsRequestsAsync().ConfigureAwait(false); - HtmlDocument htmlDocument = await Bot.ArchiWebHandler.GetConfirmations(DeviceID, confirmationHash, time).ConfigureAwait(false); + IDocument htmlDocument = await Bot.ArchiWebHandler.GetConfirmations(DeviceID, confirmationHash, time).ConfigureAwait(false); - HtmlNodeCollection confirmationNodes = htmlDocument?.DocumentNode.SelectNodes("//div[@class='mobileconf_list_entry']"); - - if (confirmationNodes == null) { + if (htmlDocument == null) { return null; } HashSet result = new HashSet(); + List confirmationNodes = htmlDocument.SelectNodes("//div[@class='mobileconf_list_entry']"); - foreach (HtmlNode confirmationNode in confirmationNodes) { - string idText = confirmationNode.GetAttributeValue("data-confid", null); + if (confirmationNodes.Count == 0) { + return result; + } + + foreach (IElement confirmationNode in confirmationNodes) { + string idText = confirmationNode.GetAttributeValue("data-confid"); if (string.IsNullOrEmpty(idText)) { Bot.ArchiLogger.LogNullError(nameof(idText)); @@ -180,7 +183,7 @@ namespace ArchiSteamFarm { return null; } - string keyText = confirmationNode.GetAttributeValue("data-key", null); + string keyText = confirmationNode.GetAttributeValue("data-key"); if (string.IsNullOrEmpty(keyText)) { Bot.ArchiLogger.LogNullError(nameof(keyText)); @@ -194,7 +197,7 @@ namespace ArchiSteamFarm { return null; } - string typeText = confirmationNode.GetAttributeValue("data-type", null); + string typeText = confirmationNode.GetAttributeValue("data-type"); if (string.IsNullOrEmpty(typeText)) { Bot.ArchiLogger.LogNullError(nameof(typeText)); diff --git a/ArchiSteamFarm/SteamSaleEvent.cs b/ArchiSteamFarm/SteamSaleEvent.cs index 607f0b9b5..57ca0f729 100644 --- a/ArchiSteamFarm/SteamSaleEvent.cs +++ b/ArchiSteamFarm/SteamSaleEvent.cs @@ -23,8 +23,8 @@ using System; using System.Collections.Immutable; using System.Threading; using System.Threading.Tasks; +using AngleSharp.Dom; using ArchiSteamFarm.Localization; -using HtmlAgilityPack; using JetBrains.Annotations; namespace ArchiSteamFarm { @@ -83,20 +83,20 @@ namespace ArchiSteamFarm { } private async Task IsDiscoveryQueueAvailable() { - HtmlDocument htmlDocument = await Bot.ArchiWebHandler.GetDiscoveryQueuePage().ConfigureAwait(false); + IDocument htmlDocument = await Bot.ArchiWebHandler.GetDiscoveryQueuePage().ConfigureAwait(false); if (htmlDocument == null) { return null; } - HtmlNode htmlNode = htmlDocument.DocumentNode.SelectSingleNode("//div[@class='subtext']"); + IElement htmlNode = htmlDocument.SelectSingleNode("//div[@class='subtext']"); if (htmlNode == null) { // Valid, no cards for exploring the queue available return false; } - string text = htmlNode.InnerText; + string text = htmlNode.TextContent; if (string.IsNullOrEmpty(text)) { Bot.ArchiLogger.LogNullError(nameof(text)); diff --git a/ArchiSteamFarm/Utilities.cs b/ArchiSteamFarm/Utilities.cs index f247b2f60..7d0a2de07 100644 --- a/ArchiSteamFarm/Utilities.cs +++ b/ArchiSteamFarm/Utilities.cs @@ -27,6 +27,8 @@ using System.Net; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; +using AngleSharp.Dom; +using AngleSharp.XPath; using Humanizer; using Humanizer.Localisation; using JetBrains.Annotations; @@ -60,6 +62,17 @@ namespace ArchiSteamFarm { return args[args.Length - 1]; } + [PublicAPI] + public static string GetAttributeValue(this INode node, string attributeName) { + if ((node == null) || string.IsNullOrEmpty(attributeName)) { + ASF.ArchiLogger.LogNullError(nameof(node) + " || " + nameof(attributeName)); + + return null; + } + + return node is IElement element ? element.GetAttribute(attributeName) : null; + } + [PublicAPI] public static uint GetUnixTime() => (uint) DateTimeOffset.UtcNow.ToUnixTimeSeconds(); @@ -183,6 +196,24 @@ namespace ArchiSteamFarm { return (text.Length % 2 == 0) && text.All(Uri.IsHexDigit); } + [ItemNotNull] + [NotNull] + [PublicAPI] + public static List SelectElementNodes([NotNull] this IElement element, string xpath) => element.SelectNodes(xpath).Cast().ToList(); + + [ItemNotNull] + [NotNull] + [PublicAPI] + public static List SelectNodes([NotNull] this IDocument document, string xpath) => document.Body.SelectNodes(xpath).Cast().ToList(); + + [CanBeNull] + [PublicAPI] + public static IElement SelectSingleElementNode([NotNull] this IElement element, string xpath) => (IElement) element.SelectSingleNode(xpath); + + [CanBeNull] + [PublicAPI] + public static IElement SelectSingleNode([NotNull] this IDocument document, string xpath) => (IElement) document.Body.SelectSingleNode(xpath); + [PublicAPI] public static IEnumerable ToEnumerable(this T item) { yield return item; diff --git a/ArchiSteamFarm/WebBrowser.cs b/ArchiSteamFarm/WebBrowser.cs index 08cc92dd4..bf821fbbd 100644 --- a/ArchiSteamFarm/WebBrowser.cs +++ b/ArchiSteamFarm/WebBrowser.cs @@ -27,9 +27,10 @@ using System.Net; using System.Net.Http; using System.Threading.Tasks; using System.Xml; +using AngleSharp; +using AngleSharp.Dom; using ArchiSteamFarm.Localization; using ArchiSteamFarm.NLog; -using HtmlAgilityPack; using JetBrains.Annotations; using Newtonsoft.Json; @@ -104,9 +105,9 @@ namespace ArchiSteamFarm { return null; } - StringResponse response = await UrlGetToString(request, referer, requestOptions, maxTries).ConfigureAwait(false); + using StreamResponse response = await UrlGetToStream(request, referer, requestOptions, maxTries).ConfigureAwait(false); - return response != null ? new HtmlDocumentResponse(response) : null; + return response != null ? await HtmlDocumentResponse.Create(response).ConfigureAwait(false) : null; } [ItemCanBeNull] @@ -121,7 +122,7 @@ namespace ArchiSteamFarm { ObjectResponse result = null; for (byte i = 0; i < maxTries; i++) { - StringResponse response = await UrlGetToString(request, referer, requestOptions | ERequestOptions.ReturnClientErrors, 1).ConfigureAwait(false); + using StreamResponse response = await UrlGetToStream(request, referer, requestOptions | ERequestOptions.ReturnClientErrors, 1).ConfigureAwait(false); // ReSharper disable once UseNullPropagationWhenPossible - false check if (response == null) { @@ -136,14 +137,18 @@ namespace ArchiSteamFarm { break; } - if (string.IsNullOrEmpty(response.Content)) { + if (response.Content == null) { continue; } T obj; try { - obj = JsonConvert.DeserializeObject(response.Content); + using StreamReader streamReader = new StreamReader(response.Content); + using JsonReader jsonReader = new JsonTextReader(streamReader); + JsonSerializer serializer = new JsonSerializer(); + + obj = serializer.Deserialize(jsonReader); } catch (JsonException e) { ArchiLogger.LogGenericWarningException(e); @@ -177,7 +182,7 @@ namespace ArchiSteamFarm { XmlDocumentResponse result = null; for (byte i = 0; i < maxTries; i++) { - StringResponse response = await UrlGetToString(request, referer, requestOptions | ERequestOptions.ReturnClientErrors, 1).ConfigureAwait(false); + using StreamResponse response = await UrlGetToStream(request, referer, requestOptions | ERequestOptions.ReturnClientErrors, 1).ConfigureAwait(false); // ReSharper disable once UseNullPropagationWhenPossible - false check if (response == null) { @@ -192,14 +197,14 @@ namespace ArchiSteamFarm { break; } - if (string.IsNullOrEmpty(response.Content)) { + if (response.Content == null) { continue; } XmlDocument xmlDocument = new XmlDocument(); try { - xmlDocument.LoadXml(response.Content); + xmlDocument.Load(response.Content); } catch (XmlException e) { ArchiLogger.LogGenericWarningException(e); @@ -300,9 +305,9 @@ namespace ArchiSteamFarm { return null; } - StringResponse response = await UrlPostToString(request, data, referer, requestOptions, maxTries).ConfigureAwait(false); + using StreamResponse response = await UrlPostToStream(request, data, referer, requestOptions, maxTries).ConfigureAwait(false); - return response != null ? new HtmlDocumentResponse(response) : null; + return response != null ? await HtmlDocumentResponse.Create(response).ConfigureAwait(false) : null; } [ItemCanBeNull] @@ -317,7 +322,7 @@ namespace ArchiSteamFarm { ObjectResponse result = null; for (byte i = 0; i < maxTries; i++) { - StringResponse response = await UrlPostToString(request, data, referer, requestOptions | ERequestOptions.ReturnClientErrors, 1).ConfigureAwait(false); + using StreamResponse response = await UrlPostToStream(request, data, referer, requestOptions | ERequestOptions.ReturnClientErrors, 1).ConfigureAwait(false); if (response == null) { return null; @@ -331,14 +336,18 @@ namespace ArchiSteamFarm { break; } - if (string.IsNullOrEmpty(response.Content)) { + if (response.Content == null) { continue; } T obj; try { - obj = JsonConvert.DeserializeObject(response.Content); + using StreamReader steamReader = new StreamReader(response.Content); + using JsonReader jsonReader = new JsonTextReader(steamReader); + JsonSerializer serializer = new JsonSerializer(); + + obj = serializer.Deserialize(jsonReader); } catch (JsonException e) { ArchiLogger.LogGenericWarningException(e); @@ -376,17 +385,16 @@ namespace ArchiSteamFarm { } } - internal static HtmlDocument StringToHtmlDocument(string html) { + internal static async Task StringToHtmlDocument(string html) { if (html == null) { ASF.ArchiLogger.LogNullError(nameof(html)); return null; } - HtmlDocument htmlDocument = new HtmlDocument(); - htmlDocument.LoadHtml(html); + IBrowsingContext context = BrowsingContext.New(Configuration.Default.WithXPath()); - return htmlDocument; + return await context.OpenAsync(req => req.Content(html)).ConfigureAwait(false); } [ItemCanBeNull] @@ -403,7 +411,8 @@ namespace ArchiSteamFarm { const byte printPercentage = 10; const byte maxBatches = 99 / printPercentage; - using HttpResponseMessage response = await InternalGet(request, referer, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false); + //using HttpResponseMessage response = await InternalGet(request, referer, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false); + using StreamResponse response = await UrlGetToStream(request, referer, requestOptions).ConfigureAwait(false); if (response == null) { continue; @@ -419,12 +428,10 @@ namespace ArchiSteamFarm { ArchiLogger.LogGenericDebug("0%..."); - uint contentLength = (uint) response.Content.Headers.ContentLength.GetValueOrDefault(); - - using MemoryStream ms = new MemoryStream((int) contentLength); + using MemoryStream ms = new MemoryStream((int) response.Length); try { - using Stream contentStream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false); + using Stream contentStream = response.Content; byte batch = 0; uint readThisBatch = 0; @@ -439,17 +446,17 @@ namespace ArchiSteamFarm { await ms.WriteAsync(buffer, 0, read).ConfigureAwait(false); - if ((contentLength == 0) || (batch >= maxBatches)) { + if ((response.Length == 0) || (batch >= maxBatches)) { continue; } readThisBatch += (uint) read; - if (readThisBatch < contentLength / printPercentage) { + if (readThisBatch < response.Length / printPercentage) { continue; } - readThisBatch -= contentLength / printPercentage; + readThisBatch -= response.Length / printPercentage; ArchiLogger.LogGenericDebug((++batch * printPercentage) + "%..."); } } catch (Exception e) { @@ -507,14 +514,14 @@ namespace ArchiSteamFarm { return result; } - private async Task InternalGet(string request, string referer = null, HttpCompletionOption httpCompletionOptions = HttpCompletionOption.ResponseContentRead) { + private async Task InternalGet(string request, string referer = null, HttpCompletionOption httpCompletionOption = HttpCompletionOption.ResponseContentRead) { if (string.IsNullOrEmpty(request)) { ArchiLogger.LogNullError(nameof(request)); return null; } - return await InternalRequest(new Uri(request), HttpMethod.Get, null, referer, httpCompletionOptions).ConfigureAwait(false); + return await InternalRequest(new Uri(request), HttpMethod.Get, null, referer, httpCompletionOption).ConfigureAwait(false); } private async Task InternalHead(string request, string referer = null) { @@ -527,14 +534,14 @@ namespace ArchiSteamFarm { return await InternalRequest(new Uri(request), HttpMethod.Head, null, referer).ConfigureAwait(false); } - private async Task InternalPost(string request, IReadOnlyCollection> data = null, string referer = null) { + private async Task InternalPost(string request, IReadOnlyCollection> data = null, string referer = null, HttpCompletionOption httpCompletionOption = HttpCompletionOption.ResponseContentRead) { if (string.IsNullOrEmpty(request)) { ArchiLogger.LogNullError(nameof(request)); return null; } - return await InternalRequest(new Uri(request), HttpMethod.Post, data, referer).ConfigureAwait(false); + return await InternalRequest(new Uri(request), HttpMethod.Post, data, referer, httpCompletionOption).ConfigureAwait(false); } private async Task InternalRequest(Uri requestUri, HttpMethod httpMethod, IReadOnlyCollection> data = null, string referer = null, HttpCompletionOption httpCompletionOption = HttpCompletionOption.ResponseContentRead, byte maxRedirections = MaxTries) { @@ -643,17 +650,17 @@ namespace ArchiSteamFarm { } [ItemCanBeNull] - private async Task UrlPostToString(string request, IReadOnlyCollection> data = null, string referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries) { + private async Task UrlGetToStream(string request, string referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries) { if (string.IsNullOrEmpty(request) || (maxTries == 0)) { ArchiLogger.LogNullError(nameof(request) + " || " + nameof(maxTries)); return null; } - StringResponse result = null; + StreamResponse result = null; for (byte i = 0; i < maxTries; i++) { - using HttpResponseMessage response = await InternalPost(request, data, referer).ConfigureAwait(false); + HttpResponseMessage response = await InternalGet(request, referer, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false); if (response == null) { continue; @@ -661,13 +668,49 @@ namespace ArchiSteamFarm { if (response.StatusCode.IsClientErrorCode()) { if (requestOptions.HasFlag(ERequestOptions.ReturnClientErrors)) { - result = new StringResponse(response); + result = new StreamResponse(response); } break; } - return new StringResponse(response, await response.Content.ReadAsStringAsync().ConfigureAwait(false)); + return new StreamResponse(response, await response.Content.ReadAsStreamAsync().ConfigureAwait(false)); + } + + if (maxTries > 1) { + ArchiLogger.LogGenericWarning(string.Format(Strings.ErrorRequestFailedTooManyTimes, maxTries)); + ArchiLogger.LogGenericDebug(string.Format(Strings.ErrorFailingRequest, request)); + } + + return result; + } + + [ItemCanBeNull] + private async Task UrlPostToStream(string request, IReadOnlyCollection> data = null, string referer = null, ERequestOptions requestOptions = ERequestOptions.None, byte maxTries = MaxTries) { + if (string.IsNullOrEmpty(request) || (maxTries == 0)) { + ArchiLogger.LogNullError(nameof(request) + " || " + nameof(maxTries)); + + return null; + } + + StreamResponse result = null; + + for (byte i = 0; i < maxTries; i++) { + HttpResponseMessage response = await InternalPost(request, data, referer, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false); + + if (response == null) { + continue; + } + + if (response.StatusCode.IsClientErrorCode()) { + if (requestOptions.HasFlag(ERequestOptions.ReturnClientErrors)) { + result = new StreamResponse(response); + } + + break; + } + + return new StreamResponse(response, await response.Content.ReadAsStreamAsync().ConfigureAwait(false)); } if (maxTries > 1) { @@ -705,16 +748,20 @@ namespace ArchiSteamFarm { public sealed class HtmlDocumentResponse : BasicResponse { [PublicAPI] - public readonly HtmlDocument Content; + public readonly IDocument Content; - internal HtmlDocumentResponse([NotNull] StringResponse stringResponse) : base(stringResponse) { - if (stringResponse == null) { - throw new ArgumentNullException(nameof(stringResponse)); + private HtmlDocumentResponse(BasicResponse streamResponse, IDocument document) : base(streamResponse) => Content = document; + + [ItemNotNull] + internal static async Task Create([NotNull] StreamResponse streamResponse) { + if (streamResponse == null) { + throw new ArgumentNullException(nameof(streamResponse)); } - if (!string.IsNullOrEmpty(stringResponse.Content)) { - Content = StringToHtmlDocument(stringResponse.Content); - } + IBrowsingContext context = BrowsingContext.New(Configuration.Default.WithXPath()); + IDocument document = await context.OpenAsync(req => req.Content(streamResponse.Content, true)).ConfigureAwait(false); + + return new HtmlDocumentResponse(streamResponse, document); } } @@ -722,9 +769,9 @@ namespace ArchiSteamFarm { [PublicAPI] public readonly T Content; - internal ObjectResponse([NotNull] StringResponse stringResponse, T content) : base(stringResponse) { - if (stringResponse == null) { - throw new ArgumentNullException(nameof(stringResponse)); + internal ObjectResponse([NotNull] StreamResponse streamResponse, T content) : base(streamResponse) { + if (streamResponse == null) { + throw new ArgumentNullException(nameof(streamResponse)); } Content = content; @@ -737,9 +784,9 @@ namespace ArchiSteamFarm { [PublicAPI] public readonly XmlDocument Content; - internal XmlDocumentResponse([NotNull] StringResponse stringResponse, XmlDocument content) : base(stringResponse) { - if (stringResponse == null) { - throw new ArgumentNullException(nameof(stringResponse)); + internal XmlDocumentResponse([NotNull] StreamResponse streamResponse, XmlDocument content) : base(streamResponse) { + if (streamResponse == null) { + throw new ArgumentNullException(nameof(streamResponse)); } Content = content; @@ -757,15 +804,38 @@ namespace ArchiSteamFarm { internal sealed class BinaryResponse : BasicResponse { internal readonly byte[] Content; - internal BinaryResponse([NotNull] HttpResponseMessage httpResponseMessage, [NotNull] byte[] content) : base(httpResponseMessage) { - if ((httpResponseMessage == null) || (content == null)) { - throw new ArgumentNullException(nameof(httpResponseMessage) + " || " + nameof(content)); + internal BinaryResponse([NotNull] BasicResponse basicResponse, [NotNull] byte[] content) : base(basicResponse) { + if ((basicResponse == null) || (content == null)) { + throw new ArgumentNullException(nameof(basicResponse) + " || " + nameof(content)); } Content = content; } - internal BinaryResponse([NotNull] HttpResponseMessage httpResponseMessage) : base(httpResponseMessage) { } + internal BinaryResponse([NotNull] BasicResponse basicResponse) : base(basicResponse) { } + } + + internal sealed class StreamResponse : BasicResponse, IDisposable { + internal readonly Stream Content; + internal readonly uint Length; + private readonly HttpResponseMessage ResponseMessage; + + internal StreamResponse([NotNull] HttpResponseMessage httpResponseMessage, [NotNull] Stream content) : base(httpResponseMessage) { + if ((httpResponseMessage == null) || (content == null)) { + throw new ArgumentNullException(nameof(httpResponseMessage) + " || " + nameof(content)); + } + + Content = content; + Length = (uint) httpResponseMessage.Content.Headers.ContentLength.GetValueOrDefault(); + ResponseMessage = httpResponseMessage; + } + + internal StreamResponse([NotNull] HttpResponseMessage httpResponseMessage) : base(httpResponseMessage) { } + + public void Dispose() { + Content.Dispose(); + ResponseMessage.Dispose(); + } } internal sealed class StringResponse : BasicResponse {