Akatsuki-Misaki/ArchiSteamFarm
1
0
Fork 0
mirror of https://github.com/JustArchiNET/ArchiSteamFarm.git synced 2026-01-06 17:10:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rewrite SendMessage() functions to account for new rate-limits (#2335)

Browse Source 
* Rewrite SendMessage() functions to account for new rate-limits

* Refactor new message splitting logic into SteamChatMessage.cs

This makes it ready for unit tests

* Change the concept into UTF8-oriented logic

* Misc

* Add fix for another unit test

* Update

* Fix failing test

I SPENT HOURS ON THIS

* Misc

* Misc

* Add additional unit tests ensuring this works as designed

* Misc

* Misc

* Add one more unit test

* Rework the logic to account for new findings

* Misc

* Add unit test verifying exception on too long prefix

* Address first @Abrynos concern

Because we can

* Throw also on too long prefix in regards to newlines

* Correct wrong bytesRead calculation

This worked previously only because we actually never had enough of room for escaped chars anyway and skipped over (2 + 2 missing bytes was smaller than 5 required to make a line)

* Add unit test verifying if calculation was done properly

* Address @Ryzhehvost concern

* Handle empty newlines in the message properly

* Misc

No reason to even calculate utf8 bytes for empty lines

* Misc

* Add unit test verifying the reserved escape message bytes count

* Correct calculation of lines by taking into account \r

* Update ArchiSteamFarm/Steam/Bot.cs

Co-authored-by: Sebastian Göls <6608231+Abrynos@users.noreply.github.com>

* @Abrynos next time check if it compiles without warnings

* Update SteamChatMessage.cs

* Apply @Abrynos idea in a different way

* Rewrite bot part to remove unnecessary delegate

* Add @Ryzhehvost test

* Add debug output

* Extend @Ryzhehvost test for prefix

* Misc

* Misc refactor

* Misc

* Misc

* Add logic for limited accounts, correct for unlimited

Thanks @Ryzhehvost

* Misc

Co-authored-by: Sebastian Göls <6608231+Abrynos@users.noreply.github.com>
This commit is contained in:
Łukasz Domeradzki
2021-06-18 19:50:14 +02:00
committed by GitHub
parent 4367134380
commit 2aab56b775
4 changed files with 599 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

222
ArchiSteamFarm/Steam/Bot.cs
View File

@@ -64,7 +64,6 @@ using SteamKit2.Internal;
namespace ArchiSteamFarm.Steam {
public sealed class Bot : IAsyncDisposable {
internal const ushort CallbackSleep = 500; // In milliseconds
internal const ushort MaxMessagePrefixLength = MaxMessageLength - ReservedMessageLength - 2; // 2 for a minimum of 2 characters (escape one and real one)
internal const byte MinCardsPerBadge = 5;
internal const byte MinPlayingBlockedTTL = 60; // Delay in seconds added when account was occupied during our disconnect, to not disconnect other Steam client session too soon
@@ -72,10 +71,8 @@ namespace ArchiSteamFarm.Steam {
private const byte LoginCooldownInMinutes = 25; // Captcha disappears after around 20 minutes, so we make it 25
private const uint LoginID = 1242; // This must be the same for all ASF bots and all ASF processes
private const byte MaxInvalidPasswordFailures = WebBrowser.MaxTries; // Max InvalidPassword failures in a row before we determine that our password is invalid (because Steam wrongly returns those, of course)
private const ushort MaxMessageLength = 5000; // This is a limitation enforced by Steam
private const byte MaxTwoFactorCodeFailures = WebBrowser.MaxTries; // Max TwoFactorCodeMismatch failures in a row before we determine that our 2FA credentials are invalid (because Steam wrongly returns those, of course)
private const byte RedeemCooldownInHours = 1; // 1 hour since first redeem attempt, this is a limitation enforced by Steam
private const byte ReservedMessageLength = 2; // 2 for 2x optional …
[PublicAPI]
public static IReadOnlyDictionary<string, Bot>? BotsReadOnly => Bots;
@@ -773,82 +770,15 @@ namespace ArchiSteamFarm.Steam {
ArchiLogger.LogChatMessage(true, message, steamID: steamID);
string? steamMessagePrefix = ASF.GlobalConfig != null ? ASF.GlobalConfig.SteamMessagePrefix : GlobalConfig.DefaultSteamMessagePrefix;
ushort maxMessageLength = (ushort) (MaxMessageLength - ReservedMessageLength - (steamMessagePrefix?.Length ?? 0));
// We must escape our message prior to sending it
message = Escape(message);
await foreach (string messagePart in SteamChatMessage.GetMessageParts(message, steamMessagePrefix, IsAccountLimited).ConfigureAwait(false)) {
ArchiLogger.LogGenericDebug(messagePart);
int i = 0;
if (!await SendMessagePart(steamID, messagePart).ConfigureAwait(false)) {
ArchiLogger.LogGenericWarning(Strings.WarningFailed);
while (i < message.Length) {
int partLength;
bool copyNewline = false;
if (message.Length - i > maxMessageLength) {
int lastNewLine = message.LastIndexOf(Environment.NewLine, (i + maxMessageLength) - Environment.NewLine.Length, maxMessageLength - Environment.NewLine.Length, StringComparison.Ordinal);
if (lastNewLine > i) {
partLength = (lastNewLine - i) + Environment.NewLine.Length;
copyNewline = true;
} else {
partLength = maxMessageLength;
}
} else {
partLength = message.Length - i;
return false;
}
// If our message is of max length and ends with a single '\' then we can't split it here, it escapes the next character
if ((partLength >= maxMessageLength) && (message[(i + partLength) - 1] == '\\') && (message[(i + partLength) - 2] != '\\')) {
// Instead, we'll cut this message one char short and include the rest in next iteration
partLength--;
}
string messagePart = message.Substring(i, partLength);
messagePart = steamMessagePrefix + (i > 0 ? "…" : "") + messagePart + (maxMessageLength < message.Length - i ? "…" : "");
await MessagingSemaphore.WaitAsync().ConfigureAwait(false);
try {
bool sent = false;
for (byte j = 0; (j < WebBrowser.MaxTries) && !sent && IsConnectedAndLoggedOn; j++) {
// We add a one-second delay here to avoid Steam screwup in form of a ghost notification
// The exact cause is unknown, but the theory is that Steam is confused when dealing with more than 1 message per second from the same user
await Task.Delay(1000).ConfigureAwait(false);
EResult result = await ArchiHandler.SendMessage(steamID, messagePart).ConfigureAwait(false);
switch (result) {
case EResult.Busy:
case EResult.Fail:
case EResult.RateLimitExceeded:
case EResult.ServiceUnavailable:
case EResult.Timeout:
await Task.Delay(5000).ConfigureAwait(false);
continue;
case EResult.OK:
sent = true;
break;
default:
ArchiLogger.LogGenericError(string.Format(CultureInfo.CurrentCulture, Strings.WarningUnknownValuePleaseReport, nameof(result), result));
return false;
}
}
if (!sent) {
ArchiLogger.LogGenericWarning(Strings.WarningFailed);
return false;
}
} finally {
MessagingSemaphore.Release();
}
i += partLength - (copyNewline ? Environment.NewLine.Length : 0);
}
return true;
@@ -875,78 +805,15 @@ namespace ArchiSteamFarm.Steam {
ArchiLogger.LogChatMessage(true, message, chatGroupID, chatID);
string? steamMessagePrefix = ASF.GlobalConfig != null ? ASF.GlobalConfig.SteamMessagePrefix : GlobalConfig.DefaultSteamMessagePrefix;
ushort maxMessageLength = (ushort) (MaxMessageLength - ReservedMessageLength - (steamMessagePrefix?.Length ?? 0));
// We must escape our message prior to sending it
message = Escape(message);
await foreach (string messagePart in SteamChatMessage.GetMessageParts(message, steamMessagePrefix, IsAccountLimited).ConfigureAwait(false)) {
ArchiLogger.LogGenericDebug(messagePart);
int i = 0;
if (!await SendMessagePart(chatID, messagePart, chatGroupID).ConfigureAwait(false)) {
ArchiLogger.LogGenericWarning(Strings.WarningFailed);
while (i < message.Length) {
int partLength;
bool copyNewline = false;
if (message.Length - i > maxMessageLength) {
int lastNewLine = message.LastIndexOf(Environment.NewLine, (i + maxMessageLength) - Environment.NewLine.Length, maxMessageLength - Environment.NewLine.Length, StringComparison.Ordinal);
if (lastNewLine > i) {
partLength = (lastNewLine - i) + Environment.NewLine.Length;
copyNewline = true;
} else {
partLength = maxMessageLength;
}
} else {
partLength = message.Length - i;
return false;
}
// If our message is of max length and ends with a single '\' then we can't split it here, it escapes the next character
if ((partLength >= maxMessageLength) && (message[(i + partLength) - 1] == '\\') && (message[(i + partLength) - 2] != '\\')) {
// Instead, we'll cut this message one char short and include the rest in next iteration
partLength--;
}
string messagePart = message.Substring(i, partLength);
messagePart = steamMessagePrefix + (i > 0 ? "…" : "") + messagePart + (maxMessageLength < message.Length - i ? "…" : "");
await MessagingSemaphore.WaitAsync().ConfigureAwait(false);
try {
bool sent = false;
for (byte j = 0; (j < WebBrowser.MaxTries) && !sent && IsConnectedAndLoggedOn; j++) {
EResult result = await ArchiHandler.SendMessage(chatGroupID, chatID, messagePart).ConfigureAwait(false);
switch (result) {
case EResult.Busy:
case EResult.Fail:
case EResult.RateLimitExceeded:
case EResult.ServiceUnavailable:
case EResult.Timeout:
await Task.Delay(5000).ConfigureAwait(false);
continue;
case EResult.OK:
sent = true;
break;
default:
ArchiLogger.LogGenericError(string.Format(CultureInfo.CurrentCulture, Strings.WarningUnknownValuePleaseReport, nameof(result), result));
return false;
}
}
if (!sent) {
ArchiLogger.LogGenericWarning(Strings.WarningFailed);
return false;
}
} finally {
MessagingSemaphore.Release();
}
i += partLength - (copyNewline ? Environment.NewLine.Length : 0);
}
return true;
@@ -1874,14 +1741,6 @@ namespace ArchiSteamFarm.Steam {
SteamClient.Disconnect();
}
private static string Escape(string message) {
if (string.IsNullOrEmpty(message)) {
throw new ArgumentNullException(nameof(message));
}
return message.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("[", "\\[", StringComparison.Ordinal);
}
private async Task<Dictionary<string, string>?> GetKeysFromFile(string filePath) {
if (string.IsNullOrEmpty(filePath)) {
throw new ArgumentNullException(nameof(filePath));
@@ -2608,7 +2467,7 @@ namespace ArchiSteamFarm.Steam {
if (!string.IsNullOrEmpty(notification.message_no_bbcode)) {
message = notification.message_no_bbcode;
} else if (!string.IsNullOrEmpty(notification.message)) {
message = UnEscape(notification.message);
message = SteamChatMessage.Unescape(notification.message);
} else {
return;
}
@@ -2654,7 +2513,7 @@ namespace ArchiSteamFarm.Steam {
if (!string.IsNullOrEmpty(notification.message_no_bbcode)) {
message = notification.message_no_bbcode;
} else if (!string.IsNullOrEmpty(notification.message)) {
message = UnEscape(notification.message);
message = SteamChatMessage.Unescape(notification.message);
} else {
return;
}
@@ -3471,6 +3330,55 @@ namespace ArchiSteamFarm.Steam {
}
}
private async Task<bool> SendMessagePart(ulong steamID, string messagePart, ulong chatGroupID = 0) {
if ((steamID == 0) || ((chatGroupID == 0) && !new SteamID(steamID).IsIndividualAccount)) {
throw new ArgumentOutOfRangeException(nameof(steamID));
}
if (string.IsNullOrEmpty(messagePart)) {
throw new ArgumentNullException(nameof(messagePart));
}
if (!IsConnectedAndLoggedOn) {
return false;
}
await MessagingSemaphore.WaitAsync().ConfigureAwait(false);
try {
for (byte i = 0; (i < WebBrowser.MaxTries) && IsConnectedAndLoggedOn; i++) {
EResult result;
if (chatGroupID == 0) {
result = await ArchiHandler.SendMessage(steamID, messagePart).ConfigureAwait(false);
} else {
result = await ArchiHandler.SendMessage(chatGroupID, steamID, messagePart).ConfigureAwait(false);
}
switch (result) {
case EResult.Busy:
case EResult.Fail:
case EResult.RateLimitExceeded:
case EResult.ServiceUnavailable:
case EResult.Timeout:
await Task.Delay(5000).ConfigureAwait(false);
continue;
case EResult.OK:
return true;
default:
ArchiLogger.LogGenericError(string.Format(CultureInfo.CurrentCulture, Strings.WarningUnknownValuePleaseReport, nameof(result), result));
return false;
}
}
return false;
} finally {
MessagingSemaphore.Release();
}
}
private bool ShouldAckChatMessage(ulong steamID) {
if ((steamID == 0) || !new SteamID(steamID).IsIndividualAccount) {
throw new ArgumentOutOfRangeException(nameof(steamID));
@@ -3505,14 +3413,6 @@ namespace ArchiSteamFarm.Steam {
PlayingWasBlockedTimer = null;
}
private static string UnEscape(string message) {
if (string.IsNullOrEmpty(message)) {
throw new ArgumentNullException(nameof(message));
}
return message.Replace("\\[", "[", StringComparison.Ordinal).Replace("\\\\", "\\", StringComparison.Ordinal);
}
private (bool IsSteamParentalEnabled, string? SteamParentalCode) ValidateSteamParental(ParentalSettings settings, string? steamParentalCode = null) {
if (settings == null) {
throw new ArgumentNullException(nameof(settings));

204
ArchiSteamFarm/Steam/Integration/SteamChatMessage.cs Normal file
View File

@@ -0,0 +1,204 @@
// _ _ _ ____ _ _____
// / \ _ __ ___ | |__ (_)/ ___| | |_ ___ __ _ _ __ ___ | ___|__ _ _ __ _ __ ___
// / _ \ | '__|/ __|| '_ \ | |\___ \ | __|/ _ \ / _` || '_ ` _ \ | |_ / _` || '__|| '_ ` _ \
// / ___ \ | | | (__ | | | || | ___) || |_| __/| (_| || | | | | || _|| (_| || | | | | | | |
// /_/ \_\|_| \___||_| |_||_||____/ \__|\___| \__,_||_| |_| |_||_| \__,_||_| |_| |_| |_|
// |
// Copyright 2015-2021 Łukasz "JustArchi" Domeradzki
// Contact: JustArchi@JustArchi.net
// |
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// |
// http://www.apache.org/licenses/LICENSE-2.0
// |
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#if NETFRAMEWORK
using ArchiSteamFarm.Compatibility;
#endif
using System;
using System.Buffers;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
namespace ArchiSteamFarm.Steam.Integration {
internal static class SteamChatMessage {
internal const char ContinuationCharacter = '…'; // A character used for indicating that the next newline part is a continuation of the previous line
internal const byte ContinuationCharacterBytes = 3; // The continuation character specified above uses 3 bytes in UTF-8
internal const ushort MaxMessageBytesForLimitedAccounts = 2400; // This is a limitation enforced by Steam
internal const ushort MaxMessageBytesForUnlimitedAccounts = 6340; // This is a limitation enforced by Steam
internal const ushort MaxMessagePrefixBytes = MaxMessageBytesForLimitedAccounts - ReservedContinuationMessageBytes - ReservedEscapeMessageBytes; // Simplified calculation, nobody should be using prefixes even close to that anyway
internal const byte NewlineWeight = 61; // This defines how much weight a newline character is adding to the output, limitation enforced by Steam
internal const byte ReservedContinuationMessageBytes = ContinuationCharacterBytes * 2; // Up to 2 optional continuation characters
internal const byte ReservedEscapeMessageBytes = 5; // 2 characters total, escape one '\' of 1 byte and real one of up to 4 bytes
internal static async IAsyncEnumerable<string> GetMessageParts(string message, string? steamMessagePrefix = null, bool isAccountLimited = false) {
if (string.IsNullOrEmpty(message)) {
throw new ArgumentNullException(nameof(message));
}
int prefixBytes = 0;
int prefixLength = 0;
if (!string.IsNullOrEmpty(steamMessagePrefix)) {
// We must escape our message prefix if needed
steamMessagePrefix = Escape(steamMessagePrefix!);
prefixBytes = GetMessagePrefixBytes(steamMessagePrefix);
if (prefixBytes > MaxMessagePrefixBytes) {
throw new ArgumentOutOfRangeException(nameof(steamMessagePrefix));
}
prefixLength = steamMessagePrefix.Length;
}
int maxMessageBytes = (isAccountLimited ? MaxMessageBytesForLimitedAccounts : MaxMessageBytesForUnlimitedAccounts) - ReservedContinuationMessageBytes;
// We must escape our message prior to sending it
message = Escape(message);
int messagePartBytes = prefixBytes;
StringBuilder messagePart = new(steamMessagePrefix);
Decoder decoder = Encoding.UTF8.GetDecoder();
ArrayPool<char> charPool = ArrayPool<char>.Shared;
using StringReader stringReader = new(message);
string? line;
while ((line = await stringReader.ReadLineAsync().ConfigureAwait(false)) != null) {
// Special case for empty newline
if (line.Length == 0) {
if (messagePart.Length > prefixLength) {
messagePartBytes += NewlineWeight;
messagePart.AppendLine();
}
// Check if we reached the limit for one message
if (messagePartBytes + NewlineWeight + ReservedEscapeMessageBytes > maxMessageBytes) {
yield return messagePart.ToString();
messagePartBytes = prefixBytes;
messagePart.Clear();
messagePart.Append(steamMessagePrefix);
}
// Move on to the next line
continue;
}
byte[] lineBytes = Encoding.UTF8.GetBytes(line);
for (int lineBytesRead = 0; lineBytesRead < lineBytes.Length;) {
if (messagePart.Length > prefixLength) {
messagePartBytes += NewlineWeight;
messagePart.AppendLine();
}
int bytesToTake = Math.Min(maxMessageBytes - messagePartBytes, lineBytes.Length - lineBytesRead);
// We can never have more characters than bytes used, so this covers the worst case of 1-byte characters exclusively
char[] lineChunk = charPool.Rent(bytesToTake);
try {
// We have to reset the decoder prior to using it, as we must discard any amount of bytes read from previous incomplete character
decoder.Reset();
int charsUsed = decoder.GetChars(lineBytes, lineBytesRead, bytesToTake, lineChunk, 0, false);
switch (charsUsed) {
case <= 0:
throw new InvalidOperationException(nameof(charsUsed));
case >= 2 when (lineChunk[charsUsed - 1] == '\\') && (lineChunk[charsUsed - 2] != '\\'):
// If our message is of max length and ends with a single '\' then we can't split it here, because it escapes the next character
// Instead, we'll cut this message one char short and include the rest in the next iteration
charsUsed--;
break;
}
int bytesUsed = Encoding.UTF8.GetByteCount(lineChunk, 0, charsUsed);
if (lineBytesRead > 0) {
messagePartBytes += ContinuationCharacterBytes;
messagePart.Append(ContinuationCharacter);
}
lineBytesRead += bytesUsed;
messagePartBytes += bytesUsed;
messagePart.Append(lineChunk, 0, charsUsed);
} finally {
charPool.Return(lineChunk);
}
if (lineBytesRead < lineBytes.Length) {
messagePartBytes += ContinuationCharacterBytes;
messagePart.Append(ContinuationCharacter);
}
// Check if we still have room for one more line
if (messagePartBytes + NewlineWeight + ReservedEscapeMessageBytes <= maxMessageBytes) {
continue;
}
yield return messagePart.ToString();
messagePartBytes = prefixBytes;
messagePart.Clear();
messagePart.Append(steamMessagePrefix);
}
}
if (messagePart.Length <= prefixLength) {
yield break;
}
yield return messagePart.ToString();
}
internal static bool IsValidPrefix(string steamMessagePrefix) {
if (string.IsNullOrEmpty(steamMessagePrefix)) {
throw new ArgumentNullException(nameof(steamMessagePrefix));
}
return GetMessagePrefixBytes(Escape(steamMessagePrefix)) <= MaxMessagePrefixBytes;
}
internal static string Unescape(string message) {
if (string.IsNullOrEmpty(message)) {
throw new ArgumentNullException(nameof(message));
}
return message.Replace("\\[", "[", StringComparison.Ordinal).Replace("\\\\", "\\", StringComparison.Ordinal);
}
private static string Escape(string message) {
if (string.IsNullOrEmpty(message)) {
throw new ArgumentNullException(nameof(message));
}
return message.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("[", "\\[", StringComparison.Ordinal);
}
private static int GetMessagePrefixBytes(string escapedSteamMessagePrefix) {
if (string.IsNullOrEmpty(escapedSteamMessagePrefix)) {
throw new ArgumentNullException(nameof(escapedSteamMessagePrefix));
}
string[] prefixLines = escapedSteamMessagePrefix.Split(new[] { "\r\n", "\r", "\n" }, StringSplitOptions.None);
return prefixLines.Where(prefixLine => prefixLine.Length > 0).Sum(Encoding.UTF8.GetByteCount) + ((prefixLines.Length - 1) * NewlineWeight);
}
}
}

4
ArchiSteamFarm/Storage/GlobalConfig.cs
View File

@@ -30,7 +30,7 @@ using System.Threading.Tasks;
using ArchiSteamFarm.Core;
using ArchiSteamFarm.Helpers;
using ArchiSteamFarm.Localization;
using ArchiSteamFarm.Steam;
using ArchiSteamFarm.Steam.Integration;
using JetBrains.Annotations;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
@@ -319,7 +319,7 @@ namespace ArchiSteamFarm.Storage {
return (false, string.Format(CultureInfo.CurrentCulture, Strings.ErrorConfigPropertyInvalid, nameof(OptimizationMode), OptimizationMode));
}
if (!string.IsNullOrEmpty(SteamMessagePrefix) && (SteamMessagePrefix!.Length > Bot.MaxMessagePrefixLength)) {
if (!string.IsNullOrEmpty(SteamMessagePrefix) && !SteamChatMessage.IsValidPrefix(SteamMessagePrefix!)) {
return (false, string.Format(CultureInfo.CurrentCulture, Strings.ErrorConfigPropertyInvalid, nameof(SteamMessagePrefix), SteamMessagePrefix));
}