From f0ef4c6ba6ceb73c62a9dd5766d7864fd9a4a03d Mon Sep 17 00:00:00 2001 From: Archi Date: Sat, 3 Feb 2024 21:18:47 +0100 Subject: [PATCH] Implement basic crash protection - On start, we create/load crash file, if previous startup was less than 5 minutes ago, we also increase the counter - If counter reaches 5, we abort the process, that is, freeze the process by not loading anything other than auto-updates - In order for user to recover from above, he needs to manually remove ASF.crash file - If process exits normally without reaching counter of 5 yet, we remove crash file (reset the counter), normal exit includes SIGTERM, SIGINT, clicking [X], !exit, !restart, and anything else that allows ASF to gracefully quit - If process exits abnormally, that includes SIGKILL, unhandled exceptions, as well as power outages and anything that prevents ASF from graceful quit, we keep crash file around - Update procedure, as an exception, allows crash file removal even with counter of 5. This should allow crash file recovery for people that crashed before, without a need of manual removal. --- ArchiSteamFarm/Core/ASF.cs | 49 +++++++- .../Localization/Strings.Designer.cs | 6 + ArchiSteamFarm/Localization/Strings.resx | 6 +- ArchiSteamFarm/NLog/ArchiLogger.cs | 7 ++ ArchiSteamFarm/Program.cs | 27 ++-- ArchiSteamFarm/SharedInfo.cs | 1 + ArchiSteamFarm/Storage/CrashFile.cs | 117 ++++++++++++++++++ 7 files changed, 199 insertions(+), 14 deletions(-) create mode 100644 ArchiSteamFarm/Storage/CrashFile.cs diff --git a/ArchiSteamFarm/Core/ASF.cs b/ArchiSteamFarm/Core/ASF.cs index 27dc33a91..4ed6fd37d 100644 --- a/ArchiSteamFarm/Core/ASF.cs +++ b/ArchiSteamFarm/Core/ASF.cs @@ -101,6 +101,7 @@ public static class ASF { return fileType switch { EFileType.Config => Path.Combine(SharedInfo.ConfigDirectory, SharedInfo.GlobalConfigFileName), EFileType.Database => Path.Combine(SharedInfo.ConfigDirectory, SharedInfo.GlobalDatabaseFileName), + EFileType.Crash => Path.Combine(SharedInfo.ConfigDirectory, SharedInfo.GlobalCrashFileName), _ => throw new InvalidOperationException(nameof(fileType)) }; } @@ -110,14 +111,22 @@ public static class ASF { throw new InvalidOperationException(nameof(GlobalConfig)); } - if (!await PluginsCore.InitPlugins().ConfigureAwait(false)) { - return false; - } - WebBrowser = new WebBrowser(ArchiLogger, GlobalConfig.WebProxy, true); await UpdateAndRestart().ConfigureAwait(false); + if (!Program.IgnoreUnsupportedEnvironment && !await ProtectAgainstCrashes().ConfigureAwait(false)) { + ArchiLogger.LogGenericError(Strings.ErrorTooManyCrashes); + + return true; + } + + Program.AllowCrashFileRemoval = true; + + if (!await PluginsCore.InitPlugins().ConfigureAwait(false)) { + return false; + } + await PluginsCore.OnASFInitModules(GlobalConfig.AdditionalProperties).ConfigureAwait(false); await InitRateLimiters().ConfigureAwait(false); @@ -815,6 +824,32 @@ public static class ASF { } } + private static async Task ProtectAgainstCrashes() { + string crashFilePath = GetFilePath(EFileType.Crash); + + CrashFile crashFile = await CrashFile.CreateOrLoad(crashFilePath).ConfigureAwait(false); + + if (crashFile.StartupCount >= WebBrowser.MaxTries) { + // We've reached maximum allowed count of recent crashes, return failure + return false; + } + + DateTime now = DateTime.UtcNow; + + if (now - crashFile.LastStartup > TimeSpan.FromMinutes(5)) { + // Last crash was long ago, restart counter + crashFile.StartupCount = 1; + } else if (++crashFile.StartupCount >= WebBrowser.MaxTries) { + // We've reached maximum allowed count of recent crashes, return failure + return false; + } + + crashFile.LastStartup = now; + + // We're allowing this run to proceed + return true; + } + private static async Task RegisterBots() { if (GlobalConfig == null) { throw new InvalidOperationException(nameof(GlobalConfig)); @@ -907,6 +942,9 @@ public static class ASF { return; } + // Allow crash file recovery, if needed + Program.AllowCrashFileRemoval = true; + await RestartOrExit().ConfigureAwait(false); } @@ -1049,6 +1087,7 @@ public static class ASF { internal enum EFileType : byte { Config, - Database + Database, + Crash } } diff --git a/ArchiSteamFarm/Localization/Strings.Designer.cs b/ArchiSteamFarm/Localization/Strings.Designer.cs index 84baf24b9..9fe3e0b58 100644 --- a/ArchiSteamFarm/Localization/Strings.Designer.cs +++ b/ArchiSteamFarm/Localization/Strings.Designer.cs @@ -1226,5 +1226,11 @@ namespace ArchiSteamFarm.Localization { return ResourceManager.GetString("WarningUnsupportedOfficialPlugins", resourceCulture); } } + + public static string ErrorTooManyCrashes { + get { + return ResourceManager.GetString("ErrorTooManyCrashes", resourceCulture); + } + } } } diff --git a/ArchiSteamFarm/Localization/Strings.resx b/ArchiSteamFarm/Localization/Strings.resx index 739be404c..60b314f31 100644 --- a/ArchiSteamFarm/Localization/Strings.resx +++ b/ArchiSteamFarm/Localization/Strings.resx @@ -90,7 +90,8 @@ StackTrace: {0} will be replaced by function name, {1} will be replaced by exception message, {2} will be replaced by entire stack trace. Please note that this string should include newlines for formatting. - Exiting with nonzero error code! + Exiting with {0} error code! + {0} will be replaced by error code (number) Request failing: {0} @@ -756,4 +757,7 @@ Process uptime: {1} You're attempting to run official {0} plugin in mismatched with ASF version: {1} (expected {2}). This suggests you're doing something horribly wrong, either fix your setup or supply --ignore-unsupported-environment argument if you really know what you're doing. {0} will be replaced by plugin name, {1} will be replaced by plugin's version number, {2} will be replaced by ASF's version number. + + Your ASF has crashed too many times recently, and due to that the process initialization has been disabled. Either investigate, fix your setup, then remove ASF.crash file from your config directory, or supply --ignore-unsupported-environment argument if you really know what you're doing. + diff --git a/ArchiSteamFarm/NLog/ArchiLogger.cs b/ArchiSteamFarm/NLog/ArchiLogger.cs index eae208c0e..04bc3d5fe 100644 --- a/ArchiSteamFarm/NLog/ArchiLogger.cs +++ b/ArchiSteamFarm/NLog/ArchiLogger.cs @@ -151,6 +151,13 @@ public sealed class ArchiLogger { Logger.Log(logEventInfo); } + internal void LogFatalError(string message, [CallerMemberName] string? previousMethodName = null) { + ArgumentException.ThrowIfNullOrEmpty(message); + ArgumentException.ThrowIfNullOrEmpty(previousMethodName); + + Logger.Fatal($"{previousMethodName}() {message}"); + } + internal async Task LogFatalException(Exception exception, [CallerMemberName] string? previousMethodName = null) { ArgumentNullException.ThrowIfNull(exception); ArgumentException.ThrowIfNullOrEmpty(previousMethodName); diff --git a/ArchiSteamFarm/Program.cs b/ArchiSteamFarm/Program.cs index 5c484abc2..b6f20eac9 100644 --- a/ArchiSteamFarm/Program.cs +++ b/ArchiSteamFarm/Program.cs @@ -47,6 +47,7 @@ using SteamKit2; namespace ArchiSteamFarm; internal static class Program { + internal static bool AllowCrashFileRemoval { get; set; } internal static bool ConfigMigrate { get; private set; } = true; internal static bool ConfigWatch { get; private set; } = true; internal static bool IgnoreUnsupportedEnvironment { get; private set; } @@ -67,7 +68,7 @@ internal static class Program { internal static async Task Exit(byte exitCode = 0) { if (exitCode != 0) { - ASF.ArchiLogger.LogGenericError(Strings.ErrorExitingWithNonZeroErrorCode); + ASF.ArchiLogger.LogGenericError(string.Format(CultureInfo.CurrentCulture, Strings.ErrorExitingWithNonZeroErrorCode, exitCode)); } await Shutdown(exitCode).ConfigureAwait(false); @@ -444,7 +445,7 @@ internal static class Program { return true; } - private static async Task InitShutdownSequence() { + private static async Task InitShutdownSequence(byte exitCode = 0) { if (ShutdownSequenceInitialized) { // We've already initialized shutdown sequence before, we won't allow the caller to init shutdown sequence again // While normally this will be respected, caller might not have any say in this for example because it's the runtime terminating ASF due to fatal exception @@ -456,8 +457,8 @@ internal static class Program { ShutdownSequenceInitialized = true; + // Unregister from registered signals if (OperatingSystem.IsFreeBSD() || OperatingSystem.IsLinux() || OperatingSystem.IsMacOS()) { - // Unregister from registered signals foreach (PosixSignalRegistration registration in RegisteredPosixSignals.Values) { registration.Dispose(); } @@ -465,6 +466,19 @@ internal static class Program { RegisteredPosixSignals.Clear(); } + // Remove crash file if allowed + if ((exitCode == 0) && AllowCrashFileRemoval) { + string crashFile = ASF.GetFilePath(ASF.EFileType.Crash); + + if (File.Exists(crashFile)) { + try { + File.Delete(crashFile); + } catch (Exception e) { + ASF.ArchiLogger.LogGenericException(e); + } + } + } + // Sockets created by IPC might still be running for a short while after complete app shutdown // Ensure that IPC is stopped before we finalize shutdown sequence await ArchiKestrel.Stop().ConfigureAwait(false); @@ -497,16 +511,13 @@ internal static class Program { return await ShutdownResetEvent.Task.ConfigureAwait(false); } - private static async void OnCancelKeyPress(object? sender, ConsoleCancelEventArgs e) => await Exit(130).ConfigureAwait(false); + private static async void OnCancelKeyPress(object? sender, ConsoleCancelEventArgs e) => await Exit().ConfigureAwait(false); private static async void OnPosixSignal(PosixSignalContext signal) { ArgumentNullException.ThrowIfNull(signal); switch (signal.Signal) { case PosixSignal.SIGINT: - await Exit(130).ConfigureAwait(false); - - break; case PosixSignal.SIGTERM: await Exit().ConfigureAwait(false); @@ -704,7 +715,7 @@ internal static class Program { } private static async Task Shutdown(byte exitCode = 0) { - if (!await InitShutdownSequence().ConfigureAwait(false)) { + if (!await InitShutdownSequence(exitCode).ConfigureAwait(false)) { return; } diff --git a/ArchiSteamFarm/SharedInfo.cs b/ArchiSteamFarm/SharedInfo.cs index bc5c6abfd..342928fc7 100644 --- a/ArchiSteamFarm/SharedInfo.cs +++ b/ArchiSteamFarm/SharedInfo.cs @@ -48,6 +48,7 @@ public static class SharedInfo { internal const string GithubReleaseURL = $"https://api.github.com/repos/{GithubRepo}/releases"; internal const string GithubRepo = $"JustArchiNET/{AssemblyName}"; internal const string GlobalConfigFileName = $"{ASF}{JsonConfigExtension}"; + internal const string GlobalCrashFileName = $"{ASF}.crash"; internal const string GlobalDatabaseFileName = $"{ASF}{DatabaseExtension}"; internal const ushort InformationDelay = 10000; internal const string IPCConfigExtension = ".config"; diff --git a/ArchiSteamFarm/Storage/CrashFile.cs b/ArchiSteamFarm/Storage/CrashFile.cs new file mode 100644 index 000000000..b13b23d50 --- /dev/null +++ b/ArchiSteamFarm/Storage/CrashFile.cs @@ -0,0 +1,117 @@ +// _ _ _ ____ _ _____ +// / \ _ __ ___ | |__ (_)/ ___| | |_ ___ __ _ _ __ ___ | ___|__ _ _ __ _ __ ___ +// / _ \ | '__|/ __|| '_ \ | |\___ \ | __|/ _ \ / _` || '_ ` _ \ | |_ / _` || '__|| '_ ` _ \ +// / ___ \ | | | (__ | | | || | ___) || |_| __/| (_| || | | | | || _|| (_| || | | | | | | | +// /_/ \_\|_| \___||_| |_||_||____/ \__|\___| \__,_||_| |_| |_||_| \__,_||_| |_| |_| |_| +// | +// Copyright 2015-2024 Łukasz "JustArchi" Domeradzki +// Contact: JustArchi@JustArchi.net +// | +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// | +// http://www.apache.org/licenses/LICENSE-2.0 +// | +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Globalization; +using System.IO; +using System.Threading.Tasks; +using ArchiSteamFarm.Core; +using ArchiSteamFarm.Helpers; +using ArchiSteamFarm.Localization; +using JetBrains.Annotations; +using Newtonsoft.Json; + +namespace ArchiSteamFarm.Storage; + +internal sealed class CrashFile : SerializableFile { + internal DateTime LastStartup { + get => BackingLastStartup; + + set { + if (BackingLastStartup == value) { + return; + } + + BackingLastStartup = value; + Utilities.InBackground(Save); + } + } + + internal byte StartupCount { + get => BackingStartupCount; + + set { + if (BackingStartupCount == value) { + return; + } + + BackingStartupCount = value; + Utilities.InBackground(Save); + } + } + + [JsonProperty(Required = Required.DisallowNull)] + private DateTime BackingLastStartup; + + [JsonProperty(Required = Required.DisallowNull)] + private byte BackingStartupCount; + + private CrashFile(string filePath) : this() { + ArgumentException.ThrowIfNullOrEmpty(filePath); + + FilePath = filePath; + } + + [JsonConstructor] + private CrashFile() { } + + [UsedImplicitly] + public bool ShouldSerializeBackingLastStartup() => BackingLastStartup > DateTime.MinValue; + + [UsedImplicitly] + public bool ShouldSerializeBackingStartupCount() => BackingStartupCount > 0; + + internal static async Task CreateOrLoad(string filePath) { + ArgumentException.ThrowIfNullOrEmpty(filePath); + + if (!File.Exists(filePath)) { + return new CrashFile(filePath); + } + + CrashFile? crashFile; + + try { + string json = await File.ReadAllTextAsync(filePath).ConfigureAwait(false); + + if (string.IsNullOrEmpty(json)) { + ASF.ArchiLogger.LogGenericError(string.Format(CultureInfo.CurrentCulture, Strings.ErrorIsEmpty, nameof(json))); + + return new CrashFile(filePath); + } + + crashFile = JsonConvert.DeserializeObject(json); + } catch (Exception e) { + ASF.ArchiLogger.LogGenericException(e); + + return new CrashFile(filePath); + } + + if (crashFile == null) { + ASF.ArchiLogger.LogNullError(crashFile); + + return new CrashFile(filePath); + } + + crashFile.FilePath = filePath; + + return crashFile; + } +}