From a5aed14dd21599c1b51d3128de055237b81715f6 Mon Sep 17 00:00:00 2001 From: Toutsu Date: Tue, 26 May 2026 15:51:25 +0300 Subject: [PATCH] fix(discord): add backoff to scheduler to prevent 403 spam - SessionSchedulerService now backs off for 15 minutes after any handler failure (confirmation, one-hour reminder, join link), preventing infinite retry loops on Discord 403 Missing Access. - Added per-session ConcurrentDictionary backoff tracking with automatic cleanup on success. - Enhanced DiscordPlatformMessenger logging for SendConfirmation and SendJoinLink to aid permission diagnostics. - Added 3 regression tests for backoff behavior. Co-Authored-By: Claude Opus 4.7 --- .../Discord/DiscordPlatformMessenger.cs | 70 ++++++++++++++----- .../Scheduling/SessionSchedulerService.cs | 60 +++++++++++++++- .../SessionSchedulerServiceTests.cs | 62 ++++++++++++++++ 3 files changed, 171 insertions(+), 21 deletions(-) diff --git a/src/GmRelay.DiscordBot/Infrastructure/Discord/DiscordPlatformMessenger.cs b/src/GmRelay.DiscordBot/Infrastructure/Discord/DiscordPlatformMessenger.cs index 4bd2896..e43b9bd 100644 --- a/src/GmRelay.DiscordBot/Infrastructure/Discord/DiscordPlatformMessenger.cs +++ b/src/GmRelay.DiscordBot/Infrastructure/Discord/DiscordPlatformMessenger.cs @@ -98,17 +98,34 @@ public sealed class DiscordPlatformMessenger : IPlatformMessenger CancellationToken ct) { var channelId = GetChannelId(request.Group); - var message = await restClient.SendMessageAsync( - channelId, - new MessageProperties() - .WithEmbeds([BuildConfirmationEmbed(request)]) - .WithComponents(BuildRsvpRows(request.SessionId, disabled: false))); + try + { + var message = await restClient.SendMessageAsync( + channelId, + new MessageProperties() + .WithEmbeds([BuildConfirmationEmbed(request)]) + .WithComponents(BuildRsvpRows(request.SessionId, disabled: false))); - return new PlatformMessageRef( - PlatformKind.Discord, - request.Group.ExternalGroupId, - null, - message.Id.ToString(CultureInfo.InvariantCulture)); + logger?.LogInformation( + "Confirmation request sent to Discord channel {ChannelId}, message id {MessageId}", + channelId, + message.Id); + + return new PlatformMessageRef( + PlatformKind.Discord, + request.Group.ExternalGroupId, + null, + message.Id.ToString(CultureInfo.InvariantCulture)); + } + catch (Exception ex) + { + logger?.LogError( + ex, + "Failed to send confirmation request to Discord channel {ChannelId} for session {SessionId}", + channelId, + request.SessionId); + throw; + } } public async Task UpdateConfirmationRequestAsync(PlatformRsvpMessageUpdate update, CancellationToken ct) @@ -135,15 +152,32 @@ public sealed class DiscordPlatformMessenger : IPlatformMessenger CancellationToken ct) { var channelId = GetChannelId(notification.Group); - var message = await restClient.SendMessageAsync( - channelId, - new MessageProperties().WithEmbeds([BuildJoinLinkEmbed(notification)])); + try + { + var message = await restClient.SendMessageAsync( + channelId, + new MessageProperties().WithEmbeds([BuildJoinLinkEmbed(notification)])); - return new PlatformMessageRef( - PlatformKind.Discord, - notification.Group.ExternalGroupId, - null, - message.Id.ToString(CultureInfo.InvariantCulture)); + logger?.LogInformation( + "Join link sent to Discord channel {ChannelId}, message id {MessageId}", + channelId, + message.Id); + + return new PlatformMessageRef( + PlatformKind.Discord, + notification.Group.ExternalGroupId, + null, + message.Id.ToString(CultureInfo.InvariantCulture)); + } + catch (Exception ex) + { + logger?.LogError( + ex, + "Failed to send join link to Discord channel {ChannelId} for session {SessionId}", + channelId, + notification.SessionId); + throw; + } } public async Task SendDirectSessionNotificationAsync( diff --git a/src/GmRelay.Shared/Infrastructure/Scheduling/SessionSchedulerService.cs b/src/GmRelay.Shared/Infrastructure/Scheduling/SessionSchedulerService.cs index 28947cf..5a358be 100644 --- a/src/GmRelay.Shared/Infrastructure/Scheduling/SessionSchedulerService.cs +++ b/src/GmRelay.Shared/Infrastructure/Scheduling/SessionSchedulerService.cs @@ -1,3 +1,4 @@ +using System.Collections.Concurrent; using GmRelay.Shared.Features.Confirmation.SendConfirmation; using GmRelay.Shared.Features.Reminders.SendJoinLink; using GmRelay.Shared.Features.Reminders.SendOneHourReminder; @@ -20,6 +21,11 @@ public sealed class SessionSchedulerService( ILogger logger) : BackgroundService { private static readonly TimeSpan TickInterval = TimeSpan.FromMinutes(1); + private static readonly TimeSpan BackoffDuration = TimeSpan.FromMinutes(15); + + private readonly ConcurrentDictionary _confirmationBackoff = new(); + private readonly ConcurrentDictionary _oneHourBackoff = new(); + private readonly ConcurrentDictionary _joinLinkBackoff = new(); protected override async Task ExecuteAsync(CancellationToken stoppingToken) { @@ -71,14 +77,30 @@ public sealed class SessionSchedulerService( foreach (var sessionId in sessionIds) { + if (_confirmationBackoff.TryGetValue(sessionId, out var backoffUntil) && backoffUntil > now) + { + logger.LogDebug( + "Skipping confirmation for session {SessionId} until {Backoff}", + sessionId, + backoffUntil); + continue; + } + try { await confirmationHandler.HandleAsync(sessionId, ct); + _confirmationBackoff.TryRemove(sessionId, out _); logger.LogInformation("Confirmation sent for session {SessionId}", sessionId); } catch (Exception ex) { - logger.LogError(ex, "Failed to send confirmation for session {SessionId}", sessionId); + var nextAttempt = now.Add(BackoffDuration); + _confirmationBackoff[sessionId] = nextAttempt; + logger.LogError( + ex, + "Failed to send confirmation for session {SessionId}, backing off until {Backoff}", + sessionId, + nextAttempt); } } } @@ -98,14 +120,30 @@ public sealed class SessionSchedulerService( foreach (var sessionId in sessionIds) { + if (_oneHourBackoff.TryGetValue(sessionId, out var backoffUntil) && backoffUntil > now) + { + logger.LogDebug( + "Skipping one-hour reminder for session {SessionId} until {Backoff}", + sessionId, + backoffUntil); + continue; + } + try { await oneHourReminderHandler.HandleAsync(sessionId, ct); + _oneHourBackoff.TryRemove(sessionId, out _); logger.LogInformation("One-hour reminder processed for session {SessionId}", sessionId); } catch (Exception ex) { - logger.LogError(ex, "Failed to process one-hour reminder for session {SessionId}", sessionId); + var nextAttempt = now.Add(BackoffDuration); + _oneHourBackoff[sessionId] = nextAttempt; + logger.LogError( + ex, + "Failed to process one-hour reminder for session {SessionId}, backing off until {Backoff}", + sessionId, + nextAttempt); } } } @@ -125,14 +163,30 @@ public sealed class SessionSchedulerService( foreach (var sessionId in sessionIds) { + if (_joinLinkBackoff.TryGetValue(sessionId, out var backoffUntil) && backoffUntil > now) + { + logger.LogDebug( + "Skipping join link for session {SessionId} until {Backoff}", + sessionId, + backoffUntil); + continue; + } + try { await joinLinkHandler.HandleAsync(sessionId, ct); + _joinLinkBackoff.TryRemove(sessionId, out _); logger.LogInformation("Join link sent for session {SessionId}", sessionId); } catch (Exception ex) { - logger.LogError(ex, "Failed to send join link for session {SessionId}", sessionId); + var nextAttempt = now.Add(BackoffDuration); + _joinLinkBackoff[sessionId] = nextAttempt; + logger.LogError( + ex, + "Failed to send join link for session {SessionId}, backing off until {Backoff}", + sessionId, + nextAttempt); } } } diff --git a/tests/GmRelay.Bot.Tests/Infrastructure/Scheduling/SessionSchedulerServiceTests.cs b/tests/GmRelay.Bot.Tests/Infrastructure/Scheduling/SessionSchedulerServiceTests.cs index 7536d17..e630563 100644 --- a/tests/GmRelay.Bot.Tests/Infrastructure/Scheduling/SessionSchedulerServiceTests.cs +++ b/tests/GmRelay.Bot.Tests/Infrastructure/Scheduling/SessionSchedulerServiceTests.cs @@ -149,6 +149,68 @@ public sealed class SessionSchedulerServiceTests Assert.Null(ex); } + [Fact] + public async Task TickAsync_WhenHandlerThrows_BackoffsForDuration() + { + var sessionId = Guid.NewGuid(); + var now = new DateTimeOffset(2026, 5, 15, 10, 0, 0, TimeSpan.Zero); + _clock.UtcNow = now; + _store.SessionsNeedingConfirmation = [sessionId]; + _confirmationHandler.ThrowFor.Add(sessionId); + + var sut = CreateSut(); + await sut.TickAsync(CancellationToken.None); + Assert.Single(_confirmationHandler.Calls); + + // Second tick immediately — should be backed off + await sut.TickAsync(CancellationToken.None); + Assert.Single(_confirmationHandler.Calls); + } + + [Fact] + public async Task TickAsync_WhenHandlerThrows_AfterBackoffRetriesAgain() + { + var sessionId = Guid.NewGuid(); + var now = new DateTimeOffset(2026, 5, 15, 10, 0, 0, TimeSpan.Zero); + _clock.UtcNow = now; + _store.SessionsNeedingConfirmation = [sessionId]; + _confirmationHandler.ThrowFor.Add(sessionId); + + var sut = CreateSut(); + await sut.TickAsync(CancellationToken.None); + Assert.Single(_confirmationHandler.Calls); + + // Advance clock past backoff duration (15 min) + _clock.UtcNow = now.AddMinutes(16); + await sut.TickAsync(CancellationToken.None); + Assert.Equal(2, _confirmationHandler.Calls.Count); + } + + [Fact] + public async Task TickAsync_WhenHandlerSucceedsAfterBackoff_ClearsBackoff() + { + var sessionId = Guid.NewGuid(); + var now = new DateTimeOffset(2026, 5, 15, 10, 0, 0, TimeSpan.Zero); + _clock.UtcNow = now; + _store.SessionsNeedingConfirmation = [sessionId]; + _confirmationHandler.ThrowFor.Add(sessionId); + + var sut = CreateSut(); + await sut.TickAsync(CancellationToken.None); + Assert.Single(_confirmationHandler.Calls); + + // Remove throw condition, advance past backoff + _confirmationHandler.ThrowFor.Remove(sessionId); + _clock.UtcNow = now.AddMinutes(16); + await sut.TickAsync(CancellationToken.None); + Assert.Equal(2, _confirmationHandler.Calls.Count); + + // Next tick should still call because backoff was cleared on success + _clock.UtcNow = now.AddMinutes(17); + await sut.TickAsync(CancellationToken.None); + Assert.Equal(3, _confirmationHandler.Calls.Count); + } + private sealed class FakeSendConfirmationHandler : ISendConfirmationHandler { public List Calls { get; } = [];