fix(discord): add backoff to scheduler to prevent 403 spam
- SessionSchedulerService now backs off for 15 minutes after any handler failure (confirmation, one-hour reminder, join link), preventing infinite retry loops on Discord 403 Missing Access. - Added per-session ConcurrentDictionary backoff tracking with automatic cleanup on success. - Enhanced DiscordPlatformMessenger logging for SendConfirmation and SendJoinLink to aid permission diagnostics. - Added 3 regression tests for backoff behavior. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -98,17 +98,34 @@ public sealed class DiscordPlatformMessenger : IPlatformMessenger
|
||||
CancellationToken ct)
|
||||
{
|
||||
var channelId = GetChannelId(request.Group);
|
||||
var message = await restClient.SendMessageAsync(
|
||||
channelId,
|
||||
new MessageProperties()
|
||||
.WithEmbeds([BuildConfirmationEmbed(request)])
|
||||
.WithComponents(BuildRsvpRows(request.SessionId, disabled: false)));
|
||||
try
|
||||
{
|
||||
var message = await restClient.SendMessageAsync(
|
||||
channelId,
|
||||
new MessageProperties()
|
||||
.WithEmbeds([BuildConfirmationEmbed(request)])
|
||||
.WithComponents(BuildRsvpRows(request.SessionId, disabled: false)));
|
||||
|
||||
return new PlatformMessageRef(
|
||||
PlatformKind.Discord,
|
||||
request.Group.ExternalGroupId,
|
||||
null,
|
||||
message.Id.ToString(CultureInfo.InvariantCulture));
|
||||
logger?.LogInformation(
|
||||
"Confirmation request sent to Discord channel {ChannelId}, message id {MessageId}",
|
||||
channelId,
|
||||
message.Id);
|
||||
|
||||
return new PlatformMessageRef(
|
||||
PlatformKind.Discord,
|
||||
request.Group.ExternalGroupId,
|
||||
null,
|
||||
message.Id.ToString(CultureInfo.InvariantCulture));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger?.LogError(
|
||||
ex,
|
||||
"Failed to send confirmation request to Discord channel {ChannelId} for session {SessionId}",
|
||||
channelId,
|
||||
request.SessionId);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task UpdateConfirmationRequestAsync(PlatformRsvpMessageUpdate update, CancellationToken ct)
|
||||
@@ -135,15 +152,32 @@ public sealed class DiscordPlatformMessenger : IPlatformMessenger
|
||||
CancellationToken ct)
|
||||
{
|
||||
var channelId = GetChannelId(notification.Group);
|
||||
var message = await restClient.SendMessageAsync(
|
||||
channelId,
|
||||
new MessageProperties().WithEmbeds([BuildJoinLinkEmbed(notification)]));
|
||||
try
|
||||
{
|
||||
var message = await restClient.SendMessageAsync(
|
||||
channelId,
|
||||
new MessageProperties().WithEmbeds([BuildJoinLinkEmbed(notification)]));
|
||||
|
||||
return new PlatformMessageRef(
|
||||
PlatformKind.Discord,
|
||||
notification.Group.ExternalGroupId,
|
||||
null,
|
||||
message.Id.ToString(CultureInfo.InvariantCulture));
|
||||
logger?.LogInformation(
|
||||
"Join link sent to Discord channel {ChannelId}, message id {MessageId}",
|
||||
channelId,
|
||||
message.Id);
|
||||
|
||||
return new PlatformMessageRef(
|
||||
PlatformKind.Discord,
|
||||
notification.Group.ExternalGroupId,
|
||||
null,
|
||||
message.Id.ToString(CultureInfo.InvariantCulture));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger?.LogError(
|
||||
ex,
|
||||
"Failed to send join link to Discord channel {ChannelId} for session {SessionId}",
|
||||
channelId,
|
||||
notification.SessionId);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task SendDirectSessionNotificationAsync(
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Collections.Concurrent;
|
||||
using GmRelay.Shared.Features.Confirmation.SendConfirmation;
|
||||
using GmRelay.Shared.Features.Reminders.SendJoinLink;
|
||||
using GmRelay.Shared.Features.Reminders.SendOneHourReminder;
|
||||
@@ -20,6 +21,11 @@ public sealed class SessionSchedulerService(
|
||||
ILogger<SessionSchedulerService> logger) : BackgroundService
|
||||
{
|
||||
private static readonly TimeSpan TickInterval = TimeSpan.FromMinutes(1);
|
||||
private static readonly TimeSpan BackoffDuration = TimeSpan.FromMinutes(15);
|
||||
|
||||
private readonly ConcurrentDictionary<Guid, DateTimeOffset> _confirmationBackoff = new();
|
||||
private readonly ConcurrentDictionary<Guid, DateTimeOffset> _oneHourBackoff = new();
|
||||
private readonly ConcurrentDictionary<Guid, DateTimeOffset> _joinLinkBackoff = new();
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
@@ -71,14 +77,30 @@ public sealed class SessionSchedulerService(
|
||||
|
||||
foreach (var sessionId in sessionIds)
|
||||
{
|
||||
if (_confirmationBackoff.TryGetValue(sessionId, out var backoffUntil) && backoffUntil > now)
|
||||
{
|
||||
logger.LogDebug(
|
||||
"Skipping confirmation for session {SessionId} until {Backoff}",
|
||||
sessionId,
|
||||
backoffUntil);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await confirmationHandler.HandleAsync(sessionId, ct);
|
||||
_confirmationBackoff.TryRemove(sessionId, out _);
|
||||
logger.LogInformation("Confirmation sent for session {SessionId}", sessionId);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogError(ex, "Failed to send confirmation for session {SessionId}", sessionId);
|
||||
var nextAttempt = now.Add(BackoffDuration);
|
||||
_confirmationBackoff[sessionId] = nextAttempt;
|
||||
logger.LogError(
|
||||
ex,
|
||||
"Failed to send confirmation for session {SessionId}, backing off until {Backoff}",
|
||||
sessionId,
|
||||
nextAttempt);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -98,14 +120,30 @@ public sealed class SessionSchedulerService(
|
||||
|
||||
foreach (var sessionId in sessionIds)
|
||||
{
|
||||
if (_oneHourBackoff.TryGetValue(sessionId, out var backoffUntil) && backoffUntil > now)
|
||||
{
|
||||
logger.LogDebug(
|
||||
"Skipping one-hour reminder for session {SessionId} until {Backoff}",
|
||||
sessionId,
|
||||
backoffUntil);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await oneHourReminderHandler.HandleAsync(sessionId, ct);
|
||||
_oneHourBackoff.TryRemove(sessionId, out _);
|
||||
logger.LogInformation("One-hour reminder processed for session {SessionId}", sessionId);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogError(ex, "Failed to process one-hour reminder for session {SessionId}", sessionId);
|
||||
var nextAttempt = now.Add(BackoffDuration);
|
||||
_oneHourBackoff[sessionId] = nextAttempt;
|
||||
logger.LogError(
|
||||
ex,
|
||||
"Failed to process one-hour reminder for session {SessionId}, backing off until {Backoff}",
|
||||
sessionId,
|
||||
nextAttempt);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -125,14 +163,30 @@ public sealed class SessionSchedulerService(
|
||||
|
||||
foreach (var sessionId in sessionIds)
|
||||
{
|
||||
if (_joinLinkBackoff.TryGetValue(sessionId, out var backoffUntil) && backoffUntil > now)
|
||||
{
|
||||
logger.LogDebug(
|
||||
"Skipping join link for session {SessionId} until {Backoff}",
|
||||
sessionId,
|
||||
backoffUntil);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await joinLinkHandler.HandleAsync(sessionId, ct);
|
||||
_joinLinkBackoff.TryRemove(sessionId, out _);
|
||||
logger.LogInformation("Join link sent for session {SessionId}", sessionId);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogError(ex, "Failed to send join link for session {SessionId}", sessionId);
|
||||
var nextAttempt = now.Add(BackoffDuration);
|
||||
_joinLinkBackoff[sessionId] = nextAttempt;
|
||||
logger.LogError(
|
||||
ex,
|
||||
"Failed to send join link for session {SessionId}, backing off until {Backoff}",
|
||||
sessionId,
|
||||
nextAttempt);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -149,6 +149,68 @@ public sealed class SessionSchedulerServiceTests
|
||||
Assert.Null(ex);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TickAsync_WhenHandlerThrows_BackoffsForDuration()
|
||||
{
|
||||
var sessionId = Guid.NewGuid();
|
||||
var now = new DateTimeOffset(2026, 5, 15, 10, 0, 0, TimeSpan.Zero);
|
||||
_clock.UtcNow = now;
|
||||
_store.SessionsNeedingConfirmation = [sessionId];
|
||||
_confirmationHandler.ThrowFor.Add(sessionId);
|
||||
|
||||
var sut = CreateSut();
|
||||
await sut.TickAsync(CancellationToken.None);
|
||||
Assert.Single(_confirmationHandler.Calls);
|
||||
|
||||
// Second tick immediately — should be backed off
|
||||
await sut.TickAsync(CancellationToken.None);
|
||||
Assert.Single(_confirmationHandler.Calls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TickAsync_WhenHandlerThrows_AfterBackoffRetriesAgain()
|
||||
{
|
||||
var sessionId = Guid.NewGuid();
|
||||
var now = new DateTimeOffset(2026, 5, 15, 10, 0, 0, TimeSpan.Zero);
|
||||
_clock.UtcNow = now;
|
||||
_store.SessionsNeedingConfirmation = [sessionId];
|
||||
_confirmationHandler.ThrowFor.Add(sessionId);
|
||||
|
||||
var sut = CreateSut();
|
||||
await sut.TickAsync(CancellationToken.None);
|
||||
Assert.Single(_confirmationHandler.Calls);
|
||||
|
||||
// Advance clock past backoff duration (15 min)
|
||||
_clock.UtcNow = now.AddMinutes(16);
|
||||
await sut.TickAsync(CancellationToken.None);
|
||||
Assert.Equal(2, _confirmationHandler.Calls.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TickAsync_WhenHandlerSucceedsAfterBackoff_ClearsBackoff()
|
||||
{
|
||||
var sessionId = Guid.NewGuid();
|
||||
var now = new DateTimeOffset(2026, 5, 15, 10, 0, 0, TimeSpan.Zero);
|
||||
_clock.UtcNow = now;
|
||||
_store.SessionsNeedingConfirmation = [sessionId];
|
||||
_confirmationHandler.ThrowFor.Add(sessionId);
|
||||
|
||||
var sut = CreateSut();
|
||||
await sut.TickAsync(CancellationToken.None);
|
||||
Assert.Single(_confirmationHandler.Calls);
|
||||
|
||||
// Remove throw condition, advance past backoff
|
||||
_confirmationHandler.ThrowFor.Remove(sessionId);
|
||||
_clock.UtcNow = now.AddMinutes(16);
|
||||
await sut.TickAsync(CancellationToken.None);
|
||||
Assert.Equal(2, _confirmationHandler.Calls.Count);
|
||||
|
||||
// Next tick should still call because backoff was cleared on success
|
||||
_clock.UtcNow = now.AddMinutes(17);
|
||||
await sut.TickAsync(CancellationToken.None);
|
||||
Assert.Equal(3, _confirmationHandler.Calls.Count);
|
||||
}
|
||||
|
||||
private sealed class FakeSendConfirmationHandler : ISendConfirmationHandler
|
||||
{
|
||||
public List<Guid> Calls { get; } = [];
|
||||
|
||||
Reference in New Issue
Block a user