a5aed14dd2
- SessionSchedulerService now backs off for 15 minutes after any handler failure (confirmation, one-hour reminder, join link), preventing infinite retry loops on Discord 403 Missing Access. - Added per-session ConcurrentDictionary backoff tracking with automatic cleanup on success. - Enhanced DiscordPlatformMessenger logging for SendConfirmation and SendJoinLink to aid permission diagnostics. - Added 3 regression tests for backoff behavior. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
194 lines
6.7 KiB
C#
194 lines
6.7 KiB
C#
using System.Collections.Concurrent;
|
|
using GmRelay.Shared.Features.Confirmation.SendConfirmation;
|
|
using GmRelay.Shared.Features.Reminders.SendJoinLink;
|
|
using GmRelay.Shared.Features.Reminders.SendOneHourReminder;
|
|
using GmRelay.Shared.Platform;
|
|
using Microsoft.Extensions.Hosting;
|
|
using Microsoft.Extensions.Logging;
|
|
|
|
namespace GmRelay.Shared.Infrastructure.Scheduling;
|
|
|
|
/// <summary>
|
|
/// Stateless scheduler: wakes every 60 seconds, queries PostgreSQL for actionable sessions.
|
|
/// All state is kept in the database so worker restarts do not lose scheduled work.
|
|
/// </summary>
|
|
public sealed class SessionSchedulerService(
|
|
ISessionTriggerStore triggerStore,
|
|
ISendConfirmationHandler confirmationHandler,
|
|
ISendOneHourReminderHandler oneHourReminderHandler,
|
|
ISendJoinLinkHandler joinLinkHandler,
|
|
ISystemClock clock,
|
|
ILogger<SessionSchedulerService> logger) : BackgroundService
|
|
{
|
|
private static readonly TimeSpan TickInterval = TimeSpan.FromMinutes(1);
|
|
private static readonly TimeSpan BackoffDuration = TimeSpan.FromMinutes(15);
|
|
|
|
private readonly ConcurrentDictionary<Guid, DateTimeOffset> _confirmationBackoff = new();
|
|
private readonly ConcurrentDictionary<Guid, DateTimeOffset> _oneHourBackoff = new();
|
|
private readonly ConcurrentDictionary<Guid, DateTimeOffset> _joinLinkBackoff = new();
|
|
|
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
|
{
|
|
logger.LogInformation("Session scheduler started (interval: {Interval})", TickInterval);
|
|
|
|
using var timer = new PeriodicTimer(TickInterval);
|
|
|
|
do
|
|
{
|
|
try
|
|
{
|
|
await TickAsync(stoppingToken);
|
|
}
|
|
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
|
{
|
|
break;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
logger.LogError(ex, "Scheduler tick failed, will retry next tick");
|
|
}
|
|
}
|
|
while (await timer.WaitForNextTickAsync(stoppingToken));
|
|
|
|
logger.LogInformation("Session scheduler stopped");
|
|
}
|
|
|
|
public async Task TickAsync(CancellationToken ct)
|
|
{
|
|
var now = clock.UtcNow;
|
|
|
|
await ProcessConfirmationTriggers(now, ct);
|
|
await ProcessOneHourReminderTriggers(now, ct);
|
|
await ProcessJoinLinkTriggers(now, ct);
|
|
}
|
|
|
|
private async Task ProcessConfirmationTriggers(DateTimeOffset now, CancellationToken ct)
|
|
{
|
|
IReadOnlyList<Guid> sessionIds;
|
|
try
|
|
{
|
|
sessionIds = await triggerStore.GetSessionsNeedingConfirmationAsync(now, ct);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
logger.LogError(ex, "Failed to query confirmation triggers");
|
|
return;
|
|
}
|
|
|
|
foreach (var sessionId in sessionIds)
|
|
{
|
|
if (_confirmationBackoff.TryGetValue(sessionId, out var backoffUntil) && backoffUntil > now)
|
|
{
|
|
logger.LogDebug(
|
|
"Skipping confirmation for session {SessionId} until {Backoff}",
|
|
sessionId,
|
|
backoffUntil);
|
|
continue;
|
|
}
|
|
|
|
try
|
|
{
|
|
await confirmationHandler.HandleAsync(sessionId, ct);
|
|
_confirmationBackoff.TryRemove(sessionId, out _);
|
|
logger.LogInformation("Confirmation sent for session {SessionId}", sessionId);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
var nextAttempt = now.Add(BackoffDuration);
|
|
_confirmationBackoff[sessionId] = nextAttempt;
|
|
logger.LogError(
|
|
ex,
|
|
"Failed to send confirmation for session {SessionId}, backing off until {Backoff}",
|
|
sessionId,
|
|
nextAttempt);
|
|
}
|
|
}
|
|
}
|
|
|
|
private async Task ProcessOneHourReminderTriggers(DateTimeOffset now, CancellationToken ct)
|
|
{
|
|
IReadOnlyList<Guid> sessionIds;
|
|
try
|
|
{
|
|
sessionIds = await triggerStore.GetSessionsNeedingOneHourReminderAsync(now, ct);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
logger.LogError(ex, "Failed to query one-hour reminder triggers");
|
|
return;
|
|
}
|
|
|
|
foreach (var sessionId in sessionIds)
|
|
{
|
|
if (_oneHourBackoff.TryGetValue(sessionId, out var backoffUntil) && backoffUntil > now)
|
|
{
|
|
logger.LogDebug(
|
|
"Skipping one-hour reminder for session {SessionId} until {Backoff}",
|
|
sessionId,
|
|
backoffUntil);
|
|
continue;
|
|
}
|
|
|
|
try
|
|
{
|
|
await oneHourReminderHandler.HandleAsync(sessionId, ct);
|
|
_oneHourBackoff.TryRemove(sessionId, out _);
|
|
logger.LogInformation("One-hour reminder processed for session {SessionId}", sessionId);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
var nextAttempt = now.Add(BackoffDuration);
|
|
_oneHourBackoff[sessionId] = nextAttempt;
|
|
logger.LogError(
|
|
ex,
|
|
"Failed to process one-hour reminder for session {SessionId}, backing off until {Backoff}",
|
|
sessionId,
|
|
nextAttempt);
|
|
}
|
|
}
|
|
}
|
|
|
|
private async Task ProcessJoinLinkTriggers(DateTimeOffset now, CancellationToken ct)
|
|
{
|
|
IReadOnlyList<Guid> sessionIds;
|
|
try
|
|
{
|
|
sessionIds = await triggerStore.GetSessionsNeedingJoinLinkAsync(now, ct);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
logger.LogError(ex, "Failed to query join-link triggers");
|
|
return;
|
|
}
|
|
|
|
foreach (var sessionId in sessionIds)
|
|
{
|
|
if (_joinLinkBackoff.TryGetValue(sessionId, out var backoffUntil) && backoffUntil > now)
|
|
{
|
|
logger.LogDebug(
|
|
"Skipping join link for session {SessionId} until {Backoff}",
|
|
sessionId,
|
|
backoffUntil);
|
|
continue;
|
|
}
|
|
|
|
try
|
|
{
|
|
await joinLinkHandler.HandleAsync(sessionId, ct);
|
|
_joinLinkBackoff.TryRemove(sessionId, out _);
|
|
logger.LogInformation("Join link sent for session {SessionId}", sessionId);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
var nextAttempt = now.Add(BackoffDuration);
|
|
_joinLinkBackoff[sessionId] = nextAttempt;
|
|
logger.LogError(
|
|
ex,
|
|
"Failed to send join link for session {SessionId}, backing off until {Backoff}",
|
|
sessionId,
|
|
nextAttempt);
|
|
}
|
|
}
|
|
}
|
|
}
|