feat(ai): finalize AI magic edits epic rollout and operations

2026-02-06 22:41:51 +01:00
parent 36bed12ff9
commit 1d2242fb4d
33 changed files with 2621 additions and 18 deletions
--- a/app/Console/Commands/AiEditsRecoverStuckCommand.php
+++ b/app/Console/Commands/AiEditsRecoverStuckCommand.php
@@ -0,0 +1,179 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Jobs\PollAiEditRequest;
+use App\Jobs\ProcessAiEditRequest;
+use App\Models\AiEditRequest;
+use App\Services\AiEditing\AiEditingRuntimeConfig;
+use Illuminate\Console\Command;
+use Illuminate\Database\Eloquent\Builder;
+use Illuminate\Database\Eloquent\Relations\HasMany;
+use Illuminate\Support\Collection;
+
+class AiEditsRecoverStuckCommand extends Command
+{
+    protected $signature = 'ai-edits:recover-stuck
+        {--minutes=30 : Minimum age in minutes for queued/processing requests}
+        {--requeue : Re-dispatch stuck requests back to the queue}
+        {--fail : Mark stuck requests as failed}';
+
+    protected $description = 'Inspect stuck AI edit requests and optionally recover them by requeueing or failing.';
+
+    public function __construct(private readonly AiEditingRuntimeConfig $runtimeConfig)
+    {
+        parent::__construct();
+    }
+
+    public function handle(): int
+    {
+        $minutes = max(1, (int) $this->option('minutes'));
+        $shouldRequeue = (bool) $this->option('requeue');
+        $shouldFail = (bool) $this->option('fail');
+
+        if ($shouldRequeue && $shouldFail) {
+            $this->error('Use either --requeue or --fail, not both.');
+
+            return self::FAILURE;
+        }
+
+        $cutoff = now()->subMinutes($minutes);
+        $requests = AiEditRequest::query()
+            ->with([
+                'event:id,slug,name',
+                'providerRuns' => function (HasMany $query): void {
+                    $query->select(['id', 'request_id', 'provider_task_id', 'attempt'])
+                        ->orderByDesc('attempt');
+                },
+            ])
+            ->whereIn('status', [AiEditRequest::STATUS_QUEUED, AiEditRequest::STATUS_PROCESSING])
+            ->where(function (Builder $query) use ($cutoff): void {
+                $query
+                    ->where(function (Builder $queuedQuery) use ($cutoff): void {
+                        $queuedQuery->whereNull('started_at')
+                            ->whereNotNull('queued_at')
+                            ->where('queued_at', '<=', $cutoff);
+                    })
+                    ->orWhere(function (Builder $processingQuery) use ($cutoff): void {
+                        $processingQuery->whereNotNull('started_at')
+                            ->where('started_at', '<=', $cutoff);
+                    })
+                    ->orWhere(function (Builder $fallbackQuery) use ($cutoff): void {
+                        $fallbackQuery->whereNull('queued_at')
+                            ->whereNull('started_at')
+                            ->where('updated_at', '<=', $cutoff);
+                    });
+            })
+            ->orderBy('updated_at')
+            ->get();
+
+        if ($requests->isEmpty()) {
+            $this->info(sprintf('No stuck AI edit requests older than %d minute(s).', $minutes));
+
+            return self::SUCCESS;
+        }
+
+        $this->table(
+            ['ID', 'Event', 'Status', 'Queued/Started', 'Latest task'],
+            $requests->map(function (AiEditRequest $request): array {
+                $latestTaskId = $this->latestProviderTaskId($request) ?? '-';
+                $eventLabel = (string) ($request->event?->name ?: $request->event?->slug ?: $request->event_id);
+                $ageSource = $request->started_at ?: $request->queued_at ?: $request->updated_at;
+
+                return [
+                    (string) $request->id,
+                    $eventLabel,
+                    $request->status,
+                    $ageSource?->toIso8601String() ?? '-',
+                    $latestTaskId,
+                ];
+            })->all()
+        );
+
+        if (! $shouldRequeue && ! $shouldFail) {
+            $this->info('Dry-run only. Use --requeue to dispatch recovery jobs or --fail to terminate stuck requests.');
+
+            return self::SUCCESS;
+        }
+
+        if ($shouldFail) {
+            $count = $this->markAsFailed($requests);
+            $this->info(sprintf('Marked %d AI edit request(s) as failed.', $count));
+
+            return self::SUCCESS;
+        }
+
+        [$processDispatches, $pollDispatches] = $this->requeueRequests($requests);
+        $this->info(sprintf(
+            'Recovered %d stuck AI edit request(s): %d process dispatch(es), %d poll dispatch(es).',
+            $processDispatches + $pollDispatches,
+            $processDispatches,
+            $pollDispatches
+        ));
+
+        return self::SUCCESS;
+    }
+
+    /**
+     * @return array{0:int,1:int}
+     */
+    private function requeueRequests(Collection $requests): array
+    {
+        $queueName = $this->runtimeConfig->queueName();
+        $processDispatches = 0;
+        $pollDispatches = 0;
+
+        foreach ($requests as $request) {
+            if ($request->status === AiEditRequest::STATUS_QUEUED) {
+                ProcessAiEditRequest::dispatch($request->id)->onQueue($queueName);
+                $processDispatches++;
+
+                continue;
+            }
+
+            $providerTaskId = $this->latestProviderTaskId($request);
+            if ($providerTaskId !== null) {
+                PollAiEditRequest::dispatch($request->id, $providerTaskId, 1)->onQueue($queueName);
+                $pollDispatches++;
+
+                continue;
+            }
+
+            ProcessAiEditRequest::dispatch($request->id)->onQueue($queueName);
+            $processDispatches++;
+        }
+
+        return [$processDispatches, $pollDispatches];
+    }
+
+    private function markAsFailed(Collection $requests): int
+    {
+        $updated = 0;
+        $now = now();
+
+        foreach ($requests as $request) {
+            $request->forceFill([
+                'status' => AiEditRequest::STATUS_FAILED,
+                'failure_code' => 'operator_recovery_marked_failed',
+                'failure_message' => 'Marked as failed by ai-edits:recover-stuck.',
+                'completed_at' => $now,
+            ])->save();
+
+            $updated++;
+        }
+
+        return $updated;
+    }
+
+    private function latestProviderTaskId(AiEditRequest $request): ?string
+    {
+        foreach ($request->providerRuns as $run) {
+            $taskId = trim((string) ($run->provider_task_id ?? ''));
+            if ($taskId !== '') {
+                return $taskId;
+            }
+        }
+
+        return null;
+    }
+}