option('minutes')); $shouldRequeue = (bool) $this->option('requeue'); $shouldFail = (bool) $this->option('fail'); if ($shouldRequeue && $shouldFail) { $this->error('Use either --requeue or --fail, not both.'); return self::FAILURE; } $cutoff = now()->subMinutes($minutes); $requests = AiEditRequest::query() ->with([ 'event:id,slug,name', 'providerRuns' => function (HasMany $query): void { $query->select(['id', 'request_id', 'provider_task_id', 'attempt']) ->orderByDesc('attempt'); }, ]) ->whereIn('status', [AiEditRequest::STATUS_QUEUED, AiEditRequest::STATUS_PROCESSING]) ->where(function (Builder $query) use ($cutoff): void { $query ->where(function (Builder $queuedQuery) use ($cutoff): void { $queuedQuery->whereNull('started_at') ->whereNotNull('queued_at') ->where('queued_at', '<=', $cutoff); }) ->orWhere(function (Builder $processingQuery) use ($cutoff): void { $processingQuery->whereNotNull('started_at') ->where('started_at', '<=', $cutoff); }) ->orWhere(function (Builder $fallbackQuery) use ($cutoff): void { $fallbackQuery->whereNull('queued_at') ->whereNull('started_at') ->where('updated_at', '<=', $cutoff); }); }) ->orderBy('updated_at') ->get(); if ($requests->isEmpty()) { $this->info(sprintf('No stuck AI edit requests older than %d minute(s).', $minutes)); return self::SUCCESS; } $this->table( ['ID', 'Event', 'Status', 'Queued/Started', 'Latest task'], $requests->map(function (AiEditRequest $request): array { $latestTaskId = $this->latestProviderTaskId($request) ?? '-'; $eventLabel = (string) ($request->event?->name ?: $request->event?->slug ?: $request->event_id); $ageSource = $request->started_at ?: $request->queued_at ?: $request->updated_at; return [ (string) $request->id, $eventLabel, $request->status, $ageSource?->toIso8601String() ?? '-', $latestTaskId, ]; })->all() ); if (! $shouldRequeue && ! $shouldFail) { $this->info('Dry-run only. Use --requeue to dispatch recovery jobs or --fail to terminate stuck requests.'); return self::SUCCESS; } if ($shouldFail) { $count = $this->markAsFailed($requests); $this->info(sprintf('Marked %d AI edit request(s) as failed.', $count)); return self::SUCCESS; } [$processDispatches, $pollDispatches] = $this->requeueRequests($requests); $this->info(sprintf( 'Recovered %d stuck AI edit request(s): %d process dispatch(es), %d poll dispatch(es).', $processDispatches + $pollDispatches, $processDispatches, $pollDispatches )); return self::SUCCESS; } /** * @return array{0:int,1:int} */ private function requeueRequests(Collection $requests): array { $queueName = $this->runtimeConfig->queueName(); $processDispatches = 0; $pollDispatches = 0; foreach ($requests as $request) { if ($request->status === AiEditRequest::STATUS_QUEUED) { ProcessAiEditRequest::dispatch($request->id)->onQueue($queueName); $processDispatches++; continue; } $providerTaskId = $this->latestProviderTaskId($request); if ($providerTaskId !== null) { PollAiEditRequest::dispatch($request->id, $providerTaskId, 1)->onQueue($queueName); $pollDispatches++; continue; } ProcessAiEditRequest::dispatch($request->id)->onQueue($queueName); $processDispatches++; } return [$processDispatches, $pollDispatches]; } private function markAsFailed(Collection $requests): int { $updated = 0; $now = now(); foreach ($requests as $request) { $request->forceFill([ 'status' => AiEditRequest::STATUS_FAILED, 'failure_code' => 'operator_recovery_marked_failed', 'failure_message' => 'Marked as failed by ai-edits:recover-stuck.', 'completed_at' => $now, ])->save(); $updated++; } return $updated; } private function latestProviderTaskId(AiEditRequest $request): ?string { foreach ($request->providerRuns as $run) { $taskId = trim((string) ($run->provider_task_id ?? '')); if ($taskId !== '') { return $taskId; } } return null; } }