fotospiel-app/app/Console/Commands/AiEditsRecoverStuckCommand.php

<?php

namespace App\Console\Commands;

use App\Jobs\PollAiEditRequest;
use App\Jobs\ProcessAiEditRequest;
use App\Models\AiEditRequest;
use App\Services\AiEditing\AiEditingRuntimeConfig;
use Illuminate\Console\Command;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Relations\HasMany;
use Illuminate\Support\Collection;

class AiEditsRecoverStuckCommand extends Command
{
    protected $signature = 'ai-edits:recover-stuck
        {--minutes=30 : Minimum age in minutes for queued/processing requests}
        {--requeue : Re-dispatch stuck requests back to the queue}
        {--fail : Mark stuck requests as failed}';

    protected $description = 'Inspect stuck AI edit requests and optionally recover them by requeueing or failing.';

    public function __construct(private readonly AiEditingRuntimeConfig $runtimeConfig)
    {
        parent::__construct();
    }

    public function handle(): int
    {
        $minutes = max(1, (int) $this->option('minutes'));
        $shouldRequeue = (bool) $this->option('requeue');
        $shouldFail = (bool) $this->option('fail');

        if ($shouldRequeue && $shouldFail) {
            $this->error('Use either --requeue or --fail, not both.');

            return self::FAILURE;
        }

        $cutoff = now()->subMinutes($minutes);
        $requests = AiEditRequest::query()
            ->with([
                'event:id,slug,name',
                'providerRuns' => function (HasMany $query): void {
                    $query->select(['id', 'request_id', 'provider_task_id', 'attempt'])
                        ->orderByDesc('attempt');
                },
            ])
            ->whereIn('status', [AiEditRequest::STATUS_QUEUED, AiEditRequest::STATUS_PROCESSING])
            ->where(function (Builder $query) use ($cutoff): void {
                $query
                    ->where(function (Builder $queuedQuery) use ($cutoff): void {
                        $queuedQuery->whereNull('started_at')
                            ->whereNotNull('queued_at')
                            ->where('queued_at', '<=', $cutoff);
                    })
                    ->orWhere(function (Builder $processingQuery) use ($cutoff): void {
                        $processingQuery->whereNotNull('started_at')
                            ->where('started_at', '<=', $cutoff);
                    })
                    ->orWhere(function (Builder $fallbackQuery) use ($cutoff): void {
                        $fallbackQuery->whereNull('queued_at')
                            ->whereNull('started_at')
                            ->where('updated_at', '<=', $cutoff);
                    });
            })
            ->orderBy('updated_at')
            ->get();

        if ($requests->isEmpty()) {
            $this->info(sprintf('No stuck AI edit requests older than %d minute(s).', $minutes));

            return self::SUCCESS;
        }

        $this->table(
            ['ID', 'Event', 'Status', 'Queued/Started', 'Latest task'],
            $requests->map(function (AiEditRequest $request): array {
                $latestTaskId = $this->latestProviderTaskId($request) ?? '-';
                $eventLabel = (string) ($request->event?->name ?: $request->event?->slug ?: $request->event_id);
                $ageSource = $request->started_at ?: $request->queued_at ?: $request->updated_at;

                return [
                    (string) $request->id,
                    $eventLabel,
                    $request->status,
                    $ageSource?->toIso8601String() ?? '-',
                    $latestTaskId,
                ];
            })->all()
        );

        if (! $shouldRequeue && ! $shouldFail) {
            $this->info('Dry-run only. Use --requeue to dispatch recovery jobs or --fail to terminate stuck requests.');

            return self::SUCCESS;
        }

        if ($shouldFail) {
            $count = $this->markAsFailed($requests);
            $this->info(sprintf('Marked %d AI edit request(s) as failed.', $count));

            return self::SUCCESS;
        }

        [$processDispatches, $pollDispatches] = $this->requeueRequests($requests);
        $this->info(sprintf(
            'Recovered %d stuck AI edit request(s): %d process dispatch(es), %d poll dispatch(es).',
            $processDispatches + $pollDispatches,
            $processDispatches,
            $pollDispatches
        ));

        return self::SUCCESS;
    }

    /**
     * @return array{0:int,1:int}
     */
    private function requeueRequests(Collection $requests): array
    {
        $queueName = $this->runtimeConfig->queueName();
        $processDispatches = 0;
        $pollDispatches = 0;

        foreach ($requests as $request) {
            if ($request->status === AiEditRequest::STATUS_QUEUED) {
                ProcessAiEditRequest::dispatch($request->id)->onQueue($queueName);
                $processDispatches++;

                continue;
            }

            $providerTaskId = $this->latestProviderTaskId($request);
            if ($providerTaskId !== null) {
                PollAiEditRequest::dispatch($request->id, $providerTaskId, 1)->onQueue($queueName);
                $pollDispatches++;

                continue;
            }

            ProcessAiEditRequest::dispatch($request->id)->onQueue($queueName);
            $processDispatches++;
        }

        return [$processDispatches, $pollDispatches];
    }

    private function markAsFailed(Collection $requests): int
    {
        $updated = 0;
        $now = now();

        foreach ($requests as $request) {
            $request->forceFill([
                'status' => AiEditRequest::STATUS_FAILED,
                'failure_code' => 'operator_recovery_marked_failed',
                'failure_message' => 'Marked as failed by ai-edits:recover-stuck.',
                'completed_at' => $now,
            ])->save();

            $updated++;
        }

        return $updated;
    }

    private function latestProviderTaskId(AiEditRequest $request): ?string
    {
        foreach ($request->providerRuns as $run) {
            $taskId = trim((string) ($run->provider_task_id ?? ''));
            if ($taskId !== '') {
                return $taskId;
            }
        }

        return null;
    }
}