diff --git a/.beads/last-touched b/.beads/last-touched index 6268903..cdadab4 100644 --- a/.beads/last-touched +++ b/.beads/last-touched @@ -1 +1 @@ -fotospiel-app-v00l +fotospiel-app-e05 diff --git a/app/Console/Commands/MonitorStorageCommand.php b/app/Console/Commands/MonitorStorageCommand.php index ec7cda9..89636c4 100644 --- a/app/Console/Commands/MonitorStorageCommand.php +++ b/app/Console/Commands/MonitorStorageCommand.php @@ -46,6 +46,12 @@ class MonitorStorageCommand extends Command $assetStats = $this->buildAssetStatistics(); $thresholds = $this->capacityThresholds(); + $checksumConfig = $this->checksumAlertConfig(); + $checksumWindowMinutes = $checksumConfig['window_minutes']; + $checksumThresholds = $checksumConfig['thresholds']; + $checksumMismatches = $checksumConfig['enabled'] && $checksumWindowMinutes > 0 + ? $this->checksumMismatchCounts($checksumWindowMinutes) + : []; $alerts = []; $snapshotTargets = []; @@ -78,6 +84,7 @@ class MonitorStorageCommand extends Command ]; } + $targetChecksumMismatches = $checksumMismatches[$target->id] ?? 0; $snapshotTargets[] = [ 'id' => $target->id, 'key' => $target->key, @@ -85,13 +92,35 @@ class MonitorStorageCommand extends Command 'is_hot' => (bool) $target->is_hot, 'capacity' => $capacity, 'assets' => $assets, + 'checksum_mismatches' => [ + 'count' => $targetChecksumMismatches, + 'window_minutes' => $checksumWindowMinutes, + ], ]; } + if ($checksumConfig['enabled'] && $checksumWindowMinutes > 0) { + $totalMismatches = array_sum($checksumMismatches); + $checksumSeverity = $this->determineChecksumSeverity($totalMismatches, $checksumThresholds); + + if ($checksumSeverity !== 'ok') { + $alerts[] = [ + 'type' => 'checksum_mismatch', + 'severity' => $checksumSeverity, + 'count' => $totalMismatches, + 'window_minutes' => $checksumWindowMinutes, + ]; + } + } + $snapshot = [ 'generated_at' => now()->toIso8601String(), 'targets' => $snapshotTargets, 'alerts' => $alerts, + 'checksum' => [ + 'window_minutes' => $checksumWindowMinutes, + 'mismatch_total' => array_sum($checksumMismatches), + ], ]; $ttlMinutes = max(1, (int) config('storage-monitor.monitor.cache_minutes', 15)); @@ -191,4 +220,62 @@ class MonitorStorageCommand extends Command return 'ok'; } + + private function checksumAlertConfig(): array + { + $enabled = (bool) config('storage-monitor.checksum_validation.enabled', true); + $windowMinutes = max(0, (int) config('storage-monitor.checksum_validation.alert_window_minutes', 60)); + $warning = (int) config('storage-monitor.checksum_validation.thresholds.warning', 1); + $critical = (int) config('storage-monitor.checksum_validation.thresholds.critical', 5); + + if ($warning > $critical && $critical > 0) { + [$warning, $critical] = [$critical, $warning]; + } + + return [ + 'enabled' => $enabled, + 'window_minutes' => $windowMinutes, + 'thresholds' => [ + 'warning' => $warning, + 'critical' => $critical, + ], + ]; + } + + private function checksumMismatchCounts(int $windowMinutes): array + { + $query = EventMediaAsset::query() + ->selectRaw('media_storage_target_id, COUNT(*) as total_count') + ->where('status', 'failed') + ->where('meta->checksum_status', 'mismatch'); + + if ($windowMinutes > 0) { + $query->where('updated_at', '>=', now()->subMinutes($windowMinutes)); + } + + return $query->groupBy('media_storage_target_id') + ->get() + ->mapWithKeys(fn ($row) => [(int) $row->media_storage_target_id => (int) $row->total_count]) + ->all(); + } + + private function determineChecksumSeverity(int $count, array $thresholds): string + { + $warning = (int) ($thresholds['warning'] ?? 1); + $critical = (int) ($thresholds['critical'] ?? 5); + + if ($count <= 0) { + return 'ok'; + } + + if ($critical > 0 && $count >= $critical) { + return 'critical'; + } + + if ($warning > 0 && $count >= $warning) { + return 'warning'; + } + + return 'ok'; + } } diff --git a/app/Jobs/ArchiveEventMediaAssets.php b/app/Jobs/ArchiveEventMediaAssets.php index 33b9f37..1083ac5 100644 --- a/app/Jobs/ArchiveEventMediaAssets.php +++ b/app/Jobs/ArchiveEventMediaAssets.php @@ -71,12 +71,44 @@ class ArchiveEventMediaAssets implements ShouldQueue Storage::disk($archiveDisk)->put($archivePath, $stream); + $checksumMeta = null; + $archiveChecksum = null; + if ($this->checksumValidationEnabled()) { + $archiveChecksum = $this->computeChecksum($archiveDisk, $archivePath); + if (! $archiveChecksum) { + throw new \RuntimeException('Archive checksum unavailable'); + } + + $expectedChecksum = $asset->checksum; + if ($expectedChecksum) { + if (! hash_equals($expectedChecksum, $archiveChecksum)) { + $this->handleChecksumMismatch($asset, $expectedChecksum, $archiveChecksum, $sourceDisk, $archiveDisk); + $this->deleteArchiveCopy($archiveDisk, $archivePath); + + continue; + } + + $checksumMeta = [ + 'checksum_status' => 'verified', + 'checksum_verified_at' => now()->toIso8601String(), + ]; + } else { + $asset->checksum = $archiveChecksum; + $checksumMeta = [ + 'checksum_status' => 'seeded', + 'checksum_verified_at' => now()->toIso8601String(), + ]; + } + } + $asset->fill([ 'disk' => $archiveDisk, 'media_storage_target_id' => $archiveTargetId, 'status' => 'archived', 'archived_at' => now(), 'error_message' => null, + 'checksum' => $asset->checksum, + 'meta' => $this->mergeMeta($asset->meta, $checksumMeta), ])->save(); if ($this->deleteSource) { @@ -102,4 +134,92 @@ class ArchiveEventMediaAssets implements ShouldQueue } } } + + private function checksumValidationEnabled(): bool + { + return (bool) config('storage-monitor.checksum_validation.enabled', true); + } + + private function computeChecksum(string $disk, string $path): ?string + { + try { + $stream = Storage::disk($disk)->readStream($path); + } catch (\Throwable $e) { + Log::channel('storage-jobs')->warning('Failed to open stream for checksum', [ + 'disk' => $disk, + 'path' => $path, + 'error' => $e->getMessage(), + ]); + + return null; + } + + if (! $stream) { + return null; + } + + try { + $context = hash_init('sha256'); + $ok = hash_update_stream($context, $stream); + if ($ok === false) { + return null; + } + + return hash_final($context); + } finally { + if (is_resource($stream)) { + fclose($stream); + } + } + } + + private function handleChecksumMismatch( + EventMediaAsset $asset, + string $expectedChecksum, + string $actualChecksum, + string $sourceDisk, + string $archiveDisk, + ): void { + Log::channel('storage-jobs')->alert('Checksum mismatch detected during archive', [ + 'asset_id' => $asset->id, + 'event_id' => $asset->event_id, + 'source_disk' => $sourceDisk, + 'archive_disk' => $archiveDisk, + 'expected_checksum' => $expectedChecksum, + 'actual_checksum' => $actualChecksum, + ]); + + $asset->update([ + 'status' => 'failed', + 'error_message' => 'checksum_mismatch', + 'meta' => $this->mergeMeta($asset->meta, [ + 'checksum_status' => 'mismatch', + 'checksum_verified_at' => now()->toIso8601String(), + 'checksum_expected' => $expectedChecksum, + 'checksum_actual' => $actualChecksum, + ]), + ]); + } + + private function deleteArchiveCopy(string $archiveDisk, string $path): void + { + try { + Storage::disk($archiveDisk)->delete($path); + } catch (\Throwable $e) { + Log::channel('storage-jobs')->warning('Failed to clean up archive copy after checksum mismatch', [ + 'disk' => $archiveDisk, + 'path' => $path, + 'error' => $e->getMessage(), + ]); + } + } + + private function mergeMeta(?array $meta, ?array $updates): ?array + { + if (! $updates) { + return $meta; + } + + return array_merge($meta ?? [], $updates); + } } diff --git a/config/storage-monitor.php b/config/storage-monitor.php index 598c1bd..1c4101f 100644 --- a/config/storage-monitor.php +++ b/config/storage-monitor.php @@ -12,6 +12,15 @@ return [ 'critical' => (int) env('STORAGE_CAPACITY_CRITICAL', 90), ], + 'checksum_validation' => [ + 'enabled' => (bool) env('STORAGE_CHECKSUM_VALIDATION', true), + 'alert_window_minutes' => (int) env('STORAGE_CHECKSUM_ALERT_WINDOW_MINUTES', 60), + 'thresholds' => [ + 'warning' => (int) env('STORAGE_CHECKSUM_WARNING', 1), + 'critical' => (int) env('STORAGE_CHECKSUM_CRITICAL', 5), + ], + ], + 'monitor' => [ 'lock_seconds' => (int) env('STORAGE_MONITOR_LOCK_SECONDS', 300), 'cache_minutes' => (int) env('STORAGE_MONITOR_CACHE_MINUTES', 15), diff --git a/docs/ops/media-storage-spec.md b/docs/ops/media-storage-spec.md index 04f37c4..2f40ad9 100644 --- a/docs/ops/media-storage-spec.md +++ b/docs/ops/media-storage-spec.md @@ -59,6 +59,7 @@ This document explains how customer photo uploads move through the Fotospiel pla | Security queue | `.env SECURITY_SCAN_QUEUE` & `config/security.php` | Defaults to `media-security`. | | Archive scheduling | `config/storage-monitor.php['archive']` | Controls grace days, chunk size, locking, and dispatch caps. | | Queue health alerts | `config/storage-monitor.php['queue_health']` | Warning/critical thresholds for `media-storage` and `media-security` queues. | +| Checksum alerts | `config/storage-monitor.php['checksum_validation']` | Enables checksum verification alerts and thresholding window. | | Container volumes | `docker-compose.yml` | `app`, workers, and scheduler share the `app-code` volume so `/var/www/html/storage` is common. | ## Operational Checklist @@ -72,6 +73,7 @@ This document explains how customer photo uploads move through the Fotospiel pla - Watch `storage:monitor` output (email or logs) for capacity warnings on hot disks. - Use Horizon or Redis metrics to verify `media-storage` queue depth; thresholds live in `config/storage-monitor.php`. - Review `/var/www/html/storage/logs/storage-jobs.log` (if configured) for archival failures. + - Checksum mismatches (hot→archive) are flagged by `storage:monitor` using `checksum_validation` thresholds. - Ensure `media-security` queue stays below critical thresholds so uploads aren’t blocked awaiting security scans. - **Troubleshooting uploads** diff --git a/tests/Feature/ArchiveEventMediaAssetsTest.php b/tests/Feature/ArchiveEventMediaAssetsTest.php new file mode 100644 index 0000000..ee2d98a --- /dev/null +++ b/tests/Feature/ArchiveEventMediaAssetsTest.php @@ -0,0 +1,129 @@ + 'hot-disk', + 'name' => 'Hot Disk', + 'driver' => 'local', + 'config' => [], + 'is_hot' => true, + 'is_default' => true, + 'is_active' => true, + 'priority' => 100, + ]); + + MediaStorageTarget::create([ + 'key' => 'archive-disk', + 'name' => 'Archive Disk', + 'driver' => 'local', + 'config' => [], + 'is_hot' => false, + 'is_default' => true, + 'is_active' => true, + 'priority' => 100, + ]); + + $event = Event::factory()->create(); + $path = 'events/'.$event->id.'/photo.jpg'; + Storage::disk('hot-disk')->put($path, 'photo-body'); + + $checksum = hash('sha256', 'photo-body'); + + $asset = EventMediaAsset::create([ + 'event_id' => $event->id, + 'media_storage_target_id' => $hotTarget->id, + 'photo_id' => null, + 'variant' => 'original', + 'disk' => 'hot-disk', + 'path' => $path, + 'size_bytes' => 10, + 'checksum' => $checksum, + 'status' => 'hot', + ]); + + (new ArchiveEventMediaAssets($event->id, true))->handle(app(EventStorageManager::class)); + + $asset->refresh(); + + $this->assertSame('archived', $asset->status); + $this->assertSame('archive-disk', $asset->disk); + $this->assertSame('verified', data_get($asset->meta, 'checksum_status')); + $this->assertNotEmpty(data_get($asset->meta, 'checksum_verified_at')); + Storage::disk('archive-disk')->assertExists($path); + Storage::disk('hot-disk')->assertMissing($path); + } + + public function test_marks_asset_failed_on_checksum_mismatch(): void + { + Storage::fake('hot-disk'); + Storage::fake('archive-disk'); + + $hotTarget = MediaStorageTarget::create([ + 'key' => 'hot-disk', + 'name' => 'Hot Disk', + 'driver' => 'local', + 'config' => [], + 'is_hot' => true, + 'is_default' => true, + 'is_active' => true, + 'priority' => 100, + ]); + + MediaStorageTarget::create([ + 'key' => 'archive-disk', + 'name' => 'Archive Disk', + 'driver' => 'local', + 'config' => [], + 'is_hot' => false, + 'is_default' => true, + 'is_active' => true, + 'priority' => 100, + ]); + + $event = Event::factory()->create(); + $path = 'events/'.$event->id.'/photo.jpg'; + Storage::disk('hot-disk')->put($path, 'photo-body'); + + $asset = EventMediaAsset::create([ + 'event_id' => $event->id, + 'media_storage_target_id' => $hotTarget->id, + 'photo_id' => null, + 'variant' => 'original', + 'disk' => 'hot-disk', + 'path' => $path, + 'size_bytes' => 10, + 'checksum' => hash('sha256', 'different-body'), + 'status' => 'hot', + ]); + + (new ArchiveEventMediaAssets($event->id, true))->handle(app(EventStorageManager::class)); + + $asset->refresh(); + + $this->assertSame('failed', $asset->status); + $this->assertSame('checksum_mismatch', $asset->error_message); + $this->assertSame('mismatch', data_get($asset->meta, 'checksum_status')); + $this->assertSame('hot-disk', $asset->disk); + Storage::disk('hot-disk')->assertExists($path); + Storage::disk('archive-disk')->assertMissing($path); + } +} diff --git a/tests/Feature/Console/MonitorStorageCommandTest.php b/tests/Feature/Console/MonitorStorageCommandTest.php index c76af3a..7763103 100644 --- a/tests/Feature/Console/MonitorStorageCommandTest.php +++ b/tests/Feature/Console/MonitorStorageCommandTest.php @@ -75,4 +75,62 @@ class MonitorStorageCommandTest extends TestCase $this->assertSame(2, $snapshot['targets'][0]['assets']['total']); $this->assertGreaterThanOrEqual(1, count($snapshot['alerts'])); } + + public function test_monitor_command_flags_checksum_mismatches(): void + { + config([ + 'storage-monitor.checksum_validation.enabled' => true, + 'storage-monitor.checksum_validation.alert_window_minutes' => 60, + 'storage-monitor.checksum_validation.thresholds.warning' => 1, + 'storage-monitor.checksum_validation.thresholds.critical' => 5, + ]); + + $target = MediaStorageTarget::create([ + 'key' => 'local-ssd', + 'name' => 'Local SSD', + 'driver' => 'local', + 'config' => ['monitor_path' => storage_path('app')], + 'is_hot' => true, + 'is_default' => true, + 'is_active' => true, + 'priority' => 100, + ]); + + $event = Event::factory()->create(); + + EventMediaAsset::create([ + 'event_id' => $event->id, + 'media_storage_target_id' => $target->id, + 'photo_id' => null, + 'variant' => 'original', + 'disk' => 'local-ssd', + 'path' => 'events/'.$event->id.'/photo.jpg', + 'size_bytes' => 2048, + 'status' => 'failed', + 'meta' => [ + 'checksum_status' => 'mismatch', + 'checksum_verified_at' => now()->toIso8601String(), + ], + ]); + + $health = Mockery::mock(StorageHealthService::class); + $health->shouldReceive('getCapacity')->andReturn([ + 'status' => 'ok', + 'total' => 100, + 'free' => 90, + 'used' => 10, + 'percentage' => 10, + 'path' => storage_path('app'), + ]); + $this->app->instance(StorageHealthService::class, $health); + + $this->artisan('storage:monitor') + ->assertExitCode(0); + + $snapshot = Cache::get('storage:monitor:last'); + + $this->assertNotNull($snapshot); + $alerts = collect($snapshot['alerts'] ?? []); + $this->assertTrue($alerts->contains(fn ($alert) => ($alert['type'] ?? null) === 'checksum_mismatch')); + } }