From 8a80aae21ef2e38d24fbaa39f931695e102cca41 Mon Sep 17 00:00:00 2001 From: Gregor Klevze Date: Tue, 25 Aug 2026 07:58:47 +0200 Subject: [PATCH] Ship production optimization M1-M12.5A: queues, metrics, HTTP observability, and vector search reliability. Keep similar-ai from tripping the global circuit on a lone URL 502, clamp Qdrant search to 100, and add Server-Timing plus slow-request logging. Studio shared props, Academy S3 exists caching, heat chunking, and Redis/scheduler hygiene stay in this rollout. --- .env.example | 32 + .../DispatchCollectionMaintenanceCommand.php | 6 + .../Commands/GenerateSitemapsCommand.php | 52 +- .../Commands/PruneMetricSnapshotsCommand.php | 125 +++- .../PruneOnlineVisitorIndexCommand.php | 88 +++ .../PurgeQueuedRecComputeTagsCommand.php | 94 +++ .../Commands/RecalculateHeatCommand.php | 48 +- .../RedisCleanupLegacyPrefixCommand.php | 139 ++++ .../Commands/RedisCleanupOrphansCommand.php | 79 +++ .../Api/SimilarAiArtworksController.php | 24 +- .../Controllers/ArtworkDownloadController.php | 20 +- app/Http/Controllers/SitemapController.php | 12 + app/Http/Middleware/AddServerTiming.php | 43 ++ app/Http/Middleware/HandleInertiaRequests.php | 13 +- app/Http/Middleware/LogSlowHttpRequest.php | 74 +++ app/Http/Middleware/TrackOnlineVisitor.php | 5 + .../Concerns/RestoresAfterArtworkIdCursor.php | 27 + app/Jobs/IndexArtworkJob.php | 16 +- app/Jobs/IndexUserJob.php | 19 +- app/Jobs/RankBuildScopeListsJob.php | 18 +- app/Jobs/RecComputeSimilarByBehaviorJob.php | 113 +++- app/Jobs/RecComputeSimilarByTagsJob.php | 59 +- app/Jobs/RecComputeSimilarHybridJob.php | 87 ++- app/Providers/AppServiceProvider.php | 5 + app/Services/Academy/AcademyAccessService.php | 13 +- .../CollectionBackgroundJobService.php | 39 ++ app/Services/LeaderboardService.php | 27 +- .../Metrics/ArtworkHourlySnapshotWindow.php | 260 ++++++++ .../Profile/CreatorJourneyService.php | 6 +- .../Sitemaps/Builders/TagsSitemapBuilder.php | 34 +- .../Studio/CreatorStudioOverviewService.php | 19 +- app/Services/Studio/StudioMetricsService.php | 70 +- .../Traffic/OnlineVisitorRepository.php | 24 +- app/Services/Traffic/PresenceIndexPruner.php | 116 ++++ .../Vision/AiArtworkVectorSearchService.php | 92 ++- app/Services/Vision/VectorGatewayClient.php | 167 ++++- .../Vision/VectorGatewayException.php | 21 + .../Http/HttpPerformanceLogAnalyzer.php | 322 ++++++++++ app/Support/Http/HttpUriNormalizer.php | 96 +++ app/Support/Http/TimedInertiaSsrGateway.php | 34 + app/Support/Queues/QueuedJobClassMatcher.php | 33 + app/Support/Redis/OrphanQueueCleanup.php | 159 +++++ bootstrap/app.php | 5 + bootstrap/ssr/ssr.js | 27 +- config/collections.php | 3 + config/forum_security.php | 2 + config/horizon.php | 28 +- config/http_observability.php | 14 + config/logging.php | 8 + config/metrics.php | 20 + config/queue.php | 5 +- config/ranking.php | 5 + config/sitemaps.php | 3 + config/skinbase_ai_moderation.php | 1 + config/traffic.php | 15 + ...uplicate_artwork_metric_snapshot_index.php | 71 ++ .../conf.d/skinbase-perf-log-format.conf | 23 + deploy/nginx/sitemaps.conf | 45 +- .../skinbase-vhost-performance-log.snippet | 16 + docs/QUEUE.md | 2 + docs/cli-reference.md | 2 +- docs/optimization-m1-recommendation-jobs.md | 290 +++++++++ ...ization-m10-database-query-optimization.md | 290 +++++++++ ...imization-m11-http-endpoint-performance.md | 322 ++++++++++ ...tion-m12-http-performance-observability.md | 303 +++++++++ ...timization-m12.2-vector-gateway-circuit.md | 27 + ...12.3-vector-gateway-circuit-attribution.md | 12 + ...imization-m12.4-similar-ai-dual-circuit.md | 7 + docs/optimization-m2-mail-queue.md | 79 +++ docs/optimization-m3-sitemaps.md | 127 ++++ ...ptimization-m4-recommendation-snapshots.md | 178 +++++ ...timization-m5-metric-snapshot-retention.md | 329 ++++++++++ ...zation-m5.1-metric-snapshot-correctness.md | 195 ++++++ docs/optimization-m6-php-fpm-memory.md | 357 ++++++++++ docs/optimization-m7-default-queue-backlog.md | 186 ++++++ docs/optimization-m7.1-queue-compatibility.md | 217 +++++++ docs/optimization-m8-redis-orphan-cleanup.md | 180 ++++++ docs/optimization-m9-scheduler-overlap.md | 212 ++++++ docs/skinbase-production-baseline.md | 607 ++++++++++++++++++ phpunit.xml | 4 + resources/js/Pages/Studio/StudioDashboard.jsx | 27 +- routes/console.php | 91 ++- scripts/analyze-http-performance.php | 68 ++ scripts/collect-production-baseline.sh | 226 +++++++ scripts/sync-safe-updated.sh | 3 + .../CollectionDispatchGateTest.php | 18 + .../Forum/ForumQueueDispatchGateTest.php | 119 ++++ .../Http/ArtworkDownloadMissingFileTest.php | 24 + tests/Feature/Http/RssPresenceSkipTest.php | 11 + .../Http/SlowHttpRequestLoggingTest.php | 113 ++++ .../Http/StudioGroupsSharedPropTest.php | 26 + .../ArtworkHourlySnapshotWindowTest.php | 175 +++++ .../Metrics/LeaderboardSnapshotDeltaTest.php | 82 +++ .../Metrics/RecalculateHeatCommandTest.php | 47 ++ tests/Feature/Profile/CreatorJourneyTest.php | 49 +- .../PruneMetricSnapshotsCommandTest.php | 164 +++++ .../Queue/DefaultQueueAssignmentTest.php | 55 ++ .../PurgeQueuedRecComputeTagsCommandTest.php | 56 ++ .../RecComputeSimilarJobsTest.php | 141 ++++ .../SimilarArtworksHybridTest.php | 53 ++ .../Feature/Redis/OrphanQueueCleanupTest.php | 96 +++ .../Scheduler/ScheduleInventoryTest.php | 43 ++ tests/Feature/SitemapTest.php | 50 ++ tests/Feature/StudioTest.php | 4 +- .../Vision/VectorGatewaySearchCircuitTest.php | 373 +++++++++++ .../Vision/VectorGatewaySearchLimitTest.php | 118 ++++ tests/Unit/HorizonMailQueueTest.php | 48 ++ .../Http/HttpPerformanceLogAnalyzerTest.php | 110 ++++ .../RecComputeSerializedCompatibilityTest.php | 73 +++ .../RecComputeSimilarByBehaviorJobTest.php | 26 + .../Jobs/RecComputeSimilarHybridJobTest.php | 10 + tests/Unit/Redis/LegacyPrefixFilterTest.php | 21 + .../Traffic/OnlineVisitorIndexReadTest.php | 37 ++ .../Unit/Traffic/PresenceIndexPrunerTest.php | 73 +++ 114 files changed, 9293 insertions(+), 284 deletions(-) create mode 100644 app/Console/Commands/PruneOnlineVisitorIndexCommand.php create mode 100644 app/Console/Commands/PurgeQueuedRecComputeTagsCommand.php create mode 100644 app/Console/Commands/RedisCleanupLegacyPrefixCommand.php create mode 100644 app/Console/Commands/RedisCleanupOrphansCommand.php create mode 100644 app/Http/Middleware/AddServerTiming.php create mode 100644 app/Http/Middleware/LogSlowHttpRequest.php create mode 100644 app/Jobs/Concerns/RestoresAfterArtworkIdCursor.php create mode 100644 app/Services/Metrics/ArtworkHourlySnapshotWindow.php create mode 100644 app/Services/Traffic/PresenceIndexPruner.php create mode 100644 app/Services/Vision/VectorGatewayException.php create mode 100644 app/Support/Http/HttpPerformanceLogAnalyzer.php create mode 100644 app/Support/Http/HttpUriNormalizer.php create mode 100644 app/Support/Http/TimedInertiaSsrGateway.php create mode 100644 app/Support/Queues/QueuedJobClassMatcher.php create mode 100644 app/Support/Redis/OrphanQueueCleanup.php create mode 100644 config/http_observability.php create mode 100644 config/metrics.php create mode 100644 config/traffic.php create mode 100644 database/migrations/2026_08_23_120000_drop_duplicate_artwork_metric_snapshot_index.php create mode 100644 deploy/nginx/conf.d/skinbase-perf-log-format.conf create mode 100644 deploy/nginx/skinbase-vhost-performance-log.snippet create mode 100644 docs/optimization-m1-recommendation-jobs.md create mode 100644 docs/optimization-m10-database-query-optimization.md create mode 100644 docs/optimization-m11-http-endpoint-performance.md create mode 100644 docs/optimization-m12-http-performance-observability.md create mode 100644 docs/optimization-m12.2-vector-gateway-circuit.md create mode 100644 docs/optimization-m12.3-vector-gateway-circuit-attribution.md create mode 100644 docs/optimization-m12.4-similar-ai-dual-circuit.md create mode 100644 docs/optimization-m2-mail-queue.md create mode 100644 docs/optimization-m3-sitemaps.md create mode 100644 docs/optimization-m4-recommendation-snapshots.md create mode 100644 docs/optimization-m5-metric-snapshot-retention.md create mode 100644 docs/optimization-m5.1-metric-snapshot-correctness.md create mode 100644 docs/optimization-m6-php-fpm-memory.md create mode 100644 docs/optimization-m7-default-queue-backlog.md create mode 100644 docs/optimization-m7.1-queue-compatibility.md create mode 100644 docs/optimization-m8-redis-orphan-cleanup.md create mode 100644 docs/optimization-m9-scheduler-overlap.md create mode 100644 docs/skinbase-production-baseline.md create mode 100644 scripts/analyze-http-performance.php create mode 100644 scripts/collect-production-baseline.sh create mode 100644 tests/Feature/Collections/CollectionDispatchGateTest.php create mode 100644 tests/Feature/Forum/ForumQueueDispatchGateTest.php create mode 100644 tests/Feature/Http/ArtworkDownloadMissingFileTest.php create mode 100644 tests/Feature/Http/RssPresenceSkipTest.php create mode 100644 tests/Feature/Http/SlowHttpRequestLoggingTest.php create mode 100644 tests/Feature/Http/StudioGroupsSharedPropTest.php create mode 100644 tests/Feature/Metrics/ArtworkHourlySnapshotWindowTest.php create mode 100644 tests/Feature/Metrics/LeaderboardSnapshotDeltaTest.php create mode 100644 tests/Feature/Metrics/RecalculateHeatCommandTest.php create mode 100644 tests/Feature/PruneMetricSnapshotsCommandTest.php create mode 100644 tests/Feature/Queue/DefaultQueueAssignmentTest.php create mode 100644 tests/Feature/Queue/PurgeQueuedRecComputeTagsCommandTest.php create mode 100644 tests/Feature/Recommendations/RecComputeSimilarJobsTest.php create mode 100644 tests/Feature/Redis/OrphanQueueCleanupTest.php create mode 100644 tests/Feature/Scheduler/ScheduleInventoryTest.php create mode 100644 tests/Feature/Vision/VectorGatewaySearchCircuitTest.php create mode 100644 tests/Feature/Vision/VectorGatewaySearchLimitTest.php create mode 100644 tests/Unit/HorizonMailQueueTest.php create mode 100644 tests/Unit/Http/HttpPerformanceLogAnalyzerTest.php create mode 100644 tests/Unit/Jobs/RecComputeSerializedCompatibilityTest.php create mode 100644 tests/Unit/Jobs/RecComputeSimilarByBehaviorJobTest.php create mode 100644 tests/Unit/Redis/LegacyPrefixFilterTest.php create mode 100644 tests/Unit/Traffic/OnlineVisitorIndexReadTest.php create mode 100644 tests/Unit/Traffic/PresenceIndexPrunerTest.php diff --git a/.env.example b/.env.example index fa38c0fe..8129f6ac 100644 --- a/.env.example +++ b/.env.example @@ -29,6 +29,11 @@ LOG_STACK=single LOG_DEPRECATIONS_CHANNEL=null LOG_LEVEL=debug +# M12 — slow Laravel HTTP log (threshold only; disable for zero overhead) +HTTP_SLOW_REQUEST_LOG_ENABLED=false +HTTP_SLOW_REQUEST_MS=750 +HTTP_SLOW_REQUEST_LOG_DAYS=14 + DB_CONNECTION=mysql DB_HOST=127.0.0.1 DB_PORT=3306 @@ -61,6 +66,28 @@ SKINBASE_SESSION_DEBUG_HEADER=false BROADCAST_CONNECTION=reverb FILESYSTEM_DISK=local QUEUE_CONNECTION=redis +# Must exceed Horizon supervisor-default timeout (960s). Production used 90s +# and nightly RecComputeSimilar* jobs failed with MaxAttemptsExceeded. +REDIS_QUEUE_RETRY_AFTER=1080 + +# Collection maintenance / forum AI jobs have no Horizon consumer. +# Leave false until a dedicated supervisor exists (M7). +COLLECTIONS_V5_DISPATCH_ENABLED=false +FORUM_QUEUE_DISPATCH_ENABLED=false +ONLINE_VISITOR_INDEX_READ_LIMIT=2000 +ONLINE_VISITOR_INDEX_PRUNE_ENABLED=false +ONLINE_VISITOR_INDEX_PRUNE_SCAN_COUNT=500 +ONLINE_VISITOR_INDEX_PRUNE_MAX_BATCHES=200 +ONLINE_VISITOR_INDEX_PRUNE_SLEEP_MS=25 +# RankBuildScopeListsJob unique lock (seconds). Must exceed queue wait + 300s timeout. +RANK_SCOPE_JOB_UNIQUE_FOR=21600 + +# Hourly artwork metric snapshots (heat / rising / monthly leaderboards). +# Production currently holds ~7 days because the scheduler passed --keep-days=7. +# Monthly leaderboards and Studio 30d views need 30 days of hourly history. +ARTWORK_METRIC_HOURLY_RETENTION_DAYS=30 +ARTWORK_METRIC_HOURLY_PRUNE_CHUNK=5000 +ARTWORK_METRIC_HOURLY_PRUNE_SLEEP_MS=50 MESSAGING_REALTIME=true MESSAGING_BROADCAST_QUEUE=broadcasts @@ -123,6 +150,11 @@ VISION_VECTOR_GATEWAY_RETRIES=1 VISION_VECTOR_GATEWAY_RETRY_DELAY_MS=250 VISION_VECTOR_GATEWAY_UPSERT_ENDPOINT=/vectors/upsert VISION_VECTOR_GATEWAY_SEARCH_ENDPOINT=/vectors/search +VISION_VECTOR_GATEWAY_SEARCH_FILE_ENDPOINT=/vectors/search/file +VISION_VECTOR_GATEWAY_SEARCH_TIMEOUT=6 +VISION_VECTOR_GATEWAY_SEARCH_CONNECT_TIMEOUT=2 +VISION_VECTOR_GATEWAY_SEARCH_RETRIES=0 +VISION_VECTOR_GATEWAY_CIRCUIT_SECONDS=30 VISION_VECTOR_GATEWAY_DELETE_ENDPOINT=/vectors/delete VISION_VECTOR_GATEWAY_COLLECTIONS_ENDPOINT=/vectors/collections diff --git a/app/Console/Commands/DispatchCollectionMaintenanceCommand.php b/app/Console/Commands/DispatchCollectionMaintenanceCommand.php index 65b38088..08593a16 100644 --- a/app/Console/Commands/DispatchCollectionMaintenanceCommand.php +++ b/app/Console/Commands/DispatchCollectionMaintenanceCommand.php @@ -18,6 +18,12 @@ class DispatchCollectionMaintenanceCommand extends Command public function handle(CollectionBackgroundJobService $jobs): int { + if (! (bool) config('collections.v5.queue.dispatch_enabled', false)) { + $this->warn('Collection maintenance dispatch is disabled (COLLECTIONS_V5_DISPATCH_ENABLED). No jobs queued.'); + + return self::SUCCESS; + } + $runHealth = (bool) $this->option('health'); $runRecommendations = (bool) $this->option('recommendations'); $runDuplicates = (bool) $this->option('duplicates'); diff --git a/app/Console/Commands/GenerateSitemapsCommand.php b/app/Console/Commands/GenerateSitemapsCommand.php index d613c35e..1c2268f7 100644 --- a/app/Console/Commands/GenerateSitemapsCommand.php +++ b/app/Console/Commands/GenerateSitemapsCommand.php @@ -48,13 +48,32 @@ final class GenerateSitemapsCommand extends Command $this->newLine(); // ── Root sitemap index ──────────────────────────────────────────── + // Write several paths so nginx `location = /sitemap.xml` and child + // listings stay in sync. `sitemaps/index.xml` is the canonical generated + // index: it is a new filename, so a scheduler user can create it even + // when a stale `sitemaps/sitemap.xml` is owned by another account. $t = microtime(true); $index = $build->buildIndex(force: true, persist: false, families: $families); - $disk->put('sitemaps/sitemap.xml', $index['content']); - $written++; + $indexPaths = ['sitemaps/index.xml', 'sitemaps/sitemap.xml', 'sitemap.xml']; + $indexWritten = 0; + + foreach ($indexPaths as $path) { + if ($this->writeXml($disk, $path, $index['content'])) { + $written++; + $indexWritten++; + } + } + + if ($indexWritten === 0) { + $this->error('Failed to write any root sitemap index file. Check ownership of public/sitemap.xml and public/sitemaps/.'); + + return self::FAILURE; + } + $this->line(sprintf( - ' ✔ sitemaps/sitemap.xml %d entries %.3fs', + ' ✔ sitemap index %d entries %d path(s) %.3fs', $index['url_count'], + $indexWritten, microtime(true) - $t, )); @@ -78,7 +97,12 @@ final class GenerateSitemapsCommand extends Command } $path = 'sitemaps/' . $documentName . '.xml'; - $disk->put($path, $built['content']); + if (! $this->writeXml($disk, $path, $built['content'])) { + $this->line(sprintf(' ✖ %s write failed', $documentName . '.xml')); + $failed++; + + continue; + } $written++; $this->line(sprintf( @@ -114,7 +138,12 @@ final class GenerateSitemapsCommand extends Command } $path = 'sitemaps/' . $groupName . '.xml'; - $disk->put($path, $built['content']); + if (! $this->writeXml($disk, $path, $built['content'])) { + $this->line(sprintf(' ✖ %s.xml write failed', $groupName)); + $failed++; + + continue; + } $written++; $this->line(sprintf( @@ -159,4 +188,17 @@ final class GenerateSitemapsCommand extends Command return array_values(array_filter($enabled, fn (string $f): bool => in_array($f, $only, true))); } + + private function writeXml(\Illuminate\Contracts\Filesystem\Filesystem $disk, string $path, string $content): bool + { + $ok = $disk->put($path, $content); + + if ($ok !== true) { + $this->warn(' Could not write '.$path); + + return false; + } + + return true; + } } diff --git a/app/Console/Commands/PruneMetricSnapshotsCommand.php b/app/Console/Commands/PruneMetricSnapshotsCommand.php index fedb4036..1eb318f6 100644 --- a/app/Console/Commands/PruneMetricSnapshotsCommand.php +++ b/app/Console/Commands/PruneMetricSnapshotsCommand.php @@ -7,34 +7,137 @@ use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Log; /** - * Prune old hourly metric snapshots to prevent unbounded table growth. + * Prune old hourly metric snapshots in bounded batches. * - * Usage: php artisan nova:prune-metric-snapshots - * php artisan nova:prune-metric-snapshots --keep-days=7 + * Usage: + * php artisan nova:prune-metric-snapshots + * php artisan nova:prune-metric-snapshots --keep-days=30 --chunk=5000 --dry-run */ class PruneMetricSnapshotsCommand extends Command { protected $signature = 'nova:prune-metric-snapshots - {--keep-days=7 : Keep snapshots for this many days}'; + {--keep-days= : Keep snapshots for this many days (default: config metrics.hourly_snapshot_retention_days)} + {--chunk= : Rows to delete per batch (default: config metrics.hourly_snapshot_prune_chunk)} + {--sleep-ms= : Pause between batches in milliseconds} + {--max-batches=0 : Stop after N batches (0 = until done)} + {--dry-run : Count rows that would be deleted without deleting}'; - protected $description = 'Delete old hourly metric snapshots beyond the retention window'; + protected $description = 'Delete old hourly metric snapshots beyond the retention window (batched)'; public function handle(): int { - $keepDays = (int) $this->option('keep-days'); - $cutoff = now()->subDays($keepDays); + $keepDays = $this->resolveKeepDays(); + $chunk = $this->resolvePositiveInt('chunk', (int) config('metrics.hourly_snapshot_prune_chunk', 5000), 1); + $sleepMs = $this->resolveNonNegativeInt('sleep-ms', (int) config('metrics.hourly_snapshot_prune_sleep_ms', 50)); + $maxBatches = max(0, (int) $this->option('max-batches')); + $dryRun = (bool) $this->option('dry-run'); + $cutoff = now()->subDays($keepDays); - $deleted = DB::table('artwork_metric_snapshots_hourly') + if ($keepDays < 1) { + $this->error('keep-days must be >= 1.'); + + return self::FAILURE; + } + + $eligible = (int) DB::table('artwork_metric_snapshots_hourly') ->where('bucket_hour', '<', $cutoff) - ->delete(); + ->count(); - $this->info("Pruned {$deleted} snapshot rows older than {$keepDays} days."); + $this->info(sprintf( + '[nova:prune-metric-snapshots] cutoff=%s keep_days=%d eligible=%d chunk=%d sleep_ms=%d max_batches=%s%s', + $cutoff->toDateTimeString(), + $keepDays, + $eligible, + $chunk, + $sleepMs, + $maxBatches === 0 ? 'unlimited' : (string) $maxBatches, + $dryRun ? ' (dry-run)' : '' + )); + + if ($dryRun) { + Log::info('[nova:prune-metric-snapshots] dry-run', [ + 'eligible' => $eligible, + 'keep_days' => $keepDays, + 'cutoff' => $cutoff->toDateTimeString(), + ]); + + return self::SUCCESS; + } + + $deleted = 0; + $batches = 0; + + while (true) { + if ($maxBatches > 0 && $batches >= $maxBatches) { + $this->warn("Stopped after max-batches={$maxBatches}."); + break; + } + + $ids = DB::table('artwork_metric_snapshots_hourly') + ->where('bucket_hour', '<', $cutoff) + ->orderBy('id') + ->limit($chunk) + ->pluck('id'); + + if ($ids->isEmpty()) { + break; + } + + $batchDeleted = DB::table('artwork_metric_snapshots_hourly') + ->whereIn('id', $ids->all()) + ->delete(); + + $deleted += $batchDeleted; + $batches++; + + Log::info('[nova:prune-metric-snapshots] batch', [ + 'batch' => $batches, + 'deleted' => $batchDeleted, + 'deleted_total' => $deleted, + ]); + + if ($sleepMs > 0) { + usleep($sleepMs * 1000); + } + } + + $this->info("Pruned {$deleted} snapshot rows older than {$keepDays} days in {$batches} batch(es)."); Log::info('[nova:prune-metric-snapshots] completed', [ - 'deleted' => $deleted, + 'deleted' => $deleted, + 'batches' => $batches, 'keep_days' => $keepDays, + 'cutoff' => $cutoff->toDateTimeString(), + 'eligible_at_start' => $eligible, ]); return self::SUCCESS; } + + private function resolveKeepDays(): int + { + $option = $this->option('keep-days'); + + if ($option === null || $option === '') { + return (int) config('metrics.hourly_snapshot_retention_days', 30); + } + + return (int) $option; + } + + private function resolvePositiveInt(string $option, int $default, int $minimum): int + { + $raw = $this->option($option); + $value = ($raw === null || $raw === '') ? $default : (int) $raw; + + return max($minimum, $value); + } + + private function resolveNonNegativeInt(string $option, int $default): int + { + $raw = $this->option($option); + $value = ($raw === null || $raw === '') ? $default : (int) $raw; + + return max(0, $value); + } } diff --git a/app/Console/Commands/PruneOnlineVisitorIndexCommand.php b/app/Console/Commands/PruneOnlineVisitorIndexCommand.php new file mode 100644 index 00000000..6de35b09 --- /dev/null +++ b/app/Console/Commands/PruneOnlineVisitorIndexCommand.php @@ -0,0 +1,88 @@ +option('dry-run'); + $execute = (bool) $this->option('execute'); + if ($dryRun === $execute) { + $this->error('Pass exactly one of --dry-run or --execute.'); + + return self::FAILURE; + } + + $scanCount = $this->intOption('scan-count', (int) config('traffic.online_visitors.prune_scan_count', 500)); + $maxBatches = $this->intOption('max-batches', (int) config('traffic.online_visitors.prune_max_batches', 200)); + $maxMembers = $this->intOption('max-members', (int) config('traffic.online_visitors.prune_max_members', 100000)); + $sleepMs = $this->intOption('sleep-ms', (int) config('traffic.online_visitors.prune_sleep_ms', 25)); + $timeLimit = $this->intOption('time-limit', 0); + + $scard = (int) Redis::scard(OnlineVisitorRepository::INDEX_KEY); + + $this->info(sprintf( + '[prune-online-visitor-index] key=%s prefix=%s scard=%d scan_count=%d max_batches=%d max_members=%d sleep_ms=%d %s', + OnlineVisitorRepository::INDEX_KEY, + (string) config('database.redis.options.prefix'), + $scard, + $scanCount, + $maxBatches, + $maxMembers, + $sleepMs, + $dryRun ? 'dry-run' : 'execute' + )); + + $result = $pruner->prune( + $scanCount, + $maxBatches, + $maxMembers, + $sleepMs, + $dryRun, + $timeLimit > 0 ? $timeLimit : null, + ); + + $this->info(sprintf( + 'scanned=%d stale=%d live=%d removed=%d batches=%d', + $result['scanned'], + $result['stale'], + $result['live'], + $result['removed'], + $result['batches'] + )); + + Log::info('[prune-online-visitor-index] completed', $result + ['scard_before' => $scard]); + + return self::SUCCESS; + } + + private function intOption(string $name, int $default): int + { + $raw = $this->option($name); + if ($raw === null || $raw === '') { + return $default; + } + + return max(0, (int) $raw); + } +} diff --git a/app/Console/Commands/PurgeQueuedRecComputeTagsCommand.php b/app/Console/Commands/PurgeQueuedRecComputeTagsCommand.php new file mode 100644 index 00000000..b0ffd244 --- /dev/null +++ b/app/Console/Commands/PurgeQueuedRecComputeTagsCommand.php @@ -0,0 +1,94 @@ +option('dry-run'); + $execute = (bool) $this->option('execute'); + + if ($dryRun === $execute) { + $this->error('Pass exactly one of --dry-run or --execute.'); + + return self::FAILURE; + } + + $queue = (string) $this->option('queue'); + if (! preg_match('/^[A-Za-z0-9_-]+$/', $queue)) { + $this->error('Invalid queue name.'); + + return self::FAILURE; + } + + $key = 'queues:'.$queue; + $target = RecComputeSimilarByTagsJob::class; + $redis = Redis::connection(); + $len = (int) $redis->llen($key); + $matchedPayloads = []; + $chunk = 200; + + $this->info(sprintf( + '[purge-queued-rec-tags] queue=%s waiting=%d target=%s %s', + $queue, + $len, + $target, + $dryRun ? 'dry-run' : 'execute' + )); + + for ($start = 0; $start < $len; $start += $chunk) { + $rows = $redis->lrange($key, $start, min($start + $chunk - 1, $len - 1)); + foreach ($rows as $raw) { + if (QueuedJobClassMatcher::isClass((string) $raw, $target)) { + $matchedPayloads[] = (string) $raw; + } + } + } + + $matched = count($matchedPayloads); + $removed = 0; + + if (! $dryRun) { + foreach ($matchedPayloads as $raw) { + $removed += (int) $redis->lrem($key, 1, $raw); + } + } + + $this->info(sprintf('matched=%d removed=%d preserved_other=%s', $matched, $removed, $dryRun ? 'yes' : 'yes')); + + Log::info('[purge-queued-rec-tags] completed', [ + 'queue' => $queue, + 'waiting' => $len, + 'matched' => $matched, + 'removed' => $removed, + 'dry_run' => $dryRun, + ]); + + return self::SUCCESS; + } +} diff --git a/app/Console/Commands/RecalculateHeatCommand.php b/app/Console/Commands/RecalculateHeatCommand.php index e7ba9382..029a5bd9 100644 --- a/app/Console/Commands/RecalculateHeatCommand.php +++ b/app/Console/Commands/RecalculateHeatCommand.php @@ -57,7 +57,7 @@ class RecalculateHeatCommand extends Command $updatedCount = 0; $skippedCount = 0; - // Process in chunks using artwork IDs that have at least one snapshot in the smoothing window + // Distinct IDs only — do not hydrate the full 24h snapshot window in one query. $artworkIds = DB::table('artwork_metric_snapshots_hourly') ->whereBetween('bucket_hour', [$lookbackStart, $currentHour]) ->distinct() @@ -68,27 +68,33 @@ class RecalculateHeatCommand extends Command return self::SUCCESS; } - // Load all snapshots for the lookback window in bulk - $snapshots = DB::table('artwork_metric_snapshots_hourly') - ->whereBetween('bucket_hour', [$lookbackStart, $currentHour]) - ->whereIn('artwork_id', $artworkIds) - ->orderBy('bucket_hour') - ->get() - ->groupBy('artwork_id'); - - // Load artwork published_at dates for age factor (use published_at, fall back to created_at) - $artworkDates = DB::table('artworks') - ->whereIn('id', $artworkIds) - ->whereNull('deleted_at') - ->where('is_approved', true) - ->select('id', 'published_at', 'created_at') - ->get() - ->mapWithKeys(fn ($row) => [ - $row->id => \Carbon\Carbon::parse($row->published_at ?? $row->created_at), - ]); - - // Process in chunks foreach ($artworkIds->chunk($chunk) as $chunkIds) { + $snapshots = DB::table('artwork_metric_snapshots_hourly') + ->select([ + 'artwork_id', + 'bucket_hour', + 'views_count', + 'downloads_count', + 'favourites_count', + 'comments_count', + 'shares_count', + ]) + ->whereBetween('bucket_hour', [$lookbackStart, $currentHour]) + ->whereIn('artwork_id', $chunkIds) + ->orderBy('bucket_hour') + ->get() + ->groupBy('artwork_id'); + + $artworkDates = DB::table('artworks') + ->whereIn('id', $chunkIds) + ->whereNull('deleted_at') + ->where('is_approved', true) + ->select('id', 'published_at', 'created_at') + ->get() + ->mapWithKeys(fn ($row) => [ + $row->id => \Carbon\Carbon::parse($row->published_at ?? $row->created_at), + ]); + $upsertRows = []; foreach ($chunkIds as $artworkId) { diff --git a/app/Console/Commands/RedisCleanupLegacyPrefixCommand.php b/app/Console/Commands/RedisCleanupLegacyPrefixCommand.php new file mode 100644 index 00000000..debc5d9f --- /dev/null +++ b/app/Console/Commands/RedisCleanupLegacyPrefixCommand.php @@ -0,0 +1,139 @@ +option('dry-run'); + $execute = (bool) $this->option('execute'); + if ($dryRun === $execute) { + $this->error('Pass exactly one of --dry-run or --execute.'); + + return self::FAILURE; + } + + $maxKeys = max(1, (int) $this->option('max-keys')); + $currentPrefix = (string) config('database.redis.options.prefix'); + $horizonPrefix = (string) config('horizon.prefix'); + + $this->info(sprintf( + '[redis-cleanup-legacy-prefix] current_prefix=%s horizon_prefix=%s max_keys=%d %s', + $currentPrefix, + $horizonPrefix, + $maxKeys, + $dryRun ? 'dry-run' : 'execute' + )); + + if (in_array($currentPrefix, self::LEGACY_PREFIXES, true) || in_array($horizonPrefix, self::LEGACY_PREFIXES, true)) { + $this->error('Current process still uses a legacy prefix. Aborting.'); + + return self::FAILURE; + } + + $client = $this->unprefixedClient(); + $found = []; + $cursor = '0'; + do { + [$cursor, $keys] = $client->scan($cursor, ['COUNT' => 200, 'MATCH' => '*']); + foreach ($keys as $key) { + $key = (string) $key; + if ($this->isProtected($key)) { + continue; + } + if (! $this->isLegacy($key)) { + continue; + } + $found[] = $key; + if (count($found) >= $maxKeys) { + $cursor = '0'; + break; + } + } + } while ($cursor !== '0' && $cursor !== 0); + + $this->info('matched='.count($found)); + foreach (array_slice($found, 0, 20) as $key) { + $this->line('key='.$key); + } + if (count($found) > 20) { + $this->line('... truncated listing'); + } + + $unlinked = 0; + if (! $dryRun && $found !== []) { + foreach (array_chunk($found, 50) as $chunk) { + $unlinked += (int) $client->unlink(...$chunk); + } + } + + $this->info('unlinked='.$unlinked); + + Log::info('[redis-cleanup-legacy-prefix] completed', [ + 'matched' => count($found), + 'unlinked' => $unlinked, + 'dry_run' => $dryRun, + ]); + + return self::SUCCESS; + } + + private function isLegacy(string $key): bool + { + foreach (self::LEGACY_PREFIXES as $prefix) { + if (str_starts_with($key, $prefix)) { + return true; + } + } + + return false; + } + + private function isProtected(string $key): bool + { + foreach (self::PROTECTED_PREFIXES as $prefix) { + if (str_starts_with($key, $prefix)) { + return true; + } + } + + return false; + } + + private function unprefixedClient(): PredisClient + { + $redis = config('database.redis.default', []); + + return new PredisClient([ + 'scheme' => 'tcp', + 'host' => $redis['host'] ?? '127.0.0.1', + 'port' => (int) ($redis['port'] ?? 6379), + 'password' => $redis['password'] ?? null, + 'database' => (int) ($redis['database'] ?? 0), + ]); + } +} diff --git a/app/Console/Commands/RedisCleanupOrphansCommand.php b/app/Console/Commands/RedisCleanupOrphansCommand.php new file mode 100644 index 00000000..e61c6887 --- /dev/null +++ b/app/Console/Commands/RedisCleanupOrphansCommand.php @@ -0,0 +1,79 @@ +option('dry-run'); + $execute = (bool) $this->option('execute'); + if ($dryRun === $execute) { + $this->error('Pass exactly one of --dry-run or --execute.'); + + return self::FAILURE; + } + + $target = (string) $this->argument('target'); + $force = (bool) $this->option('force'); + + $this->info(sprintf( + '[redis-cleanup-orphans] prefix=%s connection=default target=%s force=%s mode=%s', + (string) config('database.redis.options.prefix'), + $target, + $force ? 'yes' : 'no', + $dryRun ? 'dry-run' : 'execute' + )); + + try { + $plan = $execute + ? $cleanup->execute($target, $force) + : $cleanup->plan($target, $force); + } catch (RuntimeException $e) { + $this->error($e->getMessage()); + Log::warning('[redis-cleanup-orphans] refused', ['target' => $target, 'error' => $e->getMessage()]); + + return self::FAILURE; + } + + $this->line('logical_key='.$plan['logical_key']); + $this->line('notify_key='.$plan['notify_key']); + $this->line('llen='.$plan['llen'].' reserved='.$plan['reserved'].' delayed='.$plan['delayed']); + $this->line('sampled='.$plan['sampled'].' est_bytes='.$plan['est_bytes']); + $this->line('classes='.json_encode($plan['classes'])); + if ($plan['unexpected_classes'] !== []) { + $this->warn('unexpected_classes='.json_encode($plan['unexpected_classes'])); + } + if ($plan['errors'] !== []) { + $this->error('errors='.implode(',', $plan['errors'])); + } + if ($execute) { + $this->info('unlinked='.($plan['unlinked'] ?? 0)); + } + + Log::info('[redis-cleanup-orphans] completed', [ + 'target' => $target, + 'dry_run' => $dryRun, + 'llen' => $plan['llen'], + 'errors' => $plan['errors'], + 'unlinked' => $plan['unlinked'] ?? 0, + ]); + + return ($plan['errors'] === [] || $dryRun) ? self::SUCCESS : self::FAILURE; + } +} diff --git a/app/Http/Controllers/Api/SimilarAiArtworksController.php b/app/Http/Controllers/Api/SimilarAiArtworksController.php index 4df45ca0..b3287c12 100644 --- a/app/Http/Controllers/Api/SimilarAiArtworksController.php +++ b/app/Http/Controllers/Api/SimilarAiArtworksController.php @@ -6,10 +6,12 @@ namespace App\Http\Controllers\Api; use App\Http\Controllers\Controller; use App\Models\Artwork; +use App\Services\Vision\VectorGatewayException; use App\Services\Vision\VectorService; use Illuminate\Http\JsonResponse; use Illuminate\Http\Request; -use RuntimeException; +use Illuminate\Support\Facades\Log; +use Throwable; final class SimilarAiArtworksController extends Controller { @@ -35,11 +37,12 @@ final class SimilarAiArtworksController extends Controller try { $items = $this->vectors->similarToArtwork($artwork, $limit); - } catch (RuntimeException $e) { + } catch (Throwable $e) { + $this->logSimilarityFailure($artwork->id, $e); + return response()->json([ 'data' => [], 'reason' => 'vector_gateway_error', - 'message' => $e->getMessage(), ], 502); } @@ -52,4 +55,19 @@ final class SimilarAiArtworksController extends Controller ], ]); } + + private function logSimilarityFailure(int $artworkId, Throwable $e): void + { + $gateway = $e instanceof VectorGatewayException ? $e : null; + + Log::warning('Vector similarity search failed', [ + 'artwork_id' => $artworkId, + 'route' => 'api.art.similar-ai', + 'failure_stage' => $gateway?->operation ?? 'unknown', + 'exception_class' => $e::class, + 'http_status' => $gateway?->httpStatus, + 'circuit_worthy' => $gateway?->circuitWorthy ?? false, + 'circuit_open' => app(\App\Services\Vision\VectorGatewayClient::class)->circuitOpen(), + ]); + } } diff --git a/app/Http/Controllers/ArtworkDownloadController.php b/app/Http/Controllers/ArtworkDownloadController.php index bddf29ae..fa6c1db4 100644 --- a/app/Http/Controllers/ArtworkDownloadController.php +++ b/app/Http/Controllers/ArtworkDownloadController.php @@ -58,6 +58,16 @@ final class ArtworkDownloadController extends Controller abort(404); } + if (! File::isFile($filePath)) { + Log::warning('Artwork original file missing for download.', [ + 'artwork_id' => $artwork->id, + 'ext' => $ext, + 'resolved_path' => $filePath, + ]); + + abort(404); + } + $this->recordDownload($request, $artwork->id); $this->incrementDownloadCountIfAvailable($artwork->id); @@ -70,16 +80,6 @@ final class ArtworkDownloadController extends Controller ]); } - if (! File::isFile($filePath)) { - Log::warning('Artwork original file missing for download.', [ - 'artwork_id' => $artwork->id, - 'ext' => $ext, - 'resolved_path' => $filePath, - ]); - - abort(404); - } - $downloadName = $this->buildDownloadFilename((string) $artwork->file_name, $ext); // X-Accel-Redirect is safe only when nginx is explicitly configured to diff --git a/app/Http/Controllers/SitemapController.php b/app/Http/Controllers/SitemapController.php index 53d01364..4c56ee47 100644 --- a/app/Http/Controllers/SitemapController.php +++ b/app/Http/Controllers/SitemapController.php @@ -22,6 +22,18 @@ final class SitemapController extends Controller public function index(): Response|BinaryFileResponse { + if ((bool) config('sitemaps.pre_generated.enabled', true)) { + foreach ([ + public_path('sitemaps/index.xml'), + public_path('sitemaps/sitemap.xml'), + public_path('sitemap.xml'), + ] as $path) { + if (is_file($path) && is_readable($path)) { + return $this->xmlFileResponse($path); + } + } + } + // 1. Published release (release management pipeline fallback). $published = $this->published->resolveIndex(); if ($published !== null) { diff --git a/app/Http/Middleware/AddServerTiming.php b/app/Http/Middleware/AddServerTiming.php new file mode 100644 index 00000000..69fb2722 --- /dev/null +++ b/app/Http/Middleware/AddServerTiming.php @@ -0,0 +1,43 @@ +attributes->get('http.ssr_ms'); + if (is_numeric($ssrMs) && (float) $ssrMs >= 0) { + $metrics[] = 'ssr;desc="Inertia SSR";dur='.number_format((float) $ssrMs, 1, '.', ''); + } + + $metric = implode(', ', $metrics); + $existing = $response->headers->get('Server-Timing'); + + $response->headers->set( + 'Server-Timing', + $existing ? $existing.', '.$metric : $metric, + ); + + return $response; + } +} diff --git a/app/Http/Middleware/HandleInertiaRequests.php b/app/Http/Middleware/HandleInertiaRequests.php index e02c24da..575877f7 100644 --- a/app/Http/Middleware/HandleInertiaRequests.php +++ b/app/Http/Middleware/HandleInertiaRequests.php @@ -98,6 +98,15 @@ final class HandleInertiaRequests extends Middleware ? $request->session()->get($key) : null; + if ($user !== null && ! $user->relationLoaded('profile')) { + $user->load('profile'); + } + + $studioGroups = []; + if ($user !== null && str_starts_with($request->path(), 'studio')) { + $studioGroups = app(GroupService::class)->studioOptionsForUser($user); + } + return array_merge(parent::share($request), [ 'auth' => [ 'user' => $user ? [ @@ -134,9 +143,7 @@ final class HandleInertiaRequests extends Middleware 'group_assets' => (bool) config('features.group_assets', true), 'group_activity_feed' => (bool) config('features.group_activity_feed', true), ], - 'studio_groups' => $user - ? app(GroupService::class)->studioOptionsForUser($user) - : [], + 'studio_groups' => $studioGroups, ]); } } diff --git a/app/Http/Middleware/LogSlowHttpRequest.php b/app/Http/Middleware/LogSlowHttpRequest.php new file mode 100644 index 00000000..cd68c31a --- /dev/null +++ b/app/Http/Middleware/LogSlowHttpRequest.php @@ -0,0 +1,74 @@ +environment('testing')) { + $override = config('http_observability.slow_request.test_duration_ms'); + if (is_numeric($override)) { + $durationMs = (float) $override; + } + } + + $threshold = max(1, (int) config('http_observability.slow_request.threshold_ms', 750)); + if ($durationMs < $threshold) { + return $response; + } + + $route = $request->route(); + $routeUri = is_object($route) && method_exists($route, 'uri') + ? (string) $route->uri() + : $this->uris->normalize($request->path()); + + $payload = [ + 'time' => now()->toIso8601String(), + 'method' => $request->getMethod(), + 'route_name' => is_object($route) ? $route->getName() : null, + 'route_uri' => $routeUri, + 'status' => $response->getStatusCode(), + 'duration_ms' => round($durationMs, 1), + 'peak_memory_mb' => round(memory_get_peak_usage(true) / 1048576, 2), + 'authenticated' => $request->user() !== null, + 'inertia' => $this->isInertia($request, $response), + ]; + + Log::channel('slow-http')->info(json_encode($payload, JSON_UNESCAPED_SLASHES)); + + return $response; + } + + private function isInertia(Request $request, Response $response): bool + { + if ($request->headers->has(InertiaHeader::INERTIA)) { + return true; + } + + return $response->headers->has(InertiaHeader::INERTIA) + || $response->headers->get('Vary') === 'X-Inertia' + || str_contains((string) $response->headers->get('Vary'), 'X-Inertia'); + } +} diff --git a/app/Http/Middleware/TrackOnlineVisitor.php b/app/Http/Middleware/TrackOnlineVisitor.php index 51320c55..dd9fa4cc 100644 --- a/app/Http/Middleware/TrackOnlineVisitor.php +++ b/app/Http/Middleware/TrackOnlineVisitor.php @@ -69,6 +69,11 @@ final class TrackOnlineVisitor 'email/*', 'logout', 'up', + 'rss/*', + 'rss-feeds', + 'robots.txt', + 'sitemap.xml', + 'sitemaps/*', ])) { return false; } diff --git a/app/Jobs/Concerns/RestoresAfterArtworkIdCursor.php b/app/Jobs/Concerns/RestoresAfterArtworkIdCursor.php new file mode 100644 index 00000000..cdd25d10 --- /dev/null +++ b/app/Jobs/Concerns/RestoresAfterArtworkIdCursor.php @@ -0,0 +1,27 @@ +afterArtworkId)) { + $this->afterArtworkId = null; + } + } + + protected function afterArtworkId(): ?int + { + $this->restoreAfterArtworkIdCursor(); + + return $this->afterArtworkId; + } +} diff --git a/app/Jobs/IndexArtworkJob.php b/app/Jobs/IndexArtworkJob.php index fd74d3f1..01ff03e4 100644 --- a/app/Jobs/IndexArtworkJob.php +++ b/app/Jobs/IndexArtworkJob.php @@ -6,6 +6,7 @@ namespace App\Jobs; use App\Models\Artwork; use Illuminate\Bus\Queueable; +use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; @@ -21,16 +22,29 @@ use Meilisearch\Client as MeilisearchClient; * after_commit double-dispatch problem and ensures the document lands * in the index within this job's execution, with no extra queue hop. */ -class IndexArtworkJob implements ShouldQueue +class IndexArtworkJob implements ShouldQueue, ShouldBeUniqueUntilProcessing { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; public int $tries = 3; public int $timeout = 60; + /** + * Unique while queued so Meilisearch isn't flooded. handle() loads the + * artwork from DB, so a dropped duplicate still indexes current state. + * UniqueUntilProcessing: a change after the worker starts can enqueue again. + */ + public int $uniqueFor = 120; + public function __construct(public readonly int $artworkId) { $this->afterCommit = true; + $this->onQueue((string) config('scout.queue.queue', 'search')); + } + + public function uniqueId(): string + { + return (string) $this->artworkId; } public function handle(MeilisearchClient $client): void diff --git a/app/Jobs/IndexUserJob.php b/app/Jobs/IndexUserJob.php index df0be793..b539afea 100644 --- a/app/Jobs/IndexUserJob.php +++ b/app/Jobs/IndexUserJob.php @@ -6,6 +6,7 @@ namespace App\Jobs; use App\Models\User; use Illuminate\Bus\Queueable; +use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; @@ -15,14 +16,28 @@ use Illuminate\Queue\SerializesModels; * Queued job: index (or re-index) a single User in Meilisearch. * Dispatched by UserStatsService whenever stats change. */ -class IndexUserJob implements ShouldQueue +class IndexUserJob implements ShouldQueue, ShouldBeUniqueUntilProcessing { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; public int $tries = 3; public int $timeout = 30; - public function __construct(public readonly int $userId) {} + /** + * Unique while queued. handle() loads the user from DB so the queued job + * still reflects later stats changes. uniqueFor covers a dead worker lock. + */ + public int $uniqueFor = 60; + + public function __construct(public readonly int $userId) + { + $this->onQueue((string) config('scout.queue.queue', 'search')); + } + + public function uniqueId(): string + { + return (string) $this->userId; + } public function handle(): void { diff --git a/app/Jobs/RankBuildScopeListsJob.php b/app/Jobs/RankBuildScopeListsJob.php index d7c09c1a..8fda3e90 100644 --- a/app/Jobs/RankBuildScopeListsJob.php +++ b/app/Jobs/RankBuildScopeListsJob.php @@ -6,6 +6,7 @@ namespace App\Jobs; use App\Services\RankingService; use Illuminate\Bus\Queueable; +use Illuminate\Contracts\Queue\ShouldBeUnique; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; @@ -13,19 +14,32 @@ use Illuminate\Queue\SerializesModels; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Log; -class RankBuildScopeListsJob implements ShouldQueue +class RankBuildScopeListsJob implements ShouldQueue, ShouldBeUnique { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; public int $timeout = 300; public int $tries = 2; + /** + * Unique until the job finishes or uniqueFor elapses. + * Must outlast the queue wait (hours) plus timeout (300s), not just 360s. + */ + public int $uniqueFor; + private const LIST_TYPES = ['trending', 'new_hot', 'best']; public function __construct( public readonly string $scopeType, public readonly int $scopeId, - ) {} + ) { + $this->uniqueFor = max(3600, (int) config('ranking.scope_job_unique_for', 21600)); + } + + public function uniqueId(): string + { + return $this->scopeType . ':' . $this->scopeId; + } public function handle(RankingService $ranking): void { diff --git a/app/Jobs/RecComputeSimilarByBehaviorJob.php b/app/Jobs/RecComputeSimilarByBehaviorJob.php index 0bdd3e50..2cb65f31 100644 --- a/app/Jobs/RecComputeSimilarByBehaviorJob.php +++ b/app/Jobs/RecComputeSimilarByBehaviorJob.php @@ -10,8 +10,10 @@ use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; +use App\Jobs\Concerns\RestoresAfterArtworkIdCursor; use Illuminate\Queue\SerializesModels; use Illuminate\Support\Facades\DB; +use Illuminate\Support\Facades\Log; /** * Compute behavior-based (co-like) similarity from precomputed item pairs. @@ -21,38 +23,129 @@ use Illuminate\Support\Facades\DB; */ final class RecComputeSimilarByBehaviorJob implements ShouldQueue { - use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; + use Dispatchable, InteractsWithQueue, Queueable, RestoresAfterArtworkIdCursor; + use SerializesModels { + __unserialize as private unserializeQueuedModels; + } public int $tries = 2; - public int $timeout = 600; + public int $timeout = 120; + + private ?int $afterArtworkId = null; public function __construct( private readonly ?int $artworkId = null, private readonly int $batchSize = 200, + ?int $afterArtworkId = null, ) { + $this->afterArtworkId = $afterArtworkId; $queue = (string) config('recommendations.queue', 'default'); if ($queue !== '') { $this->onQueue($queue); } } + public function __unserialize(array $values): void + { + $this->unserializeQueuedModels($values); + $this->restoreAfterArtworkIdCursor(); + } + + public function cursorAfterArtworkId(): ?int + { + return $this->afterArtworkId(); + } + public function handle(): void { + $startedAt = microtime(true); $modelVersion = (string) config('recommendations.similarity.model_version', 'sim_v1'); $resultLimit = (int) config('recommendations.similarity.result_limit', 30); $maxPerAuthor = (int) config('recommendations.similarity.max_per_author', 2); - $query = Artwork::query()->public()->published()->select('id', 'user_id'); - if ($this->artworkId !== null) { - $query->where('id', $this->artworkId); + $artwork = Artwork::query()->public()->published()->select('id', 'user_id')->find($this->artworkId); + + if ($artwork instanceof Artwork) { + $this->processArtworkSafely($artwork, $modelVersion, $resultLimit, $maxPerAuthor); + $this->logBatchComplete($startedAt, 1, false); + + return; + } + + $this->logBatchComplete($startedAt, 0, false); + + return; } - $query->chunkById($this->batchSize, function ($artworks) use ($modelVersion, $resultLimit, $maxPerAuthor) { - foreach ($artworks as $artwork) { - $this->processArtwork($artwork, $modelVersion, $resultLimit, $maxPerAuthor); - } - }); + $artworks = Artwork::query() + ->public() + ->published() + ->select('id', 'user_id') + ->when($this->afterArtworkId() !== null, fn ($query) => $query->where('id', '>', $this->afterArtworkId())) + ->orderBy('id') + ->limit($this->batchSize) + ->get(); + + if ($artworks->isEmpty()) { + $this->logBatchComplete($startedAt, 0, false); + + return; + } + + foreach ($artworks as $artwork) { + $this->processArtworkSafely($artwork, $modelVersion, $resultLimit, $maxPerAuthor); + } + + $hasMore = $artworks->count() === $this->batchSize; + if ($hasMore) { + static::dispatch(null, $this->batchSize, (int) $artworks->last()->id); + } + + $this->logBatchComplete($startedAt, $artworks->count(), $hasMore); + } + + public function failed(\Throwable $exception): void + { + Log::error('[RecComputeSimilarByBehavior] Job failed permanently.', [ + 'artwork_id' => $this->artworkId, + 'batch_size' => $this->batchSize, + 'after_artwork_id' => $this->afterArtworkId(), + 'attempts' => $this->attempts(), + 'exception_class' => $exception::class, + 'exception_message' => $exception->getMessage(), + ]); + } + + private function processArtworkSafely( + Artwork $artwork, + string $modelVersion, + int $resultLimit, + int $maxPerAuthor, + ): void { + try { + $this->processArtwork($artwork, $modelVersion, $resultLimit, $maxPerAuthor); + } catch (\Throwable $exception) { + Log::warning("[RecComputeSimilarByBehavior] Failed for artwork {$artwork->id}: {$exception->getMessage()}", [ + 'artwork_id' => $artwork->id, + 'exception_class' => $exception::class, + ]); + } + } + + /** + * @param positive-int|0 $processed + */ + private function logBatchComplete(float $startedAt, int $processed, bool $hasMore): void + { + Log::info('[RecComputeSimilarByBehavior] Batch complete.', [ + 'artwork_id' => $this->artworkId, + 'after_artwork_id' => $this->afterArtworkId(), + 'processed' => $processed, + 'has_more' => $hasMore, + 'duration_ms' => (int) round((microtime(true) - $startedAt) * 1000), + 'memory_mb' => round(memory_get_peak_usage(true) / 1048576, 1), + ]); } private function processArtwork( diff --git a/app/Jobs/RecComputeSimilarByTagsJob.php b/app/Jobs/RecComputeSimilarByTagsJob.php index acad5553..4f39ea9a 100644 --- a/app/Jobs/RecComputeSimilarByTagsJob.php +++ b/app/Jobs/RecComputeSimilarByTagsJob.php @@ -11,7 +11,9 @@ use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\Middleware\WithoutOverlapping; use Illuminate\Queue\InteractsWithQueue; +use App\Jobs\Concerns\RestoresAfterArtworkIdCursor; use Illuminate\Queue\SerializesModels; +use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Log; @@ -24,22 +26,44 @@ use Illuminate\Support\Facades\Log; */ final class RecComputeSimilarByTagsJob implements ShouldQueue { - use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; + use Dispatchable, InteractsWithQueue, Queueable, RestoresAfterArtworkIdCursor; + use SerializesModels { + __unserialize as private unserializeQueuedModels; + } public int $tries = 2; public int $timeout = 600; + /** + * Declared with a default so missing serialized payloads do not leave + * an uninitialized typed property. Constructor defaults are not applied + * on unserialize. + */ + private ?int $afterArtworkId = null; + public function __construct( private readonly ?int $artworkId = null, private readonly int $batchSize = 200, - private readonly ?int $afterArtworkId = null, + ?int $afterArtworkId = null, ) { + $this->afterArtworkId = $afterArtworkId; $queue = (string) config('recommendations.queue', 'default'); if ($queue !== '') { $this->onQueue($queue); } } + public function __unserialize(array $values): void + { + $this->unserializeQueuedModels($values); + $this->restoreAfterArtworkIdCursor(); + } + + public function cursorAfterArtworkId(): ?int + { + return $this->afterArtworkId(); + } + /** * @return array */ @@ -63,12 +87,7 @@ final class RecComputeSimilarByTagsJob implements ShouldQueue $maxPerAuthor = (int) config('recommendations.similarity.max_per_author', 2); $resultLimit = (int) config('recommendations.similarity.result_limit', 30); - // ── Tag IDF weights (global) ─────────────────────────────────────────── - $tagFreqs = DB::table('artwork_tag') - ->select('tag_id', DB::raw('COUNT(*) as cnt')) - ->groupBy('tag_id') - ->pluck('cnt', 'tag_id') - ->all(); + $tagFreqs = $this->tagFrequencies(); if ($this->artworkId !== null) { $artwork = Artwork::query()->public()->published()->select('id', 'user_id')->find($this->artworkId); @@ -86,7 +105,7 @@ final class RecComputeSimilarByTagsJob implements ShouldQueue ->public() ->published() ->select('id', 'user_id') - ->when($this->afterArtworkId !== null, fn ($query) => $query->where('id', '>', $this->afterArtworkId)) + ->when($this->afterArtworkId() !== null, fn ($query) => $query->where('id', '>', $this->afterArtworkId())) ->orderBy('id') ->limit($this->batchSize) ->get(); @@ -109,13 +128,33 @@ final class RecComputeSimilarByTagsJob implements ShouldQueue Log::error('[RecComputeSimilarByTags] Job failed permanently.', [ 'artwork_id' => $this->artworkId, 'batch_size' => $this->batchSize, - 'after_artwork_id' => $this->afterArtworkId, + 'after_artwork_id' => $this->afterArtworkId(), 'attempts' => $this->attempts(), 'exception_class' => $exception::class, 'exception_message' => $exception->getMessage(), ]); } + /** + * Global tag frequencies change slowly relative to batch jobs. + * Cache so each RecComputeSimilarByTagsJob batch does not re-scan artwork_tag. + * + * @return array + */ + private function tagFrequencies(): array + { + $modelVersion = (string) config('recommendations.similarity.model_version', 'sim_v1'); + $ttl = (int) config('recommendations.similarity.tag_idf_cache_ttl', 3600); + + return Cache::remember('rec:tag-idf:'.$modelVersion, max(60, $ttl), function () { + return DB::table('artwork_tag') + ->select('tag_id', DB::raw('COUNT(*) as cnt')) + ->groupBy('tag_id') + ->pluck('cnt', 'tag_id') + ->all(); + }); + } + private function processArtworkSafely( Artwork $artwork, array $tagFreqs, diff --git a/app/Jobs/RecComputeSimilarHybridJob.php b/app/Jobs/RecComputeSimilarHybridJob.php index e968b593..be7fdf39 100644 --- a/app/Jobs/RecComputeSimilarHybridJob.php +++ b/app/Jobs/RecComputeSimilarHybridJob.php @@ -11,6 +11,7 @@ use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\Middleware\WithoutOverlapping; use Illuminate\Queue\InteractsWithQueue; +use App\Jobs\Concerns\RestoresAfterArtworkIdCursor; use Illuminate\Queue\SerializesModels; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Log; @@ -24,24 +25,42 @@ use Illuminate\Support\Facades\Log; */ final class RecComputeSimilarHybridJob implements ShouldQueue { - use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; + use Dispatchable, InteractsWithQueue, Queueable, RestoresAfterArtworkIdCursor; + use SerializesModels { + __unserialize as private unserializeQueuedModels; + } // This recompute is idempotent and already guards per-artwork execution. // Keep retries to a minimum so transient failures do not turn into // Horizon's max-attempt exception noise. public int $tries = 1; - public int $timeout = 900; + public int $timeout = 180; + + private ?int $afterArtworkId = null; public function __construct( private readonly ?int $artworkId = null, private readonly int $batchSize = 200, + ?int $afterArtworkId = null, ) { + $this->afterArtworkId = $afterArtworkId; $queue = (string) config('recommendations.queue', 'default'); if ($queue !== '') { $this->onQueue($queue); } } + public function __unserialize(array $values): void + { + $this->unserializeQueuedModels($values); + $this->restoreAfterArtworkIdCursor(); + } + + public function cursorAfterArtworkId(): ?int + { + return $this->afterArtworkId(); + } + /** * @return array */ @@ -72,6 +91,8 @@ final class RecComputeSimilarHybridJob implements ShouldQueue ? (array) config('recommendations.similarity.weights_with_vector') : (array) config('recommendations.similarity.weights_without_vector'); + $startedAt = microtime(true); + if ($this->artworkId !== null) { $artwork = Artwork::query() ->public() @@ -80,6 +101,8 @@ final class RecComputeSimilarHybridJob implements ShouldQueue ->find($this->artworkId); if (! $artwork instanceof Artwork) { + $this->logBatchComplete($startedAt, 0, false); + return; } @@ -93,26 +116,42 @@ final class RecComputeSimilarHybridJob implements ShouldQueue $weights, ); + $this->logBatchComplete($startedAt, 1, false); + return; } - Artwork::query() + $artworks = Artwork::query() ->public() ->published() ->select('id', 'user_id') - ->chunkById($this->batchSize, function ($artworks) use ( - $modelVersion, $vectorEnabled, $resultLimit, $maxPerAuthor, $minCatsTop12, $weights - ) { - $this->processArtworkSafely( - $artworks, - $modelVersion, - $vectorEnabled, - $resultLimit, - $maxPerAuthor, - $minCatsTop12, - $weights, - ); - }); + ->when($this->afterArtworkId() !== null, fn ($query) => $query->where('id', '>', $this->afterArtworkId())) + ->orderBy('id') + ->limit($this->batchSize) + ->get(); + + if ($artworks->isEmpty()) { + $this->logBatchComplete($startedAt, 0, false); + + return; + } + + $this->processArtworkSafely( + $artworks, + $modelVersion, + $vectorEnabled, + $resultLimit, + $maxPerAuthor, + $minCatsTop12, + $weights, + ); + + $hasMore = $artworks->count() === $this->batchSize; + if ($hasMore) { + static::dispatch(null, $this->batchSize, (int) $artworks->last()->id); + } + + $this->logBatchComplete($startedAt, $artworks->count(), $hasMore); } public function failed(\Throwable $exception): void @@ -120,12 +159,28 @@ final class RecComputeSimilarHybridJob implements ShouldQueue Log::error('[RecComputeSimilarHybrid] Job failed permanently.', [ 'artwork_id' => $this->artworkId, 'batch_size' => $this->batchSize, + 'after_artwork_id' => $this->afterArtworkId(), 'attempts' => $this->attempts(), 'exception_class' => $exception::class, 'exception_message' => $exception->getMessage(), ]); } + /** + * @param positive-int|0 $processed + */ + private function logBatchComplete(float $startedAt, int $processed, bool $hasMore): void + { + Log::info('[RecComputeSimilarHybrid] Batch complete.', [ + 'artwork_id' => $this->artworkId, + 'after_artwork_id' => $this->afterArtworkId(), + 'processed' => $processed, + 'has_more' => $hasMore, + 'duration_ms' => (int) round((microtime(true) - $startedAt) * 1000), + 'memory_mb' => round(memory_get_peak_usage(true) / 1048576, 1), + ]); + } + /** * @param iterable $artworks */ diff --git a/app/Providers/AppServiceProvider.php b/app/Providers/AppServiceProvider.php index ade2d09d..c0d0ba52 100644 --- a/app/Providers/AppServiceProvider.php +++ b/app/Providers/AppServiceProvider.php @@ -58,6 +58,11 @@ class AppServiceProvider extends ServiceProvider */ public function register(): void { + $this->app->bind( + \Inertia\Ssr\Gateway::class, + \App\Support\Http\TimedInertiaSsrGateway::class, + ); + $this->app->singleton( \App\Services\Countries\CountryRemoteProviderInterface::class, \App\Services\Countries\CountryRemoteProvider::class, diff --git a/app/Services/Academy/AcademyAccessService.php b/app/Services/Academy/AcademyAccessService.php index c0cc6313..920fca6b 100644 --- a/app/Services/Academy/AcademyAccessService.php +++ b/app/Services/Academy/AcademyAccessService.php @@ -13,6 +13,7 @@ use App\Models\AcademyLessonBlock; use App\Models\AcademyPromptPack; use App\Models\AcademyPromptTemplate; use App\Models\User; +use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Storage; use Illuminate\Support\Str; use Laravel\Cashier\Subscription; @@ -280,7 +281,9 @@ final class AcademyAccessService public function coursePayload(AcademyCourse $course, ?User $viewer, array $options = []): array { $progress = is_array($options['progress'] ?? null) ? $options['progress'] : null; - $lessonCount = (int) ($course->lessons_count_cache ?: $course->courseLessons()->count()); + $lessonCount = $course->relationLoaded('courseLessons') + ? $course->courseLessons->count() + : (int) $course->lessons_count_cache; return [ 'id' => (int) $course->id, @@ -1089,7 +1092,13 @@ final class AcademyAccessService } try { - $exists = Storage::disk((string) config('uploads.object_storage.disk', 's3'))->exists($normalizedPath); + $exists = (bool) Cache::remember( + 'academy:asset-exists:'.$cacheKey, + 3600, + function () use ($normalizedPath): bool { + return Storage::disk((string) config('uploads.object_storage.disk', 's3'))->exists($normalizedPath); + }, + ); } catch (\Throwable) { $exists = false; } diff --git a/app/Services/CollectionBackgroundJobService.php b/app/Services/CollectionBackgroundJobService.php index 1fb57cab..87dc5d8f 100644 --- a/app/Services/CollectionBackgroundJobService.php +++ b/app/Services/CollectionBackgroundJobService.php @@ -16,6 +16,10 @@ class CollectionBackgroundJobService { public function dispatchQualityRefresh(Collection $collection, ?User $actor = null): array { + if (! $this->dispatchEnabled()) { + return $this->disabledPayload('quality_refresh', $collection ? collect([$collection]) : collect()); + } + RefreshCollectionQualityJob::dispatch((int) $collection->id, $actor?->id)->afterCommit(); return [ @@ -30,6 +34,10 @@ class CollectionBackgroundJobService public function dispatchHealthRefresh(?Collection $collection = null, ?User $actor = null): array { + if (! $this->dispatchEnabled()) { + return $this->disabledPayload('health_refresh', $collection ? collect([$collection]) : collect()); + } + $targets = $collection ? collect([$collection]) : $this->healthTargets(); $targets->each(fn (Collection $item) => RefreshCollectionHealthJob::dispatch((int) $item->id, $actor?->id, 'programming-eligibility')->afterCommit()); @@ -39,6 +47,10 @@ class CollectionBackgroundJobService public function dispatchRecommendationRefresh(?Collection $collection = null, ?User $actor = null, string $context = 'default'): array { + if (! $this->dispatchEnabled()) { + return $this->disabledPayload('recommendation_refresh', $collection ? collect([$collection]) : collect()); + } + $targets = $collection ? collect([$collection]) : $this->recommendationTargets(); $targets->each(fn (Collection $item) => RefreshCollectionRecommendationJob::dispatch((int) $item->id, $actor?->id, $context)->afterCommit()); @@ -48,6 +60,10 @@ class CollectionBackgroundJobService public function dispatchDuplicateScan(?Collection $collection = null, ?User $actor = null): array { + if (! $this->dispatchEnabled()) { + return $this->disabledPayload('duplicate_scan', $collection ? collect([$collection]) : collect()); + } + $targets = $collection ? collect([$collection]) : $this->duplicateTargets(); $targets->each(fn (Collection $item) => ScanCollectionDuplicateCandidatesJob::dispatch((int) $item->id, $actor?->id)->afterCommit()); @@ -128,6 +144,29 @@ class CollectionBackgroundJobService ->get(['id']); } + private function dispatchEnabled(): bool + { + return (bool) config('collections.v5.queue.dispatch_enabled', false); + } + + /** + * @param SupportCollection $targets + */ + private function disabledPayload(string $job, SupportCollection $targets): array + { + $ids = $targets->pluck('id')->map(static fn ($id): int => (int) $id)->values()->all(); + + return [ + 'status' => 'disabled', + 'job' => $job, + 'scope' => count($ids) === 1 ? 'single' : 'batch', + 'count' => 0, + 'collection_ids' => $ids, + 'items' => [], + 'message' => 'Collection queue dispatch is disabled until a Horizon consumer exists.', + ]; + } + /** * @param SupportCollection $targets */ diff --git a/app/Services/LeaderboardService.php b/app/Services/LeaderboardService.php index 13bbcaed..4073a589 100644 --- a/app/Services/LeaderboardService.php +++ b/app/Services/LeaderboardService.php @@ -5,8 +5,8 @@ declare(strict_types=1); namespace App\Services; use App\Models\Artwork; -use App\Models\ArtworkMetricSnapshotHourly; use App\Models\Group; +use App\Services\Metrics\ArtworkHourlySnapshotWindow; use App\Models\Leaderboard; use App\Models\Story; use App\Models\StoryLike; @@ -28,6 +28,7 @@ class LeaderboardService public function __construct( private readonly GroupReputationService $groupReputation, + private readonly ArtworkHourlySnapshotWindow $snapshotWindow, ) { } @@ -835,29 +836,7 @@ class LeaderboardService private function artworkSnapshotDeltas(CarbonImmutable $start): \Illuminate\Database\Query\Builder { - return ArtworkMetricSnapshotHourly::query() - ->from('artwork_metric_snapshots_hourly as snapshots') - ->where('snapshots.bucket_hour', '>=', $start) - ->select([ - 'snapshots.artwork_id', - DB::raw($this->nonNegativeSnapshotDelta('views_count', 'views_delta')), - DB::raw($this->nonNegativeSnapshotDelta('downloads_count', 'downloads_delta')), - DB::raw($this->nonNegativeSnapshotDelta('favourites_count', 'favourites_delta')), - DB::raw($this->nonNegativeSnapshotDelta('comments_count', 'comments_delta')), - ]) - ->groupBy('snapshots.artwork_id') - ->toBase(); - } - - private function nonNegativeSnapshotDelta(string $column, string $alias): string - { - $delta = sprintf('MAX(snapshots.%1$s) - MIN(snapshots.%1$s)', $column); - - if (DB::connection()->getDriverName() === 'sqlite') { - return sprintf('CASE WHEN %1$s > 0 THEN %1$s ELSE 0 END as %2$s', $delta, $alias); - } - - return sprintf('GREATEST(%s, 0) as %s', $delta, $alias); + return $this->snapshotWindow->artworkPeriodDeltas($start); } private function creatorEntities(array $ids): array diff --git a/app/Services/Metrics/ArtworkHourlySnapshotWindow.php b/app/Services/Metrics/ArtworkHourlySnapshotWindow.php new file mode 100644 index 00000000..9eea7b94 --- /dev/null +++ b/app/Services/Metrics/ArtworkHourlySnapshotWindow.php @@ -0,0 +1,260 @@ +tableCoverage($days); + $empty = $this->emptyResult($days, $coverage); + + if ($userId <= 0 || ! Schema::hasTable('artwork_metric_snapshots_hourly')) { + return $empty; + } + + $since = now()->subDays($days); + + $row = DB::query() + ->fromSub($this->artworkPeriodDeltas($since, $userId), 'deltas') + ->selectRaw('COALESCE(SUM(views_delta), 0) as views') + ->selectRaw('COALESCE(SUM(downloads_delta), 0) as downloads') + ->selectRaw('COALESCE(SUM(favourites_delta), 0) as favourites') + ->selectRaw('COALESCE(SUM(comments_delta), 0) as comments') + ->selectRaw('COALESCE(SUM(shares_delta), 0) as shares') + ->first(); + + $userBounds = DB::table('artwork_metric_snapshots_hourly as snapshots') + ->join('artworks', 'artworks.id', '=', 'snapshots.artwork_id') + ->where('artworks.user_id', $userId) + ->whereNull('artworks.deleted_at') + ->where('snapshots.bucket_hour', '>=', $since) + ->selectRaw('MIN(snapshots.bucket_hour) as oldest_bucket') + ->selectRaw('MAX(snapshots.bucket_hour) as newest_bucket') + ->first(); + + $userCoverageDays = $this->coverageDays( + $userBounds->oldest_bucket ?? null, + $userBounds->newest_bucket ?? null, + ); + + return [ + 'views' => (int) ($row->views ?? 0), + 'downloads' => (int) ($row->downloads ?? 0), + 'favourites' => (int) ($row->favourites ?? 0), + 'comments' => (int) ($row->comments ?? 0), + 'shares' => (int) ($row->shares ?? 0), + 'requested_days' => $days, + 'table_coverage_days' => $coverage['coverage_days'], + 'user_coverage_days' => $userCoverageDays, + 'window_complete' => $coverage['window_complete'], + 'oldest_bucket' => $coverage['oldest_bucket'], + 'newest_bucket' => $coverage['newest_bucket'], + 'user_oldest_bucket' => $userBounds->oldest_bucket ?? null, + 'user_newest_bucket' => $userBounds->newest_bucket ?? null, + ]; + } + + /** + * Per-artwork non-negative period deltas for cumulative snapshot columns. + */ + public function artworkPeriodDeltas(CarbonInterface|\DateTimeInterface $start, ?int $userId = null): Builder + { + $inWindow = DB::table('artwork_metric_snapshots_hourly as snapshots'); + $this->constrainOwner($inWindow, $userId); + $inWindow + ->where('snapshots.bucket_hour', '>=', $start) + ->groupBy('snapshots.artwork_id') + ->select('snapshots.artwork_id') + ->selectRaw('MAX(snapshots.views_count) as views_latest') + ->selectRaw('MIN(snapshots.views_count) as views_min') + ->selectRaw('MAX(snapshots.downloads_count) as downloads_latest') + ->selectRaw('MIN(snapshots.downloads_count) as downloads_min') + ->selectRaw('MAX(snapshots.favourites_count) as favourites_latest') + ->selectRaw('MIN(snapshots.favourites_count) as favourites_min') + ->selectRaw('MAX(snapshots.comments_count) as comments_latest') + ->selectRaw('MIN(snapshots.comments_count) as comments_min') + ->selectRaw('MAX(snapshots.shares_count) as shares_latest') + ->selectRaw('MIN(snapshots.shares_count) as shares_min'); + + $baselineFrom = \Carbon\Carbon::parse($start)->subHours(self::BASELINE_LOOKBACK_HOURS); + $baseline = DB::table('artwork_metric_snapshots_hourly as snapshots'); + $this->constrainOwner($baseline, $userId); + $baseline + ->where('snapshots.bucket_hour', '<', $start) + ->where('snapshots.bucket_hour', '>=', $baselineFrom) + ->groupBy('snapshots.artwork_id') + ->select('snapshots.artwork_id') + ->selectRaw('MAX(snapshots.views_count) as views_baseline') + ->selectRaw('MAX(snapshots.downloads_count) as downloads_baseline') + ->selectRaw('MAX(snapshots.favourites_count) as favourites_baseline') + ->selectRaw('MAX(snapshots.comments_count) as comments_baseline') + ->selectRaw('MAX(snapshots.shares_count) as shares_baseline'); + + $bornSql = $this->quotedTimestamp($start); + + return DB::query() + ->fromSub($inWindow, 'win') + ->leftJoinSub($baseline, 'base', 'base.artwork_id', '=', 'win.artwork_id') + ->join('artworks', 'artworks.id', '=', 'win.artwork_id') + ->select('win.artwork_id') + ->selectRaw($this->deltaExpression('views', $bornSql) . ' as views_delta') + ->selectRaw($this->deltaExpression('downloads', $bornSql) . ' as downloads_delta') + ->selectRaw($this->deltaExpression('favourites', $bornSql) . ' as favourites_delta') + ->selectRaw($this->deltaExpression('comments', $bornSql) . ' as comments_delta') + ->selectRaw($this->deltaExpression('shares', $bornSql) . ' as shares_delta'); + } + + /** + * @return array{oldest_bucket: ?string, newest_bucket: ?string, coverage_days: float, window_complete: bool, requested_days: int} + */ + public function tableCoverage(int $days = 30): array + { + $days = max(1, $days); + + if (! Schema::hasTable('artwork_metric_snapshots_hourly')) { + return [ + 'oldest_bucket' => null, + 'newest_bucket' => null, + 'coverage_days' => 0.0, + 'window_complete' => false, + 'requested_days' => $days, + ]; + } + + $bounds = DB::table('artwork_metric_snapshots_hourly') + ->selectRaw('MIN(bucket_hour) as oldest_bucket') + ->selectRaw('MAX(bucket_hour) as newest_bucket') + ->first(); + + $coverageDays = $this->coverageDays( + $bounds->oldest_bucket ?? null, + $bounds->newest_bucket ?? null, + ); + + return [ + 'oldest_bucket' => $bounds->oldest_bucket ?? null, + 'newest_bucket' => $bounds->newest_bucket ?? null, + 'coverage_days' => $coverageDays, + 'window_complete' => $coverageDays + (1 / 24) >= $days, + 'requested_days' => $days, + ]; + } + + private function constrainOwner(Builder $query, ?int $userId): void + { + if ($userId === null) { + return; + } + + $query->join('artworks as owner_artworks', 'owner_artworks.id', '=', 'snapshots.artwork_id') + ->where('owner_artworks.user_id', $userId) + ->whereNull('owner_artworks.deleted_at'); + } + + private function quotedTimestamp(CarbonInterface|\DateTimeInterface $start): string + { + return DB::getPdo()->quote(\Carbon\Carbon::parse($start)->toDateTimeString()); + } + + private function deltaExpression(string $metric, string $bornSql): string + { + $latest = "COALESCE(win.{$metric}_latest, 0)"; + $min = "COALESCE(win.{$metric}_min, 0)"; + $base = "base.{$metric}_baseline"; + $bornInWindow = "COALESCE(artworks.published_at, artworks.created_at) >= {$bornSql}"; + $observed = $this->clampNonNegative("{$latest} - {$min}"); + $fromBaseline = $this->clampNonNegative("{$latest} - {$base}"); + + return "CASE WHEN {$bornInWindow} THEN {$latest} WHEN {$base} IS NOT NULL THEN {$fromBaseline} ELSE {$observed} END"; + } + + private function clampNonNegative(string $expression): string + { + if (DB::connection()->getDriverName() === 'sqlite') { + return "CASE WHEN ({$expression}) > 0 THEN ({$expression}) ELSE 0 END"; + } + + return "GREATEST({$expression}, 0)"; + } + + /** + * @return array + */ + private function emptyResult(int $days, array $coverage): array + { + return [ + 'views' => 0, + 'downloads' => 0, + 'favourites' => 0, + 'comments' => 0, + 'shares' => 0, + 'requested_days' => $days, + 'table_coverage_days' => $coverage['coverage_days'], + 'user_coverage_days' => 0.0, + 'window_complete' => $coverage['window_complete'], + 'oldest_bucket' => $coverage['oldest_bucket'], + 'newest_bucket' => $coverage['newest_bucket'], + 'user_oldest_bucket' => null, + 'user_newest_bucket' => null, + ]; + } + + private function coverageDays(?string $oldest, ?string $newest): float + { + if ($oldest === null || $newest === null) { + return 0.0; + } + + $rangeStart = \Carbon\Carbon::parse($oldest); + $rangeEnd = \Carbon\Carbon::parse($newest); + + if ($rangeEnd->lessThan($rangeStart)) { + return 0.0; + } + + return round(abs($rangeStart->diffInMinutes($rangeEnd)) / 1440, 2); + } +} diff --git a/app/Services/Profile/CreatorJourneyService.php b/app/Services/Profile/CreatorJourneyService.php index b1f07d04..d2d19748 100644 --- a/app/Services/Profile/CreatorJourneyService.php +++ b/app/Services/Profile/CreatorJourneyService.php @@ -204,7 +204,7 @@ final class CreatorJourneyService [ 'title' => 'Biggest download spike', 'headline' => (string) $bestSpike['artwork']->title, - 'summary' => 'Captured the strongest one-hour download burst recorded for a public artwork.', + 'summary' => 'Captured the strongest one-hour download burst in the retained hourly snapshot window.', 'value' => (int) $bestSpike['downloads_in_hour'] . ' downloads in 1 hour', 'artwork' => $this->artworkSnapshot($bestSpike['artwork']), 'metrics' => [ @@ -425,8 +425,12 @@ final class CreatorJourneyService $publicArtworkIds = $artworks->pluck('id')->map(fn ($id): int => (int) $id)->all(); if ($publicArtworkIds !== [] && DB::getSchemaBuilder()->hasTable('artwork_metric_snapshots_hourly')) { + $lookbackDays = max(1, (int) config('metrics.hourly_snapshot_retention_days', 30)); + $since = now()->subDays($lookbackDays); + $snapshots = DB::table('artwork_metric_snapshots_hourly as ms') ->whereIn('ms.artwork_id', $publicArtworkIds) + ->where('ms.bucket_hour', '>=', $since) ->orderBy('ms.artwork_id') ->orderBy('ms.bucket_hour') ->get([ diff --git a/app/Services/Sitemaps/Builders/TagsSitemapBuilder.php b/app/Services/Sitemaps/Builders/TagsSitemapBuilder.php index bf6bdb3b..090500be 100644 --- a/app/Services/Sitemaps/Builders/TagsSitemapBuilder.php +++ b/app/Services/Sitemaps/Builders/TagsSitemapBuilder.php @@ -5,12 +5,12 @@ declare(strict_types=1); namespace App\Services\Sitemaps\Builders; use App\Models\Tag; -use App\Services\Sitemaps\AbstractSitemapBuilder; use App\Services\Sitemaps\SitemapUrl; use App\Services\Sitemaps\SitemapUrlBuilder; -use DateTimeInterface; +use Illuminate\Database\Eloquent\Builder; +use Illuminate\Database\Eloquent\Model; -final class TagsSitemapBuilder extends AbstractSitemapBuilder +final class TagsSitemapBuilder extends AbstractIdShardableSitemapBuilder { public function __construct(private readonly SitemapUrlBuilder $urls) { @@ -21,25 +21,21 @@ final class TagsSitemapBuilder extends AbstractSitemapBuilder return 'tags'; } - public function items(): array + protected function shardConfigKey(): string + { + return 'tags'; + } + + protected function mapRecord(Model $record): ?SitemapUrl + { + return $this->urls->tag($record); + } + + protected function query(): Builder { return Tag::query() ->where('is_active', true) ->where('usage_count', '>', 0) - ->whereHas('artworks', fn ($query) => $query->public()->published()) - ->orderByDesc('usage_count') - ->orderBy('slug') - ->get() - ->map(fn (Tag $tag): SitemapUrl => $this->urls->tag($tag)) - ->values() - ->all(); - } - - public function lastModified(): ?DateTimeInterface - { - return $this->dateTime(Tag::query() - ->where('is_active', true) - ->where('usage_count', '>', 0) - ->max('updated_at')); + ->whereHas('artworks', fn ($query) => $query->public()->published()); } } \ No newline at end of file diff --git a/app/Services/Studio/CreatorStudioOverviewService.php b/app/Services/Studio/CreatorStudioOverviewService.php index f8e354aa..d6552bd3 100644 --- a/app/Services/Studio/CreatorStudioOverviewService.php +++ b/app/Services/Studio/CreatorStudioOverviewService.php @@ -8,6 +8,7 @@ use App\Models\CollectionComment; use App\Models\NovaCardComment; use App\Models\StoryComment; use App\Models\User; +use App\Services\Metrics\ArtworkHourlySnapshotWindow; use App\Support\AvatarUrl; use Illuminate\Support\Facades\DB; @@ -21,6 +22,7 @@ final class CreatorStudioOverviewService private readonly CreatorStudioPreferenceService $preferences, private readonly CreatorStudioChallengeService $challenges, private readonly CreatorStudioGrowthService $growth, + private readonly ArtworkHourlySnapshotWindow $snapshotWindow, ) { } @@ -32,15 +34,24 @@ final class CreatorStudioOverviewService $challengeData = $this->challenges->build($user); $growthData = $this->growth->build($user, $preferences['analytics_range_days']); $featuredContent = $this->content->selectedItems($user, $preferences['featured_content']); + $window = $this->snapshotWindow->userWindow((int) $user->id, 30); return [ 'kpis' => [ 'total_content' => $analytics['totals']['content_count'], - 'views_30d' => $analytics['totals']['views'], - 'appreciation_30d' => $analytics['totals']['appreciation'], - 'shares_30d' => $analytics['totals']['shares'], - 'comments_30d' => $analytics['totals']['comments'], + 'views_30d' => $window['views'], + 'appreciation_30d' => $window['favourites'], + 'shares_30d' => $window['shares'], + 'comments_30d' => $window['comments'], 'followers' => $analytics['totals']['followers'], + 'snapshot_window' => [ + 'requested_days' => $window['requested_days'], + 'table_coverage_days' => $window['table_coverage_days'], + 'user_coverage_days' => $window['user_coverage_days'], + 'window_complete' => $window['window_complete'], + 'oldest_bucket' => $window['oldest_bucket'], + 'newest_bucket' => $window['newest_bucket'], + ], ], 'module_summaries' => $moduleSummaries, 'quick_create' => $this->content->quickCreate(), diff --git a/app/Services/Studio/StudioMetricsService.php b/app/Services/Studio/StudioMetricsService.php index b1e529ed..a29b54ec 100644 --- a/app/Services/Studio/StudioMetricsService.php +++ b/app/Services/Studio/StudioMetricsService.php @@ -6,6 +6,7 @@ namespace App\Services\Studio; use App\Models\Artwork; use App\Models\ArtworkStats; +use App\Services\Metrics\ArtworkHourlySnapshotWindow; use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\DB; @@ -16,10 +17,26 @@ final class StudioMetricsService { private const CACHE_TTL = 300; // 5 minutes + public function __construct( + private readonly ArtworkHourlySnapshotWindow $snapshotWindow, + ) { + } + /** * Get dashboard KPI metrics for a creator. * - * @return array{total_artworks: int, views_30d: int, favourites_30d: int, shares_30d: int, followers: int} + * 30-day counters are latest-minus-baseline deltas of cumulative hourly snapshots, never SUM. + * + * @return array{ + * total_artworks: int, + * views_30d: int, + * favourites_30d: int, + * shares_30d: int, + * downloads_30d: int, + * comments_30d: int, + * followers: int, + * snapshot_window: array + * } */ public function getDashboardKpis(int $userId): array { @@ -30,35 +47,7 @@ final class StudioMetricsService ->whereNull('deleted_at') ->count(); - // Aggregate stats from artwork_stats for this user's artworks - $statsAgg = DB::table('artwork_stats') - ->join('artworks', 'artworks.id', '=', 'artwork_stats.artwork_id') - ->where('artworks.user_id', $userId) - ->whereNull('artworks.deleted_at') - ->selectRaw(' - COALESCE(SUM(artwork_stats.views), 0) as total_views, - COALESCE(SUM(artwork_stats.favorites), 0) as total_favourites, - COALESCE(SUM(artwork_stats.shares_count), 0) as total_shares - ') - ->first(); - - // Views in last 30 days from hourly snapshots if available, fallback to totals - $views30d = 0; - try { - if (\Illuminate\Support\Facades\Schema::hasTable('artwork_metric_snapshots_hourly')) { - $views30d = (int) DB::table('artwork_metric_snapshots_hourly') - ->join('artworks', 'artworks.id', '=', 'artwork_metric_snapshots_hourly.artwork_id') - ->where('artworks.user_id', $userId) - ->where('artwork_metric_snapshots_hourly.bucket_hour', '>=', now()->subDays(30)) - ->sum('artwork_metric_snapshots_hourly.views_count'); - } - } catch (\Throwable $e) { - // Table or column doesn't exist — fall back to totals - } - - if ($views30d === 0) { - $views30d = (int) ($statsAgg->total_views ?? 0); - } + $window = $this->snapshotWindow->userWindow($userId, 30); $followers = DB::table('user_followers') ->where('user_id', $userId) @@ -66,10 +55,20 @@ final class StudioMetricsService return [ 'total_artworks' => $totalArtworks, - 'views_30d' => $views30d, - 'favourites_30d' => (int) ($statsAgg->total_favourites ?? 0), - 'shares_30d' => (int) ($statsAgg->total_shares ?? 0), - 'followers' => $followers, + 'views_30d' => $window['views'], + 'favourites_30d' => $window['favourites'], + 'shares_30d' => $window['shares'], + 'downloads_30d' => $window['downloads'], + 'comments_30d' => $window['comments'], + 'followers' => $followers, + 'snapshot_window' => [ + 'requested_days' => $window['requested_days'], + 'table_coverage_days' => $window['table_coverage_days'], + 'user_coverage_days' => $window['user_coverage_days'], + 'window_complete' => $window['window_complete'], + 'oldest_bucket' => $window['oldest_bucket'], + 'newest_bucket' => $window['newest_bucket'], + ], ]; }); } @@ -145,7 +144,6 @@ final class StudioMetricsService $cacheKey = "studio.analytics_overview.{$userId}"; return Cache::remember($cacheKey, self::CACHE_TTL, function () use ($userId) { - // Totals $totals = DB::table('artwork_stats') ->join('artworks', 'artworks.id', '=', 'artwork_stats.artwork_id') ->where('artworks.user_id', $userId) @@ -161,7 +159,6 @@ final class StudioMetricsService ') ->first(); - // Top 10 artworks by ranking score $topArtworks = Artwork::where('user_id', $userId) ->whereNull('deleted_at') ->where('is_public', true) @@ -188,7 +185,6 @@ final class StudioMetricsService 'heat_score' => (float) ($art->stats?->heat_score ?? 0), ]); - // Content type breakdown $contentBreakdown = DB::table('artworks') ->join('artwork_category', 'artwork_category.artwork_id', '=', 'artworks.id') ->join('categories', 'categories.id', '=', 'artwork_category.category_id') diff --git a/app/Services/Traffic/OnlineVisitorRepository.php b/app/Services/Traffic/OnlineVisitorRepository.php index 6d1d9676..7977a9ec 100644 --- a/app/Services/Traffic/OnlineVisitorRepository.php +++ b/app/Services/Traffic/OnlineVisitorRepository.php @@ -181,7 +181,29 @@ class OnlineVisitorRepository */ protected function readIndexMembers(): array { - return array_map('strval', Redis::smembers(self::INDEX_KEY)); + $members = []; + $cursor = '0'; + $limit = max(1, (int) config('traffic.online_visitors.index_read_limit', 2000)); + + do { + $result = Redis::sscan(self::INDEX_KEY, (int) $cursor, ['count' => 200]); + + if (! is_array($result) || count($result) < 2) { + break; + } + + $cursor = (string) $result[0]; + + foreach ($result[1] as $member) { + $members[] = (string) $member; + + if (count($members) >= $limit) { + return $members; + } + } + } while ($cursor !== '0'); + + return $members; } /** diff --git a/app/Services/Traffic/PresenceIndexPruner.php b/app/Services/Traffic/PresenceIndexPruner.php new file mode 100644 index 00000000..932b0cdf --- /dev/null +++ b/app/Services/Traffic/PresenceIndexPruner.php @@ -0,0 +1,116 @@ += $timeLimitSeconds) { + break; + } + + $result = Redis::sscan(OnlineVisitorRepository::INDEX_KEY, $cursor, ['count' => $scanCount]); + if (! is_array($result) || count($result) < 2) { + break; + } + + $cursor = (int) $result[0]; + $members = $result[1]; + $batches++; + + if ($members === []) { + if ($cursor === 0) { + break; + } + continue; + } + + $recordKeys = []; + foreach ($members as $member) { + $member = (string) $member; + $recordKeys[$member] = OnlineVisitorRepository::KEY_PREFIX.':'.$member; + } + + $exists = Redis::pipeline(function ($pipe) use ($recordKeys): void { + foreach ($recordKeys as $recordKey) { + $pipe->exists($recordKey); + } + }); + if (! is_array($exists)) { + $exists = []; + } + + $i = 0; + foreach ($recordKeys as $member => $recordKey) { + $scanned++; + $isLive = (int) ($exists[$i] ?? 0) === 1; + $i++; + if ($isLive) { + $live++; + continue; + } + $stale++; + if ($dryRun) { + continue; + } + $removed += (int) Redis::eval( + self::SREM_IF_EXPIRED, + 2, + $recordKey, + OnlineVisitorRepository::INDEX_KEY, + $member, + ); + } + + if ($sleepMs > 0) { + usleep($sleepMs * 1000); + } + + if ($cursor === 0) { + break; + } + } + + return [ + 'scanned' => $scanned, + 'stale' => $stale, + 'removed' => $removed, + 'live' => $live, + 'batches' => $batches, + 'dry_run' => $dryRun, + ]; + } +} diff --git a/app/Services/Vision/AiArtworkVectorSearchService.php b/app/Services/Vision/AiArtworkVectorSearchService.php index 8567c635..b2a6e74d 100644 --- a/app/Services/Vision/AiArtworkVectorSearchService.php +++ b/app/Services/Vision/AiArtworkVectorSearchService.php @@ -8,12 +8,18 @@ use App\Models\Artwork; use Illuminate\Http\UploadedFile; use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Http; -use RuntimeException; use Throwable; final class AiArtworkVectorSearchService { - private const MAX_SIMILAR_RESULTS = 120; + /** qdrant-svc /vectors/search* reject limit > 100. */ + private const VECTOR_GATEWAY_MAX_SEARCH_RESULTS = 100; + + /** + * Public similar-ai may request at most 99 items because we fetch one extra + * candidate so the source artwork can be excluded from the result set. + */ + private const MAX_ARTWORK_SIMILAR_RESULTS = 99; public function __construct( private readonly VectorGatewayClient $client, @@ -31,12 +37,13 @@ final class AiArtworkVectorSearchService */ public function similarToArtwork(Artwork $artwork, int $limit = 12): array { - $safeLimit = max(1, min(self::MAX_SIMILAR_RESULTS, $limit)); + $safeLimit = max(1, min(self::MAX_ARTWORK_SIMILAR_RESULTS, $limit)); $cacheKey = sprintf('rec:artwork:%d:similar-ai:%d', $artwork->id, $safeLimit); $ttl = max(60, (int) config('recommendations.ttl.similar_artworks', 30 * 60)); return Cache::remember($cacheKey, $ttl, function () use ($artwork, $safeLimit): array { - $matches = $this->searchMatchesForArtwork($artwork, $safeLimit + 1); + $gatewayLimit = min(self::VECTOR_GATEWAY_MAX_SEARCH_RESULTS, $safeLimit + 1); + $matches = $this->searchMatchesForArtwork($artwork, $gatewayLimit); return $this->resolveMatches($matches, $safeLimit, $artwork->id); }); @@ -47,7 +54,7 @@ final class AiArtworkVectorSearchService */ public function searchByUploadedImage(UploadedFile $file, int $limit = 12): array { - $safeLimit = max(1, min(self::MAX_SIMILAR_RESULTS, $limit)); + $safeLimit = max(1, min(self::VECTOR_GATEWAY_MAX_SEARCH_RESULTS, $limit)); $matches = $this->client->searchByUploadedFile($file, $safeLimit); return $this->resolveMatches($matches, $safeLimit); @@ -93,15 +100,15 @@ final class AiArtworkVectorSearchService return []; } - if ($this->client->circuitOpen()) { - throw new RuntimeException('Vector gateway temporarily unavailable (circuit open after a recent failure).'); - } + $this->client->assertCircuitClosed(); $fileFailure = null; + $fileGatewayAttempted = false; try { $payload = $this->downloadArtworkImage($artwork, $url); if ($payload !== null) { + $fileGatewayAttempted = true; return $this->client->searchByFileContents($payload['contents'], $payload['filename'], $limit); } } catch (Throwable $e) { @@ -110,24 +117,69 @@ final class AiArtworkVectorSearchService try { return $this->client->searchByUrl($url, $limit); - } catch (Throwable $e) { - throw $this->normalizeSearchFailure($fileFailure, $e); + } catch (Throwable $fallbackFailure) { + if ($this->shouldTripSimilarAiCircuit($fileGatewayAttempted, $fileFailure, $fallbackFailure)) { + $this->client->tripIfCircuitWorthy( + [$fileFailure, $fallbackFailure], + 'similar_ai', + ['artwork_id' => (int) $artwork->id], + ); + } + + throw $this->normalizeSearchFailure($fileFailure, $fallbackFailure); } } - private function normalizeSearchFailure(?Throwable $fileFailure, Throwable $fallbackFailure): RuntimeException + private function normalizeSearchFailure(?Throwable $fileFailure, Throwable $fallbackFailure): VectorGatewayException { - if ($fileFailure === null) { - return $fallbackFailure instanceof RuntimeException - ? $fallbackFailure - : new RuntimeException($fallbackFailure->getMessage(), 0, $fallbackFailure); + if ($fallbackFailure instanceof VectorGatewayException && $fileFailure === null) { + return $fallbackFailure; } - return new RuntimeException(sprintf( - 'Vector search failed via file endpoint (%s) and URL fallback (%s).', - $fileFailure->getMessage(), - $fallbackFailure->getMessage(), - ), 0, $fallbackFailure); + $status = $fallbackFailure instanceof VectorGatewayException + ? $fallbackFailure->httpStatus + : null; + $circuitWorthy = $this->isCircuitWorthyFailure($fileFailure) + && $this->isCircuitWorthyFailure($fallbackFailure); + + if ($fileFailure === null) { + return $fallbackFailure instanceof VectorGatewayException + ? $fallbackFailure + : new VectorGatewayException( + 'Vector gateway search failed.', + 'search_url', + $status, + $this->isCircuitWorthyFailure($fallbackFailure), + $fallbackFailure, + ); + } + + return new VectorGatewayException( + 'Vector gateway search failed.', + 'search_combined', + $status, + $circuitWorthy, + $fallbackFailure, + ); + } + + private function isCircuitWorthyFailure(Throwable $failure): bool + { + return $failure instanceof VectorGatewayException && $failure->circuitWorthy; + } + + private function shouldTripSimilarAiCircuit( + bool $fileGatewayAttempted, + ?Throwable $fileFailure, + Throwable $fallbackFailure, + ): bool { + return $fileGatewayAttempted + && $fileFailure instanceof VectorGatewayException + && $fileFailure->operation === 'search_file' + && $fileFailure->circuitWorthy + && $fallbackFailure instanceof VectorGatewayException + && $fallbackFailure->operation === 'search_url' + && $fallbackFailure->circuitWorthy; } /** diff --git a/app/Services/Vision/VectorGatewayClient.php b/app/Services/Vision/VectorGatewayClient.php index 47f6786d..4c473056 100644 --- a/app/Services/Vision/VectorGatewayClient.php +++ b/app/Services/Vision/VectorGatewayClient.php @@ -5,16 +5,22 @@ declare(strict_types=1); namespace App\Services\Vision; use Illuminate\Http\UploadedFile; +use Illuminate\Http\Client\ConnectionException; use Illuminate\Http\Client\PendingRequest; +use Illuminate\Http\Client\RequestException; use Illuminate\Http\Client\Response; use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Http; +use Illuminate\Support\Facades\Log; use RuntimeException; +use Throwable; final class VectorGatewayClient { private const CIRCUIT_KEY = 'vision.vector_gateway.circuit_open'; + private const MAX_SEARCH_LIMIT = 100; + public function isConfigured(): bool { return (bool) config('vision.vector_gateway.enabled', true) @@ -31,10 +37,20 @@ final class VectorGatewayClient return Cache::has(self::CIRCUIT_KEY); } - public function tripCircuit(): void + /** + * @param array $context + */ + public function tripCircuit(array $context = []): void { + $alreadyOpen = $this->circuitOpen(); $seconds = max(1, (int) config('vision.vector_gateway.circuit_breaker_seconds', 30)); Cache::put(self::CIRCUIT_KEY, true, $seconds); + + if ($alreadyOpen) { + return; + } + + $this->logCircuitOpened($context, $seconds); } public function upsertByUrl(string $imageUrl, int|string $id, array $metadata = []): array @@ -84,25 +100,21 @@ final class VectorGatewayClient */ public function searchByUrl(string $imageUrl, int $limit = 5): array { - $this->guardCircuit(); - try { $response = $this->postJson( $this->searchRequest(), $this->url((string) config('vision.vector_gateway.search_endpoint', '/vectors/search')), [ 'url' => $imageUrl, - 'limit' => max(1, $limit), + 'limit' => $this->clampSearchLimit($limit), ] ); - } catch (\Throwable $e) { - $this->tripCircuit(); - throw $e; + } catch (Throwable $e) { + throw $this->classifyThrowable('search_url', $e); } if ($response->failed()) { - $this->tripCircuit(); - throw new RuntimeException($this->failureMessage('Vector search', $response)); + throw $this->classifyHttpFailure('search_url', $response); } return $this->extractMatches($response->json()); @@ -113,25 +125,21 @@ final class VectorGatewayClient */ public function searchByFileContents(string $contents, string $filename, int $limit = 5): array { - $this->guardCircuit(); - try { $response = $this->searchRequest() ->attach('file', $contents, $filename) ->post( $this->url((string) config('vision.vector_gateway.search_file_endpoint', '/vectors/search/file')), [ - 'limit' => max(1, $limit), + 'limit' => $this->clampSearchLimit($limit), ] ); - } catch (\Throwable $e) { - $this->tripCircuit(); - throw $e; + } catch (Throwable $e) { + throw $this->classifyThrowable('search_file', $e); } if ($response->failed()) { - $this->tripCircuit(); - throw new RuntimeException($this->failureMessage('Vector search', $response)); + throw $this->classifyHttpFailure('search_file', $response); } return $this->extractMatches($response->json()); @@ -142,6 +150,8 @@ final class VectorGatewayClient */ public function searchByUploadedFile(UploadedFile $file, int $limit = 5): array { + $this->assertCircuitClosed(); + $realPath = $file->getRealPath(); if (! is_string($realPath) || $realPath === '') { throw new RuntimeException('Uploaded file has no readable temporary path for vector search.'); @@ -152,7 +162,47 @@ final class VectorGatewayClient throw new RuntimeException('Unable to read uploaded image bytes for vector search.'); } - return $this->searchByFileContents($contents, $file->getClientOriginalName() ?: 'search-image', $limit); + try { + return $this->searchByFileContents($contents, $file->getClientOriginalName() ?: 'search-image', $limit); + } catch (VectorGatewayException $e) { + $this->tripIfCircuitWorthy([$e], 'uploaded_image'); + throw $e; + } + } + + /** + * Open the circuit only when every attempted search failure is transient/gateway-wide. + * + * @param list $failures + * @param array{artwork_id?: int} $context + */ + public function tripIfCircuitWorthy(array $failures, string $source = 'unknown', array $context = []): void + { + if ($failures === []) { + return; + } + + foreach ($failures as $failure) { + if (! $failure->circuitWorthy) { + return; + } + } + + $this->tripCircuit([ + 'source' => $source, + 'artwork_id' => isset($context['artwork_id']) ? (int) $context['artwork_id'] : null, + 'failures' => array_map(static fn (VectorGatewayException $failure): array => [ + 'operation' => $failure->operation, + 'http_status' => $failure->httpStatus, + 'circuit_worthy' => $failure->circuitWorthy, + 'exception_class' => $failure::class, + ], $failures), + ]); + } + + public function assertCircuitClosed(): void + { + $this->guardCircuit(); } public function deleteByIds(array $ids): array @@ -220,13 +270,92 @@ final class VectorGatewayClient ); } + /** + * @param array $context + */ + private function logCircuitOpened(array $context, int $ttlSeconds): void + { + $payload = [ + 'source' => (string) ($context['source'] ?? 'unknown'), + 'failures' => is_array($context['failures'] ?? null) ? $context['failures'] : [], + 'circuit_ttl_seconds' => $ttlSeconds, + 'php_sapi' => PHP_SAPI, + 'running_in_console' => app()->runningInConsole(), + ]; + + if (isset($context['artwork_id']) && is_int($context['artwork_id']) && $context['artwork_id'] > 0) { + $payload['artwork_id'] = $context['artwork_id']; + } + + Log::warning('Vector gateway circuit opened', $payload); + } + + private function clampSearchLimit(int $limit): int + { + return max(1, min(self::MAX_SEARCH_LIMIT, $limit)); + } + private function guardCircuit(): void { if ($this->circuitOpen()) { - throw new RuntimeException('Vector gateway temporarily unavailable (circuit open after a recent failure).'); + throw new VectorGatewayException( + 'Vector gateway temporarily unavailable.', + 'circuit', + null, + true, + ); } } + private function classifyHttpFailure(string $operation, Response $response): VectorGatewayException + { + $status = $response->status(); + + return new VectorGatewayException( + 'Vector gateway '.$operation.' failed with HTTP '.$status.'.', + $operation, + $status, + $this->isCircuitWorthyStatus($status), + ); + } + + private function classifyThrowable(string $operation, Throwable $e): VectorGatewayException + { + if ($e instanceof VectorGatewayException) { + return $e; + } + + if ($e instanceof RequestException && $e->response instanceof Response) { + return $this->classifyHttpFailure($operation, $e->response); + } + + $circuitWorthy = $e instanceof ConnectionException || $this->looksLikeTimeout($e); + + return new VectorGatewayException( + 'Vector gateway '.$operation.' failed.', + $operation, + null, + $circuitWorthy, + $e, + ); + } + + private function isCircuitWorthyStatus(int $status): bool + { + return $status === 408 || $status === 429 || $status >= 500; + } + + private function looksLikeTimeout(Throwable $e): bool + { + $message = strtolower($e->getMessage()); + + return str_contains($message, 'timed out') + || str_contains($message, 'timeout') + || str_contains($message, 'curl error 28') + || str_contains($message, 'connection refused') + || str_contains($message, 'could not resolve'); + } + /** * @param array $payload */ diff --git a/app/Services/Vision/VectorGatewayException.php b/app/Services/Vision/VectorGatewayException.php new file mode 100644 index 00000000..c05bf46b --- /dev/null +++ b/app/Services/Vision/VectorGatewayException.php @@ -0,0 +1,21 @@ + + */ + public function analyze(string $file, array $options = []): array + { + $since = $this->sinceCutoff($options['since'] ?? null, $options['now'] ?? new \DateTimeImmutable('now')); + $minRequests = max(1, (int) ($options['min_requests'] ?? 1)); + $top = max(1, (int) ($options['top'] ?? 20)); + $statusFilter = isset($options['status']) ? strtolower((string) $options['status']) : null; + $uriFilter = $options['uri'] ?? null; + $hostFilter = isset($options['host']) ? (string) $options['host'] : ''; + $slowMs = isset($options['slow_ms']) ? (float) $options['slow_ms'] / 1000 : null; + + $skipped = 0; + $families = []; + $slowest = []; + $statusCounts = ['4xx' => 0, '5xx' => 0, '499' => 0, 'total' => 0]; + + $handle = fopen($file, 'r'); + if ($handle === false) { + throw new \RuntimeException('Unable to open performance log: '.$file); + } + + try { + while (($line = fgets($handle)) !== false) { + $line = trim($line); + if ($line === '') { + continue; + } + + $row = json_decode($line, true); + if (! is_array($row)) { + $skipped++; + continue; + } + + $time = $this->parseTime((string) ($row['time'] ?? '')); + if ($since !== null && ($time === null || $time < $since)) { + continue; + } + + $requestTime = $this->parseFloat($row['request_time'] ?? null); + if ($requestTime === null) { + $skipped++; + continue; + } + + $status = (int) ($row['status'] ?? 0); + if ($statusFilter === '5xx' && ($status < 500 || $status > 599)) { + continue; + } + if ($statusFilter === '4xx' && ($status < 400 || $status > 499)) { + continue; + } + if ($statusFilter === '499' && $status !== 499) { + continue; + } + + $host = (string) ($row['host'] ?? ''); + if ($hostFilter !== '' && $host !== $hostFilter) { + continue; + } + + $uri = $this->uris->normalize((string) ($row['uri'] ?? '/')); + if (is_string($uriFilter) && $uriFilter !== '' && ! str_starts_with($uri, $uriFilter)) { + continue; + } + + if ($slowMs !== null && $requestTime < $slowMs) { + continue; + } + + $upstreamResponse = $this->parseUpstream($row['upstream_response_time'] ?? null); + $upstreamHeader = $this->parseUpstream($row['upstream_header_time'] ?? null); + $bytes = (int) ($row['bytes'] ?? 0); + $method = (string) ($row['method'] ?? ''); + + $statusCounts['total']++; + if ($status === 499) { + $statusCounts['499']++; + } elseif ($status >= 500) { + $statusCounts['5xx']++; + } elseif ($status >= 400) { + $statusCounts['4xx']++; + } + + $familyKey = $host."\n".$uri; + $families[$familyKey] ??= [ + 'host' => $host, + 'uri' => $uri, + 'count' => 0, + 'times' => [], + 'upstream_sum' => 0.0, + 'upstream_n' => 0, + 'header_sum' => 0.0, + 'header_n' => 0, + 'bytes_sum' => 0, + '4xx' => 0, + '5xx' => 0, + '499' => 0, + '404' => 0, + ]; + $families[$familyKey]['count']++; + $families[$familyKey]['times'][] = $requestTime; + $families[$familyKey]['bytes_sum'] += $bytes; + if ($upstreamResponse !== null) { + $families[$familyKey]['upstream_sum'] += $upstreamResponse; + $families[$familyKey]['upstream_n']++; + } + if ($upstreamHeader !== null) { + $families[$familyKey]['header_sum'] += $upstreamHeader; + $families[$familyKey]['header_n']++; + } + if ($status === 499) { + $families[$familyKey]['499']++; + } elseif ($status >= 500) { + $families[$familyKey]['5xx']++; + } elseif ($status >= 400) { + $families[$familyKey]['4xx']++; + if ($status === 404) { + $families[$familyKey]['404']++; + } + } + + $slowest[] = [ + 'time' => $row['time'] ?? null, + 'host' => $host, + 'uri' => $uri, + 'method' => $method, + 'status' => $status, + 'request_time' => $requestTime, + 'upstream_response_time' => $upstreamResponse, + ]; + } + } finally { + fclose($handle); + } + + usort($slowest, static fn (array $a, array $b): int => $b['request_time'] <=> $a['request_time']); + $slowest = array_slice($slowest, 0, $top); + + $summaries = []; + foreach ($families as $family) { + if ($family['count'] < $minRequests) { + continue; + } + $times = $family['times']; + sort($times, SORT_NUMERIC); + $avg = array_sum($times) / $family['count']; + $summaries[] = [ + 'host' => $family['host'], + 'uri' => $family['uri'], + 'count' => $family['count'], + 'p50' => $this->percentile($times, 0.50), + 'p95' => $this->percentile($times, 0.95), + 'p99' => $this->percentile($times, 0.99), + 'max' => $times[array_key_last($times)], + 'avg' => round($avg, 4), + 'total' => round(array_sum($times), 4), + 'avg_upstream_response' => $family['upstream_n'] > 0 ? round($family['upstream_sum'] / $family['upstream_n'], 4) : null, + 'avg_upstream_header' => $family['header_n'] > 0 ? round($family['header_sum'] / $family['header_n'], 4) : null, + 'avg_bytes' => (int) round($family['bytes_sum'] / $family['count']), + '4xx' => $family['4xx'], + '4xx_rate' => round($family['4xx'] / $family['count'], 4), + '404' => $family['404'], + '5xx' => $family['5xx'], + '5xx_rate' => round($family['5xx'] / $family['count'], 4), + '499' => $family['499'], + ]; + } + + $byP95 = $summaries; + usort($byP95, static fn (array $a, array $b): int => ($b['p95'] ?? 0) <=> ($a['p95'] ?? 0)); + $byP99 = $summaries; + usort($byP99, static fn (array $a, array $b): int => ($b['p99'] ?? 0) <=> ($a['p99'] ?? 0)); + $byTotal = $summaries; + usort($byTotal, static fn (array $a, array $b): int => $b['total'] <=> $a['total']); + $byCount = $summaries; + usort($byCount, static fn (array $a, array $b): int => $b['count'] <=> $a['count']); + $byCost = $summaries; + usort($byCost, static fn (array $a, array $b): int => ($b['count'] * $b['avg']) <=> ($a['count'] * $a['avg'])); + $by5xx = array_values(array_filter($summaries, static fn (array $row): bool => $row['5xx'] > 0)); + usort($by5xx, static fn (array $a, array $b): int => $b['5xx'] <=> $a['5xx']); + $by404 = array_values(array_filter($summaries, static fn (array $row): bool => ($row['404'] ?? 0) > 0)); + usort($by404, static fn (array $a, array $b): int => $b['404'] <=> $a['404']); + + return [ + 'skipped' => $skipped, + 'totals' => $statusCounts, + 'by_p95' => array_slice($byP95, 0, $top), + 'by_p99' => array_slice($byP99, 0, $top), + 'by_total_time' => array_slice($byTotal, 0, $top), + 'by_count' => array_slice($byCount, 0, $top), + 'by_cost' => array_slice($byCost, 0, $top), + 'by_5xx' => array_slice($by5xx, 0, $top), + 'by_4xx' => array_slice($by404, 0, $top), + 'slowest' => $slowest, + 'percentile_method' => 'nearest-rank (ceil(p * n), 1-indexed)', + 'upstream_aggregation' => 'sum of numeric comma-separated upstream timings; "-" is null', + ]; + } + + /** + * Nearest-rank: rank = ceil(p * n), 1-indexed. + * + * @param list $sorted + */ + public function percentile(array $sorted, float $p): ?float + { + $n = count($sorted); + if ($n === 0) { + return null; + } + + $rank = (int) ceil($p * $n); + $index = max(0, min($n - 1, $rank - 1)); + + return round($sorted[$index], 4); + } + + public function parseUpstream(mixed $value): ?float + { + if ($value === null || $value === '' || $value === '-') { + return null; + } + + if (is_int($value) || is_float($value)) { + return (float) $value; + } + + $parts = preg_split('/\s*,\s*/', (string) $value) ?: []; + $sum = 0.0; + $found = false; + foreach ($parts as $part) { + if ($part === '' || $part === '-') { + continue; + } + if (! is_numeric($part)) { + continue; + } + $sum += (float) $part; + $found = true; + } + + return $found ? $sum : null; + } + + private function parseFloat(mixed $value): ?float + { + if ($value === null || $value === '' || $value === '-') { + return null; + } + if (! is_numeric($value)) { + return null; + } + + return (float) $value; + } + + private function parseTime(string $value): ?\DateTimeImmutable + { + if ($value === '') { + return null; + } + + try { + return new \DateTimeImmutable($value); + } catch (\Exception) { + return null; + } + } + + private function sinceCutoff(?string $since, \DateTimeImmutable $now): ?\DateTimeImmutable + { + if ($since === null || $since === '') { + return null; + } + + if (preg_match('/^(\d+)(h|d|m)$/', strtolower($since), $matches) !== 1) { + return null; + } + + $n = (int) $matches[1]; + $unit = $matches[2]; + $interval = match ($unit) { + 'm' => new \DateInterval('PT'.$n.'M'), + 'h' => new \DateInterval('PT'.$n.'H'), + 'd' => new \DateInterval('P'.$n.'D'), + }; + + return $now->sub($interval); + } +} diff --git a/app/Support/Http/HttpUriNormalizer.php b/app/Support/Http/HttpUriNormalizer.php new file mode 100644 index 00000000..3605066a --- /dev/null +++ b/app/Support/Http/HttpUriNormalizer.php @@ -0,0 +1,96 @@ +normalizeSegments($rest); + } + + return $this->normalizeSegments($path); + } + + private function normalizeSegments(string $path): string + { + if ($path === '' || $path === '/') { + return $path === '' ? '' : '/'; + } + + $segments = explode('/', $path); + $normalized = []; + + foreach ($segments as $index => $segment) { + if ($segment === '') { + $normalized[] = ''; + continue; + } + + $normalized[] = $this->normalizeSegment($segment, $index, $segments); + } + + $result = implode('/', $normalized); + + return $result === '' ? '/' : $result; + } + + /** + * @param array $segments + */ + private function normalizeSegment(string $segment, int $index, array $segments): string + { + if (ctype_digit($segment)) { + return '{id}'; + } + + if (preg_match('/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i', $segment) === 1) { + return '{uuid}'; + } + + if (preg_match('/^[0-9a-f]{32,64}$/i', $segment) === 1) { + return '{hash}'; + } + + if ($this->isArtworkPublicSlug($segment, $index, $segments)) { + return '{slug}'; + } + + return $segment; + } + + /** + * /art/{id}/{slug} stays distinct from /art/{id}/similar and other actions. + * + * @param array $segments + */ + private function isArtworkPublicSlug(string $segment, int $index, array $segments): bool + { + if ($index !== 3 || ($segments[1] ?? '') !== 'art') { + return false; + } + + if (! ctype_digit((string) ($segments[2] ?? ''))) { + return false; + } + + $actions = ['similar', 'view', 'download', 'edit', 'comments', 'favourite', 'favorite']; + + return ! in_array(strtolower($segment), $actions, true); + } +} diff --git a/app/Support/Http/TimedInertiaSsrGateway.php b/app/Support/Http/TimedInertiaSsrGateway.php new file mode 100644 index 00000000..ff526a9a --- /dev/null +++ b/app/Support/Http/TimedInertiaSsrGateway.php @@ -0,0 +1,34 @@ +inner->dispatch($page); + $durationMs = (hrtime(true) - $started) / 1_000_000; + + $request = request(); + if ($request instanceof Request) { + $request->attributes->set('http.ssr_ms', round($durationMs, 1)); + } + + return $response; + } +} diff --git a/app/Support/Queues/QueuedJobClassMatcher.php b/app/Support/Queues/QueuedJobClassMatcher.php new file mode 100644 index 00000000..b7acf295 --- /dev/null +++ b/app/Support/Queues/QueuedJobClassMatcher.php @@ -0,0 +1,33 @@ +, dispatch_config: string}> + */ + public const TARGETS = [ + 'forum-moderation' => [ + 'expected' => ['cPad\\Plugins\\Forum\\Jobs\\AnalyzeForumPostJob'], + 'dispatch_config' => 'skinbase_ai_moderation.queue.dispatch_enabled', + ], + 'forum-security' => [ + 'expected' => [ + 'cPad\\Plugins\\Forum\\Jobs\\FirewallActivityMonitor', + 'cPad\\Plugins\\Forum\\Jobs\\BotActivityMonitor', + ], + 'dispatch_config' => 'forum_security.queues.dispatch_enabled', + ], + 'collections' => [ + 'expected' => [ + RefreshCollectionHealthJob::class, + RefreshCollectionRecommendationJob::class, + ScanCollectionDuplicateCandidatesJob::class, + ], + 'dispatch_config' => 'collections.v5.queue.dispatch_enabled', + ], + ]; + + /** + * @return array + */ + public function inspect(string $queue, int $sampleSize = 400): array + { + $this->assertQueueName($queue); + $redis = Redis::connection(); + $waitingKey = 'queues:'.$queue; + $llen = (int) $redis->llen($waitingKey); + $delayed = (int) $redis->zcard('queues:'.$queue.':delayed'); + $reserved = (int) $redis->zcard('queues:'.$queue.':reserved'); + $notify = (int) $redis->llen('queues:'.$queue.':notify'); + $prefix = (string) config('database.redis.options.prefix', ''); + + $classes = []; + $bytes = []; + $sampled = 0; + if ($llen > 0) { + $step = max(1, (int) floor($llen / max(1, $sampleSize))); + for ($i = 0; $i < $llen && $sampled < $sampleSize; $i += $step) { + $raw = $redis->lindex($waitingKey, $i); + if (! is_string($raw) || $raw === '') { + continue; + } + $sampled++; + $class = QueuedJobClassMatcher::className($raw) ?? 'unknown'; + $classes[$class] = ($classes[$class] ?? 0) + 1; + $bytes[] = strlen($raw); + } + } + arsort($classes); + + $avg = $bytes !== [] ? (int) round(array_sum($bytes) / count($bytes)) : 0; + + return [ + 'queue' => $queue, + 'logical_key' => $waitingKey, + 'notify_key' => 'queues:'.$queue.':notify', + 'prefix' => $prefix, + 'llen' => $llen, + 'delayed' => $delayed, + 'reserved' => $reserved, + 'notify' => $notify, + 'sampled' => $sampled, + 'classes' => $classes, + 'avg_bytes' => $avg, + 'est_bytes' => $llen * $avg, + ]; + } + + /** + * @return array + */ + public function plan(string $queue, bool $force = false): array + { + if (in_array($queue, self::PROTECTED_QUEUES, true)) { + throw new RuntimeException("Refusing protected queue [{$queue}]."); + } + if (! isset(self::TARGETS[$queue])) { + throw new RuntimeException("Unknown orphan target [{$queue}]."); + } + + $meta = self::TARGETS[$queue]; + $dispatchEnabled = (bool) config($meta['dispatch_config'], false); + $inspect = $this->inspect($queue); + + $unexpected = array_values(array_diff(array_keys($inspect['classes']), $meta['expected'])); + $errors = []; + if ($dispatchEnabled) { + $errors[] = 'producer_dispatch_enabled'; + } + if ($inspect['reserved'] > 0) { + $errors[] = 'reserved_jobs_present'; + } + if ($inspect['delayed'] > 0) { + $errors[] = 'delayed_jobs_present'; + } + if ($unexpected !== [] && ! $force) { + $errors[] = 'unexpected_job_classes'; + } + + return $inspect + [ + 'expected_classes' => $meta['expected'], + 'unexpected_classes' => $unexpected, + 'dispatch_enabled' => $dispatchEnabled, + 'errors' => $errors, + 'can_execute' => $errors === [], + ]; + } + + /** + * @return array + */ + public function execute(string $queue, bool $force = false): array + { + $plan = $this->plan($queue, $force); + if (! $plan['can_execute']) { + throw new RuntimeException('Cleanup refused: '.implode(',', $plan['errors'])); + } + + $redis = Redis::connection(); + $keys = ['queues:'.$queue, 'queues:'.$queue.':notify']; + $unlinked = 0; + foreach ($keys as $key) { + $unlinked += (int) $redis->unlink($key); + } + + return $plan + ['unlinked_keys' => $keys, 'unlinked' => $unlinked]; + } + + private function assertQueueName(string $queue): void + { + if (! preg_match('/^[A-Za-z0-9_-]+$/', $queue)) { + throw new RuntimeException('Invalid queue name.'); + } + } +} diff --git a/bootstrap/app.php b/bootstrap/app.php index 0fc20f06..37eebb4d 100644 --- a/bootstrap/app.php +++ b/bootstrap/app.php @@ -33,6 +33,11 @@ return Application::configure(basePath: dirname(__DIR__)) 'stripe/*', ]); + $middleware->prepend([ + \App\Http\Middleware\LogSlowHttpRequest::class, + \App\Http\Middleware\AddServerTiming::class, + ]); + $middleware->web(append: [ \App\Http\Middleware\SecurityHeaders::class, \App\Http\Middleware\RedirectLegacyProfileSubdomain::class, diff --git a/bootstrap/ssr/ssr.js b/bootstrap/ssr/ssr.js index 81b80594..1b251a73 100644 --- a/bootstrap/ssr/ssr.js +++ b/bootstrap/ssr/ssr.js @@ -105949,8 +105949,21 @@ const kpiConfig = [ { key: "comments_30d", label: "Comments", icon: "fa-solid fa-comments" }, { key: "followers", label: "Followers", icon: "fa-solid fa-user-group" } ]; -function KpiCard({ config, value }) { - return /* @__PURE__ */ React.createElement("div", { className: "rounded-[26px] border border-white/10 bg-white/[0.03] p-5 shadow-[0_18px_40px_rgba(2,6,23,0.18)]" }, /* @__PURE__ */ React.createElement("div", { className: "flex items-center gap-3 text-slate-300" }, /* @__PURE__ */ React.createElement("div", { className: "flex h-11 w-11 items-center justify-center rounded-2xl bg-sky-300/10 text-sky-100" }, /* @__PURE__ */ React.createElement("i", { className: config.icon })), /* @__PURE__ */ React.createElement("span", { className: "text-[11px] font-semibold uppercase tracking-[0.18em] text-slate-500" }, config.label)), /* @__PURE__ */ React.createElement("p", { className: "mt-4 text-3xl font-semibold text-white tabular-nums" }, typeof value === "number" ? value.toLocaleString() : value)); +function snapshotHint(config, window2) { + if (!window2 || !["views_30d", "appreciation_30d", "shares_30d", "comments_30d"].includes(config.key)) { + return null; + } + const requested = Number(window2.requested_days || 30); + const available = Number(window2.table_coverage_days || 0); + const complete = Boolean(window2.window_complete); + if (complete) { + return `Last ${requested} days`; + } + const shown = Math.max(0, Math.round(available * 10) / 10); + return `Based on ${shown} of ${requested} days`; +} +function KpiCard({ config, value, hint }) { + return /* @__PURE__ */ React.createElement("div", { className: "rounded-[26px] border border-white/10 bg-white/[0.03] p-5 shadow-[0_18px_40px_rgba(2,6,23,0.18)]" }, /* @__PURE__ */ React.createElement("div", { className: "flex items-center gap-3 text-slate-300" }, /* @__PURE__ */ React.createElement("div", { className: "flex h-11 w-11 items-center justify-center rounded-2xl bg-sky-300/10 text-sky-100" }, /* @__PURE__ */ React.createElement("i", { className: config.icon })), /* @__PURE__ */ React.createElement("span", { className: "text-[11px] font-semibold uppercase tracking-[0.18em] text-slate-500" }, config.label)), /* @__PURE__ */ React.createElement("p", { className: "mt-4 text-3xl font-semibold text-white tabular-nums" }, typeof value === "number" ? value.toLocaleString() : value), hint ? /* @__PURE__ */ React.createElement("p", { className: "mt-2 text-[11px] leading-4 text-slate-500" }, hint) : null); } function QuickCreateCard({ item }) { return /* @__PURE__ */ React.createElement( @@ -106032,7 +106045,15 @@ function StudioDashboard() { const kpis = overview.kpis || {}; const widgetVisibility = overview.preferences?.widget_visibility || {}; const showWidget = (key) => widgetVisibility[key] !== false; - return /* @__PURE__ */ React.createElement(StudioLayout, { title: "Overview", subtitle: "Create, manage, and grow across artworks, cards, collections, and stories from one shared creator operating surface." }, /* @__PURE__ */ React.createElement("div", { className: "grid grid-cols-2 gap-4 lg:grid-cols-5" }, kpiConfig.map((config) => /* @__PURE__ */ React.createElement(KpiCard, { key: config.key, config, value: kpis?.[config.key] ?? 0 }))), /* @__PURE__ */ React.createElement("div", { className: "mt-6 grid gap-6 xl:grid-cols-[minmax(0,1fr)_360px]" }, /* @__PURE__ */ React.createElement("section", { className: "rounded-[30px] border border-white/10 bg-[radial-gradient(circle_at_top_left,_rgba(56,189,248,0.14),_transparent_35%),linear-gradient(135deg,_rgba(15,23,42,0.86),_rgba(2,6,23,0.96))] p-6" }, /* @__PURE__ */ React.createElement("div", { className: "flex items-center justify-between" }, /* @__PURE__ */ React.createElement("h2", { className: "text-lg font-semibold text-white" }, "Command center"), /* @__PURE__ */ React.createElement("a", { href: "/studio/calendar", className: "text-sm font-medium text-sky-100" }, "Open calendar")), /* @__PURE__ */ React.createElement("div", { className: "mt-5 grid gap-5 lg:grid-cols-3" }, /* @__PURE__ */ React.createElement( + return /* @__PURE__ */ React.createElement(StudioLayout, { title: "Overview", subtitle: "Create, manage, and grow across artworks, cards, collections, and stories from one shared creator operating surface." }, /* @__PURE__ */ React.createElement("div", { className: "grid grid-cols-2 gap-4 lg:grid-cols-5" }, kpiConfig.map((config) => /* @__PURE__ */ React.createElement( + KpiCard, + { + key: config.key, + config, + value: kpis?.[config.key] ?? 0, + hint: snapshotHint(config, kpis?.snapshot_window) + } + ))), /* @__PURE__ */ React.createElement("div", { className: "mt-6 grid gap-6 xl:grid-cols-[minmax(0,1fr)_360px]" }, /* @__PURE__ */ React.createElement("section", { className: "rounded-[30px] border border-white/10 bg-[radial-gradient(circle_at_top_left,_rgba(56,189,248,0.14),_transparent_35%),linear-gradient(135deg,_rgba(15,23,42,0.86),_rgba(2,6,23,0.96))] p-6" }, /* @__PURE__ */ React.createElement("div", { className: "flex items-center justify-between" }, /* @__PURE__ */ React.createElement("h2", { className: "text-lg font-semibold text-white" }, "Command center"), /* @__PURE__ */ React.createElement("a", { href: "/studio/calendar", className: "text-sm font-medium text-sky-100" }, "Open calendar")), /* @__PURE__ */ React.createElement("div", { className: "mt-5 grid gap-5 lg:grid-cols-3" }, /* @__PURE__ */ React.createElement( CommandCenterColumn, { title: "Publishing today", diff --git a/config/collections.php b/config/collections.php index befc21ad..d16c4336 100644 --- a/config/collections.php +++ b/config/collections.php @@ -83,6 +83,9 @@ return [ 'low_engagement_after_days' => (int) env('COLLECTIONS_V5_LOW_ENGAGEMENT_AFTER_DAYS', 14), ], 'queue' => [ + // No Horizon supervisor consumes this queue today. Keep dispatch + // off until a consumer exists (M7). Existing jobs are not deleted. + 'dispatch_enabled' => (bool) env('COLLECTIONS_V5_DISPATCH_ENABLED', false), 'name' => env('COLLECTIONS_V5_QUEUE', 'collections'), 'health_batch_size' => (int) env('COLLECTIONS_V5_HEALTH_BATCH_SIZE', 40), 'recommendation_batch_size' => (int) env('COLLECTIONS_V5_RECOMMENDATION_BATCH_SIZE', 40), diff --git a/config/forum_security.php b/config/forum_security.php index 03b88c96..90504ef8 100644 --- a/config/forum_security.php +++ b/config/forum_security.php @@ -15,6 +15,8 @@ return [ 'queues' => [ 'moderation' => env('FORUM_SECURITY_MODERATION_QUEUE', 'forum-moderation'), 'firewall' => env('FORUM_SECURITY_FIREWALL_QUEUE', 'forum-security'), + // Plugin jobs have no Horizon consumer. Pause enqueue until a worker exists. + 'dispatch_enabled' => (bool) env('FORUM_QUEUE_DISPATCH_ENABLED', false), ], 'firewall' => [ diff --git a/config/horizon.php b/config/horizon.php index e2ef0503..26921668 100644 --- a/config/horizon.php +++ b/config/horizon.php @@ -99,6 +99,7 @@ return [ 'waits' => [ 'redis:broadcasts' => 15, 'redis:default' => 60, + 'redis:mail' => 60, 'redis:notifications' => 90, ], @@ -209,9 +210,9 @@ return [ 'maxJobs' => 0, 'memory' => 128, 'tries' => 1, - // Long-running recommendation rebuild jobs declare timeouts up to 900s. - // Keep the worker timeout above that ceiling so Horizon does not kill - // healthy jobs before Laravel can enforce the job-level timeout. + // Must stay below config/queue.php redis.retry_after (default 1080). + // Nightly recommendation rebuilds are chunked; keep headroom for + // remaining long jobs on the default queue. 'timeout' => 960, 'nice' => 0, ], @@ -228,6 +229,21 @@ return [ 'timeout' => 90, 'nice' => 0, ], + // Mail is intentionally isolated: SMTP latency must not block + // search/recommendation workers, and mail jobs declare tries=3–5. + 'supervisor-mail' => [ + 'connection' => 'redis', + 'queue' => ['mail'], + 'balance' => 'simple', + 'autoScalingStrategy' => 'time', + 'maxProcesses' => 1, + 'maxTime' => 0, + 'maxJobs' => 0, + 'memory' => 128, + 'tries' => 5, + 'timeout' => 90, + 'nice' => 0, + ], ], 'environments' => [ @@ -242,6 +258,9 @@ return [ 'balanceMaxShift' => 1, 'balanceCooldown' => 3, ], + 'supervisor-mail' => [ + 'maxProcesses' => 2, + ], ], 'local' => [ @@ -251,6 +270,9 @@ return [ 'supervisor-messaging' => [ 'maxProcesses' => 2, ], + 'supervisor-mail' => [ + 'maxProcesses' => 1, + ], ], ], diff --git a/config/http_observability.php b/config/http_observability.php new file mode 100644 index 00000000..44186c68 --- /dev/null +++ b/config/http_observability.php @@ -0,0 +1,14 @@ + [ + 'enabled' => (bool) env('HTTP_SLOW_REQUEST_LOG_ENABLED', false), + 'threshold_ms' => (int) env('HTTP_SLOW_REQUEST_MS', 750), + /* + * Testing only. When set, replaces measured duration so tests do not sleep. + */ + 'test_duration_ms' => env('HTTP_SLOW_REQUEST_TEST_DURATION_MS'), + ], +]; diff --git a/config/logging.php b/config/logging.php index 9e998a49..afa07ddf 100644 --- a/config/logging.php +++ b/config/logging.php @@ -123,6 +123,14 @@ return [ 'handler' => NullHandler::class, ], + 'slow-http' => [ + 'driver' => 'daily', + 'path' => storage_path('logs/slow-http.log'), + 'level' => 'info', + 'days' => (int) env('HTTP_SLOW_REQUEST_LOG_DAYS', 14), + 'replace_placeholders' => false, + ], + 'emergency' => [ 'path' => storage_path('logs/laravel.log'), ], diff --git a/config/metrics.php b/config/metrics.php new file mode 100644 index 00000000..49aa2c95 --- /dev/null +++ b/config/metrics.php @@ -0,0 +1,20 @@ + (int) env('ARTWORK_METRIC_HOURLY_RETENTION_DAYS', 30), + + 'hourly_snapshot_prune_chunk' => (int) env('ARTWORK_METRIC_HOURLY_PRUNE_CHUNK', 5000), + + 'hourly_snapshot_prune_sleep_ms' => (int) env('ARTWORK_METRIC_HOURLY_PRUNE_SLEEP_MS', 50), + +]; diff --git a/config/queue.php b/config/queue.php index 79c2c0a2..7afe314d 100644 --- a/config/queue.php +++ b/config/queue.php @@ -68,7 +68,10 @@ return [ 'driver' => 'redis', 'connection' => env('REDIS_QUEUE_CONNECTION', 'default'), 'queue' => env('REDIS_QUEUE', 'default'), - 'retry_after' => (int) env('REDIS_QUEUE_RETRY_AFTER', 90), + // Must exceed the longest Horizon worker timeout (960s) plus a buffer. + // If retry_after is shorter than a running job, Redis re-queues it as a + // new attempt and Horizon (--tries=1) marks MaxAttemptsExceeded. + 'retry_after' => (int) env('REDIS_QUEUE_RETRY_AFTER', 1080), 'block_for' => null, 'after_commit' => false, ], diff --git a/config/ranking.php b/config/ranking.php index 0fd8eb37..beb16134 100644 --- a/config/ranking.php +++ b/config/ranking.php @@ -77,6 +77,11 @@ return [ 'candidate_pool' => 200, // top N candidates to run diversity filter on ], + // Unique lock for RankBuildScopeListsJob. Must exceed queue wait + job + // timeout (300s). Production wait has been ~2.5h; 6h prevents hourly + // stacking while still recovering a dead lock the same day. + 'scope_job_unique_for' => (int) env('RANK_SCOPE_JOB_UNIQUE_FOR', 21600), + // ── Anti-spam / burst-view damping ────────────────────────────────────── 'spam' => [ 'views_24h_threshold' => 2000, diff --git a/config/sitemaps.php b/config/sitemaps.php index 9704631e..bebcc0d2 100644 --- a/config/sitemaps.php +++ b/config/sitemaps.php @@ -54,6 +54,9 @@ return [ 'forum-threads' => [ 'size' => (int) env('SITEMAPS_SHARD_FORUM_THREADS_SIZE', 10000), ], + 'tags' => [ + 'size' => (int) env('SITEMAPS_SHARD_TAGS_SIZE', 25000), + ], 'collections' => [ 'size' => (int) env('SITEMAPS_SHARD_COLLECTIONS_SIZE', 10000), ], diff --git a/config/skinbase_ai_moderation.php b/config/skinbase_ai_moderation.php index a3b61a86..f6bbf56f 100644 --- a/config/skinbase_ai_moderation.php +++ b/config/skinbase_ai_moderation.php @@ -7,6 +7,7 @@ return [ 'queue' => [ 'name' => env('SKINBASE_AI_MODERATION_QUEUE', 'forum-moderation'), + 'dispatch_enabled' => (bool) env('FORUM_QUEUE_DISPATCH_ENABLED', false), ], 'preflight' => [ diff --git a/config/traffic.php b/config/traffic.php new file mode 100644 index 00000000..6740e5f1 --- /dev/null +++ b/config/traffic.php @@ -0,0 +1,15 @@ + [ + // Cap SSCAN/all() so moderation pages never SMEMBERS the 2.4M-member index. + 'index_read_limit' => (int) env('ONLINE_VISITOR_INDEX_READ_LIMIT', 2000), + 'prune_enabled' => (bool) env('ONLINE_VISITOR_INDEX_PRUNE_ENABLED', false), + 'prune_scan_count' => (int) env('ONLINE_VISITOR_INDEX_PRUNE_SCAN_COUNT', 500), + 'prune_max_batches' => (int) env('ONLINE_VISITOR_INDEX_PRUNE_MAX_BATCHES', 200), + 'prune_max_members' => (int) env('ONLINE_VISITOR_INDEX_PRUNE_MAX_MEMBERS', 100000), + 'prune_sleep_ms' => (int) env('ONLINE_VISITOR_INDEX_PRUNE_SLEEP_MS', 25), + ], +]; diff --git a/database/migrations/2026_08_23_120000_drop_duplicate_artwork_metric_snapshot_index.php b/database/migrations/2026_08_23_120000_drop_duplicate_artwork_metric_snapshot_index.php new file mode 100644 index 00000000..e65e0e95 --- /dev/null +++ b/database/migrations/2026_08_23_120000_drop_duplicate_artwork_metric_snapshot_index.php @@ -0,0 +1,71 @@ +indexExists('artwork_metric_snapshots_hourly', 'idx_artwork_bucket')) { + return; + } + + Schema::table('artwork_metric_snapshots_hourly', function (Blueprint $table) { + $table->dropIndex('idx_artwork_bucket'); + }); + } + + public function down(): void + { + if (! Schema::hasTable('artwork_metric_snapshots_hourly')) { + return; + } + + if ($this->indexExists('artwork_metric_snapshots_hourly', 'idx_artwork_bucket')) { + return; + } + + Schema::table('artwork_metric_snapshots_hourly', function (Blueprint $table) { + $table->index(['artwork_id', 'bucket_hour'], 'idx_artwork_bucket'); + }); + } + + private function indexExists(string $table, string $indexName): bool + { + if (DB::getDriverName() === 'sqlite') { + $indexes = DB::select("PRAGMA index_list({$table})"); + + foreach ($indexes as $index) { + if (($index->name ?? '') === $indexName) { + return true; + } + } + + return false; + } + + $database = Schema::getConnection()->getDatabaseName(); + $row = DB::selectOne( + 'SELECT COUNT(*) AS c FROM information_schema.statistics + WHERE table_schema = ? AND table_name = ? AND index_name = ?', + [$database, $table, $indexName] + ); + + return (int) ($row->c ?? 0) > 0; + } +}; diff --git a/deploy/nginx/conf.d/skinbase-perf-log-format.conf b/deploy/nginx/conf.d/skinbase-perf-log-format.conf new file mode 100644 index 00000000..923a9763 --- /dev/null +++ b/deploy/nginx/conf.d/skinbase-perf-log-format.conf @@ -0,0 +1,23 @@ +# M12 — structured HTTP performance log format. +# Install into /etc/nginx/conf.d/ (already included from nginx.conf http {}). +# Do not replace the existing combined access log. +# Activate with a second access_log on the Skinbase server block. +# +# Variables validated against nginx/1.26.3 on server3. +# $uri is used (not $request_uri) so query strings are not logged. +# No $remote_addr, cookies, or Authorization. + +log_format skinbase_perf escape=json + '{"time":"$time_iso8601",' + '"host":"$host",' + '"method":"$request_method",' + '"uri":"$uri",' + '"status":$status,' + '"request_time":$request_time,' + '"upstream_response_time":"$upstream_response_time",' + '"upstream_connect_time":"$upstream_connect_time",' + '"upstream_header_time":"$upstream_header_time",' + '"bytes":$body_bytes_sent,' + '"request_length":$request_length,' + '"content_type":"$sent_http_content_type",' + '"cache":"$upstream_cache_status"}'; diff --git a/deploy/nginx/sitemaps.conf b/deploy/nginx/sitemaps.conf index 7d294160..054ce18c 100644 --- a/deploy/nginx/sitemaps.conf +++ b/deploy/nginx/sitemaps.conf @@ -1,38 +1,47 @@ # --------------------------------------------------------------------------- # Nginx static sitemap file serving # --------------------------------------------------------------------------- -# Include this snippet inside your `server {}` block BEFORE the main -# `location /` (or `location ~ \.php$`) block so nginx serves the pre-built -# static XML files without ever touching PHP/FPM. +# Compatible with the production skinbase.org vhost, which has NO named +# location @php. Laravel is already wired as: # -# The GenerateSitemapsCommand writes these files on a schedule (every 6 h): -# public/sitemap.xml <- root sitemap index -# public/sitemaps/{name}.xml <- per-family / per-shard documents +# location / { +# try_files $uri $uri/ /index.php?$query_string; +# } +# location = /index.php { +# fastcgi_pass unix:/run/php/php8.4-fpm-skinbase.sock; +# ... +# } # -# When a file has not been generated yet (first deploy, cold start) the -# request falls through to @php and Laravel's SitemapController builds it -# live on the first hit. +# Do not add `try_files ... @php` — that named location does not exist on +# server3 and nginx -t / runtime would fail (or 500) when the static file +# is missing. +# +# GenerateSitemapsCommand writes: +# public/sitemaps/index.xml <- canonical generated root index +# public/sitemaps/sitemap.xml <- legacy filename (best-effort) +# public/sitemap.xml <- legacy $uri (often unwritable on prod) +# public/sitemaps/{name}.xml <- child / shard documents +# +# Place these location blocks BEFORE `location /` is not required if they are +# more specific (`=` and regex), but they must remain inside the 443 server. # --------------------------------------------------------------------------- # Root sitemap index location = /sitemap.xml { - # Serve the static file if it exists; otherwise fall through to PHP. - try_files $uri @php; + # Prefer the scheduler-writable generated index. Fall back to the legacy + # sitemaps/sitemap.xml, then Laravel front controller (not @php). + try_files /sitemaps/index.xml /sitemaps/sitemap.xml /index.php?$query_string; - # Instruct downstream caches / crawlers how long the file is fresh. - add_header Cache-Control "public, max-age=21600" always; # 6 h + add_header Cache-Control "public, max-age=21600" always; add_header Content-Type "application/xml; charset=UTF-8" always; - - # Optional: let nginx set a strong ETag automatically (default behaviour). etag on; } # Per-family / per-shard sitemap documents location ~ ^/sitemaps/[A-Za-z0-9_\-]++\.xml$ { - try_files $uri @php; + try_files $uri /index.php?$query_string; - add_header Cache-Control "public, max-age=21600" always; # 6 h + add_header Cache-Control "public, max-age=21600" always; add_header Content-Type "application/xml; charset=UTF-8" always; - etag on; } diff --git a/deploy/nginx/skinbase-vhost-performance-log.snippet b/deploy/nginx/skinbase-vhost-performance-log.snippet new file mode 100644 index 00000000..d6314ac0 --- /dev/null +++ b/deploy/nginx/skinbase-vhost-performance-log.snippet @@ -0,0 +1,16 @@ +# M12 — add INSIDE the HTTPS server { } for skinbase.org, next to the +# existing combined access_log. Keep the original line unchanged. +# +# access_log /var/log/nginx/skinbase.org-access.log; +# access_log /var/log/nginx/skinbase-performance.log skinbase_perf; +# +# Locations that already set `access_log off;` (static assets, /photo/*.jpg) +# remain excluded from both logs. +# File is created as 0640 www-data adm by nginx; logrotate /var/log/nginx/*.log +# already rotates it (daily, 14 copies). +# +# After install: +# sudo nginx -t && sudo systemctl reload nginx +# Never `restart` nginx for this change. + +access_log /var/log/nginx/skinbase-performance.log skinbase_perf; diff --git a/docs/QUEUE.md b/docs/QUEUE.md index 9dd0fbcb..7f044af9 100644 --- a/docs/QUEUE.md +++ b/docs/QUEUE.md @@ -39,6 +39,8 @@ The example worker listens on the application queues currently used by Skinbase search,forum-security,forum-moderation,vision,recommendations,discovery,mail,default ``` +Production uses **Horizon**, not this standalone `queue:work` example. Horizon supervisors must list every queue that application code pushes to. The `mail` queue is consumed by `supervisor-mail` in `config/horizon.php` (isolated from search/recommendation workers). + If you split workloads across dedicated workers, make sure the Scout/Meilisearch `search` queue and any queues configured through `VISION_QUEUE`, `RECOMMENDATIONS_QUEUE`, or `DISCOVERY_QUEUE` are explicitly covered by at least one worker process. To use it on a Debian/Ubuntu server: diff --git a/docs/cli-reference.md b/docs/cli-reference.md index d9e5bf9f..ac6c39fc 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -178,7 +178,7 @@ Examples below are representative. For the full option list of any Artisan comma | Command | Why it is used | Example | | --- | --- | --- | | `nova:metrics-snapshot-hourly` | Collect hourly metric snapshots used by rising/heat calculations | `php artisan nova:metrics-snapshot-hourly --days=1` | -| `nova:prune-metric-snapshots` | Delete old hourly metric snapshots past the retention window | `php artisan nova:prune-metric-snapshots --keep-days=90` | +| `nova:prune-metric-snapshots` | Delete old hourly metric snapshots past the retention window (batched) | `php artisan nova:prune-metric-snapshots --keep-days=30 --chunk=5000` | | `nova:recalculate-heat` | Recalculate heat and momentum scores for the rising engine | `php artisan nova:recalculate-heat --days=30 --chunk=500` | | `nova:recalculate-rankings` | Recalculate V2 ranking scores | `php artisan nova:recalculate-rankings --chunk=500 --sync-rank-scores` | diff --git a/docs/optimization-m1-recommendation-jobs.md b/docs/optimization-m1-recommendation-jobs.md new file mode 100644 index 00000000..08171c97 --- /dev/null +++ b/docs/optimization-m1-recommendation-jobs.md @@ -0,0 +1,290 @@ +# M1 — Nightly recommendation job failures + +## 1. Executive Summary + +Production Horizon marks `RecComputeSimilarByBehaviorJob` and `RecComputeSimilarHybridJob` failed every night with `MaxAttemptsExceededException` at **~02:18** and **~02:31**. + +This is **not** the old 90-second Supervisor `queue:work` timeout. Production uses Horizon workers with **timeout=960**. The Redis queue connection has **`retry_after=90`**. A still-running nightly rebuild is treated as abandoned, re-queued, and Horizon `--tries=1` fails it. + +```text +ROOT-H — Redis retry_after (90s) shorter than job runtime, interacting with Horizon tries=1 + and a single-job full-catalog scan (~50k artworks). +``` + +Local fix (not deployed): + +1. Default `queue.connections.redis.retry_after` **1080** (> Horizon 960). +2. Fan-out Behavior and Hybrid rebuilds into cursor batches of 200, matching `RecComputeSimilarByTagsJob` (already chunked in production). + +Mail queue `LLEN=3` is **deferred** (not entangled). + +--- + +## 2. Production Symptom + +```text +failed_jobs ≈ 329 +RecComputeSimilarByBehaviorJob ~02:18 daily +RecComputeSimilarHybridJob ~02:31 daily +exception: Illuminate\Queue\MaxAttemptsExceededException + "... has been attempted too many times." +``` + +Schedule (Europe/Ljubljana): + +| Job | dailyAt | +| --- | ------- | +| RecComputeSimilarByTagsJob | 02:00 | +| RecComputeSimilarByBehaviorJob | 02:15 | +| RecComputeSimilarHybridJob | 02:30 | + +Tags does **not** appear in the nightly failure list (already cursor-batched). + +--- + +## 3. Production Evidence + +PRODUCTION SERVER (`ssh server3`, read-only): + +- `config('queue.connections.redis.retry_after')` = **90** +- `config('horizon.defaults.supervisor-default.timeout')` = **960** +- Horizon worker: `--queue=search --timeout=960 --tries=1 --memory=128` +- `recommendations.queue` = `default` +- CLI `memory_limit` = `-1` (Horizon 128 MB is post-job worker recycle, not the kill mechanism at 90s) +- Failed exception text is MaxAttemptsExceeded **without** a wrapped `TimeoutExceededException` / memory error +- Timing: + +```text +Behavior 02:15 dispatch → fail ~02:18 (~180s or ~90s after the minute the scheduler actually queued it) +Hybrid 02:30 dispatch → fail ~02:31:33 (≈ retry_after 90s) +``` + +Laravel Redis queues use `retry_after` as the reservation TTL. If the job is still reserved after 90s, Redis exposes it again. The next pop has `attempts=2`. Horizon `--tries=1` then raises `MaxAttemptsExceededException` **before** the job body runs. + +That matches the Hybrid `tries=1` failure ~90 seconds after start. Behavior’s class `$tries=2` is overridden by the worker `--tries=1`. + +No production files, jobs, or services were changed during M1. + +--- + +## 4. Local vs Production Code + +| Item | Production `5af95f65` | Local `f52879ed` (before M1) | +| ---- | --------------------- | ---------------------------- | +| RecComputeSimilarByBehaviorJob | `$tries=2`, `$timeout=600`, **full `chunkById` catalog** | same | +| RecComputeSimilarHybridJob | `$tries=1`, `$timeout=900`, **full catalog**, WithoutOverlapping only per-artwork | same | +| RecComputeSimilarByTagsJob | cursor `afterArtworkId` batches of 200 | same | +| redis retry_after | 90 | 90 | + +Local did **not** already contain this fix. M1 implements it. + +--- + +## 5. Job Architecture (after M1) + +Nightly catalog rebuild (`artworkId=null`): + +```text +load ≤ batchSize artworks after cursor +→ process each +→ if batch full, dispatch next job with afterArtworkId = last id +``` + +Per-artwork rebuilds (observers) unchanged: Hybrid still uses `WithoutOverlapping` + `dontRelease()`. + +| Job | tries | timeout | batch | +| --- | ----: | ------: | ----: | +| Behavior | 2 | 120s | 200 | +| Hybrid | 1 | 180s | 200 | +| Tags | 2 | 600s | 200 (already) | + +--- + +## 6. Scheduler Flow + +Unchanged. `Schedule::job(...)->dailyAt(...)->withoutOverlapping()`. + +Behavior and Hybrid still start at 02:15 / 02:30; work continues as chained queue jobs instead of one 15+ minute reservation. + +Root crontab that invokes `schedule:run` remains unseen (sudo). Empirical evidence (nightly fails + 10:30 sitemap files) shows the scheduler **is** running. + +--- + +## 7. Horizon Configuration + +Production supervisors unchanged in M1 (no deploy). + +Local comment on `supervisor-default.timeout` now documents that **`retry_after` must exceed 960**. + +Horizon `tries=1` remains. Batch jobs should finish well under 90s; `retry_after=1080` is the safety net for any remaining long job. + +--- + +## 8. Failed Job Analysis + +Grouped: Behavior then Hybrid, every night, same exception. No payload dump. No evidence of immediate SQL exception (would usually wrap a QueryException). No SIGKILL/memory lines inspected in failed_jobs. Horizon log at inspection time showed healthy `IndexUserJob` / Scout jobs, not the 02:xx window. + +--- + +## 9. Lock / Middleware Analysis + +Full-catalog Behavior: **no** WithoutOverlapping. + +Full-catalog Hybrid: `middleware()` returns `[]` when `artworkId === null`. **Not** the failure path. + +Scheduler `withoutOverlapping()` would **skip** dispatch, not fail a job. + +**Disproved:** `$tries=1` + WithoutOverlapping release as the nightly cause. + +**Proved:** Redis reservation TTL 90s vs long-running catalog job. + +--- + +## 10. Runtime / Memory Analysis + +PHP CLI memory is unlimited. Horizon `--memory=128` typically exits **after** a job. A 50k-artwork in-memory scan could still grow, but the 90-second fail clock is Redis, not 128 MB. + +After chunking, each batch holds 200 models + pair rows, similar to the tags job that already succeeds. + +--- + +## 11. Query Analysis + +Behavior, per artwork: + +- `UNION` on `rec_item_pairs` (`a_artwork_id` / `b_artwork_id` indexed) +- `artworks` `whereIn` related ids +- `updateOrCreate` on `rec_artwork_recs` + +~50k × 3 queries in **one** reservation was unbounded vs 90s. + +No new index in M1. Chunking reduces reservation time; query shape unchanged. + +--- + +## 12. Root Cause Classification + +```text +ROOT-H — Redis retry_after=90s < running catalog job + + Horizon --tries=1 + + Behavior/Hybrid process the full catalog in one job +``` + +Not ROOT-C (memory) as primary. Not ROOT-B (900s timeout) as the **observed** fail time (~90s). After fixing retry_after alone, ROOT-B could appear next (~600–900s). Chunking prevents that. + +--- + +## 13. Fix + +1. `config/queue.php` redis `retry_after` default **1080** (`REDIS_QUEUE_RETRY_AFTER`). +2. `.env.example` documents the production requirement. +3. Behavior + Hybrid catalog runs use the same cursor fan-out as Tags. +4. Structured batch logs: `duration_ms`, `memory_mb`, `processed`, `has_more` (no PII). +5. Per-artwork try/catch on Behavior (Hybrid already had it). + +Correctness: same scoring/persist per artwork; batches are sequential by `id`. Idempotent `updateOrCreate`. + +--- + +## 14. Files Changed + +```text +config/queue.php +config/horizon.php +.env.example +app/Jobs/RecComputeSimilarByBehaviorJob.php +app/Jobs/RecComputeSimilarHybridJob.php +tests/Unit/Jobs/RecComputeSimilarByBehaviorJobTest.php +tests/Unit/Jobs/RecComputeSimilarHybridJobTest.php +tests/Feature/Recommendations/RecComputeSimilarJobsTest.php +docs/optimization-m1-recommendation-jobs.md +``` + +No migrations. + +--- + +## 15. Tests + +```text +php artisan test --filter=RecComputeSimilar +Tests: 13 passed (16 assertions) +Duration: 15.67s +``` + +Covers retry_after vs Horizon timeout, cursor dispatch, partial batch no-dispatch, hybrid idempotency. + +Full suite not re-run (known unrelated failures from earlier audits). + +--- + +## 16. Performance Impact + +Expected: many short default-queue jobs (~50k/200 ≈ 250 Behavior + 250 Hybrid) instead of two 15-minute jobs. Worker slots free between batches. Redis no longer duplicates in-flight catalog jobs. + +--- + +## 17. Correctness Impact + +Same rec lists; order of artwork processing remains increasing `id`. A crash mid-chain can resume by re-dispatching from the last completed cursor (or wait for the next night). No silent catch-all. + +--- + +## 18. Deployment Requirements + +```text +1. Deploy this release (includes config/queue.php). +2. Set production .env REDIS_QUEUE_RETRY_AFTER=1080 if you pin the value + (default in config is already 1080 if unset). +3. php artisan config:cache (normal deploy optimize). +4. Restart Horizon via the usual Supervisor restart / queue:restart. +``` + +Without restarting Horizon, workers keep old cached config. + +--- + +## 19. Post-Deploy Verification + +```text +1. php artisan horizon:status → running +2. php artisan tinker: config('queue.connections.redis.retry_after') === 1080 +3. Next night: no new MaxAttemptsExceeded for RecComputeSimilarByBehavior/Hybrid +4. rec_artwork_recs computed_at advances for similar_behavior and similar_hybrid +5. Horizon log: "[RecComputeSimilarByBehavior] Batch complete" with has_more true then false +6. failed_jobs count not increasing for these classes +7. Optional: sample duration_ms / memory_mb +``` + +Do not retry the 329 historical failures unless product wants a one-off backfill; they are the old reservation pattern. + +--- + +## 20. Rollback Plan + +```text +- Roll back the release symlink / previous release +- config:cache + Horizon restart +- No DB migration to roll back +- Cursor jobs in flight may still complete; safe +- No Redis lock cleanup required for catalog jobs +``` + +--- + +## 21. Deferred Findings + +```text +DEFER TO M2/M1B: queues:mail LLEN=3, Horizon does not consume `mail` +DEFER: sitemap index stale +DEFER: artwork_metric_snapshots_hourly size +DEFER: swap / FPM / Sentry sample rates +``` + +--- + +## 22. Remaining Unknowns + +- Exact `schedule:run` crontab user (sudo required) +- Whether any Behavior run ever reached the 600s job timeout (masked by 90s retry_after) +- Historical failed_jobs payloads not inspected (by design) diff --git a/docs/optimization-m10-database-query-optimization.md b/docs/optimization-m10-database-query-optimization.md new file mode 100644 index 00000000..a13637a5 --- /dev/null +++ b/docs/optimization-m10-database-query-optimization.md @@ -0,0 +1,290 @@ +# M10 — Database Query & Slow Query Optimization + +```text +STATUS: COMPLETE (read-only production audit + two local application fixes) +PRODUCTION: no ALTER / DROP / OPTIMIZE / MySQL config / restart +WHEN: 2026-08-23 +``` + +Production investigation used `ssh server3` as Skinbase (`skinbase@localhost`). Slow log file and `performance_schema.events_statements_summary_by_digest` were **not readable** with this grant. Cost ranking is from `SHOW GLOBAL STATUS`, `SHOW TABLE STATUS`, `SHOW INDEX`, `EXPLAIN`, and Laravel source mapping. + +--- + +## 1. Production DB health snapshot + +| Metric | Value | +| ------ | ----- | +| Version | Percona Server 8.4.11-11 | +| Uptime | ~122233 s (~34 h) | +| Threads_connected / Threads_running | 12 / 2 | +| max_connections / Max_used_connections | 120 / 26 | +| Slow_queries (cumulative) | 14301 (`long_query_time=0.5`, `min_examined_row_limit=100`) | +| Created_tmp_disk_tables | 21 (low vs memory temps) | +| Select_full_join | 30772 | +| Select_scan | ~1.15M | +| Sort_merge_passes | 107022 | +| InnoDB buffer pool | 10 GB; hit ratio ~99.9996%; pages_data ~24% | +| Innodb_row_lock_waits | 40794; avg ~4 ms; max ~449 ms | + +Buffer pool and connection headroom are healthy. Cumulative `Slow_queries` is **not** treated as proof of a current problem (uptime + historical counter). + +--- + +## 2. Slow-log configuration + +| Variable | Production | +| -------- | ---------- | +| slow_query_log | ON | +| slow_query_log_file | `/var/log/mysql/slow.log` | +| long_query_time | 0.5 | +| min_examined_row_limit | 100 | +| log_queries_not_using_indexes | (not changed; not used as a tuning lever) | + +**Do not change** these during M10. The log file is **not readable** as the app user. `pt-query-digest` was not used. + +--- + +## 3–4. Slow-log / Performance Schema top fingerprints + +**Unavailable** (`SELECT` denied on `events_statements_summary_by_digest`; slow.log unreadable). + +Substitutes: `EXPLAIN` on known high-volume SQL + code-path frequency. + +--- + +## 5–7. Top queries by total latency / average / rows examined + +Ranked by **evidence from EXPLAIN + schedule frequency**, not P_S. + +| Fingerprint | Feature | Path | rows examined (EXPLAIN) | Index | Severity | +| ----------- | ------- | ---- | ----------------------- | ----- | -------- | +| DISTINCT `artwork_id` WHERE `bucket_hour` BETWEEN ? AND ? | Rising heat | scheduled `nova:recalculate-heat` | ~2.46M covering `idx_bucket_artwork` | Using index + temp | HIGH (was hydrating all rows in PHP) | +| SELECT snapshots WHERE `bucket_hour` BETWEEN AND `artwork_id` IN (all IDs) | Rising heat | same command (pre-fix) | ~2.46M | `idx_bucket_artwork` | HIGH — **fixed locally** (chunked) | +| GROUP BY `artwork_id` 30d window | monthly leaderboards | `LeaderboardService` / `ArtworkHourlySnapshotWindow` | ~9.12M type=index `uq_artwork_bucket` | unique | MEDIUM (grows with 30d table) | +| `artwork_tag` GROUP BY `tag_id` COUNT | rec tags IDF | `RecComputeSimilarByTagsJob::handle` every batch | ~112k covering | `artwork_tag_tag_id_index` | MEDIUM — **fixed locally** (cache) | +| public list `is_public`+`is_approved` ORDER BY | HTTP browse | artwork listing | uses `idx_artworks_browse` backward | OK | NO CHANGE | +| eligible snapshot IDs | hourly snapshot | `MetricsSnapshotHourlyCommand` | `artworks_is_approved_index` + stats PK | OK | NO CHANGE | +| forum `reviewed=false` LIMIT 250 | `forum:scan-posts` | PRIMARY range | 13k-scale table | OK | NO CHANGE | +| collections lifecycle WHERE dates | `collections:sync-lifecycle` every 10m | small tables | OK | LOW / NO CHANGE | + +--- + +## 8. Mapping queries → Laravel + +| Query | Source | +| ----- | ------ | +| Heat DISTINCT + snapshot load + `artwork_stats` upsert | `app/Console/Commands/RecalculateHeatCommand.php::handle` | +| Hourly eligible IDs + upsert | `app/Console/Commands/MetricsSnapshotHourlyCommand` | +| 30d window MAX/MIN deltas | `app/Services/Metrics/ArtworkHourlySnapshotWindow.php` used by `StudioMetricsService`, `CreatorStudioOverviewService`, `LeaderboardService`, `CreatorJourneyService` | +| Rank lists LIMIT 200 | `app/Jobs/RankBuildScopeListsJob::fetchCandidates` (`rank_artwork_scores`) | +| Tag IDF GROUP BY | `app/Jobs/RecComputeSimilarByTagsJob::tagFrequencies` | +| Search document | `app/Jobs/IndexArtworkJob::handle` (`with` user, group, tags, categories.contentType, stats, awardStat) | +| Forum scan | `packages/klevze/Plugins/Forum` + schedule `forum:scan-posts --limit=250` | +| Collections sync | `app/Console/Commands/SyncCollectionLifecycleCommand.php` | + +--- + +## 9. N+1 findings + +| Area | Finding | Action | +| ---- | ------- | ------ | +| IndexArtworkJob | Already eager-loads relations used by `toSearchableArray()` | NO CHANGE | +| IndexUserJob | Review: not a heat source; search queue empty | NO CHANGE | +| Heat command | Was one giant hydrate, not N+1 | chunked load | +| Studio / Journey | Window helper issues bounded per-user SQL | NO CHANGE | +| Public artwork list | `idx_artworks_browse`; no new eager-load without evidence | NO CHANGE | +| Academy / forum cards | No production digest proving N+1 | NO CHANGE (do not globally eager-load) | + +--- + +## 10. Artwork list / detail + +Public browse uses `idx_artworks_browse` (EXPLAIN backward index scan). Category/tag pivots have supporting indexes from batch1. **NO CHANGE** to list indexes. + +--- + +## 11. Rankings / leaderboards + +`RankBuildScopeListsJob` reads `rank_artwork_scores` with LIMIT 200 after M7.1 uniqueness. DB side is cheap relative to heat/snapshots. `leaderboards:refresh` 30d GROUP BY remains the expensive reader as the snapshot table grows — keep current SQL; do **not** FORCE INDEX yet. Do **not** redesign queue architecture. + +--- + +## 12. Metric-snapshot findings + +Table ~8.87M rows, ~1855 MB (data ~740 / index ~1115). Unique `(artwork_id, bucket_hour)`. + +Indexes: + +| Index | Columns | Verdict | +| ----- | ------- | ------- | +| PRIMARY | `id` | keep | +| `uq_artwork_bucket` UNIQUE | `artwork_id`, `bucket_hour` | keep — upsert + per-artwork window | +| `idx_artwork_bucket` | `artwork_id`, `bucket_hour` | **exact duplicate of unique** — DROP candidate | +| `idx_bucket_hour` | `bucket_hour` | left prefix of `idx_bucket_artwork` — **do not drop in this milestone** | +| `idx_bucket_artwork` | `bucket_hour`, `artwork_id` | keep — heat DISTINCT covering | + +Do **not** reduce 30-day retention. Do **not** `OPTIMIZE TABLE`. Do **not** partition. + +--- + +## 13. Forum / collections + +- `forum:scan-posts --limit=250` stays **synchronous**. No new index (PRIMARY + limit). Do not re-enable forum queues. +- Collections maintenance **queue remains disabled**. `collections:sync-lifecycle` is cheap date scans. Every-10-minute sync is fine from DB. + +--- + +## 14. Academy + +No P_S digest pointing at Academy. No speculative indexes or global eager-load. + +--- + +## 15. Search / recommendations + +- `IndexArtworkJob` eager-load is sufficient (no internal N+1 of the listed relations). +- Tag IDF GROUP BY ran **once per batch job**; cached 1h locally. +- Do **not** create a rec snapshot table (M4). + +--- + +## 16. Table / index inventory (production estimates) + +| Table | Rows (approx) | Notes | +| ----- | ------------- | ----- | +| `artwork_metric_snapshots_hourly` | 8.87M | index > data; duplicate secondary | +| `artworks` | (browse index used) | `idx_artworks_browse` | +| `artwork_stats` | 1:1 artwork | PK `artwork_id`; heat upsert | +| `artwork_tag` | covering tag_id | IDF GROUP BY | +| forum posts | ~13k | scan LIMIT 250 | +| rec tables | unique artwork+type+version | M7.1 intact | + +--- + +## 17. Proposed indexes + +**None** for add. Equality+range combinations already have `uq_artwork_bucket` and `idx_bucket_artwork`. + +--- + +## 18. Proposed redundant index removals + +**DROP `idx_artwork_bucket`** only. + +Migration: `database/migrations/2026_08_23_120000_drop_duplicate_artwork_metric_snapshot_index.php` + +**Do not run on production in the application deploy.** Separate schema window. + +Percona 8.4: secondary index drop is typically `ALGORITHM=INPLACE`, `LOCK=NONE`. Disk: reclaim is InnoDB-internal, not filesystem (same as M5 prune). Rollback: re-add the duplicate index (unnecessary). + +`idx_bucket_hour`: **NO CHANGE** until a dedicated EXPLAIN proves no leftover readers after drop of the duplicate. + +--- + +## 19. Write-amplification + +~50k upserts/hour. Each extra secondary index updates on every write. + +- Duplicate `idx_artwork_bucket`: **100% write cost, 0% extra read benefit**. +- Remaining secondaries: `idx_bucket_artwork` is justified by heat DISTINCT covering. + +--- + +## 20. Pagination / COUNT + +No production evidence of OFFSET thousands on artworks browse. Do **not** rewrite pagination. Exact Studio counts remain M5.1 semantics. + +--- + +## 21. Lock / transaction findings + +Row lock waits exist but average 4 ms. Heat upsert is keyed by `artwork_id` (not a single hot row). **No isolation-level change.** Heat no longer loads 2.4M rows into PHP, which reduces transaction/memory pressure around upserts. + +--- + +## 22. MySQL config verdict + +**NO CHANGE** to `innodb_buffer_pool_size`, `max_connections`, redo, tmp_table_size, table caches, optimizer. Pool not under pressure. + +--- + +## 23. Ranked findings + +| Severity | Item | +| -------- | ---- | +| HIGH | Heat command hydrating full 24h snapshot set — **fixed** (chunked SELECT + narrower columns) | +| MEDIUM | Tag IDF GROUP BY per rec batch — **fixed** (Cache::remember 1h) | +| MEDIUM | Duplicate `idx_artwork_bucket` write amp — **migration created, not executed on prod** | +| MEDIUM | 30d leaderboard GROUP BY ~9M rows growing to ~36M — monitor; no FORCE INDEX yet | +| LOW | Sort_merge_passes / Select_scan cumulative — relate to heat DISTINCT temp; re-check after heat deploy | +| NO CHANGE | FPM max_children=14, Horizon counts, Redis, 30d retention, partition, OPTIMIZE, MySQL knobs, forum queues, rec snapshot table | + +--- + +## 24. Files changed + +- `app/Console/Commands/RecalculateHeatCommand.php` +- `app/Jobs/RecComputeSimilarByTagsJob.php` +- `tests/Feature/Metrics/RecalculateHeatCommandTest.php` +- `tests/Feature/Recommendations/RecComputeSimilarJobsTest.php` +- `database/migrations/2026_08_23_120000_drop_duplicate_artwork_metric_snapshot_index.php` +- `docs/optimization-m10-database-query-optimization.md` + +--- + +## 25. Migrations created + +`2026_08_23_120000_drop_duplicate_artwork_metric_snapshot_index.php` — **local only until a separate production schema deploy**. + +--- + +## 26. Tests / results + +```text +pest tests/Feature/Metrics/RecalculateHeatCommandTest.php PASS +phpunit --filter tag_idf RecComputeSimilarJobsTest OK +``` + +--- + +## 27. Expected benefit + +- Heat: peak PHP memory and rows hydrated drop from ~all 24h snapshots (~1–2M rows) to `--chunk` × hours (default 1000 × 25). +- Tag jobs: one `artwork_tag` GROUP BY per hour per model version instead of once per batch/chain hop. +- Duplicate index drop (later): less write/storage on 50k rows/hour; index size currently already > data. + +--- + +## 28. Application deployment sequence + +1. Deploy code (heat chunking + IDF cache). **Do not** run the DROP INDEX migration. +2. After one `nova:recalculate-heat` cycle: confirm command duration and RSS vs previous (scheduler runtime). +3. After rec tags chain: confirm `artwork_tag` GROUP BY frequency (slow log / P_S if grants added later). +4. FPM/Horizon/Redis unchanged. + +--- + +## 29. Schema deployment sequence (separate) + +1. Confirm `SHOW INDEX FROM artwork_metric_snapshots_hourly` still shows `uq_artwork_bucket` and `idx_artwork_bucket` identical columns. +2. `ALTER TABLE ... DROP INDEX idx_artwork_bucket` via the migration in a dedicated window (INPLACE, no app coupling). +3. Re-EXPLAIN heat DISTINCT — expect `idx_bucket_artwork` unchanged. +4. Do not drop `idx_bucket_hour` in the same change. + +--- + +## 30. Rollback + +- App: revert heat/IDF commits; heat semantics unchanged (same formula, same upsert keys). +- Cache: `Cache::forget('rec:tag-idf:'.$modelVersion)` or wait TTL. +- Index: re-create `idx_artwork_bucket` only if a reader somehow used the name (none found; unique covers the same left prefix). + +--- + +## Production verification plan (per fix) + +| Fix | Verify | +| --- | ------ | +| Heat chunking | scheduler runtime; rows examined still on covering index; PHP memory; `artwork_stats.heat_score` still updates | +| IDF cache | query count of `artwork_tag` GROUP BY across a tags job chain | +| DROP duplicate index (later) | EXPLAIN heat + Studio window + prune `WHERE bucket_hour <` before/after | diff --git a/docs/optimization-m11-http-endpoint-performance.md b/docs/optimization-m11-http-endpoint-performance.md new file mode 100644 index 00000000..f1a89881 --- /dev/null +++ b/docs/optimization-m11-http-endpoint-performance.md @@ -0,0 +1,322 @@ +# M11 — HTTP Endpoint Performance & Laravel Request Profiling + +```text +STATUS: COMPLETE (read-only production HTTP audit + local low-risk fixes) +PRODUCTION: no cache flush, no restarts, no nginx/PHP/MySQL/FPM/Horizon changes +WHEN: 2026-08-23 20:58–21:05 CEST +``` + +Guest curls from `server3` to `https://skinbase.org` (HTTP/2, `Accept-Encoding: br`). Two sequential requests per path. No concurrency. No cache clears. + +M10 snapshot indexes: **unchanged**. Duplicate-index DROP remains unrun; production does not have that index. + +--- + +## 1. HTTP baseline table + +Times are TTFB seconds. Size is **compressed** body (brotli) unless noted. + +| Path | Status | 1st TTFB | 2nd TTFB | Size (br) | Cache-Control | Encoding | +| ---- | ------ | -------- | -------- | --------- | ------------- | -------- | +| `/` | 200 | 0.449 | 0.413 | ~22 KB | `public, max-age=60, s-maxage=300, swr=600` | br | +| `/` uncompressed HTML | 200 | 0.444 | — | 286 KB | same | none (no AE) | +| `/explore` | 200 | 0.504 | 0.490 | ~16 KB | `no-cache, private` | br | +| `/discover/trending` | 200 | 0.449 | 0.434 | ~8.8 KB | `no-cache, private` | br | +| `/uploads/latest` | 200 | 0.573 | 0.594 | ~11.5 KB | `no-cache, private` | br | +| `/academy` | 200 | **1.283** | 0.838 | ~24 KB | `no-cache, private` | br | +| `/academy/prompts` | 200 | **1.316** | **1.171** | ~29 KB | `no-cache, private` | br | +| `/academy/prompts/popular` | 200 | 0.835 | — | — | private | br | +| `/forum` | 200 | 0.512 | 0.525 | ~15 KB | `no-cache, private` | br | +| `/collections` | **404** | 0.682 | 0.440 | ~8.7 KB | private | br | +| `/collections/featured` | 200 | **1.037** | — | — | private | br | +| `/rss/latest-uploads.xml` | 200 | 0.349 | 0.350 | ~13 KB | `no-cache, private` | br | +| `/sitemap.xml` | 200 | 0.125 | 0.064 | 377 B | `public, max-age=21600` | br | +| `/search?q=skin` | 200 | 0.467 | 0.520 | ~15 KB | private | br | +| `/statistics` | **302 → /login** | 0.309 | 0.254 | 709 B | private | — | +| `/leaderboard` | 200 | 0.439 | 0.435 | ~14 KB | private | br | +| `/staff` | 200 | 0.382 | 0.413 | ~8 KB | private | br | +| `/@killua` (profile) | 200 | **0.975** | **0.968** | 249 KB **uncompressed** | private | — | +| `/favicon.ico` | 200 | 0.063 | — | — | static | — | + +No `Server-Timing` header **before** this milestone. Added locally (`AddServerTiming`). + +--- + +## 2. Warm vs second request + +| Path | Δ | Interpretation | +| ---- | - | -------------- | +| `/` | −36 ms | Guest homepage `Cache::flexible` working | +| `/sitemap.xml` | −61 ms | Public cache / file cache | +| `/academy` | −445 ms | Application cache (Academy home payload) + SSR warm | +| `/academy/prompts` | −145 ms | Still >1 s — listing + S3 `exists` + SSR | +| Most others | ~0 | Cache already warm; remaining cost is PHP + SSR | + +Did **not** flush Redis to manufacture cold tests. + +--- + +## 3. Access-log latency analysis + +`/etc/nginx/nginx.conf` uses **default** `access_log` (combined). **No custom `log_format`.** + +Missing: `request_time`, `upstream_response_time`, `upstream_connect_time`. + +Log file `/var/log/nginx/skinbase.org-access.log` is `www-data:adm` — app user **cannot read** it. **No p95/p99 from access logs.** Do not change nginx during M11. + +--- + +## 4–5. Laravel logs / 5xx + +`storage/logs/laravel.log` ~3.5 GB. + +**CURRENT (hours around audit):** download warnings for missing originals (not 5xx). No Redis `LOADING`. No Predis memory exhaustion. + +**RECENT (same day):** CLI errors (`nova:metrics-prune` not defined, tinker parse, Enhance worker unavailable). Not HTTP user-facing. + +**HISTORICAL:** Redis LOADING / presence SMEMBERS / Predis OOM — M7/M8, not recurring in this window. + +HTTP 5xx count from nginx: **unknown** (log unread + no timing format). + +--- + +## 6. Middleware cost + +Web append: `AddServerTiming` (new), `SecurityHeaders`, `RedirectLegacyProfileSubdomain`, `TrackOnlineVisitor`, `UpdateLastVisit`, `HandleInertiaRequests`, onboarding/email-upgrade. + +| Middleware | Per request | Verdict | +| ---------- | ----------- | ------- | +| TrackOnlineVisitor | Redis GET + SETEX + SADD after response; bots included; RSS/sitemaps **now skipped** | LOW after skip | +| UpdateLastVisit | DB update throttled 300s, auth only | NO CHANGE | +| HandleInertiaRequests | auth user + **was** full `studioOptionsForUser` on every auth Inertia page | **HIGH for auth** — gated to `/studio` | +| Session | skipped for some paths via ConditionalStartSession | NO CHANGE | + +--- + +## 7. Inertia shared props + +`app/Http/Middleware/HandleInertiaRequests.php::share` + +| Prop | Was | Now | +| ---- | --- | --- | +| `auth.user` | profile lazy-load | `load('profile')` once | +| `flash` | closures | same | +| `cdn` / `features` | config | same | +| `studio_groups` | **eager Group::with(owner.profile, members) + whereHas for every logged-in Inertia page** | **only when path starts with `studio`** | + +`studio_groups` is consumed only by `resources/js/Layouts/StudioLayout.jsx`. + +--- + +## 8. SSR audit + +- Enabled: `config/inertia.php` `ssr.enabled=true`, `http://127.0.0.1:13714` +- Process: `node …/bootstrap/ssr/ssr.js` (www-data) +- Used for Inertia HTML first visits (Academy, forum, leaderboard, collections, profile) +- Home `/` is **Blade**, not SSR +- No SSR duration in logs before Server-Timing +- Do **not** disable SSR globally +- Academy/collections ~1 s likely **SSR + PHP**; second academy hit faster (cache) + +--- + +## 9. Controller mapping (slowest) + +| Path | Controller | +| ---- | ---------- | +| `/` | `App\Http\Controllers\Web\HomeController::index` → `HomepageService::all` / `allForUser` | +| `/explore` | `ExploreController::index` (Meilisearch) | +| `/discover/trending` | `DiscoverController::trending` | +| `/academy` | `Academy\AcademyHomeController::index` | +| `/academy/prompts` | `Academy\AcademyPromptController::index` | +| `/leaderboard` | `LeaderboardPageController` → cached `LeaderboardService::getLeaderboard` | +| `/collections/featured` | `CollectionDiscoveryController::featured` (6 discovery queries + SSR) | +| `/search` | `SearchController::index` → Meilisearch + groups + news LIKE | +| `/@user` | `User\ProfileController::showByUsername` | +| `/download/artwork/{id}` | `ArtworkDownloadController` | +| RSS | `RssFeedController::latestUploads` | + +--- + +## 10–11. Query counts / M10 overlap + +Local Pest: leaderboard as auth user **0** `groups` / `group_members` queries after the shared-prop gate. + +Production EXPLAIN not repeated (M10). HTTP-facing: + +- Leaderboard HTTP uses **Cache::remember**, not the 9M snapshot GROUP BY +- Heat job is scheduler, not HTTP +- Browse `/explore` uses Meilisearch then hydrate — TTFB ~500 ms + +--- + +## 12–13. Redis / presence + +Per tracked GET: `GET` record, `SETEX` 300s, `SADD` index. No `SMEMBERS` on the request path (M8 SSCAN prune remains). + +RSS/sitemaps/robots **were tracked**; now skipped. Bots still tracked on HTML (by design). + +--- + +## 14. Payload + +Homepage uncompressed ~286 KB HTML (Inertia-like JSON in Blade). Brotli ~22 KB. + +Profile `/@killua` 249 KB uncompressed HTML — large Inertia SSR document. MEDIUM, no rewrite this milestone. + +Academy compressed 24–29 KB — fine; TTFB is CPU/S3/SSR not transfer. + +--- + +## 15–21. Surface findings + +**Home:** guest flexible cache 5s/30s fresh + 1800s stale. TTFB ~410–450 ms. Personalized `allForUser` uncached (not measured; private). + +**Browse:** `/explore` ~500 ms; `no-cache, private` even for guests (CDN cannot cache). MEDIUM documentation only — personalization/maturity may require private. + +**Artwork detail:** not sampled with a canonical slug this pass (explore HTML is CDN image URLs). Code path `ArtworkController::show` / web artwork. + +**Search:** 467 ms including Meilisearch + optional news `LIKE` on content. Malformed query guard already in `SearchController`. + +**Statistics:** auth-gated 302. Rankings `/leaderboard` 439 ms from cache. + +**Profile:** ~970 ms both hits — SSR + heavy profile assembly. Documented, no rewrite. + +**Academy:** slowest public family. Causes: Inertia SSR + `Storage::exists` on S3 for prompt preview variants + `courseLessons()->count()` when `lessons_count_cache` is 0 (`?:` bug). Fixes: Redis-remember exists 1h; use cache column without extra count. + +**Forum:** 512 ms. Existing query-bound tests. Queues stay off. + +**RSS:** 350 ms, two queries (`Artwork::published()->with(user)->limit`). Redis not required for generation. Presence writes removed from RSS. + +**Download:** production currently logs missing originals **after** incrementing counters — **fixed** (404 first). + +--- + +## 22. External / storage + +Academy list `Storage::disk(s3)->exists` per variant — **was synchronous S3 on HTTP**. Now cached 1h. + +Download `File::isFile` local originals. + +Enhance worker errors are CLI/job, not public pages. + +--- + +## 23. Cache / stampede + +Homepage already `flexible` (stale-while-revalidate). Leaderboard `Cache::remember` — stampede possible but 439 ms and scheduled refresh exist. **No new caches** except Academy asset-exists. + +--- + +## 24. Bots / deep pagination + +Access logs unreadable. Historical malformed search URLs remain historical. **No UA blocking.** + +--- + +## 25. HTTP cache / compression / static + +- gzip + brotli **on** in nginx (confirmed `content-encoding: br` when AE sent) +- Static `favicon.ico` 63 ms, not Laravel +- `/build/*` access_log off in vhost +- Guest `/` already public CDN-friendly; most Inertia HTML `private, no-cache` (auth/session/SSR) + +--- + +## 26. TTFB classification + +| Endpoint | Dominant cost | +| -------- | ------------- | +| `/` | PHP bootstrap + cached payload + HTML serialize (~400 ms floor) | +| `/explore`, `/discover/*` | PHP + Meilisearch + Blade/Inertia | +| `/academy*` | **SSR + PHP + (was) S3 exists** | +| `/collections/featured` | **SSR + multiple discovery SQL** | +| `/@user` | **SSR + profile assembly** | +| `/leaderboard` | PHP + Redis cache hit | +| `/sitemap.xml` | static/cached XML | +| `/rss/*` | PHP + small SQL | + +--- + +## 27. Ranked findings + +| Severity | Item | +| -------- | ---- | +| HIGH | Auth Inertia `studio_groups` full group graph every page — **fixed** (studio path only) | +| HIGH | Download counters/events before missing-file 404 — **fixed** | +| HIGH | Academy S3 `exists` on listing/home cards — **fixed** (1h cache) | +| MEDIUM | Academy / collections / profile ~1 s (SSR). Do not disable SSR | +| MEDIUM | Guest explore/discover `Cache-Control: private` — CDN cannot cache | +| MEDIUM | `/collections` 404; real routes `/collections/featured` etc. | +| LOW | Presence on RSS — **fixed skip** | +| LOW | `lessons_count_cache ?: count()` extra SQL — **fixed** | +| NO CHANGE | FPM 14, Horizon, Redis size, snapshot indexes, forum queues, MySQL knobs | + +--- + +## 28. Files changed + +- `app/Http/Middleware/HandleInertiaRequests.php` +- `app/Http/Middleware/TrackOnlineVisitor.php` +- `app/Http/Middleware/AddServerTiming.php` (new) +- `bootstrap/app.php` +- `app/Http/Controllers/ArtworkDownloadController.php` +- `app/Services/Academy/AcademyAccessService.php` +- `tests/Feature/Http/StudioGroupsSharedPropTest.php` +- `tests/Feature/Http/ArtworkDownloadMissingFileTest.php` +- `tests/Feature/Http/RssPresenceSkipTest.php` +- `docs/optimization-m11-http-endpoint-performance.md` + +--- + +## 29. Tests + +```text +pest StudioGroupsSharedPropTest PASS +pest ArtworkDownloadMissingFileTest PASS +pest RssPresenceSkipTest PASS +``` + +--- + +## 30. Production deployment plan + +1. Deploy application code only. +2. Do **not** run M10 DROP INDEX. +3. Do **not** restart Redis/MySQL/FPM/Horizon except normal PHP-FPM reload for PHP. +4. SSR process already running; no disable. + +--- + +## 31. Post-deploy verification + +From `server3`, sequential: + +```bash +curl -sS -D - -o /dev/null --max-time 20 -H 'Accept-Encoding: br' https://skinbase.org/academy | grep -iE 'HTTP/|server-timing|cache-control' +curl -sS -o /dev/null -w 'academy ttfb:%{time_starttransfer}\n' -H 'Accept-Encoding: br' https://skinbase.org/academy +curl -sS -o /dev/null -w 'prompts ttfb:%{time_starttransfer}\n' -H 'Accept-Encoding: br' https://skinbase.org/academy/prompts +curl -sS -o /dev/null -w 'home ttfb:%{time_starttransfer}\n' -H 'Accept-Encoding: br' https://skinbase.org/ +``` + +Expect `Server-Timing: app;desc="Laravel";dur=…`. Academy TTFB should move toward the warm ~0.8 s band more often (asset-exists cache). Guest home should stay ~400–450 ms. + +--- + +## 32. Rollback + +Revert the listed PHP files. Presence skip and Server-Timing are additive. Download 404-before-increment is the correct semantics (do not roll back unless a download of a missing file must still increment stats — it should not). + +--- + +## Measurement note (acceptance) + +Public guest pages already under ~500 ms except Academy (~1.3 s), collections featured (~1.0 s), profile (~0.97 s). + +| Path | BEFORE (prod guest) | AFTER (expected) | +| ---- | ------------------- | ---------------- | +| `/` | 0.45 / 0.41 s | same | +| `/academy` | 1.28 / 0.84 s | fewer S3 round-trips; SSR remains | +| `/academy/prompts` | 1.32 / 1.17 s | fewer S3 round-trips | +| Auth non-studio Inertia | unmeasured (group graph every request) | **0 extra group SQL** (tested) | +| Missing download | counters + 404 | **404 only** (tested) | diff --git a/docs/optimization-m12-http-performance-observability.md b/docs/optimization-m12-http-performance-observability.md new file mode 100644 index 00000000..a2a99004 --- /dev/null +++ b/docs/optimization-m12-http-performance-observability.md @@ -0,0 +1,303 @@ +# M12 — Production Performance Observability & Slow Request Logging + +```text +STATUS: COMPLETE (audit + local artifacts; production nginx/FPM/env NOT modified) +WHEN: 2026-08-24 16:46 CEST +PRODUCTION: read-only audit only +``` + +This milestone is **observability**. No application behavior changes except: + +- Server-Timing now wraps the full HTTP kernel (prepend) and can add `ssr;dur=` when Inertia SSR runs +- Slow-request logging is **off until** `HTTP_SLOW_REQUEST_LOG_ENABLED=true` + +--- + +## 1. Current nginx access logging + +| Item | Production | +| ---- | ---------- | +| nginx | **1.26.3** | +| `log_format` | **none** (default `combined`) | +| http default | `access_log /var/log/nginx/access.log;` | +| Skinbase vhost | `access_log /var/log/nginx/skinbase.org-access.log;` (no format → combined) | +| `access_log off` | static `css/js/images`, `/photo/*.jpg` | +| Query strings | **yes** in combined `$request` | +| Timing vars | **absent** | +| SHA256 `nginx.conf` | `d1720f312b054a2007ffa731158c8bf4c9bd64bba35ba3971551f28cbabd2bd5` | +| SHA256 `skinbase.org.conf` | `4e7e4aa29cb406ccedf3e42613997683a92f1c8e2786bd087adab6012dd4b26f` | + +Multiple vhosts have **separate** logs. `http {}` includes `/etc/nginx/conf.d/*.conf`. + +**Cloudflare** sits in front (`server: cloudflare`, `cf-cache-status: DYNAMIC`, `00-cloudflare-realip.conf`). Origin `request_time` is nginx-on-server3, not browser RTT. + +--- + +## 2. Logrotate + +`/etc/logrotate.d/nginx`: + +``` +/var/log/nginx/*.log { + daily + rotate 14 + compress + delaycompress + create 0640 www-data adm + postrotate: invoke-rc.d nginx rotate +} +``` + +A new `/var/log/nginx/skinbase-performance.log` **is already covered**. **No extra logrotate rule.** + +--- + +## 3. Current FPM slowlog + +Pool `/etc/php/8.4/fpm/pool.d/skinbase.conf` (SHA256 `a0211876…`): + +| Setting | Value | +| ------- | ----- | +| `pm.max_children` | 14 (**do not change**) | +| `request_terminate_timeout` | 120s | +| `request_slowlog_timeout` | **10s** (already on) | +| `slowlog` | `/var/log/php8.4-fpm-skinbase-slow.log` (~23 MB) | +| `catch_workers_output` | yes | +| `pm.status_path` | `/fpm-status` localhost only | + +**FPM slowlog recommendation: NO CHANGE.** 10s is appropriate for rare stacks. Do not lower to 2s in the same deploy as nginx JSON logging. + +`/fpm-status` and `/fpm-ping` are `allow 127.0.0.1; deny all`. No public status. No `stub_status`. + +--- + +## 4. Existing Server-Timing + +Live header (server3 curl `/`): + +``` +server-timing: app;desc="Laravel";dur=96.9 +cache-control: max-age=60, public, s-maxage=300, stale-while-revalidate=600 +``` + +M11 middleware was **web-appended**, so it missed earlier kernel/session work (TTFB ~0.45 s vs Laravel ~110 ms). M12 **prepends** it globally so `app;dur` is closer to full Laravel time. Streamed/download: skip if `headers_sent()`. No SQL/Redis/cookies in the header. + +--- + +## 5. Proposed performance log format + +File: `/var/log/nginx/skinbase-performance.log` +Format name: `skinbase_perf` `escape=json` +Prepared: `deploy/nginx/conf.d/skinbase-perf-log-format.conf` +Vhost extra line: `deploy/nginx/skinbase-vhost-performance-log.snippet` + +Existing combined log **stays**. + +Static locations with `access_log off` stay off (not forced through PHP). + +`$upstream_*` is `"-"` for static; analyzer treats as null. Fastcgi PHP has numeric upstream times. `$upstream_cache_status` is typically empty (no nginx proxy_cache for Laravel) — still logged as `cache`. + +--- + +## 6. Privacy / security + +**Logged:** time, host, method, `$uri` (path only), status, timings, bytes, content-type, cache token. + +**Not logged:** IP, cookies, Authorization, query string, POST body, user id, session. + +--- + +## 7. Expected log volume + +Combined Skinbase access log: ~17 MB so far today, ~31 MB yesterday uncompressed; rotated daily 14 days. + +JSON lines ~300–400 B. Static already excluded. Estimate **~20–40 MB/day** uncompressed, **~300–600 MB** retained (14d) before gzip of older copies (~2–4 MB/day gz). Acceptable; no field trim required. + +Permissions: nginx `0640 www-data adm`. Do not chmod 777. Analyzer may need `sudo`. + +--- + +## 8–9. Laravel slow-request logger + threshold + +- Middleware: `LogSlowHttpRequest` (global prepend; **no-op when disabled**) +- Channel: `slow-http` → `storage/logs/slow-http.log` daily, **14 days** +- Fields: time, method, route_name, route_uri (**template**), status, duration_ms, peak_memory_mb, authenticated bool, inertia bool +- **Threshold: 750 ms** + +Rationale: homepage Laravel ~100 ms; guest TTFB 400–500 ms; Academy still 0.6–1.1 s. 750 ms captures Academy/profile/SSR outliers without logging every homepage. If volume is high after 24 h, raise to 1000 ms — do not sample first. + +Disable: `HTTP_SLOW_REQUEST_LOG_ENABLED=false` (default in `.env.example`). + +--- + +## 10. SSR timing + +`TimedInertiaSsrGateway` wraps Inertia `HttpGateway` (no vendor patch, SSR not disabled). + +When SSR runs, Server-Timing becomes: + +``` +app;desc="Laravel";dur=... +ssr;desc="Inertia SSR";dur=... +``` + +`app` includes SSR (nested). Blade home has no `ssr` metric. + +--- + +## 11. FPM slowlog + +**NO CHANGE** (already 10s). Optional later: 2s in a **separate** change, not bundled with nginx JSON. + +--- + +## 12–14. Analyzer, URI rules, percentiles + +`scripts/analyze-http-performance.php` + `HttpPerformanceLogAnalyzer` + +**Host filter (M12.1):** `--host=skinbase.org` exact match only. Without `--host`, rows aggregate on `host + normalized URI` (hosts are never merged). + +**URI normalization (analysis only):** numeric segments → `{id}`; `/@name` → `/@{user}`; UUID → `{uuid}`; long hex → `{hash}`; `/art/{id}/{slug}` vs `/art/{id}/similar`. + +**Percentile:** nearest-rank, rank = `ceil(p * n)`, 1-indexed. Not an average. + +**Upstream multi-value:** sum numeric comma-separated parts (`0.100, 0.220` → 0.32). `request_time` is authoritative for request latency. + +**499:** counted separately; not treated as 5xx. + +--- + +## 15–16. Tests + +```text +pest tests/Unit/Http/HttpPerformanceLogAnalyzerTest.php PASS (4) +pest tests/Feature/Http/SlowHttpRequestLoggingTest.php PASS (5) +``` + +Fixtures cover PHP, static `-`, multi-upstream, 200/302/404/499/500, malformed JSON, `/download/artwork/{id}`, `/@{user}`. + +--- + +## 17. Files changed + +- `config/http_observability.php` +- `config/logging.php` +- `.env.example` +- `phpunit.xml` +- `bootstrap/app.php` +- `app/Providers/AppServiceProvider.php` +- `app/Http/Middleware/AddServerTiming.php` +- `app/Http/Middleware/LogSlowHttpRequest.php` +- `app/Support/Http/HttpUriNormalizer.php` +- `app/Support/Http/HttpPerformanceLogAnalyzer.php` +- `app/Support/Http/TimedInertiaSsrGateway.php` +- `scripts/analyze-http-performance.php` +- `deploy/nginx/conf.d/skinbase-perf-log-format.conf` +- `deploy/nginx/skinbase-vhost-performance-log.snippet` +- tests + fixtures +- this document + +--- + +## 18. Nginx files prepared (not installed) + +See `deploy/nginx/conf.d/skinbase-perf-log-format.conf` and `skinbase-vhost-performance-log.snippet`. + +--- + +## 19. Env + +Production recommended: + +``` +HTTP_SLOW_REQUEST_LOG_ENABLED=true +HTTP_SLOW_REQUEST_MS=750 +HTTP_SLOW_REQUEST_LOG_DAYS=14 +``` + +Local default: **false**. + +--- + +## 20. Application deploy (A) + +1. `bash ./sync.sh` (do not `git pull` on production). +2. Add the three env vars to production `.env`. +3. Config refresh as **skinbase**, only if the host uses config cache: + + `sudo -u skinbase php artisan config:cache` + +4. Do not run artisan as root. Do not `cache:clear` globally unless required. +5. PHP-FPM reload only if the deploy process already does (code opcache). No Horizon/Redis/MySQL restart. + +--- + +## 21. Nginx deploy (B) — separate from app + +1. Backup: copy `nginx.conf` and `sites-enabled/skinbase.org.conf`. +2. Install `skinbase-perf-log-format.conf` into `/etc/nginx/conf.d/`. +3. Add **one** extra `access_log` line in the HTTPS server (keep combined). +4. `sudo nginx -t` — **stop if fail**. +5. `sudo systemctl reload nginx` (**not restart**). +6. One `curl` to `/`. +7. Confirm a JSON line in `skinbase-performance.log` and a combined line in `skinbase.org-access.log`. +8. Confirm no query string / no IP in JSON. +9. `sudo -u skinbase php scripts/analyze-http-performance.php --file=/var/log/nginx/skinbase-performance.log --since=1h --top=10` + (may need sudo to read `0640 www-data adm`). + +--- + +## 22. Optional FPM (C) + +**Skip.** Already logging stacks at 10s. + +--- + +## 23. Immediate verification + +```bash +curl -sS -D - -o /dev/null --max-time 20 -H 'Accept-Encoding: br' https://skinbase.org/ \ + | grep -iE 'HTTP/|server-timing|cache-control' + +sudo tail -n 1 /var/log/nginx/skinbase-performance.log +sudo tail -n 1 /var/log/nginx/skinbase.org-access.log +sudo -u skinbase tail -n 1 /opt/www/virtual/SkinbaseNova/storage/logs/slow-http-$(date +%F).log +``` + +Expect `Server-Timing` with `app;dur=` (and `ssr;dur=` on Inertia pages). Home may be **below** 750 ms and **not** appear in slow-http. + +--- + +## 24. 24h analysis + +```bash +sudo php /opt/www/virtual/SkinbaseNova/scripts/analyze-http-performance.php \ + --file=/var/log/nginx/skinbase-performance.log --since=24h --top=20 +``` + +Then `--status=5xx`, `--status=499`, `--uri=/academy`, `--slow-ms=1000`. + +Do **not** start M13 optimizations from a handful of curls. + +--- + +## 25. Rollback + +- App: `HTTP_SLOW_REQUEST_LOG_ENABLED=false` then config cache as skinbase. No code revert required for logging. +- Nginx: remove the extra `access_log` line and/or conf.d file, `nginx -t`, `reload`. Combined log untouched. +- SSR timing: revert Gateway bind only if a defect appears (SSR behavior unchanged). + +--- + +## 26. Risks + +- JSON `access_log` disk (~40 MB/day). Rotation already exists. +- Slow-http volume if many pages >750 ms (raise threshold). +- `app;dur` will **increase** vs M11’s ~110 ms because timing now includes earlier middleware (this is more accurate, not a regression). +- Cloudflare still hides client RTT. + +--- + +## 27. NO CHANGE + +FPM `pm.max_children`, Horizon, MySQL, Redis, snapshot indexes, SSR on/off, Telescope, Debugbar, SQL/Redis tracing, forum/collections queues, load tests, nginx restart, public status pages. diff --git a/docs/optimization-m12.2-vector-gateway-circuit.md b/docs/optimization-m12.2-vector-gateway-circuit.md new file mode 100644 index 00000000..1a962dc0 --- /dev/null +++ b/docs/optimization-m12.2-vector-gateway-circuit.md @@ -0,0 +1,27 @@ +# M12.2 — Vector Gateway Search Reliability & Circuit Breaker Hotfix + +```text +STATUS: COMPLETE (application-only) +PRODUCTION: no nginx / env / FPM / Redis / Horizon changes required +``` + +## Root cause + +`VectorGatewayClient::searchByFileContents()` tripped the global circuit on **any** HTTP failure or thrown exception **before** `AiArtworkVectorSearchService` could run `searchByUrl()`. The fallback then hit `guardCircuit()` and failed immediately. Intermittent 502s (~0.23–0.55s) match fail-fast circuit-open, not cold 1.5–3s searches. + +4xx from the file endpoint also opened the circuit. + +## Circuit policy + +- Combined similar-ai starts only if the circuit is **closed**. +- File search, then URL fallback. **Neither method trips or guards the circuit.** +- After **both** fail, trip only if **every** gateway failure is circuit-worthy. +- Successful fallback **leaves the circuit closed**. +- TTL remains **30s**. Search timeouts remain 2s/6s, retries 0. + +Circuit-worthy: connection/timeout, HTTP 408/429/5xx. +Not: 400/401/403/404/409/422 and other input 4xx. + +## Deploy + +`bash ./sync.sh` only. Do not reload nginx. Do not clear the circuit key. diff --git a/docs/optimization-m12.3-vector-gateway-circuit-attribution.md b/docs/optimization-m12.3-vector-gateway-circuit-attribution.md new file mode 100644 index 00000000..5a8cf356 --- /dev/null +++ b/docs/optimization-m12.3-vector-gateway-circuit-attribution.md @@ -0,0 +1,12 @@ +# M12.3 — Vector Gateway Circuit Trip Attribution + +```text +STATUS: COMPLETE (application-only) +BEHAVIOR: circuit policy unchanged +``` + +`tripCircuit()` logs one WARNING only on **closed → open**. + +Message: `Vector gateway circuit opened` + +Victims of an already-open circuit still only get `Vector similarity search failed` with `failure_stage=circuit`. diff --git a/docs/optimization-m12.4-similar-ai-dual-circuit.md b/docs/optimization-m12.4-similar-ai-dual-circuit.md new file mode 100644 index 00000000..a1e6855d --- /dev/null +++ b/docs/optimization-m12.4-similar-ai-dual-circuit.md @@ -0,0 +1,7 @@ +# M12.4 — Require Dual Gateway Failure Before Similar-AI Circuit Trip + +```text +STATUS: COMPLETE (application-only) +``` + +Similar-ai trips the global circuit only when **search_file was actually called** and **both** `search_file` and `search_url` failed circuit-worthy. Isolated URL 502 after a skipped/failed image download no longer opens the 30s breaker. diff --git a/docs/optimization-m2-mail-queue.md b/docs/optimization-m2-mail-queue.md new file mode 100644 index 00000000..29e974db --- /dev/null +++ b/docs/optimization-m2-mail-queue.md @@ -0,0 +1,79 @@ +# M2 — Mail Queue Consumption + +## Root cause + +Application mail jobs are pushed to Redis queue **`mail`**. Production Horizon supervisors only consume: + +```text +search, default +broadcasts, notifications +``` + +There is **no** `queue:work` process for `mail`. The three items have `attempts=0` — they were never reserved. + +Isolation is **intentional** in code (`onQueue('mail')`) and in `docs/QUEUE.md`. Horizon simply never subscribed. + +## The three queued items (no PII) + +Inspected with `LRANGE`/`LINDEX` via Laravel Redis; only class metadata: + +| # | displayName | maxTries | attempts | +| - | ----------- | -------: | -------: | +| 0 | `App\Jobs\SendVerificationEmailJob` | 5 | 0 | +| 1 | `App\Mail\EmailChangedSecurityAlertMail` | 3 | 0 | +| 2 | `App\Mail\EmailChangeVerificationCodeMail` | 3 | 0 | + +`pushedAt` unix times: 1777665006, 1777697989, 1784800925 (older registration/email-change mail sitting until a worker exists). + +## Local vs production + +Same gap on both: Horizon defaults omitted `mail`. Production `config('horizon.defaults.supervisor-*.queue')` confirmed `["search","default"]` and `["broadcasts","notifications"]`. + +Legacy `deploy/supervisor/skinbase-queue.conf` **does** include `mail`, but production runs Horizon, not that unit. + +## Fix + +Added Horizon `supervisor-mail`: + +```text +queue: mail +connection: redis +tries: 5 (matches SendVerificationEmailJob; rate-limit uses release()) +timeout: 90 +maxProcesses: 1 local / 2 production +``` + +Kept isolation so SMTP does not block rec/search workers and so `--tries=1` on default workers cannot fail mail jobs on `release()`. + +## Files changed + +```text +config/horizon.php +tests/Unit/HorizonMailQueueTest.php +docs/QUEUE.md +docs/optimization-m2-mail-queue.md +``` + +## Deployment + +1. Deploy release with `config/horizon.php`. +2. `php artisan config:cache` / `optimize` as usual. +3. Restart Horizon (Supervisor `skinbase-horizon` / `queue:restart`). +4. Confirm `php artisan horizon:status` and a worker with `--queue=mail`. + +No `.env` change required. No Redis delete. + +## Should the 3 messages be processed after deploy? + +**Yes, automatically** once `supervisor-mail` starts — do not delete the list. + +Risks: + +- Recipients may receive **late** verification/security emails. +- Tokens/codes may already be expired; user sees a failed verify, not a security hole from us sending mail. +- `SendVerificationEmailJob` may still send `RegistrationVerificationMail` (also `mail` queue) — second job is fine once workers run. +- Do **not** manually `queue:retry` or `lpop` before Horizon is up. + +## Tests + +`php artisan test --filter=HorizonMailQueue` diff --git a/docs/optimization-m3-sitemaps.md b/docs/optimization-m3-sitemaps.md new file mode 100644 index 00000000..78ee40bb --- /dev/null +++ b/docs/optimization-m3-sitemaps.md @@ -0,0 +1,127 @@ +# M3 — Sitemap Production Audit & Reliability + +## Current architecture + +Routes: + +```text +GET /sitemap.xml SitemapController@index +GET /sitemaps/{name}.xml SitemapController@show +GET /robots.txt RobotsTxtController (Sitemap: {APP_URL}/sitemap.xml) +``` + +Generation: `php artisan skinbase:sitemaps:generate` (scheduler 10:30 and 22:30) writes static XML on `sitemaps_public` (`public/`). + +Also: `sitemaps:publish`, `validate`, release artifacts, optional live-build fallback. + +Families (no groups/years/videos sitemaps): + +artworks (sharded 10k), academy-*, users (sharded 10k), tags (now shardable at 25k), categories, collections, cards, stories, web-stories, news, news-google, forum-*, static-pages. + +Canonical artwork URLs: `route('art.show')` `/art/{id}/{slug}`. Public/published only. + +## Exact production issue + +`/sitemap.xml` is a **stale file from 2026-05-13**, owned by **klevze** (`-rw-r--r--`). Children (artworks shards, academy, users, …) refresh daily as **skinbase**. Scheduler cannot overwrite the index (`filesystems.sitemaps_public.throw=false`, generate ignored failed `put`). Nginx `try_files $uri` serves the stale root file and **never** hits Laravel. Academy families exist on disk but are **absent from the May index**. + +`location @php` is missing in the live vhost (documented in M0). + +## Local vs production + +Generate command already writes `sitemaps/sitemap.xml` on both. Production **file ownership** plus **nginx `$uri` = public/sitemap.xml** is the operational failure. Local did not fail writes or serve a dual path. + +## Root cause + +1. Index file not writable by `skinbase`. +2. `put()` failures ignored. +3. Nginx prefers the stale `/sitemap.xml` over `/sitemaps/index.xml` (which did not exist). + +## Implementation + +- Generate writes `sitemaps/index.xml` (new, creatable), plus tries `sitemaps/sitemap.xml` and `sitemap.xml`; **fails the command if none of the index paths write**. +- Child writes report failure instead of counting a silent skip as success. +- `SitemapController::index()` prefers static `sitemaps/index.xml`. +- `deploy/nginx/sitemaps.conf`: `try_files /sitemaps/index.xml /sitemaps/sitemap.xml /index.php?$query_string;` (no `@php`) +- Tags builder is shardable (25k) so it cannot exceed 50k URLs without an index. + +## Caching + +Unchanged: nginx `max-age=21600`; Laravel file response uses `sitemaps.cache_ttl_seconds` (900). No HTML cache. Generate is the refresh. + +## Chunking + +Artworks/users/forum-threads/collections/cards/stories already sharded at 10k. Tags 25k. Protocol: 50k URLs / 50MB. Production files are all well under 50MB (largest artworks shard ~3.5MB, tags ~2MB). + +## Deployment + +Production vhost (`/etc/nginx/sites-enabled/skinbase.org.conf`) has **no** `location @php`. PHP-FPM is: + +```nginx +location / { + try_files $uri $uri/ /index.php?$query_string; +} +location = /index.php { + fastcgi_pass unix:/run/php/php8.4-fpm-skinbase.sock; + ... +} +``` + +**Do not** use `try_files … @php` (undefined named location). Use `/index.php?$query_string` like `location /`. + +Exact replacement for the two sitemap blocks (currently ~lines 91–110): + +```nginx + location = /sitemap.xml { + try_files /sitemaps/index.xml /sitemaps/sitemap.xml /index.php?$query_string; + + add_header Cache-Control "public, max-age=21600" always; + add_header Content-Type "application/xml; charset=UTF-8" always; + etag on; + } + + location ~ ^/sitemaps/[A-Za-z0-9_\-]++\.xml$ { + try_files $uri /index.php?$query_string; + + add_header Cache-Control "public, max-age=21600" always; + add_header Content-Type "application/xml; charset=UTF-8" always; + etag on; + } +``` + +Do **not** add `$uri` to the `/sitemap.xml` try_files list: `$uri` is `/sitemap.xml`, the stale klevze-owned file, and would shadow PHP if `index.xml` is missing. + +Order: + +1. Deploy app. +2. Run `php artisan skinbase:sitemaps:generate` as `skinbase` so `sitemaps/index.xml` exists. +3. Patch the vhost, then: + +```bash +sudo nginx -t && sudo systemctl reload nginx +``` + +`nginx -t` should pass: last `try_files` argument matches the existing front-controller pattern. Introducing `@php` would not. + +Optional: `chown skinbase:skinbase` the old `sitemap.xml`. + +## Legacy index files after `sitemaps/index.xml` + +| Path | Role after M3 | +| ---- | ------------- | +| `public/sitemaps/index.xml` | **Canonical.** Scheduler can create this even when the May 2026 file is unwritable. Nginx prefers it. | +| `public/sitemaps/sitemap.xml` | **Legacy fallback** for `/sitemap.xml` if `index.xml` is missing. Same inode as root `sitemap.xml` on current prod (symlink). Best-effort write; failure must not fail generate if `index.xml` succeeded. | +| `public/sitemap.xml` | **Legacy `$uri`.** On prod this is a symlink to the unwritable klevze file. **Not required** once nginx prefers `index.xml`. Generate may skip it. | + +Generate still *attempts* all three; success requires **at least one** index write (`index.xml` is enough). + +## robots.txt + +Already declares `Sitemap: https://skinbase.org/sitemap.xml`. **No robots change** for M3. + +## URLs that should not be in sitemaps + +Already excluded: private/unapproved artworks, inactive users, inactive tags, academy pricing query variants, `/pages/about` duplicate, forbidden `/admin` `/cp` etc. No extra removals in M3. + +## Tests + +`php artisan test tests/Feature/SitemapTest.php` (run after this doc). diff --git a/docs/optimization-m4-recommendation-snapshots.md b/docs/optimization-m4-recommendation-snapshots.md new file mode 100644 index 00000000..0c4cc6c2 --- /dev/null +++ b/docs/optimization-m4-recommendation-snapshots.md @@ -0,0 +1,178 @@ +# M4 — Recommendation Snapshot & Read Path + +**Snapshot table needed: NO** + +Investigation only plus a regression test. No production changes. No new tables. + +--- + +## 1. What “snapshot table” meant + +Two different things were previously named “snapshot”: + +| Table | Size (prod) | Role | +| ----- | ----------: | ---- | +| `artwork_metric_snapshots_hourly` | **1855 MB / 8.6M rows** | Heat/ranking hourly totals (M0 DB residual). **Not** similar-art serving. | +| `rec_artwork_recs` | **11.6 MB / 27.7k rows** | Precomputed similar lists (JSON `recs`) read by the API. | + +M4 is about the **recommendation read path**, not heat snapshots. A second copy of `rec_artwork_recs` would not address the 1.85 GB hourly table. That remains a later ranking/ops milestone. + +--- + +## 2. Current data model + +```text +rec_item_pairs + (a_artwork_id, b_artwork_id) UNIQUE + weight, updated_at + ~74k rows, 16 MB + Written by RecBuildItemPairsFromFavouritesJob (DELETE all, then upsert chunks) + +rec_artwork_recs + UNIQUE (artwork_id, rec_type, model_version) + recs JSON (ordered IDs), computed_at + rec_type: similar_tags | similar_behavior | similar_hybrid | similar_visual + Written by RecComputeSimilar* via updateOrCreate (one row at a time) + +user_recommendation_cache + personalized “For You” blobs (~67 rows, 2.5 MB) — separate from similar-art +``` + +Frontend/API similar-art: + +```text +GET /api/art/{id}/similar +GET /art/{id}/similar + → HybridSimilarArtworksService::forArtwork + Cache::remember rec:artwork:{id}:similar:{model} TTL 6h + SELECT rec_artwork_recs WHERE artwork_id + rec_type + model_version (const) + hydrate Artwork whereIn(ids) public+published +``` + +--- + +## 3. Production row counts (2026-08-23, read-only) + +| rec_type | rows | computed_at span | +| -------- | ---: | ---------------- | +| similar_hybrid | 13902 | 2026-03-15 → 2026-08-23 14:36 | +| similar_tags | 9479 | 2026-03-15 → 2026-08-23 14:58 | +| similar_behavior | 4851 | 2026-04-21 → 2026-08-23 14:34 | + +~50k public artworks exist; lists are **incomplete** because nightly catalog jobs were dying (M1), not because of missing snapshots. Coverage will grow after M1 deploy. + +--- + +## 4. Read/write flow + +**Read:** unique const lookup (1 row) + Redis/cache 6h + one `whereIn` hydrate. Cheap. + +**Write (M1 batches):** 200 artworks/job, `updateOrCreate` per artwork. No truncate of `rec_artwork_recs`. Unique key replaces one JSON list atomically. + +**Write (pairs):** `DELETE FROM rec_item_pairs` then rebuild every 4 hours. Readers of pairs are **rebuild jobs**, not the HTTP similar endpoint. + +**Partial visibility:** during a catalog rebuild, some artworks have today’s `computed_at`, others yesterday. HTTP cache may keep old IDs for 6h. There is no empty-table window for `rec_artwork_recs`. Mixing generations is **per artwork**, not a torn JSON array. + +**Transactions:** none wrapping the full catalog. Each `updateOrCreate` is its own statement. + +--- + +## 5. Query plans (production EXPLAIN) + +Read: + +```text +type=const +key=rec_artwork_recs_artwork_id_rec_type_model_version_unique +rows=1 +``` + +Pairs UNION used by behavior rebuild (not HTTP): + +```text +ref on unique (a_artwork_id, b_artwork_id) +ref on b_artwork_id index +UNION RESULT: Using temporary; Using filesort (tiny LIMIT 90) +``` + +No full table scan on the similar-art HTTP path. + +--- + +## 6. Bottleneck (exact) + +| Concern | Present? | +| ------- | -------- | +| Expensive HTTP reads | **No** — const unique + cache | +| Partial rec JSON during rebuild | **No** — row replace | +| Mixed old/new *across* artworks during overnight batch | **Yes, benign** with 6h cache | +| delete/reinsert on `rec_artwork_recs` | **No** | +| delete/reinsert on `rec_item_pairs` | **Yes**, 74k rows / 4h; not on the HTTP path | +| Poor indexing | **No** on read | +| Excessive Redis | One key per artwork, 6h TTL — normal | +| Dual generations mixed in one response | **No** | + +**Exact bottleneck for similar-art quality:** incomplete `rec_artwork_recs` coverage from failed nightly jobs (M1), not read-path schema. + +**Separate large table:** `artwork_metric_snapshots_hourly` 1.85 GB — ranking/heat, **out of M4 schema scope**. + +--- + +## 7. Snapshot table? + +```text +YES/NO: NO +``` + +Would double a **12 MB** table, add cutover complexity, and not fix M1 coverage or the 1.85 GB heat snapshots. + +### Simpler alternatives (do not implement in M4 unless needed later) + +1. **Deploy M1** so batches complete; coverage grows via `updateOrCreate`. +2. If `rec_item_pairs` DELETE causes empty-pair windows during overlap with behavior jobs: replace global delete with rebuild-into-temp + `RENAME TABLE` **on that 16 MB table only** — still not a rec snapshot table. +3. Heat snapshots: partition/prune `artwork_metric_snapshots_hourly` in a dedicated milestone. + +--- + +## 8. Proposed schema if YES + +N/A. + +--- + +## 9–12. Migration / cutover / cleanup / deploy + +N/A for snapshot tables. M1 deploy remains the rec-job fix. + +Rollback: none (no schema). + +--- + +## 13. Tests + +Added: `keeps other rec types and other artworks visible when one row is rebuilt` in `tests/Feature/Recommendations/SimilarArtworksHybridTest.php`. + +Existing hybrid tests already cover fallback chain, order, author cap, API `type=`. + +--- + +## 14. Files changed + +```text +tests/Feature/Recommendations/SimilarArtworksHybridTest.php +docs/optimization-m4-recommendation-snapshots.md +``` + +No migrations, no rec model changes. + +--- + +## 15. Risks + +None from schema (none added). Residual: pairs `DELETE` vs behavior schedule overlap; 6h cache lag after rebuild; heat snapshot table still large. + +--- + +## Production + +Read-only EXPLAIN + `information_schema` + grouped counts. Temp `/tmp` script removed. No deploys, no writes to rec tables. diff --git a/docs/optimization-m5-metric-snapshot-retention.md b/docs/optimization-m5-metric-snapshot-retention.md new file mode 100644 index 00000000..d5b72033 --- /dev/null +++ b/docs/optimization-m5-metric-snapshot-retention.md @@ -0,0 +1,329 @@ +# M5 — Artwork Metric Snapshot Retention & Growth + +```text +STATUS: COMPLETE (local implementation; not deployed) +PRODUCTION WRITES: none +PRODUCTION DELETES: none +OPTIMIZE TABLE: not run +PARTITIONING: not added +NEW ANALYTICS TABLE: not created +``` + +--- + +## Verdict + +```text +Pruning needed: YES +Recommended retention: 30 days +Downsampling needed: NO +Partitioning needed: NO +Indexes to add: none +Indexes to remove: optional later (redundant idx_artwork_bucket) +Expected storage vs unlimited: large reduction +Expected storage vs current 7d prune: INCREASE (~4.3×) if retention is raised to 30d +``` + +Hourly snapshots are already pruned daily with `--keep-days=7` (unbounded single `DELETE`). That window is **too short** for monthly leaderboards and Studio 30d views. M5 makes prune batched + configurable and sets default retention to **30 days**. + +--- + +## 1. Table schema (production) + +```sql +CREATE TABLE `artwork_metric_snapshots_hourly` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT, + `artwork_id` bigint unsigned NOT NULL, + `bucket_hour` datetime NOT NULL, + `views_count` bigint unsigned NOT NULL DEFAULT '0', + `downloads_count` bigint unsigned NOT NULL DEFAULT '0', + `favourites_count` bigint unsigned NOT NULL DEFAULT '0', + `comments_count` bigint unsigned NOT NULL DEFAULT '0', + `shares_count` bigint unsigned NOT NULL DEFAULT '0', + `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `uq_artwork_bucket` (`artwork_id`,`bucket_hour`), + KEY `idx_bucket_hour` (`bucket_hour`), + KEY `idx_artwork_bucket` (`artwork_id`,`bucket_hour`), + KEY `idx_bucket_artwork` (`bucket_hour`,`artwork_id`), + CONSTRAINT `artwork_metric_snapshots_hourly_artwork_id_foreign` + FOREIGN KEY (`artwork_id`) REFERENCES `artworks` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB +``` + +Not partitioned (`CREATE_OPTIONS` empty; `PARTITIONS.PARTITION_NAME` NULL). + +Duplicates: **none** (unique `(artwork_id, bucket_hour)`). `has_duplicate_groups=0`. + +--- + +## 2. Production size & range (read-only, 2026-08-23 15:07 UTC) + +| Metric | Value | +| ------ | ----- | +| Exact `COUNT(*)` | **8,918,000** | +| `information_schema` est. | 8,622,217 | +| Data | 740.03 MB | +| Index | 1115.47 MB | +| Total | **1855.50 MB** | +| InnoDB `DATA_FREE` | 121.00 MB | +| Distinct artworks | 49,822 | +| Distinct hours | 179 | +| Oldest `bucket_hour` | 2026-08-16 05:00:00 | +| Newest `bucket_hour` | 2026-08-23 15:00:00 | +| Rows per hour (stable) | **49,822** | +| Rows / day | **1,195,728** | +| AUTO_INCREMENT | 153,018,248 (historical insert churn after prune) | + +Rows older than 7 days at query time: 548,031 (~11 extra hours until the next 04:00 prune). Oldest row is 05:00 on the day after the last 04:00 cutoff — **daily prune is running**. + +--- + +## 3. Growth estimates + +Assumes current write rate stays ~49,822 rows/hour and size scales linearly (~249 MB/day from 1855.5 MB / 7.45 days). + +| Horizon | Rows | Approx size | +| ------- | ----: | ----------: | +| Current (~7.5 d) | 8.9M | 1.86 GB | +| 24 hours | 1.2M | ~0.25 GB | +| 7 days | 8.4M | ~1.75 GB | +| **30 days** | **35.9M** | **~7.5 GB** | +| 90 days | 107.6M | ~22.4 GB | +| 365 days | 436.4M | ~91 GB | +| Unlimited 1y of this rate | same as 365d | ~91 GB | + +MB/day ≈ **249 MB/day** (data+index) at current density. + +--- + +## 4. Writers (exact) + +| Writer | When | What | +| ------ | ---- | ---- | +| `nova:metrics-snapshot-hourly` (`MetricsSnapshotHourlyCommand`) | Scheduler `hourlyAt(2)` | Upsert one row per eligible artwork for `now()->startOfHour()` | +| Eligibility | `--days=60` default | `artworks.created_at` last 60 days **OR** `artwork_stats.ranking_score > 0`; approved, not deleted | + +Chunk 1000. Unique key makes reruns idempotent. **No other writers** (no jobs, no HTTP, no observers). + +That eligibility (ranking_score>0) explains ~50k IDs/hour, not only last-60-day artworks. + +--- + +## 5. Readers (exact) + +| Consumer | Lookback | Notes | +| -------- | -------- | ----- | +| `RecalculateHeatCommand` (`nova:recalculate-heat` every 15 min) | **24h** | DISTINCT artwork_id + load window; writes `artwork_stats.heat_score` | +| `HomepageService::risingRecentActivitySubquery` | **24h** | MAX-MIN deltas | +| `DiscoverController` rising | **24h** | same pattern | +| `DiscoverFeedController` (RSS) | **24h** | same pattern | +| `LeaderboardService` daily/weekly/**monthly** | **1 / 7 / ~30 days** | `MAX-MIN` cumulative counts grouped by artwork; all-time uses `artwork_stats`, not this table | +| `StudioMetricsService::getDashboardKpis` | **30 days** | `SUM(views_count)` — **semantically wrong** (views_count is cumulative totals, not hourly increments). Falls back to lifetime stats if 0. | +| `CreatorJourneyService::biggestDownloadSpike` | **all retained hours** for the creator’s public artworks | No time filter; bounded only by prune | +| `nova:prune-metric-snapshots` | cutoff | writer of deletes | + +Frontend/API do not query this table directly; they read heat/leaderboard/studio payloads derived from it. + +No daily/weekly/monthly artwork metric aggregate table exists (only collection daily stats elsewhere). + +--- + +## 6. Required historical retention (not assumed) + +| Need | Depth | +| ---- | ----- | +| Heat / rising homepage / discover / RSS | 24 hours | +| Weekly leaderboards | 7 days | +| Monthly leaderboards | **~30 days** | +| Studio “views 30d” query | 30 days (even though the SUM is a bad formula) | +| Journey download spike | prefers longer; not a product requirement for unlimited | +| Rec similar-art serving | **none** (M4) | + +**Required: 30 days** of hourly rows if monthly leaderboards and Studio 30d stay on this table. + +Unlimited / 1 year / 90 days: **not required** by application code. + +Current production `--keep-days=7` **under-retains** monthly leaderboard snapshot deltas. + +--- + +## 7. Production query plans (EXPLAIN) + +Heat DISTINCT 24h: + +```text +type=range key=idx_bucket_artwork rows~2.46M +Using where; Using index; Using temporary +``` + +Rising GROUP BY 24h: + +```text +type=range key=idx_bucket_artwork rows~2.46M +Using index condition; Using temporary +``` + +Studio 30d for `user_id=1`: + +```text +artworks: ref artworks_user_id_index (~449 rows) +snapshots: ref uq_artwork_bucket (artwork_id) ~411 rows, Using index condition +``` + +Leaderboard monthly GROUP BY (full table, 30d > retained 7d so it scans retained set): + +```text +type=index key=uq_artwork_bucket rows~8.87M Using where +``` + +Prune count `bucket_hour < now()-7d`: + +```text +type=range key=idx_bucket_artwork rows~1.1M Using where; Using index +``` + +Indexes are used. 24h range still estimates millions of rows because ~50k artworks × 24 hours ≈ 1.2M (optimizer overestimate 2.46M). Acceptable for hourly/15-min jobs; leaderboard monthly on 30d would scan ~36M rows if retention grows — still one scheduled hourly job, not request path. + +--- + +## 8. Pruning / downsample / partition / archive + +### Pruning: YES + +Already scheduled. Replace unbounded `DELETE WHERE bucket_hour < cutoff` with PK-id batches. + +### Downsampling: NO + +No existing daily artwork snapshot table. Monthly scores only need MIN/MAX of cumulative counters in the window; hourly rows are convenient. A second table is more operational cost than 30d hourly until size is painful (~7.5 GB still fits RAM buffer pool 10 GB). Revisit if table exceeds ~15–20 GB. + +### Archive: NO + +Nothing reads off-box history. + +### Partitioning: NO + +Rolling 7–30 day window + batched DELETE is simpler than `PARTITION BY RANGE (TO_DAYS(bucket_hour))` + monthly `ALTER`/`DROP PARTITION`. Not justified at 1.86 GB (or 7.5 GB). + +--- + +## 9. Delete vs InnoDB free space vs filesystem + +- `DELETE` removes rows; InnoDB keeps pages in the tablespace (`DATA_FREE` today 121 MB). +- Disk file (`ibd`) typically **does not shrink**. +- Filesystem reclaim needs `OPTIMIZE TABLE` / `ALTER TABLE ... ENGINE=InnoDB` (rebuild) — **do not run automatically on production**. +- After raising retention, size grows; after later lowering it, expect logical free space, not a smaller `.ibd` until a rebuild is planned in a maintenance window. + +--- + +## 10. Implementation (local) + +- Config `config/metrics.php` + env `ARTWORK_METRIC_HOURLY_RETENTION_DAYS` default **30**. +- `PruneMetricSnapshotsCommand`: count eligible → loop `SELECT id … LIMIT chunk` → `DELETE WHERE id IN (…)`, sleep, logs per batch, `--dry-run`, `--max-batches`. +- Scheduler: `nova:prune-metric-snapshots` **without** hardcoded `--keep-days=7`. +- Tests for keep-days, batches, idempotence, dry-run, config default, max-batches. + +### Production-safe properties + +| Requirement | How | +| ----------- | --- | +| Bounded batches | `--chunk` default 5000 | +| No giant DELETE | one `whereIn` per batch | +| Avoid long locks | small PK deletes + `--sleep-ms=50` | +| Scheduler-safe | `withoutOverlapping`; daily 04:00 | +| Idempotent | re-run deletes 0 extra rows | +| Observable | info logs per batch + completed | +| Configurable | env + `--keep-days` override | + +--- + +## 11. Indexes + +Keep: + +- `PRIMARY (id)` — batched prune +- `uq_artwork_bucket (artwork_id, bucket_hour)` — upsert + per-artwork time +- `idx_bucket_artwork (bucket_hour, artwork_id)` — heat/rising/prune range + +Optional later (not in this milestone): + +- Drop `idx_artwork_bucket` — duplicate of unique key +- Drop `idx_bucket_hour` — prefix of `idx_bucket_artwork` + +Do not add indexes. Do not drop on production in M5 (online DDL on 1.85 GB). + +--- + +## 12. Expected storage after M5 deploy + +If `ARTWORK_METRIC_HOURLY_RETENTION_DAYS=30` on production: + +- Table grows from ~1.86 GB toward **~7.5 GB** over ~3 weeks. +- That is **correctness**, not reduction. + +If production must stay small, set `ARTWORK_METRIC_HOURLY_RETENTION_DAYS=7` and accept monthly leaderboard snapshot windows of only 7 days. **Do not** leave the scheduler at 7 days while believing Studio/monthly boards have 30d of hourly history. + +Storage reduction vs never pruning: **~91 GB/year avoided**. + +--- + +## 13. Deployment (later milestone — not M5) + +1. Ship code + `config/metrics.php`. +2. Set env on server: `ARTWORK_METRIC_HOURLY_RETENTION_DAYS=30` (or 7 if size is prioritized). +3. `php artisan config:cache` as the deploy already does. +4. Do **not** run prune by hand on first deploy unless a dry-run is wanted: `php artisan nova:prune-metric-snapshots --dry-run`. +5. Do **not** `OPTIMIZE TABLE`. +6. Watch next 04:00 run logs for batch counts. + +If raising 7 → 30: prune deletes almost nothing until the table ages to 30 days. + +--- + +## 14. Rollback + +- Restore previous command (single DELETE) and scheduler `--keep-days=7`. +- Or set env back to `7` with the new command (preferred). +- No schema change to roll back. + +--- + +## 15. Tests + +- `tests/Feature/PruneMetricSnapshotsCommandTest.php` (new) +- Existing `tests/Feature/RisingEngineTest.php` prune example still valid with `--keep-days=7` + +--- + +## 16. Files changed + +```text +config/metrics.php (new) +app/Console/Commands/PruneMetricSnapshotsCommand.php (batched prune) +routes/console.php (no hardcoded 7) +.env.example +docs/cli-reference.md +docs/optimization-m5-metric-snapshot-retention.md (new) +tests/Feature/PruneMetricSnapshotsCommandTest.php (new) +``` + +--- + +## 17. Risks + +- Raising retention to 30d **increases** MySQL size and leaderboard monthly scan cost. +- Studio 30d `SUM(views_count)` remains an incorrect metric (cumulative summed across hours). Out of M5 scope. +- Journey spike still limited to retained hours. +- Batched prune of ~1.2M rows/day at chunk 5000 ≈ 240 batches; 50ms sleep ≈ 12s extra plus delete time. Fine for 04:00. +- Redundant indexes still cost ~index-heavy 1.1 GB; dropping them is a later DDL decision. + +--- + +## 18. What was not done + +- No production SQL writes/deletes +- No `OPTIMIZE TABLE` +- No partitioning +- No new aggregate table +- No change to snapshot writer eligibility (`--days=60` + ranking_score) +- M1–M4 still undeployed diff --git a/docs/optimization-m5.1-metric-snapshot-correctness.md b/docs/optimization-m5.1-metric-snapshot-correctness.md new file mode 100644 index 00000000..83aa98da --- /dev/null +++ b/docs/optimization-m5.1-metric-snapshot-correctness.md @@ -0,0 +1,195 @@ +# M5.1 — Metric Snapshot Correctness & 30-Day Readiness + +```text +STATUS: COMPLETE (local; not deployed) +PRODUCTION WRITES: none +OPTIMIZE TABLE: not run +PARTITIONING: not added +NEW TABLES: none +``` + +--- + +## Verdict + +```text +Safe to change production retention 7 → 30 after this code ships: YES +Recommended env: ARTWORK_METRIC_HOURLY_RETENTION_DAYS=30 +M5 prune batches: verified +Hourly writer vs prune overlap: staggered (prune 04:25, snapshot hourlyAt 2) +Monthly GROUP BY at ~36M rows: acceptable as a scheduled job; not request-path +``` + +Production prune is already at 30 days (`eligible=0`); **this KPI/delta code is still local and must ship before treating Studio/monthly numbers as correct 30-day metrics.** + +--- + +## 1. Cumulative-counter bugs found + +| Location | Bug | Status | +| -------- | --- | ------ | +| `StudioMetricsService::getDashboardKpis` | `SUM(views_count)` over cumulative hourly rows | **FIXED** | +| Same method | If SUM was 0, **fallback to lifetime** `artwork_stats.views` as `views_30d` | **REMOVED** | +| Same method | `favourites_30d` / `shares_30d` were **lifetime** totals | **FIXED** | +| `CreatorStudioOverviewService` (live Studio) | 30d KPI keys were **lifetime** module totals | **FIXED** (artwork hourly window + coverage) | +| `LeaderboardService` artwork daily/weekly/monthly | `MAX−MIN` **inside** the window only — undercounts **new** artworks (first snapshot already has views) | **FIXED** (latest − pre-window baseline, or 0 if born in period) | +| Heat `RecalculateHeatCommand` | Single snapshot without earlier hour → 0 heat | **intentional** (momentum, not 30d KPI) | +| Rising homepage/discover/RSS | `MAX−MIN` over 24h | **acceptable** for 24h momentum; first hour of a brand-new work is undercounted | +| Nova cards / collections `SUM(views_count)` | Other tables | **not a snapshot bug** | + +No remaining `SUM` of `artwork_metric_snapshots_hourly` cumulative columns. + +--- + +## 2. Studio `views_30d` fix + +For each artwork: + +```text +latest = MAX(counter) in [start, now] +baseline = + 0 if published_at/created_at >= start + else snapshot at/immediately before start (last 36h before start) + else MIN(counter) in window -- warm-up / missing hours; NOT lifetime +delta = GREATEST(latest - baseline, 0) +``` + +Then SUM those **deltas** across the creator’s artworks. + +- Created during the period: first snapshot of 40 + later 90 → **90** +- Older work with pre-window 200 and latest 250 → **50** +- Older work, one in-window snapshot of 400 (no baseline) → **0** (do not report lifetime) +- Zero growth stays **0** (no lifetime fallback) + +Coverage is exposed so a 7.5-day table is not labeled as a full 30 days. + +--- + +## 3. Warm-up behavior + +Production history today starts **2026-08-16** (~7.5 days). After env=30, history stays incomplete until ~2026-09-15. + +UI/API now: + +- Still compute the delta from **available** hours (do not invent 30d). +- Expose `kpis.snapshot_window`: + - `requested_days` (30) + - `table_coverage_days` + - `user_coverage_days` + - `window_complete` (true only when table span ≥ 30 days minus one hour) + - oldest/newest buckets +- Studio dashboard hint: **“Based on 7.5 of 30 days”** until complete; then **“Last 30 days”**. + +Monthly leaderboards already score from `bucket_hour >= now-1 month` (whatever rows exist). They are internally consistent but **short** until warm-up ends. No separate leaderboard badge in this milestone (not a Studio KPI). Do not treat those boards as a full calendar month until coverage is complete. + +--- + +## 4. Creator journey + +`biggestDownloadSpike` loaded **all** retained hours for every public artwork (unbounded vs retention). After 30d that is ~720 hours × N artworks. + +**Decision:** explicit lookback = `metrics.hourly_snapshot_retention_days` (default 30). + +- Career “all-time spike” cannot exist beyond retention anyway. +- Query size stays bounded when retention is raised later. +- Milestone copy now says the retained hourly window. +- Fixture snapshots moved to recent hours so the test still sees a spike. + +--- + +## 5. Monthly leaderboard at 30-day scale + +M5 EXPLAIN (production, 8.9M rows, 30d predicate covering the whole table): + +```text +type=index key=uq_artwork_bucket rows~8.87M Using where +GROUP BY artwork_id MAX-MIN +``` + +At 36M rows the plan stays a covering unique-index scan, ~4× current row volume. `leaderboards:refresh` is hourly, `withoutOverlapping`, not on the HTTP path. + +**Acceptable** for a scheduled job. If wall time exceeds a few minutes after warm-up, rewrite to range on `idx_bucket_artwork` or pre-aggregate — not required now. No load test was run on production. + +24h rising/heat already use `idx_bucket_artwork` range (~1.2M rows) and stay fine. + +--- + +## 6. Pruning (verified in code) + +| Requirement | Implementation | +| ----------- | ---------------- | +| Default 30 days | `config/metrics.php` / `ARTWORK_METRIC_HOURLY_RETENTION_DAYS` | +| Bounded deletes | `--chunk` default **5000**, PK `id IN (...)` | +| Sleep | `--sleep-ms` default **50** | +| No giant transaction | one DELETE per batch | +| No self-overlap | `withoutOverlapping(120)` | +| Logs | start, each batch, completed | +| Resume next day | cutoff is `now - keep-days`; leftover rows delete on the next run | +| Scheduler time | **04:25** (was 04:00) | + +--- + +## 7. Snapshot writer vs prune + +| Job | Time | +| --- | ---- | +| `nova:metrics-snapshot-hourly` | `hourlyAt(2)` → **04:02** upserts **current** hour (~50k rows) | +| `nova:prune-metric-snapshots` | **04:25** deletes `bucket_hour < now-30d` | + +They no longer share 04:00–04:02. Locks: prune deletes old PK/`bucket_hour` range; writer upserts `(artwork_id, current hour)`. Different unique-key tuples. Residual risk is IO only, not a correctness lock. + +`withoutOverlapping` is per schedule **name**, so the two jobs can still run together if prune overruns into 05:02. Stagger + 50ms sleeps keep that unlikely. + +--- + +## 8. Deployment sequence (later; not M5.1) + +1. Deploy this code (window helper, Studio KPIs, journey lookback, prune 04:25). +2. Set `ARTWORK_METRIC_HOURLY_RETENTION_DAYS=30`. +3. `config:cache`. +4. Optional: `php artisan nova:prune-metric-snapshots --dry-run` (read-only). +5. Do **not** OPTIMIZE TABLE. +6. Expect table growth toward ~7.5 GB over ~3 weeks; Studio shows “N of 30 days” until then. + +**Rollback:** env=7; Studio KPIs still MAX−MIN (correct on whatever history exists). + +--- + +## 9. Tests + +- `tests/Feature/Metrics/ArtworkHourlySnapshotWindowTest.php` (SUM vs delta, new artwork, missing baseline, partial history, zero growth) +- `tests/Feature/Metrics/LeaderboardSnapshotDeltaTest.php` (daily/weekly/monthly) +- `tests/Feature/StudioTest.php` (snapshot_window on dashboard) +- creator journey retention window +- existing prune tests + +--- + +## 10. Files changed + +```text +app/Services/Metrics/ArtworkHourlySnapshotWindow.php +app/Services/LeaderboardService.php +app/Services/Studio/StudioMetricsService.php +app/Services/Studio/CreatorStudioOverviewService.php +app/Services/Profile/CreatorJourneyService.php +resources/js/Pages/Studio/StudioDashboard.jsx +routes/console.php +config/metrics.php +tests/Feature/Metrics/ArtworkHourlySnapshotWindowTest.php +tests/Feature/Metrics/LeaderboardSnapshotDeltaTest.php +tests/Feature/StudioTest.php +tests/Feature/Profile/CreatorJourneyTest.php +docs/optimization-m5.1-metric-snapshot-correctness.md +``` + +--- + +## 11. Risks + +- Studio 30d KPIs now **artwork-only** (hourly snapshots). Cards/collections/stories lifetime totals no longer inflate those tiles. +- Cached `studio.kpi.{id}` 5 minutes. +- Table-level `window_complete` is global, not per-user (correct for production warm-up). +- Monthly boards remain unlabeled for incomplete months. + +M5+M5.1 together are **safe to deploy** with env=30 after this code is on the server. Do not deploy env=30 onto the old SUM path. diff --git a/docs/optimization-m6-php-fpm-memory.md b/docs/optimization-m6-php-fpm-memory.md new file mode 100644 index 00000000..06bd279e --- /dev/null +++ b/docs/optimization-m6-php-fpm-memory.md @@ -0,0 +1,357 @@ +# M6 — PHP-FPM, Memory, Swap & Worker Capacity + +```text +STATUS: COMPLETE (read-only; no production changes) +WHEN: 2026-08-23 15:29–15:32 Europe/Ljubljana (UTC 13:29) +HOST: server3, Debian 13, kernel 6.12.96, 8 cores, uptime 24d 18h +``` + +Nothing was restarted, tuned, flushed, or edited on server3. + +--- + +## Verdict + +```text +Swap: KEEP 2 GiB. Nearly full, but NOT thrashing (si/so=0, PSI memory=0). +PHP-FPM max_children=14: KEEP. Peak has used all 14; listen queue never built. +Do not raise FPM/Horizon on this host until RAM is freed or the machine is larger. +InnoDB 10 GB pool: KEEP for 30-day snapshot growth (~7.5 GB table). Currently ~24% filled. +OPcache 256 MB: KEEP (runtime used unknown; config is not tight on paper). +SSR: ~116 MB RSS; not a memory problem. Frequent SIGTERM restarts are operator/supervisor, not OOM. +Redis: 879 MB used / 2 GB max; RSS 167 MB — likely swapped. 0 evictions. +``` + +Largest memory consumers are **MySQL (8.1 GB RSS)**, **vision Docker/Python (~2.5 GB)**, **ClamAV (1.1 GB)** — not Skinbase FPM. + +--- + +## 1. Server RAM / swap + +| Metric | Value | +| ------ | ----- | +| MemTotal | 23 GiB (24,615,736 kB) | +| Mem used | 15 GiB | +| MemAvailable | **8.2 GiB** | +| Buffers+Cached | 7.2 GiB | +| AnonPages | 13.4 GiB | +| SwapTotal | 2.0 GiB | +| Swap used | **2.0 GiB (37 MiB free)** | +| SwapCached | 774 MiB | +| Committed_AS | 30.1 GiB | +| vm.swappiness | 10 | +| vm.overcommit_memory | 0 | +| Load | 5.09 / 7.07 / 8.33 (8 CPUs) | +| PSI memory | some/full avg10=**0.00** | +| PSI cpu | some avg10=4.7, avg60=14.5 | +| vmstat si/so (1s×5) | **0 / 0** | +| Processes | 243 | + +**Swap interpretation:** cold / previously used pages sitting in swap, with **774 MiB SwapCached**. Not active thrashing. Do **not** disable swap. Do **not** resize without a RAM plan. swappiness=10 is already conservative. + +--- + +## 2. Process memory (RSS, measured) + +| Process | RSS | Notes | +| ------- | --: | ----- | +| mysqld | **8087 MB** | InnoDB pool configured 10 GB | +| uvicorn/python (vision, port 8000) | **1310 + 542 + 238 + 197 + 103 + 54 MB** | Docker vision stack | +| clamd | **1107 MB** | | +| qdrant | 262 MB | vision | +| meilisearch | 395 MB | Skinbase search | +| crowdsec | 296 MB | | +| redis-server | 171 MB RSS / **879 MB** `used_memory` | swapped | +| netdata | 180 MB | | +| gitea | 156 MB | | +| php-fpm master | 48 MB | | +| php-fpm pool skinbase (×6) | **88–110 MB** each | see §4 | +| php-fpm pool www (×2) | 13 MB each | unused by Skinbase vhost | +| horizon master | 104 MB | | +| horizon supervisors (×3) | ~103 MB each | | +| horizon workers (×6 now) | 103–125 MB | `--memory=128` | +| node SSR | 116 MB | uptime minutes (restarted) | +| reverb | 106 MB | | +| nginx workers | 10–58 MB | one “shutting down” 15d | +| php8.4-fpm cgroup | current **746 MB**, peak **1575 MB** | all pools | + +--- + +## 3. PHP-FPM Skinbase pool (`php8.4-fpm-skinbase.sock`) + +From `/etc/php/8.4/fpm/pool.d/skinbase.conf`: + +| Setting | Production value | +| ------- | ---------------- | +| pm | **dynamic** | +| pm.max_children | **14** | +| pm.start_servers | 4 | +| pm.min_spare_servers | 2 | +| pm.max_spare_servers | 6 | +| pm.process_idle_timeout | unset (N/A for dynamic) | +| pm.max_requests | **500** | +| request_terminate_timeout | 120s | +| request_slowlog_timeout | 10s | +| slowlog | `/var/log/php8.4-fpm-skinbase-slow.log` | +| pm.status_path | `/fpm-status` (localhost) | +| ping.path | `/fpm-ping` | +| listen.backlog | unset → PHP default **511** | +| memory_limit | **256M** (pool admin) | +| max_execution_time | 60 | + +Separate `www` pool: `pm.max_children=5` on `/run/php/php8.4-fpm.sock`. + +--- + +## 4. FPM status (localhost `/fpm-status`, pool up ~28.7h since 2026-08-22 10:50) + +| Field | Value | +| ----- | ----- | +| accepted conn | 214,108 | +| listen queue | **0** | +| max listen queue | **0** | +| listen queue len | 0 | +| idle processes | 5 | +| active processes | 1 | +| total processes | 6 | +| max active processes | **14** | +| max children reached | **10** | +| slow requests (since start) | **2** | +| memory peak (FPM counter) | 60 MB | + +**Not currently exhausting workers.** Peak has used all 14 children. No socket backlog was recorded, so nginx did not sit behind a full listen queue — capacity is **tight at peak, idle most of the time**. + +Worker RSS (KB): 87712, 89996, 106444, 109308, 109476, 109480. + +| Stat | RSS | +| ---- | --: | +| min | 86 MB | +| median | **108 MB** | +| mean | **102 MB** | +| max | **110 MB** | +| PSS | not readable (permission) | +| oldest worker | ~7.5 min (max_requests=500 recycling) | + +`memory_limit` 256M is the ceiling, not typical RSS. **Do not size max_children from 256M × 14.** + +Safe children from measured RSS: + +```text +typical_rss ≈ 110 MB +14 × 110 MB ≈ 1.54 GB (matches systemd MemoryPeak 1.58 GB for all FPM) +14 × 256 MB ≈ 3.58 GB worst-case if every worker hits the PHP limit +``` + +Raising max_children on a host with **swap already full** and AnonPages 13.4 GiB is not justified. **Keep 14.** + +--- + +## 5. OPcache (config; FPM runtime stats not exposed) + +| Setting | Value | +| ------- | ----- | +| enable | 1 | +| memory_consumption | 256 MB | +| interned_strings_buffer | 32 MB | +| max_accelerated_files | 50,000 | +| validate_timestamps | 1 | +| revalidate_freq | 2 s | +| JIT | disable / 0 | + +CLI cannot read the FPM cache. 50k file slots and 256 MB are large for this Laravel app. **Do not increase.** Optional later (MEDIUM, not memory): `validate_timestamps=0` after atomic releases. + +--- + +## 6. Horizon (Supervisor `skinbase-horizon`) + +Production env (`config/horizon.php` on the server): + +| Supervisor | queues | maxProcesses | timeout | tries | memory | +| ---------- | ------ | ------------: | ------: | ----: | -----: | +| supervisor-default | search, default | 5 | 960 | 1 | 128 | +| supervisor-messaging | broadcasts, notifications | 3 | 90 | 1 | 128 | +| supervisor-mail | mail | 2 | 90 | 5 | 128 | + +Measured now: 1 master + 3 supervisors + 6 workers ≈ **1.15 GB RSS**. + +Peak if all maxProcesses spawn: 1+3+5+3+2 = **14 PHP procs × ~110 MB ≈ 1.5 GB**. + +Mail isolation is intact (`tries=5`, own supervisor). **Do not merge queues.** + +Redis lists (prefix as used by Laravel): + +| List | LLEN | +| ---- | ---: | +| queues:default | **796** | +| queues:mail | 0 | +| queues:search | 0 | +| queues:broadcasts | 0 | +| queues:notifications | 0 | + +**HIGH:** default queue depth 796 — worker **throughput**, not FPM RAM. Investigate job mix in a later milestone; do not add Horizon processes until RAM is free (each extra worker ≈ 110 MB and `--memory=128` is already near RSS 125 MB on default). + +Horizon last supervisor restart: 2026-08-23 14:36 (clean exit 0 / SIGTERM wait), not OOM. + +--- + +## 7. SSR + +- RSS **116 MB**, Node `/opt/www/virtual/SkinbaseNova/bootstrap/ssr/ssr.js` +- Supervisor restarts today: 14:22, 14:30, 14:47, 15:14, 15:16, 15:28 — all **SIGTERM** “waiting to stop”, then spawn. Not crash loops from memory. +- Heap not sampled (would require attaching to Node). +- **Not material** to host memory pressure. + +--- + +## 8. Redis (read-only via Laravel) + +| Field | Value | +| ----- | ----- | +| used_memory | 879 MB (peak 917 MB) | +| used_memory_rss | **167 MB** | +| maxmemory | 2.00 GB | +| policy | allkeys-lru | +| fragmentation_ratio | **0.19** | +| evicted_keys | **0** | +| expired_keys | 225,783 | +| connected_clients | 28 | +| blocked_clients | 0 | +| keys | 3773 (3026 with TTL) | +| AOF | off | +| ops/sec | ~50 | +| hit/miss | 1.46M / 1.39M | + +frag 0.19 + RSS << used_memory ⇒ **Redis pages are in swap**. Latency risk under cache bursts. 0 evictions: cache is within 2 GB. Do not flush. Do not raise maxmemory. + +--- + +## 9. MySQL / Percona (read-only) + +| Field | Value | +| ----- | ----- | +| innodb_buffer_pool_size | **10.00 GB** (10 instances) | +| pages_data / total | 156,554 / 655,360 = **23.9%** | +| pages_free | 498,726 | +| pool hit | 1 − 104,104 / 23,840,832,084 ≈ **99.9996%** | +| max_connections | 120 | +| Max_used_connections | 26 | +| Threads_connected / running | 12 / 3 | +| tmp tables / disk tmp | 1,987,237 / **21** | +| innodb_redo_log_capacity | 2 GB | +| Slow_queries | 11,997 (uptime 103,663 s) | + +10 GB pool is **oversized for today’s ~2.8 GB schema**, but M5 30-day hourly snapshots grow toward **~7.5 GB**. Keeping 10 GB avoids refitting later. The table does **not** need to sit entirely in the pool; hit ratio is already excellent. **Do not shrink now; do not grow.** + +--- + +## 10. Combined memory budget (measured / plausible peak) + +| Bucket | Steady now | Plausible peak | +| ------ | ---------: | -------------: | +| OS/page cache (reclaimable) | ~7 GB cache | shrinks under pressure | +| MySQL | 8.1 GB | ~10 GB pool | +| Vision Docker/Python/Qdrant | ~2.7 GB | similar | +| ClamAV | 1.1 GB | similar | +| Meilisearch | 0.40 GB | 0.5 GB | +| Crowdsec | 0.30 GB | similar | +| Redis RSS | 0.17 GB | 2 GB if unswapped | +| PHP-FPM Skinbase | 0.65 GB (6×110) | **1.54 GB** (14×110) | +| PHP-FPM www | 0.03 GB | 0.06 GB | +| Horizon | 1.15 GB | **1.5 GB** | +| SSR + Reverb | 0.22 GB | 0.3 GB | +| nginx | 0.20 GB | 0.25 GB | +| Netdata/Gitea/journald/docker | ~0.5 GB | similar | +| Swap | 2 GB **in use** | — | + +Steady anonymous ~14 GB + 2 GB swap explains the full swap file while **8.2 GB still Available** as cache. Peak Skinbase (FPM 14 + Horizon 14) ≈ **3.0 GB**, already observed in FPM cgroup peak 1.58 GB. + +--- + +## 11. PHP-FPM recommendation + +**NO CHANGE** to pool sizing. + +```text +pm = dynamic +pm.max_children = 14 # keep; peak already 14, listen queue 0 +pm.start_servers = 4 # keep +pm.min_spare_servers = 2 # keep +pm.max_spare_servers = 6 # keep (matches current total=6) +pm.max_requests = 500 # keep; workers recycle in minutes +``` + +Calculation: median RSS 108 MB; 14×108≈1.5 GB. Available RAM is cache, not idle anonymous. Raising children increases swap of Redis/MySQL. Lowering below 14 would clip the measured peak (`max active processes=14`). + +--- + +## 12. Swap recommendation + +| Option | Decision | +| ------ | -------- | +| Keep 2 GiB swap | **YES** | +| Resize swap | **NO** (overflow is working; host needs RAM or fewer co-tenants) | +| Change vm.swappiness | **NO** (already 10) | +| Disable swap | **NO** | + +--- + +## 13. OOM / crash safety + +| Check | Result | +| ----- | ------ | +| Kernel OOM (`journalctl -k` / dmesg) | **no lines** visible to this user | +| PHP-FPM worker memory | **YES** — `Allowed memory size of 268435456` Predis `StreamConnection.php` 2026-08-13 and 2026-08-18 | +| Redis OOM / evictions | **no** (evicted_keys=0) | +| MySQL | no evidence in readable logs | +| Horizon | clean supervisor stop 14:36, not killed | +| SSR | SIGTERM restarts, not OOM | + +--- + +## 14. Ranked findings + +| Sev | Finding | Change now? | +| --- | ------- | ----------- | +| HIGH | Swap 2 GiB ~full; Redis RSS≪used_memory | No sysctl. Free RAM (vision/clam) or larger VM later | +| HIGH | `queues:default` LLEN=**796** | Not FPM. Later queue milestone; don’t add Horizon RAM | +| HIGH | Shared host: vision ~2.7 GB + clamd 1.1 GB | Out of Skinbase app; ops placement | +| MEDIUM | FPM `max_children` reached 10×, max active 14 | Keep 14; watch listen queue | +| MEDIUM | PHP 256 MB fatals on Predis | App/job payload size, not pool size | +| MEDIUM | slowlog 10s, 139k historical lines; status slow=2 since 22 Aug | Leave timeout; log rotation ops | +| LOW | nginx worker “shutting down” 15d | Recycle nginx in a maintenance window | +| LOW | www pool unused by Skinbase vhost | Optional disable later | +| NO CHANGE | OPcache 256/32/50k | | +| NO CHANGE | InnoDB 10 GB | Needed as snapshots grow | +| NO CHANGE | Horizon isolation / mail supervisor | | +| NO CHANGE | SSR RSS | | + +--- + +## 15. Expected benefit of doing nothing to FPM + +Avoid swapping MySQL/Redis further. Current request path is not FPM-bound (`active=1`, `listen queue=0`). + +--- + +## 16. Deployment plan (only if a change is later approved) + +M6 recommends **no FPM/swap/sysctl/MySQL/Horizon count change**. + +If ops later moves vision off-box or adds RAM: + +1. Re-measure RSS and `/fpm-status`. +2. Then consider `pm.max_children` using **new** median RSS, not 256M. +3. Pool edit + `systemctl reload php8.4-fpm` (reload, not restart if possible). +4. Rollback: restore `skinbase.conf` and reload. + +Any `pm.*` or `memory_limit` change **requires FPM reload**. Swap/sysctl/MySQL buffer pool need service-level restarts — not proposed. + +--- + +## 17. Risks of increasing max_children without more RAM + +More anonymous PHP, more swap of Redis (cache latency) and InnoDB (buffer pool eviction), possible FPM 256 MB fatals still happen per-request. + +--- + +Production was not modified. diff --git a/docs/optimization-m7-default-queue-backlog.md b/docs/optimization-m7-default-queue-backlog.md new file mode 100644 index 00000000..9d1363db --- /dev/null +++ b/docs/optimization-m7-default-queue-backlog.md @@ -0,0 +1,186 @@ +# M7 — Default Queue Backlog & Redis/PHP Memory Failures + +```text +STATUS: COMPLETE (investigation + local smallest fixes; production unchanged) +WHEN: 2026-08-23 17:49 Europe/Ljubljana +``` + +No Horizon restart, Redis flush, job retry, memory_limit change, or worker-count change on server3. + +--- + +## Verdict + +```text +Default backlog class: C + D (producer > consumer; ranking/rec jobs block index jobs) +Default depth: 1208 → 1213 in 60s (net +5/min, growing) +Oldest default job: ~2.5 h +Search queue: 0 (idle workers exist on supervisor-default) +collections: 32,632 jobs, oldest 2026-04-30, NO Horizon consumer +forum-moderation: 552,888 AnalyzeForumPostJob, ~557 MB list, NO consumer +forum-security: 5,533 jobs, no consumer +presence index: SET 2,480,292 members, ~272 MB, TTL -1 +Predis 256 MB fatals: loading those huge Redis replies over HTTP/FPM +Horizon workers: do NOT increase +search+default supervisor: KEEP together; put Index* jobs on search +``` + +--- + +## 1. `queues:default` contents (metadata only) + +| Class | Count | Avg payload | +| ----- | ----: | ----------: | +| `RankBuildScopeListsJob` | **588** | 1004 B | +| `IndexUserJob` | **395** | 954 B | +| `RecComputeSimilarByTagsJob` | **169** | 1212 B | +| `IndexArtworkJob` | 40 | 918 B | +| `RecalculateRisingNovaCardsJob` | 10 | 987 B | +| `RebuildTrendingNovaCardsJob` | 3 | 977 B | +| `RankBuildListsJob` | 2 | 931 B | +| `RecBuildItemPairsFromFavouritesJob` | 1 | 1102 B | + +- Oldest pushedAt: **2026-08-23 15:19** (~2.5 h) +- Newest: 17:48 (~75 s) +- Delayed: 1 `RankBuildScopeListsJob` +- Reserved: 0 +- Attempts: 1206×0, 2×1 +- Max payload: **1212 B** — not oversized job payloads + +Net 60s: 1208 → 1213 (**accumulating**). + +--- + +## 2. Throughput + +Horizon metrics repo empty from this user (prefix split `skinbase_horizon` vs `skinbasenova_horizon`). + +Evidence: backlog growing +5 net/min while 5 default-supervisor processes exist. Ranking jobs `timeout=300s`; rec tag jobs `timeout=600s`. A few long jobs starve `IndexUserJob` (30s, user-facing search). + +--- + +## 3. Producers + +| Producer | Job | Queue | Cadence | Volume | +| -------- | --- | ----- | ------- | ------ | +| `RankBuildListsJob` hourlyAt(15) | `RankBuildScopeListsJob` per category/content type | default | hourly fan-out | **~588/hour** still sitting | +| `UserStatsService::reindex` | `IndexUserJob` | **default** (bug; Scout config is `search`) | per stats change | 395 queued | +| Artwork indexers | `IndexArtworkJob` | **default** (same bug) | publish/trending/tags | 40 queued | +| Scheduler rec nightly | `RecComputeSimilarByTagsJob` batches | default (`RECOMMENDATIONS_QUEUE`) | nightly + leftover | 169 | +| Nova card crons | Rising/Trending rebuild | default | 15 min / hourly | 13 | +| `collections:dispatch-maintenance` hourlyAt(43) | Health/rec/dup | **collections** | hourly ~110 | **32k unconsumed** | +| Forum plugin | `AnalyzeForumPostJob` | **forum-moderation** | continuous | **552,888 unconsumed** | + +--- + +## 4. Isolation + +Move **IndexArtworkJob / IndexUserJob** to `search` (already on supervisor-default, currently idle). Same worker budget, better latency. Do **not** add processes. + +Do **not** merge mail. Rec stays on default until ranking unique + M1 afterArtworkId is healthy. + +**collections** and **forum-moderation** have **no Horizon supervisor** — isolation without a consumer is a dump. Do not add workers until RAM is freed (M6). Stop/limit dispatch first. + +--- + +## 5. Predis 256 MB fatals (13 Aug, 18 Aug) + +FPM error log (Skinbase pool, HTTP): + +```text +Allowed memory size of 268435456 bytes exhausted +(tried to allocate 67108872 bytes) +vendor/predis/predis/src/Connection/StreamConnection.php:212 +``` + +Two Redis values large enough to explode a 256 MB FPM worker when Predis reads the reply: + +| Key | Type | Size | How it is read | +| --- | ---- | ---- | -------------- | +| `queues:forum-moderation` | list | **552,888 jobs ≈ 557 MB** | Horizon UI / any `LRANGE` of the queue | +| `skinbase:presence:online:index` | set | **2,480,292 members ≈ 272 MB** | `OnlineVisitorRepository::readIndexMembers()` → **`SMEMBERS`** then `all()` on moderation traffic page | + +Index members have **TTL -1**; per-visitor keys TTL 300s. Expired visitors are never removed unless `all()` finishes — it cannot, so the set grows forever. + +**Do not raise memory_limit.** Stop loading the full set/list in one Redis command. + +Legacy prefix `skinbasenova-database-queues:*` still holds leftover lists (another ~11 MB default + forum copies). + +--- + +## 6. Payload size + +Default-queue jobs are **~1 KB**. Not the memory issue. + +--- + +## 7. Largest Redis keys/prefixes (SCAN, names redacted) + +| Group | Approx | +| ----- | -----: | +| `queues:forum-moderation` | 557 MB | +| `skinbase:presence:online:index` | 272 MB | +| `queues:collections` | 35 MB | +| `skinbasenova-database-queues:default` (legacy prefix) | 11 MB | +| `queues:forum-security` | 5.5 MB | +| Horizon recent/pending (legacy prefix) | ~3 MB | + +Keyspace ~3.7k keys but a few lists/sets dominate `used_memory` (~879 MB). + +--- + +## 8. search vs default supervisor + +**Keep sharing supervisor-default.** Search workers are idle because index jobs were on `default`. After Index* → `search`, auto-balance can drain indexing without extra RAM. + +--- + +## 9. Backlog classification + +**Default: C+D** — hourly ranking fan-out + 10-minute rec jobs exceed 5 workers; index jobs wait behind them. Not a retry storm (attempts mostly 0). + +**collections / forum-*: F** — orphan queues, growing for months (collections from 2026-04-30; forum still enqueuing today). + +**failed_jobs:** 133 rows, **all today 14:37–14:56** (Horizon SIGTERM 14:36). Classes: RecCompute* `Typed property $afterArtworkId must not be accessed before initialization` on release `20260823-12*` — production rec job constructor mismatch. Enhance jobs: curl 127.0.0.1:8095 refused. **Do not retry.** + +--- + +## 10. User-facing latency + +`IndexUserJob` / `IndexArtworkJob` wait behind ranking/rec on default. Oldest index jobs share the 2.5 h window. Emails/notifications not on default (mail/notify queues empty). Uploads/vision queue empty. + +--- + +## 11. Redis latency vs swap + +ops/sec ~42, blocked clients 0, evicted 0. Slowlog parse inconclusive. Swapped Redis **will** hurt when someone reads a 557 MB list (that *is* the 256 MB fatal). Queue pop of 1 KB jobs is small. + +--- + +## 12. Smallest fix (local; not deployed) + +1. `IndexArtworkJob` + `IndexUserJob` → `scout.queue.queue` (`search`) + `ShouldBeUnique`. +2. `RankBuildScopeListsJob` `ShouldBeUnique` per scope so hourly fan-out cannot stack 588 copies. +3. Presence index: **SSCAN + cap 2000**, never `SMEMBERS` of 2.4M. + +**Not in this deploy:** delete 552k forum jobs, add Horizon supervisors, raise PHP memory, raise Horizon processes. + +**After RAM (M6):** optional `supervisor-collections` / forum workers, or stop those dispatchers. + +--- + +## 13. Tests + +`tests/Feature/Queue/DefaultQueueAssignmentTest.php` + +--- + +## 14. Deploy + +Ship the three job/presence changes. Reload Horizon so new job `onQueue` applies to **new** dispatches (existing 395 IndexUserJob stay on default until drained). No FPM memory_limit change. No Redis delete. + +Rollback: revert those files; Horizon reload. + +Risk: unique ranking skips a scope if a stale unique lock remains (`uniqueFor=360s`). Presence admin UI shows at most 2000 index members until a later prune job (write) is designed. + +Production not modified. diff --git a/docs/optimization-m7.1-queue-compatibility.md b/docs/optimization-m7.1-queue-compatibility.md new file mode 100644 index 00000000..56a7f7f3 --- /dev/null +++ b/docs/optimization-m7.1-queue-compatibility.md @@ -0,0 +1,217 @@ +# M7.1 — Queue Compatibility & Backlog Containment + +```text +STATUS: COMPLETE + FINALIZED (local; production unchanged) +``` + +--- + +## 1. RecCompute serialization root cause + +M1 added constructor-promoted `private readonly ?int $afterArtworkId = null`. + +**Constructor defaults are not applied on `unserialize`.** PHP 8 leaves a **typed property uninitialized** if it is missing from the serialized payload. First access (handle, failed(), logging) throws: + +```text +Typed property App\Jobs\RecComputeSimilar*::$afterArtworkId must not be accessed before initialization +``` + +Old queued jobs (pre-M1, two constructor args) and Laravel `SerializesModels` (skips default-null properties on serialize) both omit the key. + +**Fix:** class-level `private ?int $afterArtworkId = null` (not readonly promotion) + `__unserialize` calls `isset` restore. Getter `afterArtworkId()` never reads uninitialized state. + +Cursor batching unchanged: `WHERE id > afterArtworkId`, `ORDER BY id`, `LIMIT batch`, no OFFSET. + +--- + +## 2. Stale Tags jobs — worst case (live 2026-08-23 18:xx) + +Live `queues:default` RecCompute* (not failed_jobs): + +- **Behavior: 0 waiting** +- **Hybrid: 0 waiting** +- **Tags: 169 waiting**, all `artworkId=null` (batch mode), **all already have `afterArtworkId` set** (cursors 1284–4517). These are overlapping remainder chains, not missing-property payloads. + +What `afterArtworkId=null` would do: process first 200 public artworks ordered by id, then `dispatch(null, 200, lastId)` until the table is exhausted. One full rebuild ≈ `ceil(N/200)` jobs. With ~50k snapshot-eligible artworks that is **~250 jobs per chain**. + +If all 169 were null-cursor roots: **169 × ~250 ≈ 42,000 tag jobs** (duplicate full rebuilds). That is the theoretical worst case. + +**Actual live worst case:** they continue from mid-table. Example: 47 jobs with `after=3709` each walk the remainder (~232 batches) → **~11,000 overlapping jobs from that cursor group alone**, plus the other cursor groups. Still dangerous. + +**Do not clear `queues:default` wholesale.** Operator command (not scheduled): + +```bash +php artisan skinbase:purge-queued-rec-tags --queue=default --dry-run +php artisan skinbase:purge-queued-rec-tags --queue=default --execute +``` + +Removes **waiting** `RecComputeSimilarByTagsJob` only. Preserves ranking/index/other. Does not touch reserved or `failed_jobs`. Then tonight’s scheduler dispatches **one** clean Tags rebuild (`afterArtworkId=null` once). + +--- + +## 3. Failed-job breakdown (complete, n=133, all queue=default) + +M7 sampled 80 rows. Full table: + +| Class | Count | Cause | +| ----- | ----: | ----- | +| RecComputeSimilarHybridJob | **66** | `$afterArtworkId` uninitialized | +| RecComputeSimilarByBehaviorJob | **42** | same | +| RecComputeSimilarByTagsJob | **10** | same | +| Enhance\ProcessEnhanceJob | **8** | curl 127.0.0.1:8095 refused | +| App\Mail\AcademyAccessIssue | **7** | mail on default | +| **Total** | **133** | | + +66+42+10+8+7=133. Do not retry. + +--- + +## 4. Index* queue + +`IndexArtworkJob` / `IndexUserJob` use `config('scout.queue.queue')` (default `search`). + +`ShouldBeUniqueUntilProcessing`: + +| | IndexUser | IndexArtwork | +| --- | --- | --- | +| uniqueId | userId | artworkId | +| uniqueFor | 60s | 120s | + +handle() loads the model from DB, so a dropped duplicate while queued still indexes **current** stats. After the worker **starts**, a later change can enqueue again (not lost). Dead-worker lock expires at uniqueFor. + +--- + +## 5. Ranking dedupe + +`RankBuildScopeListsJob` `ShouldBeUnique`, `uniqueId = scopeType:scopeId`. + +**uniqueFor=21600 (6 hours)** from `ranking.scope_job_unique_for` / `RANK_SCOPE_JOB_UNIQUE_FOR`. 360s expired long before a 2.5h wait, so hourly fan-out stacked duplicates. 6h > current wait + 300s timeout; dead locks recover the same day. Different scopes stay independent. + +--- + +## 6. Presence memory + +`SMEMBERS` removed. `SSCAN` + cap `traffic.online_visitors.index_read_limit` (default 2000). Admin UI may under-count vs 2.48M stale members until prune. + +### Stale-member prune (design only — not executed) + +1. Detect: SSCAN members; `GET skinbase:presence:online:{member}`; if missing/expired JSON, member is stale. +2. Batches: SSCAN COUNT 200; `SREM` up to 200 stale ids; sleep 50ms. +3. Cost: 2.48M SSCAN + GET. At 200/batch ~12k round trips. Off-peak 5–15 min. Do not `DEL` the index key (would drop live members). +4. Cadence: hourly `withoutOverlapping`, stop after N batches/run. + +--- + +## 7. Orphan producers + +| Queue | Consumer? | Enabled? | Useful if months late? | Dispatch today? | M7.1 | +| ----- | --------- | -------- | --------------------- | --------------- | ---- | +| collections | **No** Horizon supervisor | scheduler hourlyAt(43) | Refresh is idempotent; delayed run OK | yes until gate | **`COLLECTIONS_V5_DISPATCH_ENABLED=false`** (default) | +| forum-moderation | **No** Horizon supervisor | gated | **No** for months-old posts | was yes | **`dispatchAsyncScan()` now returns without queueing when `skinbase_ai_moderation.queue.dispatch_enabled` is false.** Topic create/edit and `forum:ai-scan` (async) are gated. `forum:scan-posts` and `forum:ai-scan --sync` stay in-process. | +| forum-security | **No** | gated | stale in hours | 5.5k | **`forum:bot-scan` / `forum:firewall-scan` async dispatch gated.** `--sync` still runs. Scheduler omits those commands when dispatch is false. | + +Do not add Horizon workers (M6 RAM). + +--- + +## 8. Existing orphan jobs (do not delete) + +| Queue | Class | Disposition | +| ----- | ----- | ----------- | +| collections | Health / recommendation / duplicate | **KEEP AND PROCESS LATER** or **SAFE TO DISCARD** (recompute from current rows). Product: either. | +| forum-moderation | AnalyzeForumPostJob × 552,888 | **SAFE TO DISCARD** for age > 48h; **NEEDS PRODUCT DECISION** for last day. Analyzing May–August posts has no moderation value. | +| forum-security | ~5.5k | **SAFE TO DISCARD** (burst/firewall windows expired). | + +Later cleanup (not now): `UNLINK queues:forum-moderation` (async, non-blocking) after stopping producers; or `LTRIM`/`RPOP` in 5k batches. Never one giant `DEL` in a request. + +--- + +## 9. Legacy prefix `skinbasenova-database-queues:*` + +Current Laravel prefix is `skinbase-database-`. `skinbasenova-database-*` is leftover from an older `REDIS_PREFIX` / app name. Nothing in this codebase reads it. Horizon also has `skinbasenova_horizon:*` vs `skinbase_horizon:*`. + +Reclaimable: ~11 MB default + ~9 MB forum + ~2.5 MB collections + Horizon recent lists (~3 MB) ≈ **25–30 MB** (not the 557 MB live forum list). + +Cleanup only after confirming no process uses `skinbasenova-database-` (`HORIZON_PREFIX`, `REDIS_PREFIX` on all PHP/Horizon). Then `UNLINK` those keys. Do not delete `skinbase-database-queues:forum-moderation`. + +--- + +## 10. Files changed + +```text +app/Jobs/Concerns/RestoresAfterArtworkIdCursor.php +app/Jobs/RecComputeSimilarByTagsJob.php +app/Jobs/RecComputeSimilarByBehaviorJob.php +app/Jobs/RecComputeSimilarHybridJob.php +app/Jobs/IndexArtworkJob.php +app/Jobs/IndexUserJob.php +app/Jobs/RankBuildScopeListsJob.php +app/Services/Traffic/OnlineVisitorRepository.php +app/Services/CollectionBackgroundJobService.php +app/Console/Commands/DispatchCollectionMaintenanceCommand.php +routes/console.php +config/collections.php +config/forum_security.php +config/skinbase_ai_moderation.php +config/traffic.php +.env.example +phpunit.xml +tests/... +docs/optimization-m7.1-queue-compatibility.md +``` + +--- + +## 11. Tests + +15 passed: RecCompute unserialize, cursor batching, Index* search queue, ranking uniqueId per scope, presence SSCAN/no SMEMBERS, collection dispatch gate. + +--- + +## 12. Deploy sequence (later; not now) + +1. Ship this release (jobs, uniqueness, presence SSCAN, collection gate, purge command). +2. Production `.env`: + ```text + COLLECTIONS_V5_DISPATCH_ENABLED=false + RANK_SCOPE_JOB_UNIQUE_FOR=21600 + ``` + Do not rely on `FORUM_QUEUE_DISPATCH_ENABLED` until the Forum plugin is changed. +3. `php artisan config:cache` +4. `sudo supervisorctl restart skinbase-horizon` +5. Optional: `sudo systemctl reload php8.4-fpm` +6. **After Horizon is on the new code**, purge stale Tags (operator, not cron): + ```bash + php artisan skinbase:purge-queued-rec-tags --queue=default --dry-run + php artisan skinbase:purge-queued-rec-tags --queue=default --execute + ``` +7. Confirm `queues:default` RecComputeSimilarByTagsJob count is 0. Leave ranking/index jobs. +8. Tonight’s scheduler may dispatch **one** Tags rebuild. + +**Horizon restart: YES.** Do not retry `failed_jobs`. Do not `SMEMBERS` presence index. Do not `DEL` forum-moderation. + +--- + +## 13. Post-deploy verification + +```bash +php artisan tinker --execute="echo json_encode([ + 'default' => Redis::llen('queues:default'), + 'search' => Redis::llen('queues:search'), + 'collections' => Redis::llen('queues:collections'), + 'forum' => Redis::llen('queues:forum-moderation'), +]);" +# After some traffic, search LLEN should rise (Index* jobs), default ranking unique should stop stacking +# Do not retry failed_jobs +# Do not SMEMBERS skinbase:presence:online:index +``` + +--- + +## 14. Risks + +- 169 in-queue tag jobs become “start from id 0” batches; duplicate compute, extra load, no crash. +- Ranking unique skips a scope for up to 360s if a lock is stuck. +- Presence UI shows ≤2000 members until prune. +- Forum enqueue may continue if the plugin ignores `FORUM_QUEUE_DISPATCH_ENABLED`. +- Collection tests set dispatch enabled in phpunit; production default is false. diff --git a/docs/optimization-m8-redis-orphan-cleanup.md b/docs/optimization-m8-redis-orphan-cleanup.md new file mode 100644 index 00000000..c8a982c2 --- /dev/null +++ b/docs/optimization-m8-redis-orphan-cleanup.md @@ -0,0 +1,180 @@ +# M8 — Redis Orphan Cleanup & Presence Compaction + +```text +STATUS: COMPLETE (tooling + tests; no production UNLINK/SREM executed) +WHEN: 2026-08-23 ~18:55 Europe/Ljubljana +``` + +--- + +## 1. Production before snapshot (read-only) + +| Metric | Value | +| ------ | ----- | +| used_memory | **883.41 MB** (926,320,256) | +| used_memory_rss | **176.88 MB** | +| used_memory_peak | 917.11 MB | +| maxmemory | 2.00 GB | +| fragmentation_ratio | **0.20** (RSS ≪ used → swapped) | +| evicted_keys | 0 | +| connected_clients | 28 | +| blocked_clients | 0 | +| ops/sec | 92 | +| keys | 4838 (4100 with TTL) | +| REDIS_PREFIX | `skinbase-database-` | +| HORIZON_PREFIX | `skinbase_horizon:` | + +| Queue | llen | delayed | reserved | +| ----- | ---: | ------: | -------: | +| default | 0 | 0 | 0 | +| search | 0 | 0 | 0 | +| mail | 0 | 0 | 0 | +| collections | **32632** | 0 | 0 | +| forum-moderation | **553090** | 0 | 0 | +| forum-security | **5538** | 0 | 0 | + +Presence `SCARD` **2,480,634**. Sample 200 members: **200 stale / 0 live** (ratio 1.0). + +Legacy SCAN: `skinbasenova_horizon:` 662 keys ~7 MB; `skinbasenova-database-` 11 keys ~23 MB. + +--- + +## 2. Producer containment + +45s later: collections 32632, forum-moderation 553090, forum-security 5538 — **unchanged**. + +Config: collections/forum_ai/forum_sec **dispatch_enabled=false**. + +Newest forum payloads ~18:47 then **no growth**. **Proceed.** + +--- + +## 3. Queue class samples (LINDEX/LRANGE ends only, not full 553k) + +| Queue | Sample | Classes | Oldest | Newest | avg bytes | +| ----- | -----: | ------- | ------ | ------ | --------: | +| forum-moderation | 204 | **100% AnalyzeForumPostJob** | 2026-04-30 | 2026-08-23 18:47 | 987 | +| forum-security | 204 | FirewallActivityMonitor + BotActivityMonitor | 2026-04-30 | 2026-08-23 18:47 | 993 | +| collections | 204 | Duplicate / Health / Recommendation (~equal) | 2026-04-30 | 2026-08-23 17:43 | 1054 | + +No reserved/delayed. + +--- + +## 4. forum-moderation + +**UNLINK waiting + notify** after dry-run. + +Months-old `AnalyzeForumPostJob` has no moderation value. Command samples classes via LINDEX stride (no 553k LRANGE). Refuses if dispatch_enabled, reserved/delayed, or unexpected class (unless `--force`). + +--- + +## 5. forum-security + +**UNLINK.** Windowed monitors from April–August are expired. Same guards. + +--- + +## 6. collections + +**A. SAFE TO DISCARD AND REBUILD.** Jobs hold `collectionId` only; `handle()` loads current DB. Old queue is a 4-month backlog of the same refresh. Do not auto-delete; operator `--execute collections` when ready. Re-enable `COLLECTIONS_V5_DISPATCH_ENABLED` only after a Horizon consumer exists. + +--- + +## 7. Presence stale ratio + +Sample **100% stale** (200/200). Estimated ~2.48M ghost members, ~272 MB. + +--- + +## 8–9. Presence pruning + +`PresenceIndexPruner`: SSCAN, pipeline EXISTS, Lua `EXISTS record==0 then SREM`. Never SMEMBERS, never DEL index. + +Race: Lua is atomic vs SETEX. If a visitor is rewritten after SREM, they SADD again on next track. + +Schedule: **hourlyAt(33)** if `ONLINE_VISITOR_INDEX_PRUNE_ENABLED=true` (default **false**). Avoids :02 snapshot, :15 rank, :21 leaderboards, :25 nova, nightly rec 02:00. First catch-up: raise `--max-batches` manually (200 batches × 500 ≈ 100k members/run → ~25 hours at hourly default). + +--- + +## 10–11. Legacy prefix + +Current process uses `skinbase-database-` / `skinbase_horizon:`. Obsolete `skinbasenova-*` ~30 MB. Command uses an **unprefixed** Predis client, UNLINKs only `skinbasenova-database-` and `skinbasenova_horizon:` keys, never current prefixes. + +--- + +## 12. Expected reclaim (used_memory, RSS later) + +| Stage | ~used_memory | +| ----- | -----------: | +| forum-security | 5.5 MB | +| forum-moderation | **557 MB** | +| legacy prefix | 25–30 MB | +| presence (after full prune) | **~272 MB** | +| collections (if later) | 35 MB | +| **Total if all stages** | **~870 MB** toward a small Redis | + +RSS may stay ~177 MB until OS reclaims. No MEMORY PURGE. + +--- + +## 13. Operator commands (not run here) + +```bash +# A audit +php artisan tinker --execute="echo json_encode(['used'=>Redis::info('memory')['used_memory_human'],'mod'=>Redis::llen('queues:forum-moderation'),'sec'=>Redis::llen('queues:forum-security'),'col'=>Redis::llen('queues:collections'),'scard'=>Redis::scard('skinbase:presence:online:index')]);" + +# B forum-security +php artisan skinbase:redis-cleanup-orphans forum-security --dry-run +php artisan skinbase:redis-cleanup-orphans forum-security --execute + +# C forum-moderation +php artisan skinbase:redis-cleanup-orphans forum-moderation --dry-run +php artisan skinbase:redis-cleanup-orphans forum-moderation --execute + +# D legacy +php artisan skinbase:redis-cleanup-legacy-prefix --dry-run +php artisan skinbase:redis-cleanup-legacy-prefix --execute --max-keys=500 + +# E presence (repeat) +php artisan skinbase:prune-online-visitor-index --dry-run --max-batches=5 +php artisan skinbase:prune-online-visitor-index --execute --max-batches=200 +# then set ONLINE_VISITOR_INDEX_PRUNE_ENABLED=true and config:cache + +# F collections (optional later) +php artisan skinbase:redis-cleanup-orphans collections --dry-run +php artisan skinbase:redis-cleanup-orphans collections --execute +``` + +Abort: skip `--execute`. UNLINK is async; cannot restore payloads. Presence SREM is idempotent; live visitors re-SADD on next request. + +--- + +## 14. Files + +```text +app/Support/Redis/OrphanQueueCleanup.php +app/Services/Traffic/PresenceIndexPruner.php +app/Console/Commands/RedisCleanupOrphansCommand.php +app/Console/Commands/PruneOnlineVisitorIndexCommand.php +app/Console/Commands/RedisCleanupLegacyPrefixCommand.php +config/traffic.php +routes/console.php +.env.example +tests/Feature/Redis/OrphanQueueCleanupTest.php +tests/Unit/Traffic/PresenceIndexPrunerTest.php +tests/Unit/Redis/LegacyPrefixFilterTest.php +docs/optimization-m8-redis-orphan-cleanup.md +``` + +--- + +## 15. Risks + +- Sampled classes (400 LINDEX) might miss a rare job type; `--force` required then. +- `queues:*:notify` UNLINK with the list is required for Laravel blocking pop leftovers. +- Presence first pass is slow; enable scheduler only after a few manual batches. +- Predis fatals: do **not** LRANGE forum-moderation in tinker/Horizon UI until UNLINK. +- Collections UNLINK loses queued refresh intent; DB is source of truth. + +No FLUSH, no DEL of huge keys from HTTP, no Horizon workers, no Redis restart. diff --git a/docs/optimization-m9-scheduler-overlap.md b/docs/optimization-m9-scheduler-overlap.md new file mode 100644 index 00000000..0aeff50c --- /dev/null +++ b/docs/optimization-m9-scheduler-overlap.md @@ -0,0 +1,212 @@ +# M9 — Scheduler, Cron Overlap & Background Job Audit + +```text +STATUS: COMPLETE (audit + small local schedule hygiene) +PRODUCTION: unchanged (no cron/systemd/Supervisor edits) +WHEN: 2026-08-23 20:23–20:25 CEST +``` + +--- + +## 1. Scheduler execution mechanism + +Laravel schedule is defined only in `routes/console.php` (`bootstrap/app.php` `commands:`). `Kernel::schedule()` is empty to avoid double registration. + +**How `schedule:run` is invoked:** empirically **running** (live `schedule:finish` wrapper around `leaderboards:refresh`; `health:tick` Redis key present). User crontab empty. `/etc/cron.d` has **no** `artisan schedule:run`. systemd has **no** schedule timer. **Inferred: root crontab** (`sudo crontab -l` not readable by this user). Do not change. + +Supervisor runs **Horizon + SSR only**, not `schedule:work`. + +--- + +## 2. Duplicate-runner check + +| Check | Result | +| ----- | ------ | +| `schedule:work` processes | **none** | +| Second `schedule:run` host | single host `server3` | +| Duplicate cron.d artisan | **none** | +| Kernel + console.php double register | Kernel schedule empty; `schedule:list` unique | +| Stale schedule:work | **none** | + +**NO ISSUE** for duplicate runners. + +--- + +## 3. Laravel schedule inventory (production `schedule:list`) + +Gates: collections dispatch **false**, forum AI/sec **false**, presence prune **true**, security-report **true**. + +Forum async and collections:dispatch-maintenance **absent** from list. Presence prune **present**. + +Every-minute: posts/artworks/news/nova-cards publish, health:tick. + +Heavy hourly: metrics :02, rankings :07/:37, heat :09/:24/:39/:54, rank-build :15, forum:scan-posts :17, leaderboards :21 (mutex held, **>2 min** observed), search-reconcile :28, presence prune :33, horizon snapshot :45. + +Nightly rec: tags 02:00, behavior 02:15, hybrid 02:30 (queue jobs; M1/M7.1). + +Sitemaps: generate 10:30 and 22:30; publish `8 */6`; validate 04:45; cleanup 05:00. + +Full table: production `schedule:list` output captured 20:23 CEST (44 events). Local adds overlap names on analytics/uploads; flush minutes **18 instead of 21**. + +`uploads:cleanup` previously had **no** `withoutOverlapping` (fixed locally). Four `analytics:aggregate-*` same (fixed locally). + +Default overlap expiry: Laravel **1440 minutes** except prune-metric **120**, publish-scheduled **2**, presence prune **25**. + +--- + +## 4. System cron/timers relevant to Skinbase + +| Source | What | +| ------ | ---- | +| `/etc/cron.d/skinbase-public-index-integrity` | `*/5` root integrity script | +| `/etc/cron.d/php` | sessionclean 09,39 (systemd timer wins) | +| certbot.timer | ~22:26 | +| logrotate.timer | ~00:02 | +| apt-daily.timer | ~02:12 | +| lynis, crowdsec-hub, dpkg-db-backup | ~00:00–00:26 | +| ClamAV | daemon, not a Skinbase timer | + +--- + +## 5–6. Timeline / collisions + +| When | Tasks | Rank | +| ---- | ----- | ---- | +| **:21** | `flush-redis-stats` **and** `leaderboards:refresh` (refresh held mutex **>2 min**) | **MEDIUM** — local: flush moved to **:18** | +| :33 | `collections:sync-lifecycle` (DB) + presence prune (Redis) | **LOW** — different resources | +| :15 | rank-build-lists + homepage warm | **LOW** | +| 02:00–02:30 | rec tags→behavior→hybrid on **default queue**; apt-daily ~02:12 | **MEDIUM** (queue contention, not schedule mutex). Offsets are guessed, not a completion barrier. Do not redesign in M9. | +| 03:00–04:45 | uploads, analytics, reset-windowed, enhance, academy prune, metric prune, sitemap validate | **LOW–MEDIUM** overnight cluster, sequential minutes | +| 03:30 | reset-windowed-stats-24h **and** enhance:cleanup | **LOW** (own mutexes, enhance background) | +| every minute | 5 commands | **NO ISSUE** if each is cheap | + +--- + +## 7. Measured runtimes + +| Job | Evidence | +| --- | -------- | +| leaderboards:refresh | process **etime 2m23s** still running; schedule mutex listed | +| IndexUserJob / MakeSearchable | Horizon 16–108 **ms** | +| sitemaps generate | prior ~65s; `public/sitemaps/sitemap.xml` mtime 14:51 (release/publish, not 10:30) | +| presence prune | first scheduled run **~81k** members (M8 verify); hourlyAt(33) keep | +| rec nightly | historical MaxAttempts 02:18 / 02:31 (M1; mitigated in code, not schedule) | +| health:tick | Redis SETEX; key `1787509506` present | + +Did not run expensive commands for benchmarks. + +--- + +## 8. withoutOverlapping + +Missing (fixed locally): `uploads:cleanup`, four analytics aggregates. + +TTL 1440m default: crash suppresses a daily job up to 24h — acceptable for daily. Presence **25m** matches catch-up batches. Metric prune **120m**. Publish **2m**. + +leaderboards uses default 1440 while runtime ~2–3 min — OK. + +--- + +## 9. runInBackground + +Used for publish, rankings, metrics, heat, sitemaps, forum scan-posts, presence prune, leaderboards, etc. + +Observed: `leaderboards:refresh` spawned via `schedule:finish` and stdout to `/dev/null`. Failures may not fail `schedule:run`. **Do not remove globally.** Mutex still applied (Has Mutex on list). + +--- + +## 10. Every-minute + +`health:tick`: one Redis SETEX, TTL 300. **NO CHANGE.** + +Publish-scheduled ×3 + posts: **withoutOverlapping(2)** / default. Keep. + +`posts:warm-trending` odd minutes: overlap protected. + +--- + +## 11. Recommendation + +02:00 tags, 02:15 behavior, 02:30 hybrid, everyFourHours pairs at **:00**. M1/M7.1 job code intact. **No completion wait** between stages — hybrid may run while tag batches still on default. Historical failures were retry_after (M1), not this offset. **No architecture change.** + +--- + +## 12. Ranking / leaderboards + +`nova:recalculate-rankings` :07/:37 background. `rank-build-lists` :15 (dispatches unique scopes, 6h lock). `leaderboards:refresh` :21 **DB-heavy, >2 min**. Duplicate scope stacking addressed in M7.1. **NO further frequency change.** + +--- + +## 13. Metrics + +Snapshot :02 (~50k upserts). Prune **04:25** overlap 120 — keep (outside :02). Studio 30d is HTTP not schedule. + +--- + +## 14. Presence + +Enabled on production. hourlyAt(33), overlap 25. Redis-heavy; :33 vs lifecycle DB. **Keep. Do not accelerate.** + +--- + +## 15. Forum / collections + +`forum:scan-posts --limit=250` hourly :17, in-process, overlap, background. Async forum **absent**. Collections dispatch **absent**. `collections:sync-lifecycle` every 10 min (3,13,23,33,43,53) — invite/lifecycle SQL, **not** the orphan queue. **NO CHANGE** to frequency without row-count evidence. + +--- + +## 16. Sitemap + +generate 10:30/22:30 background overlap. publish every 6h at :08. validate 04:45. cleanup 05:00. Two daily generates remain justified (fresh shards). **Do not regress M3.** + +--- + +## 17. Horizon snapshot + +hourlyAt(45). Current prefix `skinbase_horizon:`. Horizon log shows live job timings — metrics path works. **NO CHANGE.** + +--- + +## 18. OS maintenance + +apt-daily ~02:12 vs rec 02:00–02:30 (**MEDIUM** IO). logrotate ~00:02. Do not edit OS timers. + +--- + +## 19. Deploy + +`sync.sh` → `deploy-production.sh`, restarts Horizon/SSR. Background `schedule:finish` children can outlive symlink (leaderboards pattern). Mutex in cache/Redis (not files found on disk). **No deploy script change** — no proven double scheduler on release switch. + +--- + +## 20. Ranked findings + +| Sev | Finding | Action | +| --- | ------- | ------ | +| MEDIUM | flush-redis-stats and leaderboards both :21; leaderboards >2 min | **Local:** flush **:18** | +| LOW | uploads:cleanup / analytics aggregates lacked withoutOverlapping | **Local:** added | +| MEDIUM | rec stages overlap on default queue by clock | Document only | +| LOW | :33 lifecycle + presence | Different stores; keep | +| NO CHANGE | health:tick, presence hour, metric retention, forum/collections gates, sitemap times, Horizon snapshot, worker counts | + +--- + +## 21. Recommended vs current (only implemented) + +| Task | Current | Recommended | +| ---- | ------- | ----------- | +| flush-redis-stats | 1,11,**21**,31,41,51 | 1,11,**18**,31,41,51 | +| uploads:cleanup | daily 03:00, no overlap | + `withoutOverlapping` + name | +| analytics aggregates | daily, no overlap | + `withoutOverlapping` + names | + +Rollback: revert those three edits in `routes/console.php`. + +--- + +## 22–25. Files / tests / deploy + +`routes/console.php`, `docs/optimization-m9-scheduler-overlap.md`, `tests/Feature/Scheduler/ScheduleInventoryTest.php`. + +Deploy: ship `routes/console.php`; no cron edit; no Horizon restart required for schedule (next `schedule:run` loads new events). Rollback: revert file. + +**Do not** increase workers/FPM/Redis/MySQL/swap or re-enable orphan queues. diff --git a/docs/skinbase-production-baseline.md b/docs/skinbase-production-baseline.md new file mode 100644 index 00000000..e3241e0b --- /dev/null +++ b/docs/skinbase-production-baseline.md @@ -0,0 +1,607 @@ +# Skinbase.org — M0 Production Baseline & Runtime Audit + +```text +STATUS: COMPLETE +``` + +Primary origin runtime unknowns were resolved via **read-only `ssh server3`**. Some items remain UNKNOWN (MySQL `slow.log` not readable by this user; root crontab not readable without sudo; `performance_schema` denied to `skinbase@localhost`; `iostat` not installed). + +**Nothing was modified on server3.** No restarts, no cache clears, no SQL writes, no deploys. + +Evidence classes used below: + +```text +LOCAL CODE EVIDENCE +PRODUCTION SERVER EVIDENCE +PUBLIC CLOUDFLARE/HTTP EVIDENCE +``` + +--- + +## Executive Summary + +```text +Production OS: Debian 13 (trixie), kernel 6.12.96+deb13-cloud-amd64, x86_64 +CPU: AMD EPYC, 8 cores / 8 threads (1 socket) +RAM: 23 GiB; ~8.6 GiB available; SWAP 2.0 GiB **almost fully used** +Storage: 400G virtio disk `/dev/sda1` ext4 ~185G/394G (49%); rotational=0 (SSD/NVMe-like) +nginx: 1.26.3; HTTP/2 on; gzip+brotli in nginx.conf; vhost skinbase.org.conf +PHP: 8.4.24 (FPM + CLI) +PHP-FPM: pool `skinbase`, dynamic, max_children=14, memory_limit=256M, terminate=120s +MySQL: 8.4.11-11 local; DB `skinbase` 2758 MB; buffer pool 10 GB +Redis: 8.10.1 local 127.0.0.1:6379; 878 MB used / 2 GB max; allkeys-lru +Meilisearch: local systemd; health available; RSS ~401 MB +Queue manager: Supervisor → `php artisan horizon` (NOT standalone queue:work) +Cache store: **redis** (PRODUCTION; not repo default database) +Session store: **redis** +CDN: Cloudflare + cdn.skinbase.org RGW/S3 +Download acceleration: **OFF** (`download_accel_enabled=false`; no X-Accel location in vhost) +Sitemap static serving: **ACTIVE** (nginx try_files + shared files). Index file stale (May 13). +Debug mode: OFF +Debugbar production: NOT PRESENT (packages-dev=0, no debugbar routes) +``` + +```text +PRODUCTION_APP_PATH=/opt/www/virtual/SkinbaseNova + → symlink → /opt/www/virtual/SkinbaseNova.releases/current + → /opt/www/virtual/SkinbaseNova.releases/releases/20260801-172803-5af95f65-dirty + +Local commit: f52879edbb19ecc5d3e807bd4872016f3071dad7 (develop) +Production commit: 5af95f65 (release name 5af95f65-dirty, 2026-08-01) +Same revision: no +Production git: not a git checkout (no .git in release) +``` + +```text +Active P0 findings: + QUEUE-001 (90s vs 900s): MITIGATED — Horizon default workers timeout=960 + QUEUE rec-job failures: ACTIVE P0 — RecComputeSimilar* fail every night MaxAttemptsExceeded + SITEMAP-001 live-build: NOT the current incident — static files exist + SITEMAP index stale: ACTIVE P1/P0-SEO — sitemap.xml mtime 2026-05-13 while shards refresh daily + STAT-001: NOT currently P0 at 2666 views/24h; design still sync writes (P1) + DB-001: indexes PRESENT; residual UNKNOWN without slow.log. Snapshot table 1.85 GB is the dominant store. + +Active P1 findings: swap exhaustion; mail queue unconsumed (LLEN=3); upload 20M vs app 50M; Sentry traces 100%; FPM 10s slowlog volume; CF HTML DYNAMIC; asset TTL 30d not 1y immutable + +Major current bottleneck: MySQL row-read volume + 1.85 GB hourly snapshots; nightly rec jobs failing; memory pressure (swap full) + +Largest remaining unknown: contents of /var/log/mysql/slow.log (permission) + +M1 readiness: YES for planning. Do not implement in M0. +``` + +--- + +## 1. Repository State (LOCAL) + +```text +branch: develop +HEAD: f52879edbb19ecc5d3e807bd4872016f3071dad7 +commit: f52879ed Current state with latest updates +tree: clean at start of this SSH pass (ahead of origin/develop by 1) +``` + +No reset/stash/clean on local or production. + +--- + +## 2. Production Host (PRODUCTION SERVER) + +```text +hostname: server3 +FQDN: server3.klevze.si +OS: Debian GNU/Linux 13 (trixie) 13.6 +kernel: 6.12.96+deb13-cloud-amd64 +arch: x86_64 +uptime: 24 days, 16:58 (sampled 2026-08-23 13:43 CEST) +``` + +--- + +## 3. Hardware Baseline (PRODUCTION SERVER) + +```text +CPU model: AMD EPYC Processor (with IBPB) +sockets: 1 +cores: 8 +threads: 8 (1 thread/core) +RAM: 23 GiB +swap: 2.0 GiB, ~2.0 GiB used, 2.7 MiB free +disk: /dev/sda 400G → sda1 399.9G on / +filesystem: ~394G, 185G used, 49% +rotational: 0 → SSD/cloud SSD (not HDD) +``` + +`dmidecode` not used (would need sudo). + +--- + +## 4. Current System Load (PRODUCTION SERVER) + +Short snapshot only (limitation: not peak-hour). + +```text +load average: 0.85, 2.47, 2.88 +vmstat 5s: idle ~76–83%, wa ~0–1%, r=3–6 +CPU notables: mysqld ~20%, containerd/dockerd ~45% each (other workloads on same host), redis ~5% +RAM: 14 GiB used + 7 GiB cache; 8.6 GiB available +swap: fully used +``` + +Largest RSS: + +| Process | RSS | Note | +| ------- | --: | ---- | +| mysqld | ~7.7 GiB | 32.7% | +| python uvicorn × several | up to 1.3 GiB | vision/ML stack | +| clamd | ~1.1 GiB | | +| meilisearch | ~397–401 MiB | | +| redis | ~164 MiB | | +| php-fpm skinbase | ~95–100 MiB each | 6 workers sampled | +| node ssr.js | ~122 MiB | | + +Classification: **WATCH / CONCERNING** (swap exhausted; shared host with Docker/ML/Qdrant/Gitea/Crowdsec). Not a Skinbase-only box. + +--- + +## 5. Process Inventory (PRODUCTION SERVER) + +| Component | Reality | +| --------- | ------- | +| Horizon | **YES** — Supervisor `skinbase-horizon`, pid running ~1d | +| standalone `queue:work` | **NO** | +| Supervisor | **YES** — horizon + ssr | +| systemd queue workers | **NO** | +| Inertia SSR | **YES** — `node bootstrap/ssr/ssr.js` as www-data | +| Reverb | **YES** — systemd `skinbase-reverb.service`, 127.0.0.1:8080 | +| Meilisearch | **YES** — local systemd | +| Redis | **local** 127.0.0.1:6379 | +| MySQL | **local** `/usr/sbin/mysqld` | +| PHP-FPM | php8.4, pool `skinbase` + unused pool `www` | +| nginx | master + 8 workers, www-data | +| Docker/ML | uvicorn:8000, qdrant, gitea — **co-tenant** | + +--- + +## 6. nginx (PRODUCTION SERVER) + +```text +version: nginx/1.26.3 +vhost: /etc/nginx/sites-enabled/skinbase.org.conf +root: /opt/www/virtual/SkinbaseNova/public +listen: 80 → 301 HTTPS; 443 ssl; http2 on +PHP: unix:/run/php/php8.4-fpm-skinbase.sock +client_max_body_size: 24m +gzip: on (nginx.conf) +brotli: on (nginx.conf) +real IP: /etc/nginx/conf.d/00-cloudflare-realip.conf ACTIVE +search: location = /search + conf.d/13-skinbase-search-protection.conf (20r/m, abusive query map) +HTTP/3: not in this vhost (Cloudflare alt-svc h3 is edge-only) +``` + +Snippet vs live: + +| Snippet | Status | +| ------- | ------ | +| sitemaps.conf (`max-age=21600`, try_files) | **ACTIVE** (inlined in vhost) | +| static-cache.conf 1y immutable `/build/assets` | **NOT ACTIVE** — generic `expires 30d` for css/js/images | +| download-accel.conf | **NOT ACTIVE** | +| search-rate-limit.conf (repo) | **EQUIVALENT ACTIVE** as `13-skinbase-search-protection.conf` | +| upstream-error-pages.conf | **NOT seen** in vhost | + +**Bug/risk:** sitemap locations use `try_files $uri @php` but **no `location @php`** exists in the vhost. Missing files will not fall through to Laravel as the snippet comments claim. + +No `nginx -T` via sudo (password required). Readable vhost was enough. + +--- + +## 7–9. HTTP / CDN (PUBLIC CLOUDFLARE/HTTP + PRODUCTION) + +Unchanged from edge probes, now correlated with origin: + +- Homepage Cache-Control matches Laravel guest headers; CF **DYNAMIC** +- Build assets CF **HIT**, origin `expires 30d` +- CDN webp `max-age=31536000` HIT + Polish +- Brotli **CONFIRMED** at edge; origin brotli **on** +- HTTP/2 **CONFIRMED** at origin (`http2 on`); this Windows curl was HTTP/1.1 to CF +- HTTP/3 **offered by CF** (`alt-svc`), not configured on origin vhost + +--- + +## 10–12. PHP / OPcache / FPM (PRODUCTION SERVER) + +```text +PHP: 8.4.24 +FPM master: /etc/php/8.4/fpm/php-fpm.conf +pool: [skinbase] user=skinbase +listen: /run/php/php8.4-fpm-skinbase.sock +pm: dynamic +max_children: 14 +start_servers: 4 +min/max spare: 2 / 6 +max_requests: 500 +request_terminate_timeout: 120s +slowlog: 10s → /var/log/php8.4-fpm-skinbase-slow.log (23 MB, 139556 lines) +memory_limit: 256M (pool) +max_execution: 60s (pool) +upload/post: 20M / 24M (pool) ← app allows 50M images / 200M archives +CLI upload_max: 2M (irrelevant to FPM) +extensions: opcache, redis, pcntl, posix, intl, mbstring, curl, gd, pdo_mysql +imagick: NOT in php -m +opcache (FPM php.ini): memory 256MB, interned 16, max_files 20000, validate_timestamps=1, revalidate_freq=2, jit=off +opcache runtime hit rate: UNKNOWN (no status scrape of /fpm-status from remote; localhost-only) +``` + +Workers sampled: **6** skinbase processes, **avg RSS ~95 MB**, total ~570 MB. + +Theoretical ceiling: `14 × 256 MB = 3.6 GB` if every worker hit `memory_limit`. Typical: `14 × 95 MB ≈ 1.3 GB`. + +Available RAM 8.6 GiB **but swap already full** — do not raise max_children in M0. + +--- + +## 13–15. Laravel Production Configuration (PRODUCTION SERVER) + +`php artisan about` + tinker `config()` (no `.env` dump): + +```text +APP_ENV=production +APP_DEBUG=false +CACHE_STORE=redis +SESSION_DRIVER=redis +QUEUE_CONNECTION=redis +SCOUT_DRIVER=meilisearch +FILESYSTEM_DISK=local +UPLOAD_QUEUE_DERIVATIVES=false +DOWNLOAD_ACCEL_ENABLED=false +DOWNLOAD_ACCEL_PATH=/internal/originals (path set, flag off, nginx location missing) +SITEMAPS_BUILD_ON_REQUEST=true +SITEMAPS_FALLBACK_TO_LIVE_BUILD=true +SITEMAPS_PREGENERATED_ENABLED=true +SITEMAPS_PREFER_PUBLISHED_RELEASE=true +SITEMAPS_STATIC_PUBLISH_ENABLED=true +REDIS_CLIENT=predis +homepage.cache_store=homepage +homepage.guest_payload_ttl_seconds=1800 +recommendations.queue=default +vision.queue=default +discovery.queue=default +broadcasting=reverb +horizon.path=horizon +Sentry enabled, sample rate errors 100%, performance 100% +``` + +Laravel caches: **config, events, routes, views ALL CACHED**. + +Debugbar/Telescope/Clockwork routes: **none**. Composer `packages-dev=0` → **--no-dev install**. + +--- + +## 16–22. MySQL (PRODUCTION SERVER) + +```text +location: local mysqld +version: 8.4.11-11 +database: skinbase +size: 2758.2 MB (data 1230.4 + indexes 1527.8) +buffer pool: 10 GB, 10 instances +max_conn: 120 (Max_used 25) +slow_query_log: ON, long_query_time=0.5s, file /var/log/mysql/slow.log (not readable here) +Slow_queries: 11342 in Uptime 97330s (~27h) ≈ 0.12/s +Threads: connected 3, running 2 +InnoDB hit: 1 - 103791/18420885623 ≈ 99.999% +Rows read: 9.70e9 in ~27h (high — job/scan load) +tmp tables: 455567 created, 21 on disk +``` + +Largest tables (information_schema estimates): + +| Table | ~rows | total MB | +| ----- | -----: | -------: | +| artwork_metric_snapshots_hourly | 8.62M | **1855.5** | +| artwork_downloads | 579k | 167.3 | +| forum_bot_logs | 267k | 141.8 | +| artworks | 50k | 94.2 | +| user_activities | 240k | 69.3 | +| artwork_comments | 172k | 62.2 | +| artwork_view_events | 161k | 25.4 | +| rank_artwork_scores | 50k | 22.4 | +| rec_item_pairs | 74k | 16.0 | +| artwork_stats | 49k | 15.2 | + +`cache`/`sessions` tables nearly empty (drivers are Redis). `jobs` table empty (Redis queues). + +### Indexes (batch1) — PRODUCTION + +| Index | Status | +| ----- | ------ | +| artworks.idx_public_approved_published_id | **PRESENT** | +| artworks.idx_public_approved_user_id | **PRESENT** | +| artworks FULLTEXT title+description | **PRESENT** | +| snapshots.idx_bucket_artwork | **PRESENT** | +| rank_artwork_scores idx_mv_trending/new_hot/best | **PRESENT** | +| tags.artworks_count + idx_tags_artworks_count | **PRESENT** | + +### DB-001 re-evaluation + +```text +Classification: UNKNOWN as “still 78%”, NOT “unindexed public scans” +``` + +April indexes **are deployed**. Dominant storage is hourly snapshots ≈ 50k artworks × 24h × 7d. Heat/ranking jobs that join this table can still dominate `Innodb_rows_read` even with `idx_bucket_artwork`. Cannot confirm fingerprints without `slow.log` / performance_schema (denied). + +Do not add more indexes in M0. + +--- + +## 23–25. Redis / Cache / Sessions (PRODUCTION SERVER) + +```text +redis_version: 8.10.1 +used: 877.55 MB (peak 917 MB) +maxmemory: 2.00 G, policy allkeys-lru +evicted_keys: 0 +clients: 20 +ops/sec: 44 (instant) +keyspace hits/misses: 1.40M / 1.30M (~52% hit) +db0: 1932 keys; db1: 6641; db2: 55176 (all with TTL) +``` + +| System | Redis in production? | +| ------ | -------------------- | +| Cache default | **YES** | +| Homepage store `homepage` | configured (failover redis→database) | +| Sessions | **YES** | +| Queues / Horizon | **YES** | +| Presence | code uses Redis (not separately counted) | +| Stats deltas | flush command scheduled; view path still MySQL sync | + +CACHE-001 from the code audit (**database cache default**) is **NOT ACTIVE in production**. + +--- + +## 26–32. Horizon / Queues / QUEUE-001 (PRODUCTION SERVER) + +Horizon **running**. Supervisors: + +| Supervisor | queues | timeout | maxProcesses | +| ---------- | ------ | ------: | -----------: | +| supervisor-default | search, default | **960** | 5 | +| supervisor-messaging | broadcasts, notifications | **90** | 3 | + +`recommendations.queue` / `vision.queue` / `discovery.queue` = **`default`** → consumed by 960s workers. + +Standalone supervisor `queue:work --timeout=90` from `deploy/supervisor/skinbase-queue.conf` is **NOT the production worker**. + +Failed jobs: **329** rows (`queue:failed` listed many; table_rows estimate 159). Dominant: + +```text +RecComputeSimilarByBehaviorJob ~02:18 daily MaxAttemptsExceededException +RecComputeSimilarHybridJob ~02:31 daily MaxAttemptsExceededException +``` + +Horizon worker `tries=1`, job `$timeout=900`, worker timeout 960. Failures are **not** explained by the old 90s worker. Likely 900s job timeout, 128 MB worker memory, or lock/overlap. **Do not retry in M0.** + +Queue LLEN (Redis): all listed queues 0 except **`queues:mail` = 3**. Horizon does **not** listen to `mail`. Mail jobs can stall. + +```text +QUEUE-001 (90 vs 900): MITIGATED +Nightly rec job failures: ACTIVE P0 (related, different mechanism) +mail queue unconsumed: P1 +``` + +--- + +## 31–32. Scheduler (PRODUCTION SERVER) + +`php artisan schedule:list` matches `routes/console.php` (generate sitemaps 10:30/22:30, rec jobs 02:00–02:30, etc.). + +**How `schedule:run` is invoked:** user crontab empty; sudo crontab **not readable**; no systemd timer named laravel/schedule. **Empirically running** (sitemap shards 10:30 today; rec jobs fail 02:18/02:31 daily). Likely root crontab. Do not change. + +02:00–05:00 window (code + evidence it fires): rec jobs, ranking/heat, analytics, prune snapshots, sitemap validate — plus nightly rec **failures**. + +--- + +## 33. Meilisearch (PRODUCTION SERVER) + +```text +process: /usr/local/bin/meilisearch --config-file-path /etc/meilisearch.toml +health: {"status":"available"} +version/stats: 401 without key (not printed) +RSS: ~401 MB +``` + +No reindex. + +--- + +## 34–36. Storage / Downloads / Sitemaps + +- `FILESYSTEM_DISK=local`; CDN for public derivatives. +- Download accel **off**; originals would stream via PHP if used. +- Sitemaps live under shared `.../shared/public/sitemaps/` (20M, 27 xml files). +- **Shards + academy + users + forum-threads mtime 2026-08-23 10:30–10:31** +- **`sitemap.xml` mtime 2026-05-13 21:39, 1906 bytes** — generate/publish does **not** update the index file. +- May index omits academy-* families that now exist on disk. + +SITEMAP-001 live-build-on-missing: files exist, so crawlers are not building XML in PHP for the index. `@php` named location missing anyway. + +```text +SITEMAP-001 original (PHP live-build stampede): NOT ACTIVE right now +Stale sitemap index vs daily shards: ACTIVE (SEO/ops) +``` + +--- + +## 37–38. Views / Downloads + +```text +artwork_view_events last 1h: 72 +artwork_view_events last 24h: 2666 +artwork_downloads last 24h: 4718 +view_events table: ~161k rows, 25.4 MB +downloads table: ~579k rows, 167 MB +``` + +STAT-001 architecture (sync INSERT+UPDATE, `defer=false`) is **still the code on this release**. At **~0.03 views/s** it is not the current DB bottleneck. Classify **P1** (architecture / growth), not active P0 load. + +Downloads were not fetched (would write). + +--- + +## 39–47. Latency / Traffic / Resources + +Origin access-log QPS not aggregated (would include IPs — skipped). Edge n=1 TTFBs from earlier remain **PUBLIC** evidence, not origin APM. + +FPM slowlog: 139k historical lines, last file mtime Aug 23 05:05, timeout 10s. Scripts are `index.php` only (no URI dump here). + +Resource baseline: see §4. **Swap full = CONCERNING.** + +--- + +## 48. Lighthouse / CWV + +Not re-run. Prior lab file is skinbase.top 2026-03-23 — do not use as this baseline. + +--- + +## 49. Observability + +- Sentry **on**, 100% error **and** performance sample (P1 cost). +- Netdata on host. +- Horizon dashboard path `horizon` (auth not verified; do not probe). +- `/fpm-status` localhost only. +- `/stats/` basic-auth (not accessed). +- MySQL slow.log exists but unreadable to this SSH user. + +--- + +## 50. Re-evaluated P0 Findings + +| ID | Verdict | Why | +| -- | ------- | --- | +| QUEUE-001 90s worker vs 900s job | **MITIGATED** | Horizon default timeout **960**; rec queues aliased to `default` | +| RecCompute* nightly MaxAttemptsExceeded | **ACTIVE P0** | every night 02:18/02:31 on `redis@default`; 329 failed jobs | +| STAT-001 sync view writes | **P1** at current 2666/day | still sync in config/code; not the load leader | +| SITEMAP-001 request-time build | **NOT ACTIVE** | static files present; `@php` missing | +| Stale sitemap.xml (May vs Aug shards) | **ACTIVE P1** (SEO) | generate writes children, not index | +| DB-001 unindexed aggregates | **MITIGATED for missing indexes** | batch1 **PRESENT**. Residual scan cost **UNKNOWN**; snapshots 1.85 GB | + +--- + +## 51. Re-evaluated P1 Findings + +- Production **Redis** cache/session (repo default database is wrong for prod). +- Upload 20M/24M FPM vs app 50M/200M. +- `UPLOAD_QUEUE_DERIVATIVES=false` — sync image work on publish. +- `mail` Redis queue not consumed by Horizon. +- Sentry performance sampling 100%. +- CF HTML DYNAMIC despite public Cache-Control. +- Asset TTL 30d not 1y immutable. +- Swap 2G full; shared ML/Docker host. +- FPM slowlog 10s historically large. +- Local git **newer** than production release (f52879ed vs 5af95f65). +- Session cookies on `/search` and 404 (code-consistent). +- Duplicate security headers (app middleware + nginx snippet). + +--- + +## 52. Baseline Metric Table + +| Metric | Value | Source | +| ------ | ----- | ------ | +| DB size | 2758 MB | information_schema | +| Snapshots table | 1855 MB / ~8.6M rows | information_schema | +| Artworks | ~50k / 94 MB | information_schema | +| Views 24h | 2666 | count on 161k table | +| Downloads 24h | 4718 | count | +| Slow_queries / 27h | 11342 | SHOW STATUS | +| InnoDB pool hit | ~100% | STATUS | +| Redis used | 878 MB / 2 GB | INFO | +| Redis hit ratio | ~52% | INFO | +| FPM workers | 6 / max 14, ~95 MB RSS | ps | +| Horizon | running, timeout 960/90 | ps + artisan | +| Failed jobs | 329 | DB | +| Homepage TTFB (edge) | 146 ms n=1 | PUBLIC | +| Production release | 20260801-172803-5af95f65-dirty | filesystem | + +--- + +## 53. Production Risk Matrix + +| Risk | Evidence | Impact | Confidence | +| ---- | -------- | ------ | ---------- | +| Nightly rec jobs never succeed | failed_jobs every day | stale similar-art | high | +| Hourly snapshot table 1.85 GB | information_schema | heat/rank I/O | high | +| Swap full + co-tenant ML | free/ps | latency spikes | high | +| Stale sitemap index | mtime May 13 vs shards today | crawl/SEO | high | +| mail queue not consumed | LLEN=3, Horizon queues | delayed mail | high | +| Missing @php location | nginx vhost | missing shard → error not Laravel | high | +| Cannot read slow.log | permissions | DB-001 residual | high | + +--- + +## 54. Optimization Targets (do not implement now) + +1. Rec job failures (timeout/memory/overlap) — after measuring why MaxAttemptsExceeded. +2. Snapshot retention / heat query EXPLAIN once slow.log is readable. +3. Publish `sitemap.xml` index with shards. +4. Consume `mail` or stop using that queue name. +5. Download X-Accel if originals still hit FPM. +6. Align upload limits. +7. Memory/swap / co-tenancy. +8. Sentry sample rates. +9. CF HTML cache policy (product decision). + +--- + +## 55. M1–M4 Readiness + +```text +M1 planning: READY (production drivers known; P0 recast) +M1 implementation: not this milestone +Blockers for DB-001 closeout: mysql slow.log or GRANTs on performance_schema +``` + +--- + +## 56. Unknowns + +- Root crontab contents (scheduler invocation path) +- MySQL slow.log fingerprints **now** +- OPcache hit rate (FPM status localhost-only) +- Meilisearch document counts (auth) +- Peak QPS / access-log (IPs omitted on purpose) +- Why rec jobs MaxAttemptsExceeded (timeout vs memory vs exception) — do not dump payloads +- HTTP/3 to origin (not configured) +- iostat (command missing) + +--- + +## 57. Repository Changes + +```text +Created earlier: +- scripts/collect-production-baseline.sh + +Updated: +- docs/skinbase-production-baseline.md + +Modified application: +- none + +Database changes: +- none + +Server configuration changes: +- none + +Service restarts: +- none + +Cache clears: +- none + +Queue operations: +- none (failed jobs listed only, not retried) +``` diff --git a/phpunit.xml b/phpunit.xml index e4b91466..9856bf80 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -37,5 +37,9 @@ + + + + diff --git a/resources/js/Pages/Studio/StudioDashboard.jsx b/resources/js/Pages/Studio/StudioDashboard.jsx index 3ab8fd0d..dd89cce8 100644 --- a/resources/js/Pages/Studio/StudioDashboard.jsx +++ b/resources/js/Pages/Studio/StudioDashboard.jsx @@ -12,7 +12,24 @@ const kpiConfig = [ { key: 'followers', label: 'Followers', icon: 'fa-solid fa-user-group' }, ] -function KpiCard({ config, value }) { +function snapshotHint(config, window) { + if (!window || !['views_30d', 'appreciation_30d', 'shares_30d', 'comments_30d'].includes(config.key)) { + return null + } + + const requested = Number(window.requested_days || 30) + const available = Number(window.table_coverage_days || 0) + const complete = Boolean(window.window_complete) + + if (complete) { + return `Last ${requested} days` + } + + const shown = Math.max(0, Math.round(available * 10) / 10) + return `Based on ${shown} of ${requested} days` +} + +function KpiCard({ config, value, hint }) { return (
@@ -22,6 +39,7 @@ function KpiCard({ config, value }) { {config.label}

{typeof value === 'number' ? value.toLocaleString() : value}

+ {hint ?

{hint}

: null}
) } @@ -235,7 +253,12 @@ export default function StudioDashboard() {
{kpiConfig.map((config) => ( - + ))}
diff --git a/routes/console.php b/routes/console.php index 8219acf7..1420d244 100644 --- a/routes/console.php +++ b/routes/console.php @@ -44,16 +44,31 @@ Schedule::command('skinbase:reset-windowed-stats --period=7d') ->withoutOverlapping(); // Daily maintenance -Schedule::command('uploads:cleanup')->dailyAt('03:00'); +Schedule::command('uploads:cleanup') + ->dailyAt('03:00') + ->name('uploads-cleanup') + ->withoutOverlapping(); Schedule::command('enhance:cleanup --force') ->dailyAt('03:30') ->name('enhance-cleanup') ->withoutOverlapping() ->runInBackground(); -Schedule::command('analytics:aggregate-similar-artworks')->dailyAt('03:10'); -Schedule::command('analytics:aggregate-feed')->dailyAt('03:20'); -Schedule::command('analytics:aggregate-discovery-feedback')->dailyAt('03:25'); -Schedule::command('analytics:aggregate-tag-interactions')->dailyAt('03:35'); +Schedule::command('analytics:aggregate-similar-artworks') + ->dailyAt('03:10') + ->name('analytics-aggregate-similar-artworks') + ->withoutOverlapping(); +Schedule::command('analytics:aggregate-feed') + ->dailyAt('03:20') + ->name('analytics-aggregate-feed') + ->withoutOverlapping(); +Schedule::command('analytics:aggregate-discovery-feedback') + ->dailyAt('03:25') + ->name('analytics-aggregate-discovery-feedback') + ->withoutOverlapping(); +Schedule::command('analytics:aggregate-tag-interactions') + ->dailyAt('03:35') + ->name('analytics-aggregate-tag-interactions') + ->withoutOverlapping(); Schedule::command('academy:analytics-rollup') ->hourlyAt(12) ->name('academy-analytics-rollup') @@ -74,8 +89,9 @@ Schedule::command('academy:analytics-prune-events --days=180') // Drain Redis artwork-stat delta queue so MySQL counters stay fresh. // Offset this off the :00/:10 boundaries so it does not pile onto publish jobs. +// :18 not :21 so this does not start with leaderboards:refresh (observed >2 min). Schedule::command('skinbase:flush-redis-stats') - ->cron('1,11,21,31,41,51 * * * *') + ->cron('1,11,18,31,41,51 * * * *') ->name('flush-redis-stats') ->withoutOverlapping(); @@ -226,11 +242,13 @@ Schedule::job(new \App\Jobs\Sitemaps\CleanupSitemapReleasesJob()) ->name('sitemaps-cleanup') ->withoutOverlapping(); -Schedule::command('collections:dispatch-maintenance') - ->hourlyAt(43) - ->name('dispatch-collection-maintenance') - ->withoutOverlapping() - ->runInBackground(); +if ((bool) config('collections.v5.queue.dispatch_enabled', false)) { + Schedule::command('collections:dispatch-maintenance') + ->hourlyAt(43) + ->name('dispatch-collection-maintenance') + ->withoutOverlapping() + ->runInBackground(); +} Schedule::job(new \App\Jobs\RankBuildListsJob()) ->hourlyAt(15) @@ -253,10 +271,11 @@ Schedule::job(new \App\Jobs\RecalculateRisingNovaCardsJob()) ->name('nova-cards-rising-cache-refresh') ->withoutOverlapping(); -Schedule::command('nova:prune-metric-snapshots --keep-days=7') - ->dailyAt('04:00') +// After hourly snapshot at :02 so prune does not share the 04:02 upsert window. +Schedule::command('nova:prune-metric-snapshots') + ->dailyAt('04:25') ->name('prune-metric-snapshots') - ->withoutOverlapping(); + ->withoutOverlapping(120); Schedule::command('skinbase:sync-countries') ->monthlyOn(1, '03:40') @@ -269,17 +288,13 @@ Schedule::command('health:tick') ->name('health-scheduler-tick') ->withoutOverlapping(); -Schedule::command('forum:ai-scan') - ->hourlyAt(16) - ->name('forum-ai-scan') - ->withoutOverlapping() - ->runInBackground(); - -Schedule::command('forum:bot-scan') - ->hourlyAt(22) - ->name('forum-bot-scan') - ->withoutOverlapping() - ->runInBackground(); +if ((bool) config('skinbase_ai_moderation.queue.dispatch_enabled', false)) { + Schedule::command('forum:ai-scan') + ->hourlyAt(16) + ->name('forum-ai-scan') + ->withoutOverlapping() + ->runInBackground(); +} Schedule::command('forum:scan-posts --limit=250') ->hourlyAt(17) @@ -287,13 +302,29 @@ Schedule::command('forum:scan-posts --limit=250') ->withoutOverlapping() ->runInBackground(); -Schedule::command('forum:firewall-scan') - ->hourlyAt(40) - ->name('forum-firewall-scan') - ->withoutOverlapping() - ->runInBackground(); +if ((bool) config('forum_security.queues.dispatch_enabled', false)) { + Schedule::command('forum:bot-scan') + ->hourlyAt(22) + ->name('forum-bot-scan') + ->withoutOverlapping() + ->runInBackground(); + + Schedule::command('forum:firewall-scan') + ->hourlyAt(40) + ->name('forum-firewall-scan') + ->withoutOverlapping() + ->runInBackground(); +} Schedule::command('horizon:snapshot') ->hourlyAt(45) ->name('horizon-snapshot') ->withoutOverlapping(); + +if ((bool) config('traffic.online_visitors.prune_enabled', false)) { + Schedule::command('skinbase:prune-online-visitor-index --execute') + ->hourlyAt(33) + ->name('prune-online-visitor-index') + ->withoutOverlapping(25) + ->runInBackground(); +} diff --git a/scripts/analyze-http-performance.php b/scripts/analyze-http-performance.php new file mode 100644 index 00000000..7a839f0f --- /dev/null +++ b/scripts/analyze-http-performance.php @@ -0,0 +1,68 @@ + '/var/log/nginx/skinbase-performance.log', + 'since' => '24h', + 'min_requests' => 1, + 'top' => 20, + 'status' => null, + 'uri' => null, + 'host' => null, + 'slow_ms' => null, +]; + +foreach (array_slice($argv, 1) as $arg) { + if (! str_starts_with($arg, '--')) { + fwrite(STDERR, "Unknown argument: {$arg}\n"); + exit(2); + } + [$key, $value] = array_pad(explode('=', substr($arg, 2), 2), 2, '1'); + $map = [ + 'file' => 'file', + 'since' => 'since', + 'min-requests' => 'min_requests', + 'top' => 'top', + 'status' => 'status', + 'uri' => 'uri', + 'host' => 'host', + 'slow-ms' => 'slow_ms', + ]; + if (! isset($map[$key])) { + fwrite(STDERR, "Unknown option: --{$key}\n"); + exit(2); + } + $options[$map[$key]] = $value; +} + +$file = (string) $options['file']; +if (! is_readable($file)) { + fwrite(STDERR, "Cannot read {$file}\n"); + exit(1); +} + +$analyzer = new HttpPerformanceLogAnalyzer(new HttpUriNormalizer()); +$result = $analyzer->analyze($file, [ + 'since' => $options['since'], + 'min_requests' => (int) $options['min_requests'], + 'top' => (int) $options['top'], + 'status' => $options['status'], + 'uri' => $options['uri'], + 'host' => $options['host'], + 'slow_ms' => $options['slow_ms'] !== null ? (float) $options['slow_ms'] : null, +]); + +echo json_encode($result, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES), PHP_EOL; diff --git a/scripts/collect-production-baseline.sh b/scripts/collect-production-baseline.sh new file mode 100644 index 00000000..a18073b6 --- /dev/null +++ b/scripts/collect-production-baseline.sh @@ -0,0 +1,226 @@ +#!/usr/bin/env bash +# Skinbase.org M0 production baseline collector +# READ-ONLY / non-destructive / secret-safe +# +# Run on the production application host from the live app directory, e.g.: +# cd /opt/www/virtual/SkinbaseNova && sudo -u www-data bash scripts/collect-production-baseline.sh +# +# Never prints: passwords, tokens, keys, APP_KEY, DSNs, cookies, session payloads, .env dumps. +# Does not: write to MySQL, flush Redis, restart services, clear caches, run jobs, migrate, deploy. + +set -u +set -o pipefail + +umask 077 + +APPROVED_ENV_KEYS='^(APP_ENV|APP_DEBUG|CACHE_STORE|SESSION_DRIVER|QUEUE_CONNECTION|SCOUT_DRIVER|FILESYSTEM_DISK|UPLOAD_QUEUE_DERIVATIVES|DOWNLOAD_ACCEL_ENABLED|DOWNLOAD_ACCEL_PATH|SITEMAPS_BUILD_ON_REQUEST|SITEMAPS_FALLBACK_TO_LIVE_BUILD|SITEMAPS_PREGENERATED_ENABLED|SITEMAPS_PREFER_PUBLISHED_RELEASE|REDIS_CLIENT|HOMEPAGE_CACHE_STORE|BROADCAST_CONNECTION|SCOUT_QUEUE_NAME|HORIZON_PATH)$' + +redact() { + sed -E \ + -e 's/(password|passwd|secret|token|apikey|api_key|access_key|private_key|dsn|authorization)[=:][[:space:]]*[^[:space:]]+/\1=[REDACTED]/Ig' \ + -e 's#mysql://[^[:space:]]+#mysql://[REDACTED]#g' \ + -e 's#redis://[^[:space:]]+#redis://[REDACTED]#g' +} + +run() { + local name="$1" + shift + echo "=== ${name} ===" >>"${LOG}" + echo "\$ $*" >>"${LOG}" + if "$@" >>"${OUT}/${name}.txt" 2>>"${OUT}/${name}.err"; then + echo "OK ${name}" | tee -a "${LOG}" + else + echo "FAIL ${name} exit=$?" | tee -a "${LOG}" + fi +} + +sql() { + local name="$1" + local query="$2" + echo "=== sql:${name} ===" >>"${LOG}" + if command -v php >/dev/null 2>&1 && [[ -f artisan ]]; then + php artisan db:show --counts=false >/dev/null 2>&1 || true + php -r ' + require "vendor/autoload.php"; + $app = require "bootstrap/app.php"; + $kernel = $app->make(Illuminate\Contracts\Console\Kernel::class); + $kernel->bootstrap(); + $q = $argv[1]; + try { + $rows = Illuminate\Support\Facades\DB::select($q); + echo json_encode($rows, JSON_PRETTY_PRINT), PHP_EOL; + } catch (Throwable $e) { + fwrite(STDERR, "SQL_ERROR " . $e->getMessage() . PHP_EOL); + exit(1); + } + ' -- "$query" >"${OUT}/sql-${name}.json" 2>"${OUT}/sql-${name}.err" && echo "OK sql:${name}" | tee -a "${LOG}" || echo "FAIL sql:${name}" | tee -a "${LOG}" + else + echo "SKIP sql:${name} (no php/artisan)" | tee -a "${LOG}" + fi +} + +ROOT="$(pwd)" +if [[ -f artisan ]]; then + ROOT="$(pwd)" +elif [[ -f ../artisan ]]; then + cd .. + ROOT="$(pwd)" +fi + +STAMP="$(date -u +%Y%m%dT%H%M%SZ)" +OUT="${ROOT}/storage/app/optimization-baseline/${STAMP}" +mkdir -p "${OUT}" +LOG="${OUT}/collector.log" +touch "${LOG}" +echo "Skinbase M0 collector start ${STAMP}" | tee -a "${LOG}" +echo "cwd=${ROOT}" | tee -a "${LOG}" + +# --- host --- +run 01-hostname hostname +run 01b-hostname-f hostname -f +run 02-uname uname -a +run 03-os-release cat /etc/os-release +run 04-lscpu lscpu +run 05-free free -h +run 06-lsblk lsblk +run 07-df df -h +run 08-uptime uptime +run 09-vmstat vmstat 1 5 + +if command -v iostat >/dev/null 2>&1; then + run 10-iostat iostat -x 1 5 +else + echo "SKIP iostat" | tee -a "${LOG}" +fi + +run 11-ps-cpu ps aux --sort=-%cpu +run 12-ps-mem ps aux --sort=-%mem +# keep only heads in summary copies +if [[ -f "${OUT}/11-ps-cpu.txt" ]]; then head -n 30 "${OUT}/11-ps-cpu.txt" >"${OUT}/11-ps-cpu-top30.txt"; fi +if [[ -f "${OUT}/12-ps-mem.txt" ]]; then head -n 30 "${OUT}/12-ps-mem.txt" >"${OUT}/12-ps-mem-top30.txt"; fi + +run 13-ps-stack bash -c "ps aux | egrep 'nginx|php-fpm|redis|mysql|mysqld|meili|horizon|queue:work|supervisord|ssr.js|reverb|artisan' | grep -v grep || true" +run 14-systemctl-running systemctl --type=service --state=running --no-pager + +# --- nginx --- +run 15-nginx-v nginx -v +run 16-nginx-V nginx -V +if command -v nginx >/dev/null 2>&1; then + if nginx -T >/dev/null 2>&1; then + nginx -T 2>"${OUT}/17-nginx-T.err" | redact >"${OUT}/17-nginx-T.txt" || true + echo "OK 17-nginx-T" | tee -a "${LOG}" + elif sudo -n nginx -T >/dev/null 2>&1; then + sudo -n nginx -T 2>"${OUT}/17-nginx-T.err" | redact >"${OUT}/17-nginx-T.txt" || true + echo "OK 17-nginx-T via sudo -n" | tee -a "${LOG}" + else + echo "SKIP 17-nginx-T (need permission)" | tee -a "${LOG}" + fi +fi + +# --- php --- +run 18-php-v php -v +run 19-php-ini php --ini +run 20-php-m php -m +run 21-php-limits bash -c "php -i | egrep 'memory_limit|max_execution_time|max_input_time|upload_max_filesize|post_max_size|max_file_uploads|realpath_cache_size|realpath_cache_ttl' || true" +run 22-php-opcache bash -c "php -i | grep -i opcache || true" + +run 23-php-fpm-ps bash -c "ps aux | grep php-fpm | grep -v grep || true" +run 24-php-units systemctl list-units --no-pager +# filter later + +# copy fpm pool configs if readable +mkdir -p "${OUT}/php-fpm" +for f in /etc/php/*/fpm/pool.d/*.conf /etc/php-fpm.d/*.conf /usr/local/etc/php-fpm.d/*.conf; do + if [[ -r "$f" ]]; then + cp "$f" "${OUT}/php-fpm/$(basename "$f")" || true + fi +done + +# --- laravel non-secret env --- +if [[ -f .env ]]; then + grep -E "${APPROVED_ENV_KEYS}" .env >"${OUT}/25-env-approved.txt" 2>"${OUT}/25-env-approved.err" || true + echo "OK 25-env-approved (filtered)" | tee -a "${LOG}" +else + echo "SKIP 25-env-approved (no .env in cwd)" | tee -a "${LOG}" +fi + +if [[ -f artisan ]]; then + php artisan about --only=environment,cache,drivers,storage 2>"${OUT}/26-artisan-about.err" | redact >"${OUT}/26-artisan-about.txt" || echo "FAIL artisan about" | tee -a "${LOG}" + php artisan horizon:status >"${OUT}/27-horizon-status.txt" 2>"${OUT}/27-horizon-status.err" || true + php artisan queue:failed --json >"${OUT}/28-queue-failed.json" 2>"${OUT}/28-queue-failed.err" || php artisan queue:failed >"${OUT}/28-queue-failed.txt" 2>"${OUT}/28-queue-failed.err" || true + php artisan schedule:list >"${OUT}/29-schedule-list.txt" 2>"${OUT}/29-schedule-list.err" || true + php artisan route:list --columns=method,uri,name,action 2>/dev/null | egrep -i 'debugbar|telescope|clockwork' >"${OUT}/30-debug-routes.txt" || true +fi + +# laravel cache files +ls -la bootstrap/cache >"${OUT}/31-bootstrap-cache.txt" 2>"${OUT}/31-bootstrap-cache.err" || true +ls -la storage/framework/views | head -n 20 >"${OUT}/32-compiled-views.txt" 2>"${OUT}/32-compiled-views.err" || true +ls -la public/sitemaps | head -n 50 >"${OUT}/33-public-sitemaps.txt" 2>"${OUT}/33-public-sitemaps.err" || true +ls -la public/sitemap.xml public/robots.txt >"${OUT}/34-public-seo-files.txt" 2>"${OUT}/34-public-seo-files.err" || true + +# composer install mode +if [[ -f vendor/composer/installed.json ]]; then + php -r '$j=json_decode(file_get_contents("vendor/composer/installed.json"), true); echo "dev-package-count="; $n=0; foreach (($j["packages-dev"] ?? []) as $p) { $n++; } echo $n, PHP_EOL; echo "packages-count=", count($j["packages"] ?? []), PHP_EOL;' >"${OUT}/35-composer-dev-state.txt" 2>"${OUT}/35-composer-dev-state.err" || true +fi + +# --- mysql --- +run 36-mysql-version mysql --version +sql 37-version "SELECT VERSION() AS version" +sql 38-vars "SHOW VARIABLES WHERE Variable_name IN ('innodb_buffer_pool_size','innodb_buffer_pool_instances','max_connections','thread_cache_size','tmp_table_size','max_heap_table_size','slow_query_log','slow_query_log_file','long_query_time','log_queries_not_using_indexes','performance_schema')" +sql 39-status "SHOW GLOBAL STATUS WHERE Variable_name IN ('Threads_connected','Threads_running','Max_used_connections','Slow_queries','Queries','Questions','Uptime')" +sql 40-dbsize "SELECT table_schema, ROUND(SUM(data_length + index_length)/1024/1024,1) AS total_mb, ROUND(SUM(data_length)/1024/1024,1) AS data_mb, ROUND(SUM(index_length)/1024/1024,1) AS index_mb FROM information_schema.tables WHERE table_schema = DATABASE() GROUP BY table_schema" +sql 41-largest "SELECT table_name, table_rows, ROUND(data_length/1024/1024,1) AS data_mb, ROUND(index_length/1024/1024,1) AS index_mb, ROUND((data_length+index_length)/1024/1024,1) AS total_mb FROM information_schema.tables WHERE table_schema = DATABASE() ORDER BY data_length+index_length DESC LIMIT 50" +sql 42-cache-sessions "SELECT table_name, table_rows, ROUND((data_length+index_length)/1024/1024,1) AS mb FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name IN ('cache','cache_locks','sessions')" +sql 43-innodb-reads "SHOW GLOBAL STATUS LIKE 'Innodb_buffer_pool_read%'" +sql 44-innodb-rows "SHOW GLOBAL STATUS LIKE 'Innodb_rows_%'" +sql 45-tmp "SHOW GLOBAL STATUS LIKE 'Created_tmp%'" +sql 46-handler "SHOW GLOBAL STATUS LIKE 'Handler_read%'" + +for tbl in artworks artwork_stats artwork_metric_snapshots_hourly rank_artwork_scores tags artwork_view_events artwork_downloads; do + sql "index-${tbl}" "SHOW INDEX FROM \`${tbl}\`" +done + +# --- redis --- +if command -v redis-cli >/dev/null 2>&1; then + redis-cli INFO server >"${OUT}/50-redis-server.txt" 2>"${OUT}/50-redis-server.err" || true + redis-cli INFO memory >"${OUT}/51-redis-memory.txt" 2>"${OUT}/51-redis-memory.err" || true + redis-cli INFO stats >"${OUT}/52-redis-stats.txt" 2>"${OUT}/52-redis-stats.err" || true + redis-cli INFO clients >"${OUT}/53-redis-clients.txt" 2>"${OUT}/53-redis-clients.err" || true + redis-cli INFO keyspace >"${OUT}/54-redis-keyspace.txt" 2>"${OUT}/54-redis-keyspace.err" || true + for q in default search vision recommendations discovery mail notifications broadcasts forum-security forum-moderation; do + echo -n "queues:${q}: " >>"${OUT}/55-redis-queue-llen.txt" + redis-cli LLEN "queues:${q}" >>"${OUT}/55-redis-queue-llen.txt" 2>>"${OUT}/55-redis-queue-llen.err" || echo "fail" >>"${OUT}/55-redis-queue-llen.txt" + done + echo "OK redis-cli probes" | tee -a "${LOG}" +else + echo "SKIP redis-cli" | tee -a "${LOG}" +fi + +# --- processes / workers --- +run 56-queue-ps bash -c "ps aux | grep -E 'queue:work|horizon|ssr.js|reverb' | grep -v grep || true" +run 57-supervisor bash -c "command -v supervisorctl >/dev/null && supervisorctl status || echo 'no supervisorctl'" +run 58-cron-user bash -c "crontab -l || echo 'no user crontab'" +run 59-systemd-timers systemctl list-timers --no-pager + +# --- meilisearch --- +run 60-meili-ps bash -c "ps aux | grep -i meili | grep -v grep || true" +if command -v curl >/dev/null 2>&1; then + curl -sS --max-time 5 http://127.0.0.1:7700/health >"${OUT}/61-meili-health.txt" 2>"${OUT}/61-meili-health.err" || true + curl -sS --max-time 5 http://127.0.0.1:7700/version >"${OUT}/62-meili-version.txt" 2>"${OUT}/62-meili-version.err" || true +fi + +# --- public HTTP from the server itself --- +if command -v curl >/dev/null 2>&1; then + curl -sI --max-time 15 https://skinbase.org/ >"${OUT}/70-head-home.txt" 2>"${OUT}/70-head-home.err" || true + curl -sI --max-time 15 https://skinbase.org/sitemap.xml >"${OUT}/71-head-sitemap.txt" 2>"${OUT}/71-head-sitemap.err" || true + curl -sI --max-time 15 https://skinbase.org/robots.txt >"${OUT}/72-head-robots.txt" 2>"${OUT}/72-head-robots.err" || true +fi + +# strip cookies if any slipped into HTTP dumps +for f in "${OUT}"/*.txt; do + [[ -f "$f" ]] || continue + sed -i -E 's/^(Set-Cookie:).*/\1 [REDACTED]/I' "$f" 2>/dev/null || true +done + +echo "DONE output=${OUT}" | tee -a "${LOG}" +echo "${OUT}" diff --git a/scripts/sync-safe-updated.sh b/scripts/sync-safe-updated.sh index e0e802d6..b18255b0 100644 --- a/scripts/sync-safe-updated.sh +++ b/scripts/sync-safe-updated.sh @@ -329,6 +329,9 @@ build_rsync_args() { --exclude "/var/php-sessions" --exclude "/oldSite" --exclude "/vendor" + --exclude ".config" + --exclude ".copilot" + --exclude ".vscode" -e "$ssh_bin" ) } diff --git a/tests/Feature/Collections/CollectionDispatchGateTest.php b/tests/Feature/Collections/CollectionDispatchGateTest.php new file mode 100644 index 00000000..63a112ee --- /dev/null +++ b/tests/Feature/Collections/CollectionDispatchGateTest.php @@ -0,0 +1,18 @@ + false]); + Queue::fake(); + + $collection = Collection::factory()->create(); + $result = app(CollectionBackgroundJobService::class)->dispatchHealthRefresh($collection); + + expect($result['status'])->toBe('disabled') + ->and($result['count'])->toBe(0); + + Queue::assertNothingPushed(); +}); diff --git a/tests/Feature/Forum/ForumQueueDispatchGateTest.php b/tests/Feature/Forum/ForumQueueDispatchGateTest.php new file mode 100644 index 00000000..7f9b81c7 --- /dev/null +++ b/tests/Feature/Forum/ForumQueueDispatchGateTest.php @@ -0,0 +1,119 @@ + false]); + Queue::fake(); + + $post = new ForumPost(); + $post->id = 42; + + $queued = app(ForumModerationService::class)->dispatchAsyncScan($post); + + expect($queued)->toBeFalse(); + Queue::assertNotPushed(AnalyzeForumPostJob::class); +}); + +it('dispatches AnalyzeForumPostJob when AI queue dispatch is enabled', function () { + config(['skinbase_ai_moderation.queue.dispatch_enabled' => true]); + Queue::fake(); + + $post = new ForumPost(); + $post->id = 43; + + $queued = app(ForumModerationService::class)->dispatchAsyncScan($post); + + expect($queued)->toBeTrue(); + Queue::assertPushedOn( + (string) config('skinbase_ai_moderation.queue.name', 'forum-moderation'), + AnalyzeForumPostJob::class, + ); +}); + +it('does not dispatch BotActivityMonitor when security queue dispatch is disabled', function () { + config(['forum_security.queues.dispatch_enabled' => false]); + Queue::fake(); + + $this->artisan('forum:bot-scan')->assertSuccessful(); + + Queue::assertNotPushed(BotActivityMonitor::class); +}); + +it('dispatches BotActivityMonitor when security queue dispatch is enabled', function () { + config(['forum_security.queues.dispatch_enabled' => true]); + Queue::fake(); + + $this->artisan('forum:bot-scan')->assertSuccessful(); + + Queue::assertPushed(BotActivityMonitor::class); +}); + +it('does not dispatch FirewallActivityMonitor when security queue dispatch is disabled', function () { + config(['forum_security.queues.dispatch_enabled' => false]); + Queue::fake(); + + $this->artisan('forum:firewall-scan')->assertSuccessful(); + + Queue::assertNotPushed(FirewallActivityMonitor::class); +}); + +it('dispatches FirewallActivityMonitor when security queue dispatch is enabled', function () { + config(['forum_security.queues.dispatch_enabled' => true]); + Queue::fake(); + + $this->artisan('forum:firewall-scan')->assertSuccessful(); + + Queue::assertPushed(FirewallActivityMonitor::class); +}); + +it('runs forum bot scan synchronously without queueing when --sync is passed', function () { + config(['forum_security.queues.dispatch_enabled' => false]); + Queue::fake(); + + $this->mock(BotProtectionService::class, function ($mock): void { + $mock->shouldReceive('scanRecentActivity')->once()->andReturn(['processed' => 0]); + }); + + $this->artisan('forum:bot-scan --sync')->assertSuccessful(); + + Queue::assertNothingPushed(); +}); + +it('runs forum firewall scan synchronously without queueing when --sync is passed', function () { + config(['forum_security.queues.dispatch_enabled' => false]); + Queue::fake(); + + $this->mock(ForumSecurityFirewallService::class, function ($mock): void { + $mock->shouldReceive('scanRecentThreats')->once()->andReturn(['processed' => 0]); + }); + + $this->artisan('forum:firewall-scan --sync')->assertSuccessful(); + + Queue::assertNothingPushed(); +}); + +it('does not enqueue AnalyzeForumPostJob from forum:ai-scan without --sync when dispatch is disabled', function () { + config(['skinbase_ai_moderation.queue.dispatch_enabled' => false]); + Queue::fake(); + + $this->artisan('forum:ai-scan')->assertSuccessful(); + + Queue::assertNotPushed(AnalyzeForumPostJob::class); +}); + +it('keeps synchronous forum:scan-posts on the schedule and omits async forum enqueue commands when dispatch is disabled', function () { + $this->artisan('schedule:list') + ->expectsOutputToContain('forum:scan-posts') + ->doesntExpectOutputToContain('forum:ai-scan') + ->doesntExpectOutputToContain('forum:bot-scan') + ->doesntExpectOutputToContain('forum:firewall-scan') + ->assertSuccessful(); +}); diff --git a/tests/Feature/Http/ArtworkDownloadMissingFileTest.php b/tests/Feature/Http/ArtworkDownloadMissingFileTest.php new file mode 100644 index 00000000..1aed234b --- /dev/null +++ b/tests/Feature/Http/ArtworkDownloadMissingFileTest.php @@ -0,0 +1,24 @@ +create([ + 'is_public' => true, + 'is_approved' => true, + 'published_at' => now()->subDay(), + 'file_name' => 'missing-original.jpg', + 'file_ext' => 'jpg', + 'hash' => str_repeat('ab', 32), + ]); + + $this->mock(ArtworkStatsService::class, function ($mock) { + $mock->shouldReceive('incrementDownloads')->never(); + }); + + $this->get('/download/artwork/'.$artwork->id)->assertNotFound(); + + expect(DB::table('artwork_downloads')->where('artwork_id', $artwork->id)->count())->toBe(0); +}); diff --git a/tests/Feature/Http/RssPresenceSkipTest.php b/tests/Feature/Http/RssPresenceSkipTest.php new file mode 100644 index 00000000..f2e52c4e --- /dev/null +++ b/tests/Feature/Http/RssPresenceSkipTest.php @@ -0,0 +1,11 @@ +mock(OnlineVisitorRepository::class, function ($mock): void { + $mock->shouldReceive('track')->never(); + }); + + $this->get('/rss/latest-uploads.xml')->assertOk(); +}); diff --git a/tests/Feature/Http/SlowHttpRequestLoggingTest.php b/tests/Feature/Http/SlowHttpRequestLoggingTest.php new file mode 100644 index 00000000..722b696e --- /dev/null +++ b/tests/Feature/Http/SlowHttpRequestLoggingTest.php @@ -0,0 +1,113 @@ + true, + 'http_observability.slow_request.threshold_ms' => 750, + 'logging.channels.slow-http.driver' => 'single', + 'logging.channels.slow-http.path' => slowHttpLogPath(), + 'logging.channels.slow-http.level' => 'info', + ]); + Log::forgetChannel('slow-http'); + File::delete(slowHttpLogPath()); +} + +afterEach(function (): void { + File::delete(slowHttpLogPath()); +}); + +it('does not write a slow-http record for a fast request', function () { + configureSlowHttpLog(); + config(['http_observability.slow_request.test_duration_ms' => 120]); + + $this->get('/leaderboard')->assertOk(); + + expect(File::exists(slowHttpLogPath()))->toBeFalse(); +}); + +it('writes one structured slow-http record using the route template', function () { + configureSlowHttpLog(); + config(['http_observability.slow_request.test_duration_ms' => 900]); + + $response = $this->get('/leaderboard'); + $response->assertOk(); + expect($response->headers->get('Server-Timing'))->toContain('app;desc="Laravel"'); + + $raw = File::get(slowHttpLogPath()); + expect(substr_count(trim($raw), "\n"))->toBe(0); + + $line = trim($raw); + preg_match('/\{.*\}\s*$/', $line, $matches); + expect($matches[0] ?? null)->not->toBeNull(); + $json = json_decode($matches[0], true, flags: JSON_THROW_ON_ERROR); + + expect($json['method'])->toBe('GET') + ->and($json['status'])->toBe(200) + ->and($json['duration_ms'])->toEqual(900) + ->and($json['route_uri'])->not->toContain('?') + ->and($json['route_uri'])->not->toContain('leaderboard/') + ->and($json)->not->toHaveKey('query') + ->and($json)->not->toHaveKey('ip') + ->and($json)->not->toHaveKey('email') + ->and($json)->not->toHaveKey('user_id') + ->and($json['authenticated'])->toBeFalse(); +}); + +it('does not log when slow-request logging is disabled', function () { + configureSlowHttpLog(); + config([ + 'http_observability.slow_request.enabled' => false, + 'http_observability.slow_request.test_duration_ms' => 5000, + ]); + + $this->get('/leaderboard')->assertOk()->assertHeader('Server-Timing'); + + expect(File::exists(slowHttpLogPath()))->toBeFalse(); +}); + +it('does not break a missing-file download response and still sends Server-Timing', function () { + configureSlowHttpLog(); + config(['http_observability.slow_request.test_duration_ms' => 100]); + + $artwork = Artwork::factory()->create([ + 'is_public' => true, + 'is_approved' => true, + 'published_at' => now()->subDay(), + 'file_ext' => 'jpg', + 'hash' => str_repeat('cd', 32), + ]); + + $this->get('/download/artwork/'.$artwork->id) + ->assertNotFound() + ->assertHeader('Server-Timing'); +}); + +it('does not log query strings or user identifiers on an authenticated slow request', function () { + configureSlowHttpLog(); + config(['http_observability.slow_request.test_duration_ms' => 800]); + + $user = User::factory()->create([ + 'email' => 'slow-log-secret@example.com', + 'username' => 'slowlogsecret', + ]); + + $this->actingAs($user)->get('/leaderboard?period=weekly&debug=1')->assertOk(); + + $raw = File::get(slowHttpLogPath()); + expect($raw)->not->toContain('slow-log-secret@example.com') + ->and($raw)->not->toContain('period=weekly') + ->and($raw)->not->toContain('debug=1') + ->and($raw)->not->toContain('"user_id"') + ->and($raw)->not->toContain($user->email); +}); diff --git a/tests/Feature/Http/StudioGroupsSharedPropTest.php b/tests/Feature/Http/StudioGroupsSharedPropTest.php new file mode 100644 index 00000000..4b79bc3f --- /dev/null +++ b/tests/Feature/Http/StudioGroupsSharedPropTest.php @@ -0,0 +1,26 @@ +create(); + + Group::factory()->create([ + 'owner_user_id' => $user->id, + 'status' => Group::LIFECYCLE_ACTIVE, + ]); + + $groupQueries = 0; + DB::listen(function ($query) use (&$groupQueries): void { + $sql = strtolower($query->sql); + if (str_contains($sql, 'from `groups`') || str_contains($sql, 'from groups') || str_contains($sql, 'group_members')) { + $groupQueries++; + } + }); + + $this->actingAs($user)->get('/leaderboard')->assertOk(); + + expect($groupQueries)->toBe(0); +}); diff --git a/tests/Feature/Metrics/ArtworkHourlySnapshotWindowTest.php b/tests/Feature/Metrics/ArtworkHourlySnapshotWindowTest.php new file mode 100644 index 00000000..ed6fd40c --- /dev/null +++ b/tests/Feature/Metrics/ArtworkHourlySnapshotWindowTest.php @@ -0,0 +1,175 @@ + \App\Models\Artwork::factory()->create([ + 'user_id' => $userId, + 'is_approved' => true, + 'is_public' => true, + 'published_at' => now()->subDays(40), + 'deleted_at' => null, + ])); +} + +function insertHourlySnapshot(int $artworkId, $bucketHour, int $views, int $favourites = 0, int $shares = 0, int $comments = 0, int $downloads = 0): void +{ + ArtworkMetricSnapshotHourly::create([ + 'artwork_id' => $artworkId, + 'bucket_hour' => $bucketHour, + 'views_count' => $views, + 'downloads_count' => $downloads, + 'favourites_count' => $favourites, + 'comments_count' => $comments, + 'shares_count' => $shares, + ]); +} + +it('computes 30d views as max-min per artwork not sum of cumulatives', function () { + $user = \App\Models\User::factory()->create(); + $artwork = snapshotArtworkForUser($user->id); + + insertHourlySnapshot($artwork->id, now()->subDays(10)->startOfHour(), 100, 10, 1, 2, 4); + insertHourlySnapshot($artwork->id, now()->subDays(1)->startOfHour(), 130, 16, 4, 5, 9); + + $window = app(ArtworkHourlySnapshotWindow::class)->userWindow($user->id, 30); + + expect($window['views'])->toBe(30) + ->and($window['favourites'])->toBe(6) + ->and($window['shares'])->toBe(3) + ->and($window['comments'])->toBe(3) + ->and($window['downloads'])->toBe(5) + ->and($window['views'])->not->toBe(230); +}); + +it('sums window deltas across a creator\'s artworks', function () { + $user = \App\Models\User::factory()->create(); + $a = snapshotArtworkForUser($user->id); + $b = snapshotArtworkForUser($user->id); + + insertHourlySnapshot($a->id, now()->subDays(5)->startOfHour(), 50); + insertHourlySnapshot($a->id, now()->subHour()->startOfHour(), 80); + insertHourlySnapshot($b->id, now()->subDays(5)->startOfHour(), 10); + insertHourlySnapshot($b->id, now()->subHour()->startOfHour(), 15); + + $window = app(ArtworkHourlySnapshotWindow::class)->userWindow($user->id, 30); + + expect($window['views'])->toBe(35); +}); + +it('does not treat zero growth as lifetime totals', function () { + $user = \App\Models\User::factory()->create(); + $artwork = snapshotArtworkForUser($user->id); + + DB::table('artwork_stats')->updateOrInsert( + ['artwork_id' => $artwork->id], + ['views' => 9999, 'favorites' => 50, 'shares_count' => 7] + ); + + insertHourlySnapshot($artwork->id, now()->subDays(2)->startOfHour(), 40); + insertHourlySnapshot($artwork->id, now()->subHour()->startOfHour(), 40); + + Cache::flush(); + $kpis = app(StudioMetricsService::class)->getDashboardKpis($user->id); + + expect($kpis['views_30d'])->toBe(0) + ->and($kpis['favourites_30d'])->toBe(0) + ->and($kpis['snapshot_window']['window_complete'])->toBeFalse(); +}); + +it('marks a short table history as an incomplete 30-day window', function () { + $user = \App\Models\User::factory()->create(); + $artwork = snapshotArtworkForUser($user->id); + + insertHourlySnapshot($artwork->id, now()->subDays(2)->startOfHour(), 10); + insertHourlySnapshot($artwork->id, now()->subHour()->startOfHour(), 18); + + $window = app(ArtworkHourlySnapshotWindow::class)->userWindow($user->id, 30); + + expect($window['views'])->toBe(8) + ->and($window['window_complete'])->toBeFalse() + ->and($window['table_coverage_days'])->toBeLessThan(30) + ->and($window['requested_days'])->toBe(30); +}); + +it('counts all views for artwork created during the window', function () { + $user = \App\Models\User::factory()->create(); + $artwork = \App\Models\Artwork::withoutEvents(fn () => \App\Models\Artwork::factory()->create([ + 'user_id' => $user->id, + 'is_approved' => true, + 'is_public' => true, + 'published_at' => now()->subDays(3), + 'created_at' => now()->subDays(3), + 'deleted_at' => null, + ])); + + insertHourlySnapshot($artwork->id, now()->subDays(3)->startOfHour(), 40); + insertHourlySnapshot($artwork->id, now()->subHour()->startOfHour(), 90); + + $window = app(ArtworkHourlySnapshotWindow::class)->userWindow($user->id, 30); + + expect($window['views'])->toBe(90); +}); + +it('uses a pre-window snapshot as baseline for older artwork', function () { + $user = \App\Models\User::factory()->create(); + $artwork = snapshotArtworkForUser($user->id); + + insertHourlySnapshot($artwork->id, now()->subDays(30)->subHour()->startOfHour(), 200); + insertHourlySnapshot($artwork->id, now()->subDays(10)->startOfHour(), 220); + insertHourlySnapshot($artwork->id, now()->subHour()->startOfHour(), 250); + + $window = app(ArtworkHourlySnapshotWindow::class)->userWindow($user->id, 30); + + expect($window['views'])->toBe(50); +}); + +it('does not treat a single in-window snapshot of older artwork as lifetime views', function () { + $user = \App\Models\User::factory()->create(); + $artwork = snapshotArtworkForUser($user->id); + + insertHourlySnapshot($artwork->id, now()->subDays(2)->startOfHour(), 400); + + $window = app(ArtworkHourlySnapshotWindow::class)->userWindow($user->id, 30); + + expect($window['views'])->toBe(0); +}); + +it('treats a single snapshot of a new artwork as growth from zero', function () { + $user = \App\Models\User::factory()->create(); + $artwork = \App\Models\Artwork::withoutEvents(fn () => \App\Models\Artwork::factory()->create([ + 'user_id' => $user->id, + 'is_approved' => true, + 'is_public' => true, + 'published_at' => now()->subHours(2), + 'created_at' => now()->subHours(2), + 'deleted_at' => null, + ])); + + insertHourlySnapshot($artwork->id, now()->subHour()->startOfHour(), 12); + + $window = app(ArtworkHourlySnapshotWindow::class)->userWindow($user->id, 30); + + expect($window['views'])->toBe(12); +}); + +it('ignores creator-journey snapshots older than the retention window', function () { + $user = \App\Models\User::factory()->create(); + $artwork = snapshotArtworkForUser($user->id); + + insertHourlySnapshot($artwork->id, now()->subDays(40)->startOfHour(), 10, 0, 0, 0, 1); + insertHourlySnapshot($artwork->id, now()->subDays(39)->startOfHour(), 10, 0, 0, 0, 500); + + $lookbackDays = (int) config('metrics.hourly_snapshot_retention_days', 30); + $count = DB::table('artwork_metric_snapshots_hourly') + ->where('artwork_id', $artwork->id) + ->where('bucket_hour', '>=', now()->subDays($lookbackDays)) + ->count(); + + expect($count)->toBe(0); +}); diff --git a/tests/Feature/Metrics/LeaderboardSnapshotDeltaTest.php b/tests/Feature/Metrics/LeaderboardSnapshotDeltaTest.php new file mode 100644 index 00000000..c68b672a --- /dev/null +++ b/tests/Feature/Metrics/LeaderboardSnapshotDeltaTest.php @@ -0,0 +1,82 @@ + Artwork::factory()->create(array_merge([ + 'is_approved' => true, + 'is_public' => true, + 'published_at' => now()->subDays(40), + 'deleted_at' => null, + ], $attrs))); +} + +function leaderboardSnapshot(int $artworkId, $hour, int $views): void +{ + ArtworkMetricSnapshotHourly::create([ + 'artwork_id' => $artworkId, + 'bucket_hour' => $hour, + 'views_count' => $views, + 'downloads_count' => 0, + 'favourites_count' => 0, + 'comments_count' => 0, + 'shares_count' => 0, + ]); +} + +it('scores monthly artwork leaderboards from period view deltas not cumulative sums', function () { + $old = leaderboardArtwork(); + $bornInMonth = leaderboardArtwork([ + 'published_at' => now()->subDays(4), + 'created_at' => now()->subDays(4), + ]); + + leaderboardSnapshot($old->id, now()->subDays(20)->startOfHour(), 100); + leaderboardSnapshot($old->id, now()->subHour()->startOfHour(), 140); + leaderboardSnapshot($bornInMonth->id, now()->subDays(3)->startOfHour(), 25); + leaderboardSnapshot($bornInMonth->id, now()->subHour()->startOfHour(), 55); + + app(LeaderboardService::class)->calculateArtworkLeaderboard(Leaderboard::PERIOD_MONTHLY); + + $rows = Leaderboard::query() + ->where('type', Leaderboard::TYPE_ARTWORK) + ->where('period', Leaderboard::PERIOD_MONTHLY) + ->get() + ->keyBy('entity_id'); + + expect((int) $rows[$old->id]->score)->toBe(40) + ->and((int) $rows[$bornInMonth->id]->score)->toBe(55) + ->and((int) $rows[$old->id]->score)->not->toBe(240); +}); + +it('scores daily and weekly artwork leaderboards from the same delta rules', function () { + $artwork = leaderboardArtwork(); + + leaderboardSnapshot($artwork->id, now()->subDays(7)->subHour()->startOfHour(), 10); + leaderboardSnapshot($artwork->id, now()->subDays(3)->startOfHour(), 18); + leaderboardSnapshot($artwork->id, now()->subDay()->subHour()->startOfHour(), 18); + leaderboardSnapshot($artwork->id, now()->subHours(2)->startOfHour(), 21); + + $service = app(LeaderboardService::class); + $service->calculateArtworkLeaderboard(Leaderboard::PERIOD_WEEKLY); + $service->calculateArtworkLeaderboard(Leaderboard::PERIOD_DAILY); + + $weekly = Leaderboard::query() + ->where('type', Leaderboard::TYPE_ARTWORK) + ->where('period', Leaderboard::PERIOD_WEEKLY) + ->where('entity_id', $artwork->id) + ->value('score'); + + $daily = Leaderboard::query() + ->where('type', Leaderboard::TYPE_ARTWORK) + ->where('period', Leaderboard::PERIOD_DAILY) + ->where('entity_id', $artwork->id) + ->value('score'); + + expect((int) $weekly)->toBe(11) + ->and((int) $daily)->toBe(3); +}); diff --git a/tests/Feature/Metrics/RecalculateHeatCommandTest.php b/tests/Feature/Metrics/RecalculateHeatCommandTest.php new file mode 100644 index 00000000..0f608b22 --- /dev/null +++ b/tests/Feature/Metrics/RecalculateHeatCommandTest.php @@ -0,0 +1,47 @@ +create(); + $artwork = Artwork::withoutEvents(fn () => Artwork::factory()->create([ + 'user_id' => $user->id, + 'is_approved' => true, + 'is_public' => true, + 'published_at' => now()->subDays(2), + 'deleted_at' => null, + ])); + + $currentHour = now()->startOfHour(); + $prevHour = $currentHour->copy()->subHour(); + + ArtworkMetricSnapshotHourly::create([ + 'artwork_id' => $artwork->id, + 'bucket_hour' => $prevHour, + 'views_count' => 10, + 'downloads_count' => 1, + 'favourites_count' => 1, + 'comments_count' => 0, + 'shares_count' => 0, + ]); + + ArtworkMetricSnapshotHourly::create([ + 'artwork_id' => $artwork->id, + 'bucket_hour' => $currentHour, + 'views_count' => 40, + 'downloads_count' => 4, + 'favourites_count' => 3, + 'comments_count' => 1, + 'shares_count' => 1, + ]); + + $this->artisan('nova:recalculate-heat', ['--chunk' => 1])->assertExitCode(0); + + $row = DB::table('artwork_stats')->where('artwork_id', $artwork->id)->first(); + + expect($row)->not->toBeNull() + ->and((float) $row->heat_score)->toBeGreaterThan(0); +}); diff --git a/tests/Feature/Profile/CreatorJourneyTest.php b/tests/Feature/Profile/CreatorJourneyTest.php index 330ee768..d212f1ef 100644 --- a/tests/Feature/Profile/CreatorJourneyTest.php +++ b/tests/Feature/Profile/CreatorJourneyTest.php @@ -135,7 +135,7 @@ function seedCreatorJourneyFixture(): User DB::table('artwork_metric_snapshots_hourly')->insert([ [ 'artwork_id' => $breakthroughArtwork->id, - 'bucket_hour' => Carbon::parse('2024-05-10 10:00:00'), + 'bucket_hour' => now()->subHours(5)->startOfHour(), 'views_count' => 1200, 'downloads_count' => 40, 'favourites_count' => 60, @@ -145,7 +145,7 @@ function seedCreatorJourneyFixture(): User ], [ 'artwork_id' => $breakthroughArtwork->id, - 'bucket_hour' => Carbon::parse('2024-05-10 11:00:00'), + 'bucket_hour' => now()->subHours(4)->startOfHour(), 'views_count' => 1320, 'downloads_count' => 64, 'favourites_count' => 68, @@ -155,7 +155,7 @@ function seedCreatorJourneyFixture(): User ], [ 'artwork_id' => $lateYearArtwork->id, - 'bucket_hour' => Carbon::parse('2024-11-20 19:00:00'), + 'bucket_hour' => now()->subHours(3)->startOfHour(), 'views_count' => 300, 'downloads_count' => 10, 'favourites_count' => 12, @@ -165,7 +165,7 @@ function seedCreatorJourneyFixture(): User ], [ 'artwork_id' => $lateYearArtwork->id, - 'bucket_hour' => Carbon::parse('2024-11-20 20:00:00'), + 'bucket_hour' => now()->subHours(2)->startOfHour(), 'views_count' => 360, 'downloads_count' => 14, 'favourites_count' => 15, @@ -276,4 +276,45 @@ it('hydrates the public profile page with creator journey props', function () { ->where('journey.timeline', fn ($timeline) => collect($timeline)->pluck('headline')->contains('First Spectrum Pack')) ->where('journeyApiUrl', route('api.profile.journey', ['username' => strtolower((string) $creator->username)])) ); +}); + +it('ignores download spikes older than the snapshot retention window', function () { + $creator = seedCreatorJourneyFixture(); + $oldArtwork = Artwork::query()->where('user_id', $creator->id)->where('slug', 'sky-one')->firstOrFail(); + + DB::table('artwork_metric_snapshots_hourly')->insert([ + [ + 'artwork_id' => $oldArtwork->id, + 'bucket_hour' => now()->subDays(40)->startOfHour(), + 'views_count' => 10, + 'downloads_count' => 1, + 'favourites_count' => 0, + 'comments_count' => 0, + 'shares_count' => 0, + 'created_at' => now(), + ], + [ + 'artwork_id' => $oldArtwork->id, + 'bucket_hour' => now()->subDays(40)->addHour()->startOfHour(), + 'views_count' => 10, + 'downloads_count' => 900, + 'favourites_count' => 0, + 'comments_count' => 0, + 'shares_count' => 0, + 'created_at' => now(), + ], + ]); + + app(CreatorJourneyService::class)->rebuildForUser($creator); + + $spike = DB::table('creator_milestones') + ->where('user_id', $creator->id) + ->where('type', 'biggest_download_spike') + ->first(); + + $payload = json_decode((string) $spike->payload_json, true); + + expect($spike)->not->toBeNull() + ->and((int) data_get($payload, 'metrics.downloads_in_hour'))->toBe(24) + ->and((int) data_get($payload, 'metrics.downloads_in_hour'))->not->toBe(899); }); \ No newline at end of file diff --git a/tests/Feature/PruneMetricSnapshotsCommandTest.php b/tests/Feature/PruneMetricSnapshotsCommandTest.php new file mode 100644 index 00000000..55106ee0 --- /dev/null +++ b/tests/Feature/PruneMetricSnapshotsCommandTest.php @@ -0,0 +1,164 @@ + true, + 'is_public' => true, + 'published_at' => now()->subDays(40), + ]); + + ArtworkMetricSnapshotHourly::create([ + 'artwork_id' => $artwork->id, + 'bucket_hour' => now()->subDays(40)->startOfHour(), + 'views_count' => 10, + 'downloads_count' => 1, + 'favourites_count' => 0, + 'comments_count' => 0, + 'shares_count' => 0, + ]); + + ArtworkMetricSnapshotHourly::create([ + 'artwork_id' => $artwork->id, + 'bucket_hour' => now()->subDays(10)->startOfHour(), + 'views_count' => 20, + 'downloads_count' => 2, + 'favourites_count' => 0, + 'comments_count' => 0, + 'shares_count' => 0, + ]); + + ArtworkMetricSnapshotHourly::create([ + 'artwork_id' => $artwork->id, + 'bucket_hour' => now()->subHour()->startOfHour(), + 'views_count' => 30, + 'downloads_count' => 3, + 'favourites_count' => 0, + 'comments_count' => 0, + 'shares_count' => 0, + ]); + + $this->artisan('nova:prune-metric-snapshots --keep-days=30 --sleep-ms=0') + ->assertSuccessful(); + + $hours = ArtworkMetricSnapshotHourly::where('artwork_id', $artwork->id) + ->orderBy('bucket_hour') + ->pluck('views_count') + ->all(); + + expect($hours)->toBe([20, 30]); +}); + +it('deletes in bounded batches and is idempotent', function () { + $artwork = createArtworkWithoutObserver([ + 'is_approved' => true, + 'is_public' => true, + 'published_at' => now()->subDays(20), + ]); + + foreach (range(1, 5) as $daysAgo) { + ArtworkMetricSnapshotHourly::create([ + 'artwork_id' => $artwork->id, + 'bucket_hour' => now()->subDays($daysAgo + 10)->startOfHour(), + 'views_count' => $daysAgo, + 'downloads_count' => 0, + 'favourites_count' => 0, + 'comments_count' => 0, + 'shares_count' => 0, + ]); + } + + $this->artisan('nova:prune-metric-snapshots --keep-days=7 --chunk=2 --sleep-ms=0') + ->assertSuccessful(); + + expect(ArtworkMetricSnapshotHourly::where('artwork_id', $artwork->id)->count())->toBe(0); + + $this->artisan('nova:prune-metric-snapshots --keep-days=7 --chunk=2 --sleep-ms=0') + ->assertSuccessful(); + + expect(ArtworkMetricSnapshotHourly::where('artwork_id', $artwork->id)->count())->toBe(0); +}); + +it('dry-run does not delete rows', function () { + $artwork = createArtworkWithoutObserver([ + 'is_approved' => true, + 'is_public' => true, + 'published_at' => now()->subDays(20), + ]); + + ArtworkMetricSnapshotHourly::create([ + 'artwork_id' => $artwork->id, + 'bucket_hour' => now()->subDays(14)->startOfHour(), + 'views_count' => 1, + 'downloads_count' => 0, + 'favourites_count' => 0, + 'comments_count' => 0, + 'shares_count' => 0, + ]); + + $this->artisan('nova:prune-metric-snapshots --keep-days=7 --dry-run') + ->assertSuccessful(); + + expect(ArtworkMetricSnapshotHourly::where('artwork_id', $artwork->id)->count())->toBe(1); +}); + +it('uses config retention when keep-days is omitted', function () { + config(['metrics.hourly_snapshot_retention_days' => 7]); + + $artwork = createArtworkWithoutObserver([ + 'is_approved' => true, + 'is_public' => true, + 'published_at' => now()->subDays(20), + ]); + + ArtworkMetricSnapshotHourly::create([ + 'artwork_id' => $artwork->id, + 'bucket_hour' => now()->subDays(10)->startOfHour(), + 'views_count' => 1, + 'downloads_count' => 0, + 'favourites_count' => 0, + 'comments_count' => 0, + 'shares_count' => 0, + ]); + + ArtworkMetricSnapshotHourly::create([ + 'artwork_id' => $artwork->id, + 'bucket_hour' => now()->subDays(2)->startOfHour(), + 'views_count' => 2, + 'downloads_count' => 0, + 'favourites_count' => 0, + 'comments_count' => 0, + 'shares_count' => 0, + ]); + + $this->artisan('nova:prune-metric-snapshots --sleep-ms=0') + ->assertSuccessful(); + + expect(ArtworkMetricSnapshotHourly::where('artwork_id', $artwork->id)->count())->toBe(1); +}); + +it('honours max-batches so a run cannot delete the whole table at once', function () { + $artwork = createArtworkWithoutObserver([ + 'is_approved' => true, + 'is_public' => true, + 'published_at' => now()->subDays(20), + ]); + + foreach (range(1, 4) as $n) { + ArtworkMetricSnapshotHourly::create([ + 'artwork_id' => $artwork->id, + 'bucket_hour' => now()->subDays(10 + $n)->startOfHour(), + 'views_count' => $n, + 'downloads_count' => 0, + 'favourites_count' => 0, + 'comments_count' => 0, + 'shares_count' => 0, + ]); + } + + $this->artisan('nova:prune-metric-snapshots --keep-days=7 --chunk=1 --max-batches=2 --sleep-ms=0') + ->assertSuccessful(); + + expect(ArtworkMetricSnapshotHourly::where('artwork_id', $artwork->id)->count())->toBe(2); +}); diff --git a/tests/Feature/Queue/DefaultQueueAssignmentTest.php b/tests/Feature/Queue/DefaultQueueAssignmentTest.php new file mode 100644 index 00000000..8a2579e1 --- /dev/null +++ b/tests/Feature/Queue/DefaultQueueAssignmentTest.php @@ -0,0 +1,55 @@ +queue)->toBe('search') + ->and((new IndexUserJob(22))->queue)->toBe('search'); + + Queue::fake(); + IndexUserJob::dispatch(22); + Queue::assertPushedOn('search', IndexUserJob::class); +}); + +it('collapses duplicate index jobs with unique ids', function () { + $artwork = new IndexArtworkJob(15); + $user = new IndexUserJob(22); + + expect($artwork)->toBeInstanceOf(ShouldBeUniqueUntilProcessing::class) + ->and($artwork->uniqueId())->toBe('15') + ->and($artwork->uniqueFor)->toBe(120) + ->and($user)->toBeInstanceOf(ShouldBeUniqueUntilProcessing::class) + ->and($user->uniqueId())->toBe('22') + ->and($user->uniqueFor)->toBe(60); +}); + +it('uniques ranking scope rebuilds so hourly fan-out cannot stack the same scope', function () { + $job = new RankBuildScopeListsJob('category', 44); + + expect($job)->toBeInstanceOf(ShouldBeUnique::class) + ->and($job->uniqueId())->toBe('category:44') + ->and($job->uniqueFor)->toBe(21600) + ->and($job->uniqueFor)->toBeGreaterThan(360) + ->and((new RankBuildScopeListsJob('content_type', 3))->uniqueId())->toBe('content_type:3') + ->and((new RankBuildScopeListsJob('category', 44))->uniqueId()) + ->not->toBe((new RankBuildScopeListsJob('category', 45))->uniqueId()); +}); + +it('does not enqueue the same ranking scope again after 360s while the unique lock remains', function () { + Queue::fake(); + $this->freezeTime(); + + RankBuildScopeListsJob::dispatch('category', 88); + $this->travel(361)->seconds(); + RankBuildScopeListsJob::dispatch('category', 88); + RankBuildScopeListsJob::dispatch('category', 89); + + Queue::assertPushed(RankBuildScopeListsJob::class, 2); + Queue::assertPushed(RankBuildScopeListsJob::class, fn (RankBuildScopeListsJob $job): bool => $job->uniqueId() === 'category:88'); + Queue::assertPushed(RankBuildScopeListsJob::class, fn (RankBuildScopeListsJob $job): bool => $job->uniqueId() === 'category:89'); +}); diff --git a/tests/Feature/Queue/PurgeQueuedRecComputeTagsCommandTest.php b/tests/Feature/Queue/PurgeQueuedRecComputeTagsCommandTest.php new file mode 100644 index 00000000..a9b8e101 --- /dev/null +++ b/tests/Feature/Queue/PurgeQueuedRecComputeTagsCommandTest.php @@ -0,0 +1,56 @@ + RecComputeSimilarByTagsJob::class, + 'data' => ['commandName' => RecComputeSimilarByTagsJob::class], + ]); + $other = json_encode([ + 'displayName' => IndexUserJob::class, + 'data' => ['commandName' => IndexUserJob::class], + ]); + + expect(QueuedJobClassMatcher::isClass($tags, RecComputeSimilarByTagsJob::class))->toBeTrue() + ->and(QueuedJobClassMatcher::isClass($other, RecComputeSimilarByTagsJob::class))->toBeFalse(); +}); + +it('dry-run counts stale tags jobs and does not remove them', function () { + $tags = json_encode(['displayName' => RecComputeSimilarByTagsJob::class, 'uuid' => 'aaa']); + $other = json_encode(['displayName' => IndexUserJob::class, 'uuid' => 'bbb']); + + $redis = Mockery::mock(); + $redis->shouldReceive('llen')->once()->with('queues:default')->andReturn(2); + $redis->shouldReceive('lrange')->once()->andReturn([$other, $tags]); + $redis->shouldReceive('lrem')->never(); + Redis::partialMock()->shouldReceive('connection')->andReturn($redis); + + $this->artisan('skinbase:purge-queued-rec-tags --dry-run') + ->expectsOutputToContain('matched=1') + ->assertSuccessful(); +}); + +it('execute removes only tags jobs and preserves unrelated payloads', function () { + $tagsA = json_encode(['displayName' => RecComputeSimilarByTagsJob::class, 'uuid' => 't1']); + $tagsB = json_encode(['displayName' => RecComputeSimilarByTagsJob::class, 'uuid' => 't2']); + $other = json_encode(['displayName' => IndexUserJob::class, 'uuid' => 'keep']); + + $redis = Mockery::mock(); + $redis->shouldReceive('llen')->once()->with('queues:default')->andReturn(3); + $redis->shouldReceive('lrange')->once()->andReturn([$other, $tagsA, $tagsB]); + $redis->shouldReceive('lrem')->twice()->andReturn(1); + Redis::partialMock()->shouldReceive('connection')->andReturn($redis); + + $this->artisan('skinbase:purge-queued-rec-tags --execute') + ->expectsOutputToContain('matched=2') + ->assertSuccessful(); +}); + +it('refuses to run without dry-run or execute', function () { + $this->artisan('skinbase:purge-queued-rec-tags') + ->assertFailed(); +}); diff --git a/tests/Feature/Recommendations/RecComputeSimilarJobsTest.php b/tests/Feature/Recommendations/RecComputeSimilarJobsTest.php new file mode 100644 index 00000000..c2b431aa --- /dev/null +++ b/tests/Feature/Recommendations/RecComputeSimilarJobsTest.php @@ -0,0 +1,141 @@ +assertGreaterThan( + (int) config('horizon.defaults.supervisor-default.timeout'), + (int) config('queue.connections.redis.retry_after'), + ); + } + + public function test_behavior_job_processes_a_batch_and_dispatches_the_next_cursor(): void + { + $user = User::factory()->create(); + $artworks = Artwork::factory()->count(3)->create(['user_id' => $user->id]); + + DB::table('rec_item_pairs')->insert([ + [ + 'a_artwork_id' => $artworks[0]->id, + 'b_artwork_id' => $artworks[1]->id, + 'weight' => 1.5, + 'updated_at' => now(), + ], + [ + 'a_artwork_id' => $artworks[0]->id, + 'b_artwork_id' => $artworks[2]->id, + 'weight' => 1.0, + 'updated_at' => now(), + ], + ]); + + Queue::fake(); + + (new RecComputeSimilarByBehaviorJob(null, 2))->handle(); + + $this->assertNotNull( + RecArtworkRec::query() + ->where('artwork_id', $artworks[0]->id) + ->where('rec_type', 'similar_behavior') + ->first(), + ); + + $expectedCursor = (int) $artworks->sortBy('id')->values()[1]->id; + + Queue::assertPushed( + RecComputeSimilarByBehaviorJob::class, + fn (RecComputeSimilarByBehaviorJob $job): bool => $job->cursorAfterArtworkId() === $expectedCursor, + ); + } + + public function test_behavior_job_does_not_dispatch_when_the_batch_is_partial(): void + { + Artwork::factory()->count(2)->create(); + + Queue::fake(); + + (new RecComputeSimilarByBehaviorJob(null, 10))->handle(); + + Queue::assertNothingPushed(); + } + + public function test_hybrid_job_is_idempotent_for_a_single_artwork(): void + { + $user = User::factory()->create(); + $source = Artwork::factory()->create(['user_id' => $user->id]); + $related = Artwork::factory()->create(['user_id' => $user->id]); + + RecArtworkRec::query()->create([ + 'artwork_id' => $source->id, + 'rec_type' => 'similar_tags', + 'model_version' => (string) config('recommendations.similarity.model_version', 'sim_v1'), + 'recs' => [$related->id], + 'computed_at' => now(), + ]); + + (new RecComputeSimilarHybridJob($source->id))->handle(); + (new RecComputeSimilarHybridJob($source->id))->handle(); + + $this->assertSame( + 1, + RecArtworkRec::query() + ->where('artwork_id', $source->id) + ->where('rec_type', 'similar_hybrid') + ->count(), + ); + } + + public function test_hybrid_batch_dispatches_continuation_when_full(): void + { + Artwork::factory()->count(3)->create(); + + Queue::fake(); + + (new RecComputeSimilarHybridJob(null, 2))->handle(); + + Queue::assertPushed(RecComputeSimilarHybridJob::class); + } + + public function test_tag_idf_group_by_is_cached_across_handle_calls(): void + { + Cache::flush(); + + Artwork::factory()->create([ + 'is_public' => true, + 'is_approved' => true, + 'published_at' => now(), + ]); + + $idfQueries = 0; + DB::listen(function ($query) use (&$idfQueries): void { + if (str_contains(strtolower($query->sql), 'artwork_tag') + && str_contains(strtolower($query->sql), 'group by')) { + $idfQueries++; + } + }); + + (new RecComputeSimilarByTagsJob(null, 10))->handle(); + (new RecComputeSimilarByTagsJob(null, 10))->handle(); + + $this->assertSame(1, $idfQueries); + } +} diff --git a/tests/Feature/Recommendations/SimilarArtworksHybridTest.php b/tests/Feature/Recommendations/SimilarArtworksHybridTest.php index f3e0cf12..d9577f74 100644 --- a/tests/Feature/Recommendations/SimilarArtworksHybridTest.php +++ b/tests/Feature/Recommendations/SimilarArtworksHybridTest.php @@ -396,6 +396,59 @@ it('passes type query param from API endpoint', function () { ->assertJsonCount(1, 'data'); }); +it('keeps other rec types and other artworks visible when one row is rebuilt', function () { + $source = createPublicArtwork(); + $other = createPublicArtwork(); + $hybridA = createPublicArtwork(); + $hybridB = createPublicArtwork(); + $tagsA = createPublicArtwork(); + $behaviorNew = createPublicArtwork(); + + RecArtworkRec::create([ + 'artwork_id' => $source->id, + 'rec_type' => 'similar_hybrid', + 'model_version' => 'sim_v1', + 'recs' => [$hybridA->id], + 'computed_at' => now()->subDay(), + ]); + RecArtworkRec::create([ + 'artwork_id' => $source->id, + 'rec_type' => 'similar_tags', + 'model_version' => 'sim_v1', + 'recs' => [$tagsA->id], + 'computed_at' => now()->subDay(), + ]); + RecArtworkRec::create([ + 'artwork_id' => $other->id, + 'rec_type' => 'similar_hybrid', + 'model_version' => 'sim_v1', + 'recs' => [$hybridB->id], + 'computed_at' => now()->subDay(), + ]); + + RecArtworkRec::query()->updateOrCreate( + [ + 'artwork_id' => $source->id, + 'rec_type' => 'similar_behavior', + 'model_version' => 'sim_v1', + ], + [ + 'recs' => [$behaviorNew->id], + 'computed_at' => now(), + ], + ); + + expect(RecArtworkRec::query()->where('artwork_id', $source->id)->where('rec_type', 'similar_hybrid')->value('recs')) + ->toEqual([$hybridA->id]); + expect(RecArtworkRec::query()->where('artwork_id', $source->id)->where('rec_type', 'similar_tags')->value('recs')) + ->toEqual([$tagsA->id]); + expect(RecArtworkRec::query()->where('artwork_id', $other->id)->where('rec_type', 'similar_hybrid')->value('recs')) + ->toEqual([$hybridB->id]); + + $service = app(HybridSimilarArtworksService::class); + expect($service->forArtwork($source->id, 12)->pluck('id')->all())->toEqual([$hybridA->id]); +}); + // ─── Cosine normalized pair weights ──────────────────────────────────────────── it('produces cosine-normalized weights in pair builder', function () { diff --git a/tests/Feature/Redis/OrphanQueueCleanupTest.php b/tests/Feature/Redis/OrphanQueueCleanupTest.php new file mode 100644 index 00000000..794469e8 --- /dev/null +++ b/tests/Feature/Redis/OrphanQueueCleanupTest.php @@ -0,0 +1,96 @@ +shouldReceive('llen')->with(Mockery::on(fn ($k) => str_ends_with((string) $k, ':notify')))->andReturn($llen); + $redis->shouldReceive('llen')->andReturn($llen); + $redis->shouldReceive('zcard')->andReturnUsing(function ($key) use ($reserved, $delayed) { + return str_contains((string) $key, 'reserved') ? $reserved : $delayed; + }); + $redis->shouldReceive('lindex')->andReturnUsing(function ($key, $i) use ($payloads) { + return $payloads[$i % max(1, count($payloads))] ?? null; + }); + Redis::shouldReceive('connection')->andReturn($redis); + + return $redis; +} + +it('dry-run inspects forum-moderation without unlinking', function () { + config(['skinbase_ai_moderation.queue.dispatch_enabled' => false]); + $payload = json_encode(['displayName' => 'cPad\\Plugins\\Forum\\Jobs\\AnalyzeForumPostJob']); + $redis = orphanRedisMock(5, [$payload]); + $redis->shouldReceive('unlink')->never(); + + $plan = app(OrphanQueueCleanup::class)->plan('forum-moderation'); + + expect($plan['can_execute'])->toBeTrue() + ->and($plan['llen'])->toBe(5) + ->and($plan['classes'])->toHaveKey('cPad\\Plugins\\Forum\\Jobs\\AnalyzeForumPostJob'); +}); + +it('refuses forum cleanup when producer dispatch is enabled', function () { + config(['skinbase_ai_moderation.queue.dispatch_enabled' => true]); + $payload = json_encode(['displayName' => 'cPad\\Plugins\\Forum\\Jobs\\AnalyzeForumPostJob']); + orphanRedisMock(2, [$payload]); + + $plan = app(OrphanQueueCleanup::class)->plan('forum-moderation'); + + expect($plan['can_execute'])->toBeFalse() + ->and($plan['errors'])->toContain('producer_dispatch_enabled'); +}); + +it('refuses protected default/search/mail queues', function () { + expect(fn () => app(OrphanQueueCleanup::class)->plan('default')) + ->toThrow(\RuntimeException::class); +}); + +it('refuses unexpected job classes unless force is set', function () { + config(['skinbase_ai_moderation.queue.dispatch_enabled' => false]); + $payload = json_encode(['displayName' => IndexUserJob::class]); + orphanRedisMock(2, [$payload]); + + $plan = app(OrphanQueueCleanup::class)->plan('forum-moderation'); + expect($plan['errors'])->toContain('unexpected_job_classes'); + + $forced = app(OrphanQueueCleanup::class)->plan('forum-moderation', true); + expect($forced['errors'])->not->toContain('unexpected_job_classes'); +}); + +it('execute unlinks only the intended orphan queue keys', function () { + config(['skinbase_ai_moderation.queue.dispatch_enabled' => false]); + $payload = json_encode(['displayName' => 'cPad\\Plugins\\Forum\\Jobs\\AnalyzeForumPostJob']); + $redis = orphanRedisMock(3, [$payload]); + $redis->shouldReceive('unlink')->once()->with('queues:forum-moderation')->andReturn(1); + $redis->shouldReceive('unlink')->once()->with('queues:forum-moderation:notify')->andReturn(1); + + $result = app(OrphanQueueCleanup::class)->execute('forum-moderation'); + + expect($result['unlinked'])->toBe(2) + ->and($result['unlinked_keys'])->toBe(['queues:forum-moderation', 'queues:forum-moderation:notify']); +}); + +it('plans forum-security cleanup for monitor jobs', function () { + config(['forum_security.queues.dispatch_enabled' => false]); + $payload = json_encode(['displayName' => 'cPad\\Plugins\\Forum\\Jobs\\FirewallActivityMonitor']); + orphanRedisMock(4, [$payload]); + + $plan = app(OrphanQueueCleanup::class)->plan('forum-security'); + + expect($plan['can_execute'])->toBeTrue(); +}); + +it('artisan dry-run does not unlink', function () { + config(['skinbase_ai_moderation.queue.dispatch_enabled' => false]); + $payload = json_encode(['displayName' => 'cPad\\Plugins\\Forum\\Jobs\\AnalyzeForumPostJob']); + $redis = orphanRedisMock(1, [$payload]); + $redis->shouldReceive('unlink')->never(); + Redis::shouldReceive('connection')->andReturn($redis); + + $this->artisan('skinbase:redis-cleanup-orphans forum-moderation --dry-run') + ->assertSuccessful(); +}); diff --git a/tests/Feature/Scheduler/ScheduleInventoryTest.php b/tests/Feature/Scheduler/ScheduleInventoryTest.php new file mode 100644 index 00000000..421624e8 --- /dev/null +++ b/tests/Feature/Scheduler/ScheduleInventoryTest.php @@ -0,0 +1,43 @@ +events()); + $flush = $events->first(fn ($event) => str_contains((string) ($event->command ?? $event->description ?? ''), 'flush-redis-stats') + || ($event->mutexName() === 'flush-redis-stats') + || str_contains($event->getSummaryForDisplay(), 'flush-redis-stats')); + + expect($flush)->not->toBeNull() + ->and($flush->expression)->toBe('1,11,18,31,41,51 * * * *'); + + $boards = $events->first(fn ($event) => str_contains($event->getSummaryForDisplay(), 'leaderboards:refresh') + || str_contains((string) ($event->command ?? ''), 'leaderboards:refresh')); + + expect($boards)->not->toBeNull() + ->and($boards->expression)->toContain('21'); +}); + +it('keeps recommendation nightly offsets and does not register disabled forum async producers', function () { + $display = collect(app(Schedule::class)->events()) + ->map(fn ($event) => $event->getSummaryForDisplay()) + ->implode("\n"); + + expect($display)->toContain('rec-compute-tags') + ->and($display)->toContain('rec-compute-behavior') + ->and($display)->toContain('rec-compute-hybrid') + ->and($display)->toContain('forum-post-scan') + ->and($display)->not->toContain('forum-ai-scan') + ->and($display)->not->toContain('forum-bot-scan') + ->and($display)->not->toContain('forum-firewall-scan'); +}); + +it('does not register presence prune unless enabled', function () { + expect((bool) config('traffic.online_visitors.prune_enabled'))->toBeFalse(); + + $display = collect(app(Schedule::class)->events()) + ->map(fn ($event) => $event->getSummaryForDisplay()) + ->implode("\n"); + + expect($display)->not->toContain('prune-online-visitor-index'); +}); diff --git a/tests/Feature/SitemapTest.php b/tests/Feature/SitemapTest.php index d78acb71..dde5ef99 100644 --- a/tests/Feature/SitemapTest.php +++ b/tests/Feature/SitemapTest.php @@ -431,6 +431,56 @@ it('can rollback sitemap delivery to the previous published release', function ( ->not->toContain('/art/' . $newArtwork->id . '/'); }); +it('generate command writes a canonical index.xml and public sitemap.xml', function (): void { + Storage::fake('sitemaps_public'); + seedSitemapFixtures(); + + expect(Artisan::call('skinbase:sitemaps:generate', ['--disk' => 'sitemaps_public']))->toBe(0); + + $disk = Storage::disk('sitemaps_public'); + $disk->assertExists('sitemaps/index.xml'); + $disk->assertExists('sitemaps/sitemap.xml'); + $disk->assertExists('sitemap.xml'); + + $index = $disk->get('sitemaps/index.xml'); + expect($index) + ->toContain('toContain('/sitemaps/artworks') + ->toContain('/sitemaps/static-pages.xml'); + + expect($disk->get('sitemap.xml'))->toBe($index); + expect(strlen($index))->toBeLessThan(50 * 1024 * 1024); +}); + +it('serves the generated sitemaps/index.xml for /sitemap.xml when present', function (): void { + config([ + 'sitemaps.pre_generated.enabled' => true, + 'sitemaps.delivery.prefer_published_release' => false, + 'sitemaps.delivery.fallback_to_live_build' => false, + ]); + + $indexPath = public_path('sitemaps/index.xml'); + $dir = dirname($indexPath); + + if (! is_dir($dir)) { + mkdir($dir, 0777, true); + } + + $marker = 'http://skinbase26.test/sitemaps/generated-index-marker.xml'; + file_put_contents($indexPath, ''.$marker.''); + + try { + $response = $this->get('/sitemap.xml')->assertOk(); + $content = method_exists($response, 'streamedContent') + ? ($response->streamedContent() ?: $response->getContent()) + : $response->getContent(); + + expect($content)->toContain($marker); + } finally { + @unlink($indexPath); + } +}); + /** * @return array */ diff --git a/tests/Feature/StudioTest.php b/tests/Feature/StudioTest.php index 7f702368..1b0a4add 100644 --- a/tests/Feature/StudioTest.php +++ b/tests/Feature/StudioTest.php @@ -71,7 +71,9 @@ test('studio dashboard loads for authenticated user', function () { ->component('Studio/StudioDashboard') ->has('overview.active_challenges') ->has('overview.creator_health') - ->has('overview.featured_status')); + ->has('overview.featured_status') + ->has('overview.kpis.snapshot_window') + ->where('overview.kpis.snapshot_window.window_complete', false)); }); test('studio dashboard honors saved landing page preference', function () { diff --git a/tests/Feature/Vision/VectorGatewaySearchCircuitTest.php b/tests/Feature/Vision/VectorGatewaySearchCircuitTest.php new file mode 100644 index 00000000..51497847 --- /dev/null +++ b/tests/Feature/Vision/VectorGatewaySearchCircuitTest.php @@ -0,0 +1,373 @@ +set('vision.vector_gateway.enabled', true); + config()->set('vision.vector_gateway.base_url', 'https://vision.klevze.net'); + config()->set('vision.vector_gateway.api_key', 'test-key'); + config()->set('vision.vector_gateway.search_endpoint', '/vectors/search'); + config()->set('vision.vector_gateway.search_file_endpoint', '/vectors/search/file'); + config()->set('vision.vector_gateway.circuit_breaker_seconds', 30); + config()->set('cdn.files_url', 'https://files.skinbase.org'); + config()->set('app.url', 'https://skinbase.test'); + Storage::fake('public'); + Cache::flush(); +}); + +function similarPair(): array +{ + $source = Artwork::factory()->create([ + 'title' => 'Circuit source', + 'hash' => 'aa11bb22cc33', + 'thumb_ext' => 'webp', + 'is_public' => true, + 'is_approved' => true, + 'published_at' => now()->subHour(), + ]); + $match = Artwork::factory()->create([ + 'title' => 'Circuit match', + 'hash' => 'dd44ee55ff66', + 'thumb_ext' => 'webp', + 'is_public' => true, + 'is_approved' => true, + 'published_at' => now()->subHour(), + ]); + + return [$source, $match]; +} + +function fakeSuccessfulUrlSearch(Artwork $source, Artwork $match): array +{ + return [ + 'https://files.skinbase.org/*' => Http::response('image-bytes', 200, ['Content-Type' => 'image/webp']), + 'https://vision.klevze.net/vectors/search' => Http::response([ + 'results' => [ + ['id' => $source->id, 'score' => 1.0], + ['id' => $match->id, 'score' => 0.91], + ], + ], 200), + ]; +} + +it('does not call URL search when file search succeeds and leaves the circuit closed', function (): void { + [$source, $match] = similarPair(); + + Http::fake([ + 'https://files.skinbase.org/*' => Http::response('image-bytes', 200, ['Content-Type' => 'image/webp']), + 'https://vision.klevze.net/vectors/search/file' => Http::response([ + 'results' => [ + ['id' => $source->id, 'score' => 1.0], + ['id' => $match->id, 'score' => 0.91], + ], + ], 200), + 'https://vision.klevze.net/vectors/search' => Http::response(['error' => 'should not be called'], 500), + ]); + + getJson('/api/art/'.$source->id.'/similar-ai')->assertOk()->assertJsonPath('data.0.id', $match->id); + + Http::assertNotSent(fn ($request): bool => $request->url() === 'https://vision.klevze.net/vectors/search'); + expect(app(VectorGatewayClient::class)->circuitOpen())->toBeFalse(); +}); + +it('falls back to URL search after file 400 and keeps the circuit closed', function (): void { + [$source, $match] = similarPair(); + + Http::fake(array_merge(fakeSuccessfulUrlSearch($source, $match), [ + 'https://vision.klevze.net/vectors/search/file' => Http::response(['error' => 'bad image'], 400), + ])); + + getJson('/api/art/'.$source->id.'/similar-ai')->assertOk()->assertJsonPath('data.0.id', $match->id); + Http::assertSent(fn ($request): bool => $request->url() === 'https://vision.klevze.net/vectors/search'); + expect(app(VectorGatewayClient::class)->circuitOpen())->toBeFalse(); +}); + +it('falls back to URL search after file 500 and keeps the circuit closed', function (): void { + [$source, $match] = similarPair(); + $logged = []; + Event::listen(MessageLogged::class, function (MessageLogged $event) use (&$logged): void { + $logged[] = $event; + }); + + Http::fake(array_merge(fakeSuccessfulUrlSearch($source, $match), [ + 'https://vision.klevze.net/vectors/search/file' => Http::response(['error' => 'upstream'], 500), + ])); + + getJson('/api/art/'.$source->id.'/similar-ai')->assertOk()->assertJsonPath('data.0.id', $match->id); + expect(app(VectorGatewayClient::class)->circuitOpen())->toBeFalse() + ->and(openedCircuitLogs($logged))->toHaveCount(0); +}); + +it('falls back to URL search after a file connection failure and keeps the circuit closed', function (): void { + [$source, $match] = similarPair(); + + Http::fake(array_merge(fakeSuccessfulUrlSearch($source, $match), [ + 'https://vision.klevze.net/vectors/search/file' => function () { + throw new ConnectionException('cURL error 28: Operation timed out'); + }, + ])); + + getJson('/api/art/'.$source->id.'/similar-ai')->assertOk()->assertJsonPath('data.0.id', $match->id); + expect(app(VectorGatewayClient::class)->circuitOpen())->toBeFalse(); +}); + +function openedCircuitLogs(array $logged): array +{ + return array_values(array_filter( + $logged, + static fn (MessageLogged $event): bool => $event->message === 'Vector gateway circuit opened', + )); +} + +it('opens the circuit when both search strategies fail transiently', function (): void { + [$source] = similarPair(); + $logged = []; + Event::listen(MessageLogged::class, function (MessageLogged $event) use (&$logged): void { + $logged[] = $event; + }); + + Http::fake([ + 'https://files.skinbase.org/*' => Http::response('image-bytes', 200, ['Content-Type' => 'image/webp']), + 'https://vision.klevze.net/vectors/search/file' => Http::response(['error' => 'down'], 503), + 'https://vision.klevze.net/vectors/search' => Http::response(['error' => 'down'], 503), + ]); + + getJson('/api/art/'.$source->id.'/similar-ai') + ->assertStatus(502) + ->assertJson([ + 'data' => [], + 'reason' => 'vector_gateway_error', + ]) + ->assertJsonMissingPath('message'); + + expect(app(VectorGatewayClient::class)->circuitOpen())->toBeTrue(); + + $opened = openedCircuitLogs($logged); + expect($opened)->toHaveCount(1) + ->and($opened[0]->level)->toBe('warning') + ->and($opened[0]->context['source'] ?? null)->toBe('similar_ai') + ->and($opened[0]->context['artwork_id'] ?? null)->toBe($source->id) + ->and($opened[0]->context['circuit_ttl_seconds'] ?? null)->toBe(30) + ->and($opened[0]->context)->toHaveKey('php_sapi') + ->and($opened[0]->context)->toHaveKey('running_in_console'); + + $operations = collect($opened[0]->context['failures'] ?? [])->pluck('operation')->all(); + $statuses = collect($opened[0]->context['failures'] ?? [])->pluck('http_status')->all(); + expect($operations)->toContain('search_file') + ->and($operations)->toContain('search_url') + ->and($statuses)->toBe([503, 503]); + + $encoded = json_encode($opened[0]->context); + expect($encoded)->not->toContain('test-key') + ->and($encoded)->not->toContain('image-bytes') + ->and($encoded)->not->toContain('vision.klevze.net') + ->and($encoded)->not->toContain('files.skinbase.org'); +}); + +it('does not open the circuit when both strategies fail with permanent 4xx', function (): void { + [$source] = similarPair(); + $logged = []; + Event::listen(MessageLogged::class, function (MessageLogged $event) use (&$logged): void { + $logged[] = $event; + }); + + Http::fake([ + 'https://files.skinbase.org/*' => Http::response('image-bytes', 200, ['Content-Type' => 'image/webp']), + 'https://vision.klevze.net/vectors/search/file' => Http::response(['error' => 'bad file'], 400), + 'https://vision.klevze.net/vectors/search' => Http::response(['error' => 'bad url'], 422), + ]); + + getJson('/api/art/'.$source->id.'/similar-ai')->assertStatus(502); + expect(app(VectorGatewayClient::class)->circuitOpen())->toBeFalse() + ->and(openedCircuitLogs($logged))->toHaveCount(0); +}); + +it('fails fast when the circuit is already open without downloading or calling the gateway', function (): void { + [$source] = similarPair(); + Cache::put('vision.vector_gateway.circuit_open', true, 30); + $logged = []; + Event::listen(MessageLogged::class, function (MessageLogged $event) use (&$logged): void { + $logged[] = $event; + }); + + Http::fake(); + + getJson('/api/art/'.$source->id.'/similar-ai') + ->assertStatus(502) + ->assertJsonPath('reason', 'vector_gateway_error') + ->assertJsonMissingPath('message'); + + Http::assertNothingSent(); + expect(openedCircuitLogs($logged))->toHaveCount(0); + + $failFast = collect($logged)->first(fn (MessageLogged $event): bool => $event->message === 'Vector similarity search failed'); + expect($failFast)->not->toBeNull() + ->and($failFast->context['failure_stage'] ?? null)->toBe('circuit'); +}); + +it('logs one structured warning on final similarity failure without leaking internals', function (): void { + [$source] = similarPair(); + $logged = []; + Event::listen(MessageLogged::class, function (MessageLogged $event) use (&$logged): void { + $logged[] = $event; + }); + + Http::fake([ + 'https://files.skinbase.org/*' => Http::response('image-bytes', 200, ['Content-Type' => 'image/webp']), + 'https://vision.klevze.net/vectors/search/file' => Http::response(['error' => 'x'], 500), + 'https://vision.klevze.net/vectors/search' => Http::response(['error' => 'y'], 500), + ]); + + getJson('/api/art/'.$source->id.'/similar-ai')->assertStatus(502); + + $match = collect($logged)->first(fn (MessageLogged $event): bool => str_contains((string) $event->message, 'Vector similarity search failed')); + expect($match)->not->toBeNull() + ->and($match->level)->toBe('warning') + ->and($match->context['artwork_id'] ?? null)->toBe($source->id) + ->and($match->context['route'] ?? null)->toBe('api.art.similar-ai') + ->and($match->context)->toHaveKey('circuit_worthy') + ->and($match->context)->toHaveKey('circuit_open') + ->and(json_encode($match->context))->not->toContain('test-key') + ->and(json_encode($match->context))->not->toContain('image-bytes'); +}); + +it('does not trip the circuit when file gateway was never attempted and URL returns 502', function (): void { + [$source] = similarPair(); + $logged = []; + Event::listen(MessageLogged::class, function (MessageLogged $event) use (&$logged): void { + $logged[] = $event; + }); + + Http::fake([ + 'https://files.skinbase.org/*' => Http::response('not-found', 404), + 'https://vision.klevze.net/vectors/search' => Http::response(['error' => 'bad gateway'], 502), + 'https://vision.klevze.net/vectors/search/file' => Http::response(['error' => 'should not be called'], 500), + ]); + + getJson('/api/art/'.$source->id.'/similar-ai') + ->assertStatus(502) + ->assertJsonPath('reason', 'vector_gateway_error'); + + Http::assertNotSent(fn ($request): bool => $request->url() === 'https://vision.klevze.net/vectors/search/file'); + expect(app(VectorGatewayClient::class)->circuitOpen())->toBeFalse() + ->and(openedCircuitLogs($logged))->toHaveCount(0); +}); + +it('does not trip the circuit when image download throws before the file gateway', function (): void { + [$source] = similarPair(); + $logged = []; + Event::listen(MessageLogged::class, function (MessageLogged $event) use (&$logged): void { + $logged[] = $event; + }); + + Http::fake([ + 'https://files.skinbase.org/*' => function () { + throw new RuntimeException('cdn unavailable'); + }, + 'https://vision.klevze.net/vectors/search' => Http::response(['error' => 'bad gateway'], 502), + ]); + + getJson('/api/art/'.$source->id.'/similar-ai')->assertStatus(502); + expect(app(VectorGatewayClient::class)->circuitOpen())->toBeFalse() + ->and(openedCircuitLogs($logged))->toHaveCount(0); +}); + +it('does not trip the circuit when file gateway is 422 and URL is 502', function (): void { + [$source] = similarPair(); + $logged = []; + Event::listen(MessageLogged::class, function (MessageLogged $event) use (&$logged): void { + $logged[] = $event; + }); + + Http::fake([ + 'https://files.skinbase.org/*' => Http::response('image-bytes', 200, ['Content-Type' => 'image/webp']), + 'https://vision.klevze.net/vectors/search/file' => Http::response(['error' => 'unprocessable'], 422), + 'https://vision.klevze.net/vectors/search' => Http::response(['error' => 'bad gateway'], 502), + ]); + + getJson('/api/art/'.$source->id.'/similar-ai')->assertStatus(502); + expect(app(VectorGatewayClient::class)->circuitOpen())->toBeFalse() + ->and(openedCircuitLogs($logged))->toHaveCount(0); +}); + +it('does not trip the circuit when file gateway is 503 and URL is 422', function (): void { + [$source] = similarPair(); + $logged = []; + Event::listen(MessageLogged::class, function (MessageLogged $event) use (&$logged): void { + $logged[] = $event; + }); + + Http::fake([ + 'https://files.skinbase.org/*' => Http::response('image-bytes', 200, ['Content-Type' => 'image/webp']), + 'https://vision.klevze.net/vectors/search/file' => Http::response(['error' => 'down'], 503), + 'https://vision.klevze.net/vectors/search' => Http::response(['error' => 'bad url'], 422), + ]); + + getJson('/api/art/'.$source->id.'/similar-ai')->assertStatus(502); + expect(app(VectorGatewayClient::class)->circuitOpen())->toBeFalse() + ->and(openedCircuitLogs($logged))->toHaveCount(0); +}); + +it('attributes uploaded-image transient failures as uploaded_image', function (): void { + $logged = []; + Event::listen(MessageLogged::class, function (MessageLogged $event) use (&$logged): void { + $logged[] = $event; + }); + + Http::fake([ + 'https://vision.klevze.net/vectors/search/file' => Http::response(['error' => 'down'], 503), + ]); + + $file = UploadedFile::fake()->image('query.png', 64, 64); + + expect(fn () => app(AiArtworkVectorSearchService::class)->searchByUploadedImage($file, 12)) + ->toThrow(VectorGatewayException::class); + + expect(app(VectorGatewayClient::class)->circuitOpen())->toBeTrue(); + + $opened = openedCircuitLogs($logged); + expect($opened)->toHaveCount(1) + ->and($opened[0]->context['source'] ?? null)->toBe('uploaded_image') + ->and($opened[0]->context['failures'][0]['operation'] ?? null)->toBe('search_file') + ->and($opened[0]->context['failures'][0]['http_status'] ?? null)->toBe(503) + ->and(json_encode($opened[0]->context))->not->toContain('test-key'); +}); + +it('serves similar-ai from cache without contacting the gateway', function (): void { + [$source, $match] = similarPair(); + + Http::fake([ + 'https://files.skinbase.org/*' => Http::response('image-bytes', 200, ['Content-Type' => 'image/webp']), + 'https://vision.klevze.net/vectors/search/file' => Http::response([ + 'results' => [ + ['id' => $source->id, 'score' => 1.0], + ['id' => $match->id, 'score' => 0.91], + ], + ], 200), + ]); + + getJson('/api/art/'.$source->id.'/similar-ai')->assertOk(); + + Http::fake(function () { + throw new RuntimeException('gateway should not be called on cache hit'); + }); + + getJson('/api/art/'.$source->id.'/similar-ai') + ->assertOk() + ->assertJsonPath('data.0.id', $match->id); +}); diff --git a/tests/Feature/Vision/VectorGatewaySearchLimitTest.php b/tests/Feature/Vision/VectorGatewaySearchLimitTest.php new file mode 100644 index 00000000..295d821c --- /dev/null +++ b/tests/Feature/Vision/VectorGatewaySearchLimitTest.php @@ -0,0 +1,118 @@ +set('vision.vector_gateway.enabled', true); + config()->set('vision.vector_gateway.base_url', 'https://vision.klevze.net'); + config()->set('vision.vector_gateway.api_key', 'test-key'); + config()->set('vision.vector_gateway.search_endpoint', '/vectors/search'); + config()->set('vision.vector_gateway.search_file_endpoint', '/vectors/search/file'); + config()->set('cdn.files_url', 'https://files.skinbase.org'); + Storage::fake('public'); + Cache::flush(); +}); + +function limitSourceArtwork(): Artwork +{ + return Artwork::factory()->create([ + 'title' => 'Limit source', + 'hash' => 'aabbccddeeff', + 'thumb_ext' => 'webp', + 'is_public' => true, + 'is_approved' => true, + 'published_at' => now()->subHour(), + ]); +} + +function fakeFileSearchOk(): void +{ + Http::fake(function ($request) { + $url = $request->url(); + if (str_contains($url, '/vectors/search')) { + return Http::response(['results' => []], 200); + } + + return Http::response('image-bytes', 200, ['Content-Type' => 'image/webp']); + }); +} + +function gatewaySearchLimit($request): ?int +{ + $url = $request->url(); + if (! str_contains($url, '/vectors/search')) { + return null; + } + + $data = $request->data(); + if (isset($data['limit']) && is_numeric($data['limit'])) { + return (int) $data['limit']; + } + + foreach ($data as $part) { + if (is_array($part) && ($part['name'] ?? null) === 'limit' && is_numeric($part['contents'] ?? null)) { + return (int) $part['contents']; + } + } + + $body = $request->body(); + if (preg_match('/name="limit".*?\r\n\r\n(\d+)/s', $body, $matches) === 1) { + return (int) $matches[1]; + } + if (preg_match('/"limit"\s*:\s*([0-9]+)/', $body, $matches) === 1) { + return (int) $matches[1]; + } + + return null; +} + +it('sends gateway limit 100 when similar-ai is requested at 99', function (): void { + $source = limitSourceArtwork(); + fakeFileSearchOk(); + + app(AiArtworkVectorSearchService::class)->similarToArtwork($source, 99); + + Http::assertSent(fn ($request): bool => gatewaySearchLimit($request) === 100); +}); + +it('clamps similar-ai requests of 100 and 120 so the gateway never sees more than 100', function (): void { + $source = limitSourceArtwork(); + + foreach ([100, 120, 121] as $requested) { + fakeFileSearchOk(); + Cache::flush(); + + app(AiArtworkVectorSearchService::class)->similarToArtwork($source, $requested); + + Http::assertSent(fn ($request): bool => gatewaySearchLimit($request) === 100); + } +}); + +it('sends gateway limit 13 for the default similar-ai request of 12', function (): void { + $source = limitSourceArtwork(); + fakeFileSearchOk(); + + app(AiArtworkVectorSearchService::class)->similarToArtwork($source, 12); + + Http::assertSent(fn ($request): bool => gatewaySearchLimit($request) === 13); +}); + +it('clamps uploaded-image search to 100 without adding an extra candidate', function (): void { + fakeFileSearchOk(); + + app(AiArtworkVectorSearchService::class)->searchByUploadedImage( + UploadedFile::fake()->image('query.png', 32, 32), + 120, + ); + + Http::assertSent(fn ($request): bool => gatewaySearchLimit($request) === 100); +}); diff --git a/tests/Unit/HorizonMailQueueTest.php b/tests/Unit/HorizonMailQueueTest.php new file mode 100644 index 00000000..84a982d9 --- /dev/null +++ b/tests/Unit/HorizonMailQueueTest.php @@ -0,0 +1,48 @@ +assertIsArray($defaults); + $this->assertSame(['mail'], $defaults['queue']); + $this->assertSame('redis', $defaults['connection']); + $this->assertGreaterThanOrEqual(5, (int) $defaults['tries']); + $this->assertLessThanOrEqual( + (int) config('queue.connections.redis.retry_after'), + (int) $defaults['timeout'], + ); + } + + public function test_production_and_local_horizon_environments_enable_the_mail_supervisor(): void + { + $this->assertArrayHasKey('supervisor-mail', config('horizon.environments.production')); + $this->assertArrayHasKey('supervisor-mail', config('horizon.environments.local')); + } + + public function test_verification_and_security_mailables_use_the_mail_queue(): void + { + $this->assertSame('mail', (new SendVerificationEmailJob(1, 'user@example.test', 'token', 1, '127.0.0.1'))->queue); + $this->assertSame('mail', (new RegistrationVerificationMail('token'))->queue); + $this->assertSame('mail', (new EmailChangeVerificationCodeMail('123456', 15))->queue); + $this->assertSame('mail', (new EmailChangedSecurityAlertMail('old@example.test'))->queue); + } + + public function test_default_and_messaging_supervisors_do_not_steal_the_mail_queue(): void + { + $this->assertNotContains('mail', config('horizon.defaults.supervisor-default.queue')); + $this->assertNotContains('mail', config('horizon.defaults.supervisor-messaging.queue')); + } +} diff --git a/tests/Unit/Http/HttpPerformanceLogAnalyzerTest.php b/tests/Unit/Http/HttpPerformanceLogAnalyzerTest.php new file mode 100644 index 00000000..9393ed81 --- /dev/null +++ b/tests/Unit/Http/HttpPerformanceLogAnalyzerTest.php @@ -0,0 +1,110 @@ +normalize('/download/artwork/17776'))->toBe('/download/artwork/{id}') + ->and($n->normalize('/download/artwork/42026'))->toBe('/download/artwork/{id}') + ->and($n->normalize('/@killua'))->toBe('/@{user}') + ->and($n->normalize('/@killua/artworks'))->toBe('/@{user}/artworks') + ->and($n->normalize('/academy/prompts/123'))->toBe('/academy/prompts/{id}') + ->and($n->normalize('/forum/some-thread/456'))->toBe('/forum/some-thread/{id}') + ->and($n->normalize('/art/53867/light-mark-docks'))->toBe('/art/{id}/{slug}') + ->and($n->normalize('/art/3858/similar'))->toBe('/art/{id}/similar') + ->and($n->normalize('/api/art/69876/view'))->toBe('/api/art/{id}/view') + ->and($n->normalize('/api/rank/category/583'))->toBe('/api/rank/category/{id}') + ->and($n->normalize('/search?q=skin&page=2'))->toBe('/search'); +}); + +it('parses mixed nginx performance lines and skips malformed JSON', function () { + $analyzer = new HttpPerformanceLogAnalyzer(new HttpUriNormalizer()); + $now = new DateTimeImmutable('2026-08-24T16:00:00+02:00'); + + $result = $analyzer->analyze(perfFixture('sample.log'), [ + 'since' => '24h', + 'now' => $now, + 'top' => 10, + 'min_requests' => 1, + ]); + + expect($result['skipped'])->toBe(1) + ->and($result['totals']['total'])->toBe(9) + ->and($result['totals']['5xx'])->toBe(1) + ->and($result['totals']['499'])->toBe(1) + ->and($result['percentile_method'])->toContain('nearest-rank'); + + $academy = collect($result['by_count'])->firstWhere('uri', '/academy'); + expect($academy['count'])->toBe(2) + ->and($academy['p95'])->toBeGreaterThan(0); + + $static = collect($result['by_count'])->firstWhere('uri', '/favicon.ico'); + expect($static['avg_upstream_response'])->toBeNull(); + + $download = collect($result['slowest'])->firstWhere('uri', '/download/artwork/{id}'); + expect($download['upstream_response_time'])->toBe(0.32); + + $profile = collect($result['by_count'])->firstWhere('uri', '/@{user}'); + expect($profile)->not->toBeNull(); +}); + +it('uses nearest-rank percentiles rather than the average', function () { + $analyzer = new HttpPerformanceLogAnalyzer(new HttpUriNormalizer()); + $sorted = [0.1, 0.2, 0.3, 0.4, 1.0]; + + expect($analyzer->percentile($sorted, 0.50))->toBe(0.3) + ->and($analyzer->percentile($sorted, 0.95))->toBe(1.0) + ->and($analyzer->percentile($sorted, 0.95))->not->toBe(0.4); +}); + +it('sums comma-separated upstream timings and treats dash as missing', function () { + $analyzer = new HttpPerformanceLogAnalyzer(new HttpUriNormalizer()); + + expect($analyzer->parseUpstream('-'))->toBeNull() + ->and($analyzer->parseUpstream('0.451'))->toBe(0.451) + ->and($analyzer->parseUpstream('0.100, 0.220'))->toBe(0.32); +}); + +it('filters hosts exactly and does not merge wildcard hosts', function () { + $analyzer = new HttpPerformanceLogAnalyzer(new HttpUriNormalizer()); + $now = new DateTimeImmutable('2026-08-24T16:00:00+02:00'); + $opts = ['since' => '24h', 'now' => $now, 'top' => 20, 'min_requests' => 1]; + + $filtered = $analyzer->analyze(perfFixture('hosts.log'), $opts + ['host' => 'skinbase.org']); + $hosts = array_unique(array_column($filtered['by_count'], 'host')); + + expect($filtered['skipped'])->toBe(1) + ->and($filtered['totals']['total'])->toBe(8) + ->and($hosts)->toBe(['skinbase.org']) + ->and(collect($filtered['by_count'])->pluck('host')->all())->not->toContain('www.skinbase.org') + ->and(collect($filtered['by_count'])->pluck('host')->all())->not->toContain('thumb.skinbase.org') + ->and(collect($filtered['by_count'])->pluck('host')->all())->not->toContain('hub.skinbase.org'); + + $unfiltered = $analyzer->analyze(perfFixture('hosts.log'), $opts); + $artRows = collect($unfiltered['by_count'])->where('uri', '/art/{id}/{slug}')->values(); + + expect($artRows)->toHaveCount(4) + ->and($artRows->pluck('host')->sort()->values()->all())->toBe([ + 'hub.skinbase.org', + 'skinbase.org', + 'thumb.skinbase.org', + 'www.skinbase.org', + ]) + ->and($artRows->firstWhere('host', 'skinbase.org')['count'])->toBe(1) + ->and($artRows->firstWhere('host', 'www.skinbase.org')['avg'])->toBe(0.9); + + expect(collect($filtered['by_count'])->firstWhere('uri', '/art/{id}/{slug}')['host'])->toBe('skinbase.org') + ->and(collect($filtered['by_count'])->firstWhere('uri', '/art/{id}/similar'))->not->toBeNull() + ->and(collect($filtered['by_count'])->firstWhere('uri', '/download/artwork/{id}'))->not->toBeNull() + ->and(collect($filtered['by_count'])->firstWhere('uri', '/api/art/{id}/view'))->not->toBeNull() + ->and(collect($filtered['by_count'])->firstWhere('uri', '/api/rank/category/{id}'))->not->toBeNull() + ->and(collect($filtered['by_count'])->firstWhere('uri', '/academy/prompts/{id}'))->not->toBeNull() + ->and(collect($filtered['by_count'])->firstWhere('uri', '/@{user}'))->not->toBeNull(); +}); diff --git a/tests/Unit/Jobs/RecComputeSerializedCompatibilityTest.php b/tests/Unit/Jobs/RecComputeSerializedCompatibilityTest.php new file mode 100644 index 00000000..68554f58 --- /dev/null +++ b/tests/Unit/Jobs/RecComputeSerializedCompatibilityTest.php @@ -0,0 +1,73 @@ +unserializeWithoutAfterArtworkId(new RecComputeSimilarByTagsJob(null, 200, 99)); + + $this->assertNull($job->cursorAfterArtworkId()); + } + + public function test_old_serialized_behavior_job_without_after_artwork_id_can_run(): void + { + $job = $this->unserializeWithoutAfterArtworkId(new RecComputeSimilarByBehaviorJob(null, 200, 42)); + + $this->assertNull($job->cursorAfterArtworkId()); + } + + public function test_old_serialized_hybrid_job_without_after_artwork_id_can_run(): void + { + $job = $this->unserializeWithoutAfterArtworkId(new RecComputeSimilarHybridJob(null, 200, 7)); + + $this->assertNull($job->cursorAfterArtworkId()); + } + + public function test_new_cursor_jobs_keep_after_artwork_id(): void + { + $job = unserialize(serialize(new RecComputeSimilarByBehaviorJob(null, 200, 15))); + + $this->assertSame(15, $job->cursorAfterArtworkId()); + } + + public function test_laravel_serialize_roundtrip_keeps_queueable_and_cursor_fields(): void + { + $original = new RecComputeSimilarByTagsJob(5, 50, 11); + $original->onQueue('default'); + + $copy = unserialize(serialize($original)); + + $this->assertInstanceOf(RecComputeSimilarByTagsJob::class, $copy); + $this->assertSame(11, $copy->cursorAfterArtworkId()); + $this->assertSame('default', $copy->queue); + $this->assertSame(600, $copy->timeout); + $this->assertSame(2, $copy->tries); + } + + /** + * @template T of RecComputeSimilarByTagsJob|RecComputeSimilarByBehaviorJob|RecComputeSimilarHybridJob + * @param T $job + * @return T + */ + private function unserializeWithoutAfterArtworkId(object $job): object + { + $values = $job->__serialize(); + $class = $job::class; + unset($values["\0{$class}\0afterArtworkId"]); + + $restored = (new ReflectionClass($class))->newInstanceWithoutConstructor(); + $restored->__unserialize($values); + + return $restored; + } +} diff --git a/tests/Unit/Jobs/RecComputeSimilarByBehaviorJobTest.php b/tests/Unit/Jobs/RecComputeSimilarByBehaviorJobTest.php new file mode 100644 index 00000000..f7bb7a8c --- /dev/null +++ b/tests/Unit/Jobs/RecComputeSimilarByBehaviorJobTest.php @@ -0,0 +1,26 @@ +assertLessThan( + (int) config('queue.connections.redis.retry_after'), + $job->timeout, + ); + } + + public function test_batch_jobs_do_not_use_without_overlapping_middleware(): void + { + $this->assertFalse(method_exists(new RecComputeSimilarByBehaviorJob(), 'middleware')); + } +} diff --git a/tests/Unit/Jobs/RecComputeSimilarHybridJobTest.php b/tests/Unit/Jobs/RecComputeSimilarHybridJobTest.php index fa3c6711..f8d605fd 100644 --- a/tests/Unit/Jobs/RecComputeSimilarHybridJobTest.php +++ b/tests/Unit/Jobs/RecComputeSimilarHybridJobTest.php @@ -32,4 +32,14 @@ final class RecComputeSimilarHybridJobTest extends TestCase $this->assertSame([], $job->middleware()); } + + public function test_job_timeout_stays_below_queue_retry_after(): void + { + $job = new RecComputeSimilarHybridJob(); + + $this->assertLessThan( + (int) config('queue.connections.redis.retry_after'), + $job->timeout, + ); + } } diff --git a/tests/Unit/Redis/LegacyPrefixFilterTest.php b/tests/Unit/Redis/LegacyPrefixFilterTest.php new file mode 100644 index 00000000..858c20ac --- /dev/null +++ b/tests/Unit/Redis/LegacyPrefixFilterTest.php @@ -0,0 +1,21 @@ +toBeTrue(); + expect(str_starts_with($key, 'skinbase-database-') || str_starts_with($key, 'skinbase_horizon:'))->toBeFalse(); + } + foreach ($current as $key) { + expect(str_starts_with($key, 'skinbasenova-database-') || str_starts_with($key, 'skinbasenova_horizon:'))->toBeFalse(); + } +}); diff --git a/tests/Unit/Traffic/OnlineVisitorIndexReadTest.php b/tests/Unit/Traffic/OnlineVisitorIndexReadTest.php new file mode 100644 index 00000000..65317222 --- /dev/null +++ b/tests/Unit/Traffic/OnlineVisitorIndexReadTest.php @@ -0,0 +1,37 @@ +once() + ->andReturn([0, ['visitor-a', 'visitor-b']]); + Redis::shouldReceive('get')->twice()->andReturn(null); + Redis::shouldReceive('srem')->once(); + Redis::shouldReceive('smembers')->never(); + + $repository = new OnlineVisitorRepository(app(BotClassifier::class)); + $records = $repository->all(); + + $this->assertSame([], $records); + } + + public function test_source_does_not_call_smembers_on_the_presence_index(): void + { + $source = file_get_contents(base_path('app/Services/Traffic/OnlineVisitorRepository.php')); + + $this->assertNotFalse($source); + $this->assertStringNotContainsString('smembers', strtolower($source)); + $this->assertStringContainsString('sscan', strtolower($source)); + } +} diff --git a/tests/Unit/Traffic/PresenceIndexPrunerTest.php b/tests/Unit/Traffic/PresenceIndexPrunerTest.php new file mode 100644 index 00000000..bf76656a --- /dev/null +++ b/tests/Unit/Traffic/PresenceIndexPrunerTest.php @@ -0,0 +1,73 @@ +once()->andReturn([0, ['stale-1', 'live-1']]); + Redis::shouldReceive('pipeline')->once()->andReturnUsing(function ($cb) { + $pipe = Mockery::mock(); + $pipe->shouldReceive('exists')->twice(); + $cb($pipe); + + return [0, 1]; + }); + Redis::shouldReceive('eval')->once()->andReturn(1); + + $result = app(PresenceIndexPruner::class)->prune(200, 1, 10, 0, false); + + expect($result['scanned'])->toBe(2) + ->and($result['stale'])->toBe(1) + ->and($result['live'])->toBe(1) + ->and($result['removed'])->toBe(1); +}); + +it('dry-run counts stale members without srem/eval', function () { + Redis::shouldReceive('sscan')->once()->andReturn([0, ['stale-1']]); + Redis::shouldReceive('pipeline')->once()->andReturnUsing(function ($cb) { + $pipe = Mockery::mock(); + $pipe->shouldReceive('exists')->once(); + $cb($pipe); + + return [0]; + }); + Redis::shouldReceive('eval')->never(); + + $result = app(PresenceIndexPruner::class)->prune(200, 5, 100, 0, true); + + expect($result['stale'])->toBe(1) + ->and($result['removed'])->toBe(0) + ->and($result['dry_run'])->toBeTrue(); +}); + +it('respects max-batches and max-members', function () { + Redis::shouldReceive('sscan')->once()->andReturn([1, ['a', 'b']]); + Redis::shouldReceive('pipeline')->once()->andReturnUsing(function ($cb) { + $pipe = Mockery::mock(); + $pipe->shouldReceive('exists')->twice(); + $cb($pipe); + + return [0, 0]; + }); + Redis::shouldReceive('eval')->never(); + + $result = app(PresenceIndexPruner::class)->prune(200, 1, 2, 0, true); + + expect($result['batches'])->toBe(1) + ->and($result['scanned'])->toBe(2); +}); + +it('does not use smembers on the presence index', function () { + $source = file_get_contents(base_path('app/Services/Traffic/PresenceIndexPruner.php')); + expect(strtolower((string) $source))->not->toContain('smembers') + ->and(strtolower((string) $source))->toContain('sscan'); +}); + +it('never deletes the presence index key', function () { + $source = (string) file_get_contents(base_path('app/Services/Traffic/PresenceIndexPruner.php')); + expect($source)->toContain('SREM') + ->and($source)->not->toContain('unlink') + ->and($source)->not->toContain('->del(') + ->and($source)->not->toContain('FLUSH'); +});