Ship production optimization M1-M12.5A: queues, metrics, HTTP observability, and vector search reliability.

Keep similar-ai from tripping the global circuit on a lone URL 502, clamp Qdrant search to 100, and add Server-Timing plus slow-request logging. Studio shared props, Academy S3 exists caching, heat chunking, and Redis/scheduler hygiene stay in this rollout.

app/Services/Academy/AcademyAccessService.php

@@ -13,6 +13,7 @@ use App\Models\AcademyLessonBlock;
 use App\Models\AcademyPromptPack;
 use App\Models\AcademyPromptTemplate;
 use App\Models\User;
+use Illuminate\Support\Facades\Cache;
 use Illuminate\Support\Facades\Storage;
 use Illuminate\Support\Str;
 use Laravel\Cashier\Subscription;
@@ -280,7 +281,9 @@ final class AcademyAccessService
     public function coursePayload(AcademyCourse $course, ?User $viewer, array $options = []): array
     {
         $progress = is_array($options['progress'] ?? null) ? $options['progress'] : null;
-        $lessonCount = (int) ($course->lessons_count_cache ?: $course->courseLessons()->count());
+        $lessonCount = $course->relationLoaded('courseLessons')
+            ? $course->courseLessons->count()
+            : (int) $course->lessons_count_cache;
 
         return [
             'id' => (int) $course->id,
@@ -1089,7 +1092,13 @@ final class AcademyAccessService
         }
 
         try {
-            $exists = Storage::disk((string) config('uploads.object_storage.disk', 's3'))->exists($normalizedPath);
+            $exists = (bool) Cache::remember(
+                'academy:asset-exists:'.$cacheKey,
+                3600,
+                function () use ($normalizedPath): bool {
+                    return Storage::disk((string) config('uploads.object_storage.disk', 's3'))->exists($normalizedPath);
+                },
+            );
         } catch (\Throwable) {
             $exists = false;
         }

app/Services/CollectionBackgroundJobService.php

@@ -16,6 +16,10 @@ class CollectionBackgroundJobService
 {
     public function dispatchQualityRefresh(Collection $collection, ?User $actor = null): array
     {
+        if (! $this->dispatchEnabled()) {
+            return $this->disabledPayload('quality_refresh', $collection ? collect([$collection]) : collect());
+        }
+
         RefreshCollectionQualityJob::dispatch((int) $collection->id, $actor?->id)->afterCommit();
 
         return [
@@ -30,6 +34,10 @@ class CollectionBackgroundJobService
 
     public function dispatchHealthRefresh(?Collection $collection = null, ?User $actor = null): array
     {
+        if (! $this->dispatchEnabled()) {
+            return $this->disabledPayload('health_refresh', $collection ? collect([$collection]) : collect());
+        }
+
         $targets = $collection ? collect([$collection]) : $this->healthTargets();
 
         $targets->each(fn (Collection $item) => RefreshCollectionHealthJob::dispatch((int) $item->id, $actor?->id, 'programming-eligibility')->afterCommit());
@@ -39,6 +47,10 @@ class CollectionBackgroundJobService
 
     public function dispatchRecommendationRefresh(?Collection $collection = null, ?User $actor = null, string $context = 'default'): array
     {
+        if (! $this->dispatchEnabled()) {
+            return $this->disabledPayload('recommendation_refresh', $collection ? collect([$collection]) : collect());
+        }
+
         $targets = $collection ? collect([$collection]) : $this->recommendationTargets();
 
         $targets->each(fn (Collection $item) => RefreshCollectionRecommendationJob::dispatch((int) $item->id, $actor?->id, $context)->afterCommit());
@@ -48,6 +60,10 @@ class CollectionBackgroundJobService
 
     public function dispatchDuplicateScan(?Collection $collection = null, ?User $actor = null): array
     {
+        if (! $this->dispatchEnabled()) {
+            return $this->disabledPayload('duplicate_scan', $collection ? collect([$collection]) : collect());
+        }
+
         $targets = $collection ? collect([$collection]) : $this->duplicateTargets();
 
         $targets->each(fn (Collection $item) => ScanCollectionDuplicateCandidatesJob::dispatch((int) $item->id, $actor?->id)->afterCommit());
@@ -128,6 +144,29 @@ class CollectionBackgroundJobService
             ->get(['id']);
     }
 
+    private function dispatchEnabled(): bool
+    {
+        return (bool) config('collections.v5.queue.dispatch_enabled', false);
+    }
+
+    /**
+     * @param  SupportCollection<int, Collection>  $targets
+     */
+    private function disabledPayload(string $job, SupportCollection $targets): array
+    {
+        $ids = $targets->pluck('id')->map(static fn ($id): int => (int) $id)->values()->all();
+
+        return [
+            'status' => 'disabled',
+            'job' => $job,
+            'scope' => count($ids) === 1 ? 'single' : 'batch',
+            'count' => 0,
+            'collection_ids' => $ids,
+            'items' => [],
+            'message' => 'Collection queue dispatch is disabled until a Horizon consumer exists.',
+        ];
+    }
+
     /**
      * @param  SupportCollection<int, Collection>  $targets
      */

app/Services/LeaderboardService.php

@@ -5,8 +5,8 @@ declare(strict_types=1);
 namespace App\Services;
 
 use App\Models\Artwork;
-use App\Models\ArtworkMetricSnapshotHourly;
 use App\Models\Group;
+use App\Services\Metrics\ArtworkHourlySnapshotWindow;
 use App\Models\Leaderboard;
 use App\Models\Story;
 use App\Models\StoryLike;
@@ -28,6 +28,7 @@ class LeaderboardService
 
     public function __construct(
         private readonly GroupReputationService $groupReputation,
+        private readonly ArtworkHourlySnapshotWindow $snapshotWindow,
     ) {
     }
 
@@ -835,29 +836,7 @@ class LeaderboardService
 
     private function artworkSnapshotDeltas(CarbonImmutable $start): \Illuminate\Database\Query\Builder
     {
-        return ArtworkMetricSnapshotHourly::query()
-            ->from('artwork_metric_snapshots_hourly as snapshots')
-            ->where('snapshots.bucket_hour', '>=', $start)
-            ->select([
-                'snapshots.artwork_id',
-                DB::raw($this->nonNegativeSnapshotDelta('views_count', 'views_delta')),
-                DB::raw($this->nonNegativeSnapshotDelta('downloads_count', 'downloads_delta')),
-                DB::raw($this->nonNegativeSnapshotDelta('favourites_count', 'favourites_delta')),
-                DB::raw($this->nonNegativeSnapshotDelta('comments_count', 'comments_delta')),
-            ])
-            ->groupBy('snapshots.artwork_id')
-            ->toBase();
-    }
-
-    private function nonNegativeSnapshotDelta(string $column, string $alias): string
-    {
-        $delta = sprintf('MAX(snapshots.%1$s) - MIN(snapshots.%1$s)', $column);
-
-        if (DB::connection()->getDriverName() === 'sqlite') {
-            return sprintf('CASE WHEN %1$s > 0 THEN %1$s ELSE 0 END as %2$s', $delta, $alias);
-        }
-
-        return sprintf('GREATEST(%s, 0) as %s', $delta, $alias);
+        return $this->snapshotWindow->artworkPeriodDeltas($start);
     }
 
     private function creatorEntities(array $ids): array

app/Services/Metrics/ArtworkHourlySnapshotWindow.php

@@ -0,0 +1,260 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Services\Metrics;
+
+use Carbon\CarbonInterface;
+use Illuminate\Database\Query\Builder;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Schema;
+
+/**
+ * Windowed deltas from cumulative hourly artwork metric snapshots.
+ *
+ * Each snapshot stores running totals. Period growth is:
+ *   latest in window
+ *   - baseline at or immediately before the window start
+ *
+ * If the artwork was created/published during the window, baseline is 0.
+ * If an older artwork has no pre-window snapshot (warm-up / missing hours),
+ * fall back to MIN in the window — never SUM cumulatives, never lifetime totals.
+ */
+final class ArtworkHourlySnapshotWindow
+{
+    /**
+     * How far before the window start to look for a baseline snapshot.
+     * Keeps the baseline scan bounded at 30-day table scale.
+     */
+    public const BASELINE_LOOKBACK_HOURS = 36;
+
+    /**
+     * @return array{
+     *     views: int,
+     *     downloads: int,
+     *     favourites: int,
+     *     comments: int,
+     *     shares: int,
+     *     requested_days: int,
+     *     table_coverage_days: float,
+     *     user_coverage_days: float,
+     *     window_complete: bool,
+     *     oldest_bucket: ?string,
+     *     newest_bucket: ?string,
+     *     user_oldest_bucket: ?string,
+     *     user_newest_bucket: ?string,
+     * }
+     */
+    public function userWindow(int $userId, int $days = 30): array
+    {
+        $days = max(1, $days);
+        $coverage = $this->tableCoverage($days);
+        $empty = $this->emptyResult($days, $coverage);
+
+        if ($userId <= 0 || ! Schema::hasTable('artwork_metric_snapshots_hourly')) {
+            return $empty;
+        }
+
+        $since = now()->subDays($days);
+
+        $row = DB::query()
+            ->fromSub($this->artworkPeriodDeltas($since, $userId), 'deltas')
+            ->selectRaw('COALESCE(SUM(views_delta), 0) as views')
+            ->selectRaw('COALESCE(SUM(downloads_delta), 0) as downloads')
+            ->selectRaw('COALESCE(SUM(favourites_delta), 0) as favourites')
+            ->selectRaw('COALESCE(SUM(comments_delta), 0) as comments')
+            ->selectRaw('COALESCE(SUM(shares_delta), 0) as shares')
+            ->first();
+
+        $userBounds = DB::table('artwork_metric_snapshots_hourly as snapshots')
+            ->join('artworks', 'artworks.id', '=', 'snapshots.artwork_id')
+            ->where('artworks.user_id', $userId)
+            ->whereNull('artworks.deleted_at')
+            ->where('snapshots.bucket_hour', '>=', $since)
+            ->selectRaw('MIN(snapshots.bucket_hour) as oldest_bucket')
+            ->selectRaw('MAX(snapshots.bucket_hour) as newest_bucket')
+            ->first();
+
+        $userCoverageDays = $this->coverageDays(
+            $userBounds->oldest_bucket ?? null,
+            $userBounds->newest_bucket ?? null,
+        );
+
+        return [
+            'views' => (int) ($row->views ?? 0),
+            'downloads' => (int) ($row->downloads ?? 0),
+            'favourites' => (int) ($row->favourites ?? 0),
+            'comments' => (int) ($row->comments ?? 0),
+            'shares' => (int) ($row->shares ?? 0),
+            'requested_days' => $days,
+            'table_coverage_days' => $coverage['coverage_days'],
+            'user_coverage_days' => $userCoverageDays,
+            'window_complete' => $coverage['window_complete'],
+            'oldest_bucket' => $coverage['oldest_bucket'],
+            'newest_bucket' => $coverage['newest_bucket'],
+            'user_oldest_bucket' => $userBounds->oldest_bucket ?? null,
+            'user_newest_bucket' => $userBounds->newest_bucket ?? null,
+        ];
+    }
+
+    /**
+     * Per-artwork non-negative period deltas for cumulative snapshot columns.
+     */
+    public function artworkPeriodDeltas(CarbonInterface|\DateTimeInterface $start, ?int $userId = null): Builder
+    {
+        $inWindow = DB::table('artwork_metric_snapshots_hourly as snapshots');
+        $this->constrainOwner($inWindow, $userId);
+        $inWindow
+            ->where('snapshots.bucket_hour', '>=', $start)
+            ->groupBy('snapshots.artwork_id')
+            ->select('snapshots.artwork_id')
+            ->selectRaw('MAX(snapshots.views_count) as views_latest')
+            ->selectRaw('MIN(snapshots.views_count) as views_min')
+            ->selectRaw('MAX(snapshots.downloads_count) as downloads_latest')
+            ->selectRaw('MIN(snapshots.downloads_count) as downloads_min')
+            ->selectRaw('MAX(snapshots.favourites_count) as favourites_latest')
+            ->selectRaw('MIN(snapshots.favourites_count) as favourites_min')
+            ->selectRaw('MAX(snapshots.comments_count) as comments_latest')
+            ->selectRaw('MIN(snapshots.comments_count) as comments_min')
+            ->selectRaw('MAX(snapshots.shares_count) as shares_latest')
+            ->selectRaw('MIN(snapshots.shares_count) as shares_min');
+
+        $baselineFrom = \Carbon\Carbon::parse($start)->subHours(self::BASELINE_LOOKBACK_HOURS);
+        $baseline = DB::table('artwork_metric_snapshots_hourly as snapshots');
+        $this->constrainOwner($baseline, $userId);
+        $baseline
+            ->where('snapshots.bucket_hour', '<', $start)
+            ->where('snapshots.bucket_hour', '>=', $baselineFrom)
+            ->groupBy('snapshots.artwork_id')
+            ->select('snapshots.artwork_id')
+            ->selectRaw('MAX(snapshots.views_count) as views_baseline')
+            ->selectRaw('MAX(snapshots.downloads_count) as downloads_baseline')
+            ->selectRaw('MAX(snapshots.favourites_count) as favourites_baseline')
+            ->selectRaw('MAX(snapshots.comments_count) as comments_baseline')
+            ->selectRaw('MAX(snapshots.shares_count) as shares_baseline');
+
+        $bornSql = $this->quotedTimestamp($start);
+
+        return DB::query()
+            ->fromSub($inWindow, 'win')
+            ->leftJoinSub($baseline, 'base', 'base.artwork_id', '=', 'win.artwork_id')
+            ->join('artworks', 'artworks.id', '=', 'win.artwork_id')
+            ->select('win.artwork_id')
+            ->selectRaw($this->deltaExpression('views', $bornSql) . ' as views_delta')
+            ->selectRaw($this->deltaExpression('downloads', $bornSql) . ' as downloads_delta')
+            ->selectRaw($this->deltaExpression('favourites', $bornSql) . ' as favourites_delta')
+            ->selectRaw($this->deltaExpression('comments', $bornSql) . ' as comments_delta')
+            ->selectRaw($this->deltaExpression('shares', $bornSql) . ' as shares_delta');
+    }
+
+    /**
+     * @return array{oldest_bucket: ?string, newest_bucket: ?string, coverage_days: float, window_complete: bool, requested_days: int}
+     */
+    public function tableCoverage(int $days = 30): array
+    {
+        $days = max(1, $days);
+
+        if (! Schema::hasTable('artwork_metric_snapshots_hourly')) {
+            return [
+                'oldest_bucket' => null,
+                'newest_bucket' => null,
+                'coverage_days' => 0.0,
+                'window_complete' => false,
+                'requested_days' => $days,
+            ];
+        }
+
+        $bounds = DB::table('artwork_metric_snapshots_hourly')
+            ->selectRaw('MIN(bucket_hour) as oldest_bucket')
+            ->selectRaw('MAX(bucket_hour) as newest_bucket')
+            ->first();
+
+        $coverageDays = $this->coverageDays(
+            $bounds->oldest_bucket ?? null,
+            $bounds->newest_bucket ?? null,
+        );
+
+        return [
+            'oldest_bucket' => $bounds->oldest_bucket ?? null,
+            'newest_bucket' => $bounds->newest_bucket ?? null,
+            'coverage_days' => $coverageDays,
+            'window_complete' => $coverageDays + (1 / 24) >= $days,
+            'requested_days' => $days,
+        ];
+    }
+
+    private function constrainOwner(Builder $query, ?int $userId): void
+    {
+        if ($userId === null) {
+            return;
+        }
+
+        $query->join('artworks as owner_artworks', 'owner_artworks.id', '=', 'snapshots.artwork_id')
+            ->where('owner_artworks.user_id', $userId)
+            ->whereNull('owner_artworks.deleted_at');
+    }
+
+    private function quotedTimestamp(CarbonInterface|\DateTimeInterface $start): string
+    {
+        return DB::getPdo()->quote(\Carbon\Carbon::parse($start)->toDateTimeString());
+    }
+
+    private function deltaExpression(string $metric, string $bornSql): string
+    {
+        $latest = "COALESCE(win.{$metric}_latest, 0)";
+        $min = "COALESCE(win.{$metric}_min, 0)";
+        $base = "base.{$metric}_baseline";
+        $bornInWindow = "COALESCE(artworks.published_at, artworks.created_at) >= {$bornSql}";
+        $observed = $this->clampNonNegative("{$latest} - {$min}");
+        $fromBaseline = $this->clampNonNegative("{$latest} - {$base}");
+
+        return "CASE WHEN {$bornInWindow} THEN {$latest} WHEN {$base} IS NOT NULL THEN {$fromBaseline} ELSE {$observed} END";
+    }
+
+    private function clampNonNegative(string $expression): string
+    {
+        if (DB::connection()->getDriverName() === 'sqlite') {
+            return "CASE WHEN ({$expression}) > 0 THEN ({$expression}) ELSE 0 END";
+        }
+
+        return "GREATEST({$expression}, 0)";
+    }
+
+    /**
+     * @return array<string, mixed>
+     */
+    private function emptyResult(int $days, array $coverage): array
+    {
+        return [
+            'views' => 0,
+            'downloads' => 0,
+            'favourites' => 0,
+            'comments' => 0,
+            'shares' => 0,
+            'requested_days' => $days,
+            'table_coverage_days' => $coverage['coverage_days'],
+            'user_coverage_days' => 0.0,
+            'window_complete' => $coverage['window_complete'],
+            'oldest_bucket' => $coverage['oldest_bucket'],
+            'newest_bucket' => $coverage['newest_bucket'],
+            'user_oldest_bucket' => null,
+            'user_newest_bucket' => null,
+        ];
+    }
+
+    private function coverageDays(?string $oldest, ?string $newest): float
+    {
+        if ($oldest === null || $newest === null) {
+            return 0.0;
+        }
+
+        $rangeStart = \Carbon\Carbon::parse($oldest);
+        $rangeEnd = \Carbon\Carbon::parse($newest);
+
+        if ($rangeEnd->lessThan($rangeStart)) {
+            return 0.0;
+        }
+
+        return round(abs($rangeStart->diffInMinutes($rangeEnd)) / 1440, 2);
+    }
+}

app/Services/Profile/CreatorJourneyService.php

@@ -204,7 +204,7 @@ final class CreatorJourneyService
                 [
                     'title' => 'Biggest download spike',
                     'headline' => (string) $bestSpike['artwork']->title,
-                    'summary' => 'Captured the strongest one-hour download burst recorded for a public artwork.',
+                    'summary' => 'Captured the strongest one-hour download burst in the retained hourly snapshot window.',
                     'value' => (int) $bestSpike['downloads_in_hour'] . ' downloads in 1 hour',
                     'artwork' => $this->artworkSnapshot($bestSpike['artwork']),
                     'metrics' => [
@@ -425,8 +425,12 @@ final class CreatorJourneyService
         $publicArtworkIds = $artworks->pluck('id')->map(fn ($id): int => (int) $id)->all();
 
         if ($publicArtworkIds !== [] && DB::getSchemaBuilder()->hasTable('artwork_metric_snapshots_hourly')) {
+            $lookbackDays = max(1, (int) config('metrics.hourly_snapshot_retention_days', 30));
+            $since = now()->subDays($lookbackDays);
+
             $snapshots = DB::table('artwork_metric_snapshots_hourly as ms')
                 ->whereIn('ms.artwork_id', $publicArtworkIds)
+                ->where('ms.bucket_hour', '>=', $since)
                 ->orderBy('ms.artwork_id')
                 ->orderBy('ms.bucket_hour')
                 ->get([

app/Services/Sitemaps/Builders/TagsSitemapBuilder.php

@@ -5,12 +5,12 @@ declare(strict_types=1);
 namespace App\Services\Sitemaps\Builders;
 
 use App\Models\Tag;
-use App\Services\Sitemaps\AbstractSitemapBuilder;
 use App\Services\Sitemaps\SitemapUrl;
 use App\Services\Sitemaps\SitemapUrlBuilder;
-use DateTimeInterface;
+use Illuminate\Database\Eloquent\Builder;
+use Illuminate\Database\Eloquent\Model;
 
-final class TagsSitemapBuilder extends AbstractSitemapBuilder
+final class TagsSitemapBuilder extends AbstractIdShardableSitemapBuilder
 {
     public function __construct(private readonly SitemapUrlBuilder $urls)
     {
@@ -21,25 +21,21 @@ final class TagsSitemapBuilder extends AbstractSitemapBuilder
         return 'tags';
     }
 
-    public function items(): array
+    protected function shardConfigKey(): string
+    {
+        return 'tags';
+    }
+
+    protected function mapRecord(Model $record): ?SitemapUrl
+    {
+        return $this->urls->tag($record);
+    }
+
+    protected function query(): Builder
     {
         return Tag::query()
             ->where('is_active', true)
             ->where('usage_count', '>', 0)
-            ->whereHas('artworks', fn ($query) => $query->public()->published())
-            ->orderByDesc('usage_count')
-            ->orderBy('slug')
-            ->get()
-            ->map(fn (Tag $tag): SitemapUrl => $this->urls->tag($tag))
-            ->values()
-            ->all();
-    }
-
-    public function lastModified(): ?DateTimeInterface
-    {
-        return $this->dateTime(Tag::query()
-            ->where('is_active', true)
-            ->where('usage_count', '>', 0)
-            ->max('updated_at'));
+            ->whereHas('artworks', fn ($query) => $query->public()->published());
     }
 }

app/Services/Studio/CreatorStudioOverviewService.php

@@ -8,6 +8,7 @@ use App\Models\CollectionComment;
 use App\Models\NovaCardComment;
 use App\Models\StoryComment;
 use App\Models\User;
+use App\Services\Metrics\ArtworkHourlySnapshotWindow;
 use App\Support\AvatarUrl;
 use Illuminate\Support\Facades\DB;
 
@@ -21,6 +22,7 @@ final class CreatorStudioOverviewService
         private readonly CreatorStudioPreferenceService $preferences,
         private readonly CreatorStudioChallengeService $challenges,
         private readonly CreatorStudioGrowthService $growth,
+        private readonly ArtworkHourlySnapshotWindow $snapshotWindow,
     ) {
     }
 
@@ -32,15 +34,24 @@ final class CreatorStudioOverviewService
         $challengeData = $this->challenges->build($user);
         $growthData = $this->growth->build($user, $preferences['analytics_range_days']);
         $featuredContent = $this->content->selectedItems($user, $preferences['featured_content']);
+        $window = $this->snapshotWindow->userWindow((int) $user->id, 30);
 
         return [
             'kpis' => [
                 'total_content' => $analytics['totals']['content_count'],
-                'views_30d' => $analytics['totals']['views'],
-                'appreciation_30d' => $analytics['totals']['appreciation'],
-                'shares_30d' => $analytics['totals']['shares'],
-                'comments_30d' => $analytics['totals']['comments'],
+                'views_30d' => $window['views'],
+                'appreciation_30d' => $window['favourites'],
+                'shares_30d' => $window['shares'],
+                'comments_30d' => $window['comments'],
                 'followers' => $analytics['totals']['followers'],
+                'snapshot_window' => [
+                    'requested_days' => $window['requested_days'],
+                    'table_coverage_days' => $window['table_coverage_days'],
+                    'user_coverage_days' => $window['user_coverage_days'],
+                    'window_complete' => $window['window_complete'],
+                    'oldest_bucket' => $window['oldest_bucket'],
+                    'newest_bucket' => $window['newest_bucket'],
+                ],
             ],
             'module_summaries' => $moduleSummaries,
             'quick_create' => $this->content->quickCreate(),

app/Services/Studio/StudioMetricsService.php

@@ -6,6 +6,7 @@ namespace App\Services\Studio;
 
 use App\Models\Artwork;
 use App\Models\ArtworkStats;
+use App\Services\Metrics\ArtworkHourlySnapshotWindow;
 use Illuminate\Support\Facades\Cache;
 use Illuminate\Support\Facades\DB;
 
@@ -16,10 +17,26 @@ final class StudioMetricsService
 {
     private const CACHE_TTL = 300; // 5 minutes
 
+    public function __construct(
+        private readonly ArtworkHourlySnapshotWindow $snapshotWindow,
+    ) {
+    }
+
     /**
      * Get dashboard KPI metrics for a creator.
      *
-     * @return array{total_artworks: int, views_30d: int, favourites_30d: int, shares_30d: int, followers: int}
+     * 30-day counters are latest-minus-baseline deltas of cumulative hourly snapshots, never SUM.
+     *
+     * @return array{
+     *     total_artworks: int,
+     *     views_30d: int,
+     *     favourites_30d: int,
+     *     shares_30d: int,
+     *     downloads_30d: int,
+     *     comments_30d: int,
+     *     followers: int,
+     *     snapshot_window: array<string, mixed>
+     * }
      */
     public function getDashboardKpis(int $userId): array
     {
@@ -30,35 +47,7 @@ final class StudioMetricsService
                 ->whereNull('deleted_at')
                 ->count();
 
-            // Aggregate stats from artwork_stats for this user's artworks
-            $statsAgg = DB::table('artwork_stats')
-                ->join('artworks', 'artworks.id', '=', 'artwork_stats.artwork_id')
-                ->where('artworks.user_id', $userId)
-                ->whereNull('artworks.deleted_at')
-                ->selectRaw('
-                    COALESCE(SUM(artwork_stats.views), 0) as total_views,
-                    COALESCE(SUM(artwork_stats.favorites), 0) as total_favourites,
-                    COALESCE(SUM(artwork_stats.shares_count), 0) as total_shares
-                ')
-                ->first();
-
-            // Views in last 30 days from hourly snapshots if available, fallback to totals
-            $views30d = 0;
-            try {
-                if (\Illuminate\Support\Facades\Schema::hasTable('artwork_metric_snapshots_hourly')) {
-                    $views30d = (int) DB::table('artwork_metric_snapshots_hourly')
-                        ->join('artworks', 'artworks.id', '=', 'artwork_metric_snapshots_hourly.artwork_id')
-                        ->where('artworks.user_id', $userId)
-                        ->where('artwork_metric_snapshots_hourly.bucket_hour', '>=', now()->subDays(30))
-                        ->sum('artwork_metric_snapshots_hourly.views_count');
-                }
-            } catch (\Throwable $e) {
-                // Table or column doesn't exist — fall back to totals
-            }
-
-            if ($views30d === 0) {
-                $views30d = (int) ($statsAgg->total_views ?? 0);
-            }
+            $window = $this->snapshotWindow->userWindow($userId, 30);
 
             $followers = DB::table('user_followers')
                 ->where('user_id', $userId)
@@ -66,10 +55,20 @@ final class StudioMetricsService
 
             return [
                 'total_artworks' => $totalArtworks,
-                'views_30d'      => $views30d,
-                'favourites_30d' => (int) ($statsAgg->total_favourites ?? 0),
-                'shares_30d'     => (int) ($statsAgg->total_shares ?? 0),
-                'followers'      => $followers,
+                'views_30d' => $window['views'],
+                'favourites_30d' => $window['favourites'],
+                'shares_30d' => $window['shares'],
+                'downloads_30d' => $window['downloads'],
+                'comments_30d' => $window['comments'],
+                'followers' => $followers,
+                'snapshot_window' => [
+                    'requested_days' => $window['requested_days'],
+                    'table_coverage_days' => $window['table_coverage_days'],
+                    'user_coverage_days' => $window['user_coverage_days'],
+                    'window_complete' => $window['window_complete'],
+                    'oldest_bucket' => $window['oldest_bucket'],
+                    'newest_bucket' => $window['newest_bucket'],
+                ],
             ];
         });
     }
@@ -145,7 +144,6 @@ final class StudioMetricsService
         $cacheKey = "studio.analytics_overview.{$userId}";
 
         return Cache::remember($cacheKey, self::CACHE_TTL, function () use ($userId) {
-            // Totals
             $totals = DB::table('artwork_stats')
                 ->join('artworks', 'artworks.id', '=', 'artwork_stats.artwork_id')
                 ->where('artworks.user_id', $userId)
@@ -161,7 +159,6 @@ final class StudioMetricsService
                 ')
                 ->first();
 
-            // Top 10 artworks by ranking score
             $topArtworks = Artwork::where('user_id', $userId)
                 ->whereNull('deleted_at')
                 ->where('is_public', true)
@@ -188,7 +185,6 @@ final class StudioMetricsService
                     'heat_score'    => (float) ($art->stats?->heat_score ?? 0),
                 ]);
 
-            // Content type breakdown
             $contentBreakdown = DB::table('artworks')
                 ->join('artwork_category', 'artwork_category.artwork_id', '=', 'artworks.id')
                 ->join('categories', 'categories.id', '=', 'artwork_category.category_id')

app/Services/Traffic/OnlineVisitorRepository.php

@@ -181,7 +181,29 @@ class OnlineVisitorRepository
      */
     protected function readIndexMembers(): array
     {
-        return array_map('strval', Redis::smembers(self::INDEX_KEY));
+        $members = [];
+        $cursor = '0';
+        $limit = max(1, (int) config('traffic.online_visitors.index_read_limit', 2000));
+
+        do {
+            $result = Redis::sscan(self::INDEX_KEY, (int) $cursor, ['count' => 200]);
+
+            if (! is_array($result) || count($result) < 2) {
+                break;
+            }
+
+            $cursor = (string) $result[0];
+
+            foreach ($result[1] as $member) {
+                $members[] = (string) $member;
+
+                if (count($members) >= $limit) {
+                    return $members;
+                }
+            }
+        } while ($cursor !== '0');
+
+        return $members;
     }
 
     /**

app/Services/Traffic/PresenceIndexPruner.php

@@ -0,0 +1,116 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Services\Traffic;
+
+use Illuminate\Support\Facades\Redis;
+
+final class PresenceIndexPruner
+{
+    private const SREM_IF_EXPIRED = <<<'LUA'
+if redis.call('EXISTS', KEYS[1]) == 0 then
+  return redis.call('SREM', KEYS[2], ARGV[1])
+end
+return 0
+LUA;
+
+    /**
+     * @return array{scanned:int,stale:int,removed:int,live:int,batches:int,dry_run:bool}
+     */
+    public function prune(
+        int $scanCount,
+        int $maxBatches,
+        int $maxMembers,
+        int $sleepMs,
+        bool $dryRun,
+        ?int $timeLimitSeconds = null,
+    ): array {
+        $scanCount = max(10, $scanCount);
+        $maxBatches = max(1, $maxBatches);
+        $maxMembers = max(1, $maxMembers);
+        $started = microtime(true);
+        $cursor = 0;
+        $scanned = 0;
+        $stale = 0;
+        $removed = 0;
+        $live = 0;
+        $batches = 0;
+
+        while ($batches < $maxBatches && $scanned < $maxMembers) {
+            if ($timeLimitSeconds !== null && (microtime(true) - $started) >= $timeLimitSeconds) {
+                break;
+            }
+
+            $result = Redis::sscan(OnlineVisitorRepository::INDEX_KEY, $cursor, ['count' => $scanCount]);
+            if (! is_array($result) || count($result) < 2) {
+                break;
+            }
+
+            $cursor = (int) $result[0];
+            $members = $result[1];
+            $batches++;
+
+            if ($members === []) {
+                if ($cursor === 0) {
+                    break;
+                }
+                continue;
+            }
+
+            $recordKeys = [];
+            foreach ($members as $member) {
+                $member = (string) $member;
+                $recordKeys[$member] = OnlineVisitorRepository::KEY_PREFIX.':'.$member;
+            }
+
+            $exists = Redis::pipeline(function ($pipe) use ($recordKeys): void {
+                foreach ($recordKeys as $recordKey) {
+                    $pipe->exists($recordKey);
+                }
+            });
+            if (! is_array($exists)) {
+                $exists = [];
+            }
+
+            $i = 0;
+            foreach ($recordKeys as $member => $recordKey) {
+                $scanned++;
+                $isLive = (int) ($exists[$i] ?? 0) === 1;
+                $i++;
+                if ($isLive) {
+                    $live++;
+                    continue;
+                }
+                $stale++;
+                if ($dryRun) {
+                    continue;
+                }
+                $removed += (int) Redis::eval(
+                    self::SREM_IF_EXPIRED,
+                    2,
+                    $recordKey,
+                    OnlineVisitorRepository::INDEX_KEY,
+                    $member,
+                );
+            }
+
+            if ($sleepMs > 0) {
+                usleep($sleepMs * 1000);
+            }
+
+            if ($cursor === 0) {
+                break;
+            }
+        }
+
+        return [
+            'scanned' => $scanned,
+            'stale' => $stale,
+            'removed' => $removed,
+            'live' => $live,
+            'batches' => $batches,
+            'dry_run' => $dryRun,
+        ];
+    }
+}

app/Services/Vision/AiArtworkVectorSearchService.php

@@ -8,12 +8,18 @@ use App\Models\Artwork;
 use Illuminate\Http\UploadedFile;
 use Illuminate\Support\Facades\Cache;
 use Illuminate\Support\Facades\Http;
-use RuntimeException;
 use Throwable;
 
 final class AiArtworkVectorSearchService
 {
-    private const MAX_SIMILAR_RESULTS = 120;
+    /** qdrant-svc /vectors/search* reject limit > 100. */
+    private const VECTOR_GATEWAY_MAX_SEARCH_RESULTS = 100;
+
+    /**
+     * Public similar-ai may request at most 99 items because we fetch one extra
+     * candidate so the source artwork can be excluded from the result set.
+     */
+    private const MAX_ARTWORK_SIMILAR_RESULTS = 99;
 
     public function __construct(
         private readonly VectorGatewayClient $client,
@@ -31,12 +37,13 @@ final class AiArtworkVectorSearchService
      */
     public function similarToArtwork(Artwork $artwork, int $limit = 12): array
     {
-        $safeLimit = max(1, min(self::MAX_SIMILAR_RESULTS, $limit));
+        $safeLimit = max(1, min(self::MAX_ARTWORK_SIMILAR_RESULTS, $limit));
         $cacheKey = sprintf('rec:artwork:%d:similar-ai:%d', $artwork->id, $safeLimit);
         $ttl = max(60, (int) config('recommendations.ttl.similar_artworks', 30 * 60));
 
         return Cache::remember($cacheKey, $ttl, function () use ($artwork, $safeLimit): array {
-            $matches = $this->searchMatchesForArtwork($artwork, $safeLimit + 1);
+            $gatewayLimit = min(self::VECTOR_GATEWAY_MAX_SEARCH_RESULTS, $safeLimit + 1);
+            $matches = $this->searchMatchesForArtwork($artwork, $gatewayLimit);
 
             return $this->resolveMatches($matches, $safeLimit, $artwork->id);
         });
@@ -47,7 +54,7 @@ final class AiArtworkVectorSearchService
      */
     public function searchByUploadedImage(UploadedFile $file, int $limit = 12): array
     {
-        $safeLimit = max(1, min(self::MAX_SIMILAR_RESULTS, $limit));
+        $safeLimit = max(1, min(self::VECTOR_GATEWAY_MAX_SEARCH_RESULTS, $limit));
         $matches = $this->client->searchByUploadedFile($file, $safeLimit);
 
         return $this->resolveMatches($matches, $safeLimit);
@@ -93,15 +100,15 @@ final class AiArtworkVectorSearchService
             return [];
         }
 
-        if ($this->client->circuitOpen()) {
-            throw new RuntimeException('Vector gateway temporarily unavailable (circuit open after a recent failure).');
-        }
+        $this->client->assertCircuitClosed();
 
         $fileFailure = null;
+        $fileGatewayAttempted = false;
 
         try {
             $payload = $this->downloadArtworkImage($artwork, $url);
             if ($payload !== null) {
+                $fileGatewayAttempted = true;
                 return $this->client->searchByFileContents($payload['contents'], $payload['filename'], $limit);
             }
         } catch (Throwable $e) {
@@ -110,24 +117,69 @@ final class AiArtworkVectorSearchService
 
         try {
             return $this->client->searchByUrl($url, $limit);
-        } catch (Throwable $e) {
-            throw $this->normalizeSearchFailure($fileFailure, $e);
+        } catch (Throwable $fallbackFailure) {
+            if ($this->shouldTripSimilarAiCircuit($fileGatewayAttempted, $fileFailure, $fallbackFailure)) {
+                $this->client->tripIfCircuitWorthy(
+                    [$fileFailure, $fallbackFailure],
+                    'similar_ai',
+                    ['artwork_id' => (int) $artwork->id],
+                );
+            }
+
+            throw $this->normalizeSearchFailure($fileFailure, $fallbackFailure);
         }
     }
 
-    private function normalizeSearchFailure(?Throwable $fileFailure, Throwable $fallbackFailure): RuntimeException
+    private function normalizeSearchFailure(?Throwable $fileFailure, Throwable $fallbackFailure): VectorGatewayException
     {
-        if ($fileFailure === null) {
-            return $fallbackFailure instanceof RuntimeException
-                ? $fallbackFailure
-                : new RuntimeException($fallbackFailure->getMessage(), 0, $fallbackFailure);
+        if ($fallbackFailure instanceof VectorGatewayException && $fileFailure === null) {
+            return $fallbackFailure;
         }
 
-        return new RuntimeException(sprintf(
-            'Vector search failed via file endpoint (%s) and URL fallback (%s).',
-            $fileFailure->getMessage(),
-            $fallbackFailure->getMessage(),
-        ), 0, $fallbackFailure);
+        $status = $fallbackFailure instanceof VectorGatewayException
+            ? $fallbackFailure->httpStatus
+            : null;
+        $circuitWorthy = $this->isCircuitWorthyFailure($fileFailure)
+            && $this->isCircuitWorthyFailure($fallbackFailure);
+
+        if ($fileFailure === null) {
+            return $fallbackFailure instanceof VectorGatewayException
+                ? $fallbackFailure
+                : new VectorGatewayException(
+                    'Vector gateway search failed.',
+                    'search_url',
+                    $status,
+                    $this->isCircuitWorthyFailure($fallbackFailure),
+                    $fallbackFailure,
+                );
+        }
+
+        return new VectorGatewayException(
+            'Vector gateway search failed.',
+            'search_combined',
+            $status,
+            $circuitWorthy,
+            $fallbackFailure,
+        );
+    }
+
+    private function isCircuitWorthyFailure(Throwable $failure): bool
+    {
+        return $failure instanceof VectorGatewayException && $failure->circuitWorthy;
+    }
+
+    private function shouldTripSimilarAiCircuit(
+        bool $fileGatewayAttempted,
+        ?Throwable $fileFailure,
+        Throwable $fallbackFailure,
+    ): bool {
+        return $fileGatewayAttempted
+            && $fileFailure instanceof VectorGatewayException
+            && $fileFailure->operation === 'search_file'
+            && $fileFailure->circuitWorthy
+            && $fallbackFailure instanceof VectorGatewayException
+            && $fallbackFailure->operation === 'search_url'
+            && $fallbackFailure->circuitWorthy;
     }
 
     /**

app/Services/Vision/VectorGatewayClient.php

@@ -5,16 +5,22 @@ declare(strict_types=1);
 namespace App\Services\Vision;
 
 use Illuminate\Http\UploadedFile;
+use Illuminate\Http\Client\ConnectionException;
 use Illuminate\Http\Client\PendingRequest;
+use Illuminate\Http\Client\RequestException;
 use Illuminate\Http\Client\Response;
 use Illuminate\Support\Facades\Cache;
 use Illuminate\Support\Facades\Http;
+use Illuminate\Support\Facades\Log;
 use RuntimeException;
+use Throwable;
 
 final class VectorGatewayClient
 {
     private const CIRCUIT_KEY = 'vision.vector_gateway.circuit_open';
 
+    private const MAX_SEARCH_LIMIT = 100;
+
     public function isConfigured(): bool
     {
         return (bool) config('vision.vector_gateway.enabled', true)
@@ -31,10 +37,20 @@ final class VectorGatewayClient
         return Cache::has(self::CIRCUIT_KEY);
     }
 
-    public function tripCircuit(): void
+    /**
+     * @param  array<string, mixed>  $context
+     */
+    public function tripCircuit(array $context = []): void
     {
+        $alreadyOpen = $this->circuitOpen();
         $seconds = max(1, (int) config('vision.vector_gateway.circuit_breaker_seconds', 30));
         Cache::put(self::CIRCUIT_KEY, true, $seconds);
+
+        if ($alreadyOpen) {
+            return;
+        }
+
+        $this->logCircuitOpened($context, $seconds);
     }
 
     public function upsertByUrl(string $imageUrl, int|string $id, array $metadata = []): array
@@ -84,25 +100,21 @@ final class VectorGatewayClient
      */
     public function searchByUrl(string $imageUrl, int $limit = 5): array
     {
-        $this->guardCircuit();
-
         try {
             $response = $this->postJson(
                 $this->searchRequest(),
                 $this->url((string) config('vision.vector_gateway.search_endpoint', '/vectors/search')),
                 [
                     'url' => $imageUrl,
-                    'limit' => max(1, $limit),
+                    'limit' => $this->clampSearchLimit($limit),
                 ]
             );
-        } catch (\Throwable $e) {
-            $this->tripCircuit();
-            throw $e;
+        } catch (Throwable $e) {
+            throw $this->classifyThrowable('search_url', $e);
         }
 
         if ($response->failed()) {
-            $this->tripCircuit();
-            throw new RuntimeException($this->failureMessage('Vector search', $response));
+            throw $this->classifyHttpFailure('search_url', $response);
         }
 
         return $this->extractMatches($response->json());
@@ -113,25 +125,21 @@ final class VectorGatewayClient
      */
     public function searchByFileContents(string $contents, string $filename, int $limit = 5): array
     {
-        $this->guardCircuit();
-
         try {
             $response = $this->searchRequest()
                 ->attach('file', $contents, $filename)
                 ->post(
                     $this->url((string) config('vision.vector_gateway.search_file_endpoint', '/vectors/search/file')),
                     [
-                        'limit' => max(1, $limit),
+                        'limit' => $this->clampSearchLimit($limit),
                     ]
                 );
-        } catch (\Throwable $e) {
-            $this->tripCircuit();
-            throw $e;
+        } catch (Throwable $e) {
+            throw $this->classifyThrowable('search_file', $e);
         }
 
         if ($response->failed()) {
-            $this->tripCircuit();
-            throw new RuntimeException($this->failureMessage('Vector search', $response));
+            throw $this->classifyHttpFailure('search_file', $response);
         }
 
         return $this->extractMatches($response->json());
@@ -142,6 +150,8 @@ final class VectorGatewayClient
      */
     public function searchByUploadedFile(UploadedFile $file, int $limit = 5): array
     {
+        $this->assertCircuitClosed();
+
         $realPath = $file->getRealPath();
         if (! is_string($realPath) || $realPath === '') {
             throw new RuntimeException('Uploaded file has no readable temporary path for vector search.');
@@ -152,7 +162,47 @@ final class VectorGatewayClient
             throw new RuntimeException('Unable to read uploaded image bytes for vector search.');
         }
 
-        return $this->searchByFileContents($contents, $file->getClientOriginalName() ?: 'search-image', $limit);
+        try {
+            return $this->searchByFileContents($contents, $file->getClientOriginalName() ?: 'search-image', $limit);
+        } catch (VectorGatewayException $e) {
+            $this->tripIfCircuitWorthy([$e], 'uploaded_image');
+            throw $e;
+        }
+    }
+
+    /**
+     * Open the circuit only when every attempted search failure is transient/gateway-wide.
+     *
+     * @param  list<VectorGatewayException>  $failures
+     * @param  array{artwork_id?: int}  $context
+     */
+    public function tripIfCircuitWorthy(array $failures, string $source = 'unknown', array $context = []): void
+    {
+        if ($failures === []) {
+            return;
+        }
+
+        foreach ($failures as $failure) {
+            if (! $failure->circuitWorthy) {
+                return;
+            }
+        }
+
+        $this->tripCircuit([
+            'source' => $source,
+            'artwork_id' => isset($context['artwork_id']) ? (int) $context['artwork_id'] : null,
+            'failures' => array_map(static fn (VectorGatewayException $failure): array => [
+                'operation' => $failure->operation,
+                'http_status' => $failure->httpStatus,
+                'circuit_worthy' => $failure->circuitWorthy,
+                'exception_class' => $failure::class,
+            ], $failures),
+        ]);
+    }
+
+    public function assertCircuitClosed(): void
+    {
+        $this->guardCircuit();
     }
 
     public function deleteByIds(array $ids): array
@@ -220,13 +270,92 @@ final class VectorGatewayClient
             );
     }
 
+    /**
+     * @param  array<string, mixed>  $context
+     */
+    private function logCircuitOpened(array $context, int $ttlSeconds): void
+    {
+        $payload = [
+            'source' => (string) ($context['source'] ?? 'unknown'),
+            'failures' => is_array($context['failures'] ?? null) ? $context['failures'] : [],
+            'circuit_ttl_seconds' => $ttlSeconds,
+            'php_sapi' => PHP_SAPI,
+            'running_in_console' => app()->runningInConsole(),
+        ];
+
+        if (isset($context['artwork_id']) && is_int($context['artwork_id']) && $context['artwork_id'] > 0) {
+            $payload['artwork_id'] = $context['artwork_id'];
+        }
+
+        Log::warning('Vector gateway circuit opened', $payload);
+    }
+
+    private function clampSearchLimit(int $limit): int
+    {
+        return max(1, min(self::MAX_SEARCH_LIMIT, $limit));
+    }
+
     private function guardCircuit(): void
     {
         if ($this->circuitOpen()) {
-            throw new RuntimeException('Vector gateway temporarily unavailable (circuit open after a recent failure).');
+            throw new VectorGatewayException(
+                'Vector gateway temporarily unavailable.',
+                'circuit',
+                null,
+                true,
+            );
         }
     }
 
+    private function classifyHttpFailure(string $operation, Response $response): VectorGatewayException
+    {
+        $status = $response->status();
+
+        return new VectorGatewayException(
+            'Vector gateway '.$operation.' failed with HTTP '.$status.'.',
+            $operation,
+            $status,
+            $this->isCircuitWorthyStatus($status),
+        );
+    }
+
+    private function classifyThrowable(string $operation, Throwable $e): VectorGatewayException
+    {
+        if ($e instanceof VectorGatewayException) {
+            return $e;
+        }
+
+        if ($e instanceof RequestException && $e->response instanceof Response) {
+            return $this->classifyHttpFailure($operation, $e->response);
+        }
+
+        $circuitWorthy = $e instanceof ConnectionException || $this->looksLikeTimeout($e);
+
+        return new VectorGatewayException(
+            'Vector gateway '.$operation.' failed.',
+            $operation,
+            null,
+            $circuitWorthy,
+            $e,
+        );
+    }
+
+    private function isCircuitWorthyStatus(int $status): bool
+    {
+        return $status === 408 || $status === 429 || $status >= 500;
+    }
+
+    private function looksLikeTimeout(Throwable $e): bool
+    {
+        $message = strtolower($e->getMessage());
+
+        return str_contains($message, 'timed out')
+            || str_contains($message, 'timeout')
+            || str_contains($message, 'curl error 28')
+            || str_contains($message, 'connection refused')
+            || str_contains($message, 'could not resolve');
+    }
+
     /**
      * @param array<string, mixed> $payload
      */

app/Services/Vision/VectorGatewayException.php

@@ -0,0 +1,21 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Services\Vision;
+
+use RuntimeException;
+use Throwable;
+
+final class VectorGatewayException extends RuntimeException
+{
+    public function __construct(
+        string $message,
+        public readonly string $operation,
+        public readonly ?int $httpStatus = null,
+        public readonly bool $circuitWorthy = false,
+        ?Throwable $previous = null,
+    ) {
+        parent::__construct($message, 0, $previous);
+    }
+}