Ship production optimization M1-M12.5A: queues, metrics, HTTP observability, and vector search reliability.

Keep similar-ai from tripping the global circuit on a lone URL 502, clamp Qdrant search to 100, and add Server-Timing plus slow-request logging. Studio shared props, Academy S3 exists caching, heat chunking, and Redis/scheduler hygiene stay in this rollout.

tests/Unit/HorizonMailQueueTest.php

@@ -0,0 +1,48 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Tests\Unit;
+
+use App\Jobs\SendVerificationEmailJob;
+use App\Mail\EmailChangedSecurityAlertMail;
+use App\Mail\EmailChangeVerificationCodeMail;
+use App\Mail\RegistrationVerificationMail;
+use Tests\TestCase;
+
+final class HorizonMailQueueTest extends TestCase
+{
+    public function test_horizon_defines_a_mail_supervisor_that_consumes_only_mail(): void
+    {
+        $defaults = config('horizon.defaults.supervisor-mail');
+
+        $this->assertIsArray($defaults);
+        $this->assertSame(['mail'], $defaults['queue']);
+        $this->assertSame('redis', $defaults['connection']);
+        $this->assertGreaterThanOrEqual(5, (int) $defaults['tries']);
+        $this->assertLessThanOrEqual(
+            (int) config('queue.connections.redis.retry_after'),
+            (int) $defaults['timeout'],
+        );
+    }
+
+    public function test_production_and_local_horizon_environments_enable_the_mail_supervisor(): void
+    {
+        $this->assertArrayHasKey('supervisor-mail', config('horizon.environments.production'));
+        $this->assertArrayHasKey('supervisor-mail', config('horizon.environments.local'));
+    }
+
+    public function test_verification_and_security_mailables_use_the_mail_queue(): void
+    {
+        $this->assertSame('mail', (new SendVerificationEmailJob(1, '[email protected]', 'token', 1, '127.0.0.1'))->queue);
+        $this->assertSame('mail', (new RegistrationVerificationMail('token'))->queue);
+        $this->assertSame('mail', (new EmailChangeVerificationCodeMail('123456', 15))->queue);
+        $this->assertSame('mail', (new EmailChangedSecurityAlertMail('[email protected]'))->queue);
+    }
+
+    public function test_default_and_messaging_supervisors_do_not_steal_the_mail_queue(): void
+    {
+        $this->assertNotContains('mail', config('horizon.defaults.supervisor-default.queue'));
+        $this->assertNotContains('mail', config('horizon.defaults.supervisor-messaging.queue'));
+    }
+}

tests/Unit/Http/HttpPerformanceLogAnalyzerTest.php

@@ -0,0 +1,110 @@
+<?php
+
+use App\Support\Http\HttpPerformanceLogAnalyzer;
+use App\Support\Http\HttpUriNormalizer;
+
+function perfFixture(string $name): string
+{
+    return dirname(__DIR__, 2).'/fixtures/http-performance/'.$name;
+}
+
+it('normalizes numeric, profile, and download URIs', function () {
+    $n = new HttpUriNormalizer();
+
+    expect($n->normalize('/download/artwork/17776'))->toBe('/download/artwork/{id}')
+        ->and($n->normalize('/download/artwork/42026'))->toBe('/download/artwork/{id}')
+        ->and($n->normalize('/@killua'))->toBe('/@{user}')
+        ->and($n->normalize('/@killua/artworks'))->toBe('/@{user}/artworks')
+        ->and($n->normalize('/academy/prompts/123'))->toBe('/academy/prompts/{id}')
+        ->and($n->normalize('/forum/some-thread/456'))->toBe('/forum/some-thread/{id}')
+        ->and($n->normalize('/art/53867/light-mark-docks'))->toBe('/art/{id}/{slug}')
+        ->and($n->normalize('/art/3858/similar'))->toBe('/art/{id}/similar')
+        ->and($n->normalize('/api/art/69876/view'))->toBe('/api/art/{id}/view')
+        ->and($n->normalize('/api/rank/category/583'))->toBe('/api/rank/category/{id}')
+        ->and($n->normalize('/search?q=skin&page=2'))->toBe('/search');
+});
+
+it('parses mixed nginx performance lines and skips malformed JSON', function () {
+    $analyzer = new HttpPerformanceLogAnalyzer(new HttpUriNormalizer());
+    $now = new DateTimeImmutable('2026-08-24T16:00:00+02:00');
+
+    $result = $analyzer->analyze(perfFixture('sample.log'), [
+        'since' => '24h',
+        'now' => $now,
+        'top' => 10,
+        'min_requests' => 1,
+    ]);
+
+    expect($result['skipped'])->toBe(1)
+        ->and($result['totals']['total'])->toBe(9)
+        ->and($result['totals']['5xx'])->toBe(1)
+        ->and($result['totals']['499'])->toBe(1)
+        ->and($result['percentile_method'])->toContain('nearest-rank');
+
+    $academy = collect($result['by_count'])->firstWhere('uri', '/academy');
+    expect($academy['count'])->toBe(2)
+        ->and($academy['p95'])->toBeGreaterThan(0);
+
+    $static = collect($result['by_count'])->firstWhere('uri', '/favicon.ico');
+    expect($static['avg_upstream_response'])->toBeNull();
+
+    $download = collect($result['slowest'])->firstWhere('uri', '/download/artwork/{id}');
+    expect($download['upstream_response_time'])->toBe(0.32);
+
+    $profile = collect($result['by_count'])->firstWhere('uri', '/@{user}');
+    expect($profile)->not->toBeNull();
+});
+
+it('uses nearest-rank percentiles rather than the average', function () {
+    $analyzer = new HttpPerformanceLogAnalyzer(new HttpUriNormalizer());
+    $sorted = [0.1, 0.2, 0.3, 0.4, 1.0];
+
+    expect($analyzer->percentile($sorted, 0.50))->toBe(0.3)
+        ->and($analyzer->percentile($sorted, 0.95))->toBe(1.0)
+        ->and($analyzer->percentile($sorted, 0.95))->not->toBe(0.4);
+});
+
+it('sums comma-separated upstream timings and treats dash as missing', function () {
+    $analyzer = new HttpPerformanceLogAnalyzer(new HttpUriNormalizer());
+
+    expect($analyzer->parseUpstream('-'))->toBeNull()
+        ->and($analyzer->parseUpstream('0.451'))->toBe(0.451)
+        ->and($analyzer->parseUpstream('0.100, 0.220'))->toBe(0.32);
+});
+
+it('filters hosts exactly and does not merge wildcard hosts', function () {
+    $analyzer = new HttpPerformanceLogAnalyzer(new HttpUriNormalizer());
+    $now = new DateTimeImmutable('2026-08-24T16:00:00+02:00');
+    $opts = ['since' => '24h', 'now' => $now, 'top' => 20, 'min_requests' => 1];
+
+    $filtered = $analyzer->analyze(perfFixture('hosts.log'), $opts + ['host' => 'skinbase.org']);
+    $hosts = array_unique(array_column($filtered['by_count'], 'host'));
+
+    expect($filtered['skipped'])->toBe(1)
+        ->and($filtered['totals']['total'])->toBe(8)
+        ->and($hosts)->toBe(['skinbase.org'])
+        ->and(collect($filtered['by_count'])->pluck('host')->all())->not->toContain('www.skinbase.org')
+        ->and(collect($filtered['by_count'])->pluck('host')->all())->not->toContain('thumb.skinbase.org')
+        ->and(collect($filtered['by_count'])->pluck('host')->all())->not->toContain('hub.skinbase.org');
+
+    $unfiltered = $analyzer->analyze(perfFixture('hosts.log'), $opts);
+    $artRows = collect($unfiltered['by_count'])->where('uri', '/art/{id}/{slug}')->values();
+
+    expect($artRows)->toHaveCount(4)
+        ->and($artRows->pluck('host')->sort()->values()->all())->toBe([
+            'hub.skinbase.org',
+            'skinbase.org',
+            'thumb.skinbase.org',
+            'www.skinbase.org',
+        ])
+        ->and($artRows->firstWhere('host', 'skinbase.org')['count'])->toBe(1)
+        ->and($artRows->firstWhere('host', 'www.skinbase.org')['avg'])->toBe(0.9);
+
+    expect(collect($filtered['by_count'])->firstWhere('uri', '/art/{id}/{slug}')['host'])->toBe('skinbase.org')
+        ->and(collect($filtered['by_count'])->firstWhere('uri', '/art/{id}/similar'))->not->toBeNull()
+        ->and(collect($filtered['by_count'])->firstWhere('uri', '/download/artwork/{id}'))->not->toBeNull()
+        ->and(collect($filtered['by_count'])->firstWhere('uri', '/api/art/{id}/view'))->not->toBeNull()
+        ->and(collect($filtered['by_count'])->firstWhere('uri', '/api/rank/category/{id}'))->not->toBeNull()
+        ->and(collect($filtered['by_count'])->firstWhere('uri', '/academy/prompts/{id}'))->not->toBeNull()
+        ->and(collect($filtered['by_count'])->firstWhere('uri', '/@{user}'))->not->toBeNull();
+});

tests/Unit/Jobs/RecComputeSerializedCompatibilityTest.php

@@ -0,0 +1,73 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Tests\Unit\Jobs;
+
+use App\Jobs\RecComputeSimilarByBehaviorJob;
+use App\Jobs\RecComputeSimilarByTagsJob;
+use App\Jobs\RecComputeSimilarHybridJob;
+use ReflectionClass;
+use Tests\TestCase;
+
+final class RecComputeSerializedCompatibilityTest extends TestCase
+{
+    public function test_old_serialized_tags_job_without_after_artwork_id_can_run(): void
+    {
+        $job = $this->unserializeWithoutAfterArtworkId(new RecComputeSimilarByTagsJob(null, 200, 99));
+
+        $this->assertNull($job->cursorAfterArtworkId());
+    }
+
+    public function test_old_serialized_behavior_job_without_after_artwork_id_can_run(): void
+    {
+        $job = $this->unserializeWithoutAfterArtworkId(new RecComputeSimilarByBehaviorJob(null, 200, 42));
+
+        $this->assertNull($job->cursorAfterArtworkId());
+    }
+
+    public function test_old_serialized_hybrid_job_without_after_artwork_id_can_run(): void
+    {
+        $job = $this->unserializeWithoutAfterArtworkId(new RecComputeSimilarHybridJob(null, 200, 7));
+
+        $this->assertNull($job->cursorAfterArtworkId());
+    }
+
+    public function test_new_cursor_jobs_keep_after_artwork_id(): void
+    {
+        $job = unserialize(serialize(new RecComputeSimilarByBehaviorJob(null, 200, 15)));
+
+        $this->assertSame(15, $job->cursorAfterArtworkId());
+    }
+
+    public function test_laravel_serialize_roundtrip_keeps_queueable_and_cursor_fields(): void
+    {
+        $original = new RecComputeSimilarByTagsJob(5, 50, 11);
+        $original->onQueue('default');
+
+        $copy = unserialize(serialize($original));
+
+        $this->assertInstanceOf(RecComputeSimilarByTagsJob::class, $copy);
+        $this->assertSame(11, $copy->cursorAfterArtworkId());
+        $this->assertSame('default', $copy->queue);
+        $this->assertSame(600, $copy->timeout);
+        $this->assertSame(2, $copy->tries);
+    }
+
+    /**
+     * @template T of RecComputeSimilarByTagsJob|RecComputeSimilarByBehaviorJob|RecComputeSimilarHybridJob
+     * @param  T  $job
+     * @return T
+     */
+    private function unserializeWithoutAfterArtworkId(object $job): object
+    {
+        $values = $job->__serialize();
+        $class = $job::class;
+        unset($values["\0{$class}\0afterArtworkId"]);
+
+        $restored = (new ReflectionClass($class))->newInstanceWithoutConstructor();
+        $restored->__unserialize($values);
+
+        return $restored;
+    }
+}

tests/Unit/Jobs/RecComputeSimilarByBehaviorJobTest.php

@@ -0,0 +1,26 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Tests\Unit\Jobs;
+
+use App\Jobs\RecComputeSimilarByBehaviorJob;
+use Tests\TestCase;
+
+final class RecComputeSimilarByBehaviorJobTest extends TestCase
+{
+    public function test_job_timeout_stays_below_queue_retry_after(): void
+    {
+        $job = new RecComputeSimilarByBehaviorJob();
+
+        $this->assertLessThan(
+            (int) config('queue.connections.redis.retry_after'),
+            $job->timeout,
+        );
+    }
+
+    public function test_batch_jobs_do_not_use_without_overlapping_middleware(): void
+    {
+        $this->assertFalse(method_exists(new RecComputeSimilarByBehaviorJob(), 'middleware'));
+    }
+}

tests/Unit/Jobs/RecComputeSimilarHybridJobTest.php

@@ -32,4 +32,14 @@ final class RecComputeSimilarHybridJobTest extends TestCase
 
         $this->assertSame([], $job->middleware());
     }
+
+    public function test_job_timeout_stays_below_queue_retry_after(): void
+    {
+        $job = new RecComputeSimilarHybridJob();
+
+        $this->assertLessThan(
+            (int) config('queue.connections.redis.retry_after'),
+            $job->timeout,
+        );
+    }
 }

tests/Unit/Redis/LegacyPrefixFilterTest.php

@@ -0,0 +1,21 @@
+<?php
+
+it('classifies legacy keys without touching current prefixes', function () {
+    $legacy = [
+        'skinbasenova-database-queues:default',
+        'skinbasenova_horizon:recent_jobs',
+    ];
+    $current = [
+        'skinbase-database-queues:default',
+        'skinbase_horizon:recent_jobs',
+        'queues:default',
+    ];
+
+    foreach ($legacy as $key) {
+        expect(str_starts_with($key, 'skinbasenova-database-') || str_starts_with($key, 'skinbasenova_horizon:'))->toBeTrue();
+        expect(str_starts_with($key, 'skinbase-database-') || str_starts_with($key, 'skinbase_horizon:'))->toBeFalse();
+    }
+    foreach ($current as $key) {
+        expect(str_starts_with($key, 'skinbasenova-database-') || str_starts_with($key, 'skinbasenova_horizon:'))->toBeFalse();
+    }
+});

tests/Unit/Traffic/OnlineVisitorIndexReadTest.php

@@ -0,0 +1,37 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Tests\Unit\Traffic;
+
+use App\Services\Traffic\BotClassifier;
+use App\Services\Traffic\OnlineVisitorRepository;
+use Illuminate\Support\Facades\Redis;
+use Tests\TestCase;
+
+final class OnlineVisitorIndexReadTest extends TestCase
+{
+    public function test_all_uses_sscan_and_never_smembers(): void
+    {
+        Redis::shouldReceive('sscan')
+            ->once()
+            ->andReturn([0, ['visitor-a', 'visitor-b']]);
+        Redis::shouldReceive('get')->twice()->andReturn(null);
+        Redis::shouldReceive('srem')->once();
+        Redis::shouldReceive('smembers')->never();
+
+        $repository = new OnlineVisitorRepository(app(BotClassifier::class));
+        $records = $repository->all();
+
+        $this->assertSame([], $records);
+    }
+
+    public function test_source_does_not_call_smembers_on_the_presence_index(): void
+    {
+        $source = file_get_contents(base_path('app/Services/Traffic/OnlineVisitorRepository.php'));
+
+        $this->assertNotFalse($source);
+        $this->assertStringNotContainsString('smembers', strtolower($source));
+        $this->assertStringContainsString('sscan', strtolower($source));
+    }
+}

tests/Unit/Traffic/PresenceIndexPrunerTest.php

@@ -0,0 +1,73 @@
+<?php
+
+use App\Services\Traffic\OnlineVisitorRepository;
+use App\Services\Traffic\PresenceIndexPruner;
+use Illuminate\Support\Facades\Redis;
+
+it('removes stale presence members and keeps live ones', function () {
+    Redis::shouldReceive('sscan')->once()->andReturn([0, ['stale-1', 'live-1']]);
+    Redis::shouldReceive('pipeline')->once()->andReturnUsing(function ($cb) {
+        $pipe = Mockery::mock();
+        $pipe->shouldReceive('exists')->twice();
+        $cb($pipe);
+
+        return [0, 1];
+    });
+    Redis::shouldReceive('eval')->once()->andReturn(1);
+
+    $result = app(PresenceIndexPruner::class)->prune(200, 1, 10, 0, false);
+
+    expect($result['scanned'])->toBe(2)
+        ->and($result['stale'])->toBe(1)
+        ->and($result['live'])->toBe(1)
+        ->and($result['removed'])->toBe(1);
+});
+
+it('dry-run counts stale members without srem/eval', function () {
+    Redis::shouldReceive('sscan')->once()->andReturn([0, ['stale-1']]);
+    Redis::shouldReceive('pipeline')->once()->andReturnUsing(function ($cb) {
+        $pipe = Mockery::mock();
+        $pipe->shouldReceive('exists')->once();
+        $cb($pipe);
+
+        return [0];
+    });
+    Redis::shouldReceive('eval')->never();
+
+    $result = app(PresenceIndexPruner::class)->prune(200, 5, 100, 0, true);
+
+    expect($result['stale'])->toBe(1)
+        ->and($result['removed'])->toBe(0)
+        ->and($result['dry_run'])->toBeTrue();
+});
+
+it('respects max-batches and max-members', function () {
+    Redis::shouldReceive('sscan')->once()->andReturn([1, ['a', 'b']]);
+    Redis::shouldReceive('pipeline')->once()->andReturnUsing(function ($cb) {
+        $pipe = Mockery::mock();
+        $pipe->shouldReceive('exists')->twice();
+        $cb($pipe);
+
+        return [0, 0];
+    });
+    Redis::shouldReceive('eval')->never();
+
+    $result = app(PresenceIndexPruner::class)->prune(200, 1, 2, 0, true);
+
+    expect($result['batches'])->toBe(1)
+        ->and($result['scanned'])->toBe(2);
+});
+
+it('does not use smembers on the presence index', function () {
+    $source = file_get_contents(base_path('app/Services/Traffic/PresenceIndexPruner.php'));
+    expect(strtolower((string) $source))->not->toContain('smembers')
+        ->and(strtolower((string) $source))->toContain('sscan');
+});
+
+it('never deletes the presence index key', function () {
+    $source = (string) file_get_contents(base_path('app/Services/Traffic/PresenceIndexPruner.php'));
+    expect($source)->toContain('SREM')
+        ->and($source)->not->toContain('unlink')
+        ->and($source)->not->toContain('->del(')
+        ->and($source)->not->toContain('FLUSH');
+});