Ship production optimization M1-M12.5A: queues, metrics, HTTP observability, and vector search reliability.

Keep similar-ai from tripping the global circuit on a lone URL 502, clamp Qdrant search to 100, and add Server-Timing plus slow-request logging. Studio shared props, Academy S3 exists caching, heat chunking, and Redis/scheduler hygiene stay in this rollout.
This commit is contained in:
2026-08-25 07:58:47 +02:00
parent f52879edbb
commit 8a80aae21e
114 changed files with 9293 additions and 284 deletions
@@ -18,6 +18,12 @@ class DispatchCollectionMaintenanceCommand extends Command
public function handle(CollectionBackgroundJobService $jobs): int
{
if (! (bool) config('collections.v5.queue.dispatch_enabled', false)) {
$this->warn('Collection maintenance dispatch is disabled (COLLECTIONS_V5_DISPATCH_ENABLED). No jobs queued.');
return self::SUCCESS;
}
$runHealth = (bool) $this->option('health');
$runRecommendations = (bool) $this->option('recommendations');
$runDuplicates = (bool) $this->option('duplicates');
@@ -48,13 +48,32 @@ final class GenerateSitemapsCommand extends Command
$this->newLine();
// ── Root sitemap index ────────────────────────────────────────────
// Write several paths so nginx `location = /sitemap.xml` and child
// listings stay in sync. `sitemaps/index.xml` is the canonical generated
// index: it is a new filename, so a scheduler user can create it even
// when a stale `sitemaps/sitemap.xml` is owned by another account.
$t = microtime(true);
$index = $build->buildIndex(force: true, persist: false, families: $families);
$disk->put('sitemaps/sitemap.xml', $index['content']);
$written++;
$indexPaths = ['sitemaps/index.xml', 'sitemaps/sitemap.xml', 'sitemap.xml'];
$indexWritten = 0;
foreach ($indexPaths as $path) {
if ($this->writeXml($disk, $path, $index['content'])) {
$written++;
$indexWritten++;
}
}
if ($indexWritten === 0) {
$this->error('Failed to write any root sitemap index file. Check ownership of public/sitemap.xml and public/sitemaps/.');
return self::FAILURE;
}
$this->line(sprintf(
' <info>✔</info> sitemaps/sitemap.xml %d entries <comment>%.3fs</comment>',
' <info>✔</info> sitemap index %d entries %d path(s) <comment>%.3fs</comment>',
$index['url_count'],
$indexWritten,
microtime(true) - $t,
));
@@ -78,7 +97,12 @@ final class GenerateSitemapsCommand extends Command
}
$path = 'sitemaps/' . $documentName . '.xml';
$disk->put($path, $built['content']);
if (! $this->writeXml($disk, $path, $built['content'])) {
$this->line(sprintf(' <fg=red>✖</> %s write failed', $documentName . '.xml'));
$failed++;
continue;
}
$written++;
$this->line(sprintf(
@@ -114,7 +138,12 @@ final class GenerateSitemapsCommand extends Command
}
$path = 'sitemaps/' . $groupName . '.xml';
$disk->put($path, $built['content']);
if (! $this->writeXml($disk, $path, $built['content'])) {
$this->line(sprintf(' <fg=red>✖</> %s.xml write failed', $groupName));
$failed++;
continue;
}
$written++;
$this->line(sprintf(
@@ -159,4 +188,17 @@ final class GenerateSitemapsCommand extends Command
return array_values(array_filter($enabled, fn (string $f): bool => in_array($f, $only, true)));
}
private function writeXml(\Illuminate\Contracts\Filesystem\Filesystem $disk, string $path, string $content): bool
{
$ok = $disk->put($path, $content);
if ($ok !== true) {
$this->warn(' Could not write '.$path);
return false;
}
return true;
}
}
@@ -7,34 +7,137 @@ use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
/**
* Prune old hourly metric snapshots to prevent unbounded table growth.
* Prune old hourly metric snapshots in bounded batches.
*
* Usage: php artisan nova:prune-metric-snapshots
* php artisan nova:prune-metric-snapshots --keep-days=7
* Usage:
* php artisan nova:prune-metric-snapshots
* php artisan nova:prune-metric-snapshots --keep-days=30 --chunk=5000 --dry-run
*/
class PruneMetricSnapshotsCommand extends Command
{
protected $signature = 'nova:prune-metric-snapshots
{--keep-days=7 : Keep snapshots for this many days}';
{--keep-days= : Keep snapshots for this many days (default: config metrics.hourly_snapshot_retention_days)}
{--chunk= : Rows to delete per batch (default: config metrics.hourly_snapshot_prune_chunk)}
{--sleep-ms= : Pause between batches in milliseconds}
{--max-batches=0 : Stop after N batches (0 = until done)}
{--dry-run : Count rows that would be deleted without deleting}';
protected $description = 'Delete old hourly metric snapshots beyond the retention window';
protected $description = 'Delete old hourly metric snapshots beyond the retention window (batched)';
public function handle(): int
{
$keepDays = (int) $this->option('keep-days');
$cutoff = now()->subDays($keepDays);
$keepDays = $this->resolveKeepDays();
$chunk = $this->resolvePositiveInt('chunk', (int) config('metrics.hourly_snapshot_prune_chunk', 5000), 1);
$sleepMs = $this->resolveNonNegativeInt('sleep-ms', (int) config('metrics.hourly_snapshot_prune_sleep_ms', 50));
$maxBatches = max(0, (int) $this->option('max-batches'));
$dryRun = (bool) $this->option('dry-run');
$cutoff = now()->subDays($keepDays);
$deleted = DB::table('artwork_metric_snapshots_hourly')
if ($keepDays < 1) {
$this->error('keep-days must be >= 1.');
return self::FAILURE;
}
$eligible = (int) DB::table('artwork_metric_snapshots_hourly')
->where('bucket_hour', '<', $cutoff)
->delete();
->count();
$this->info("Pruned {$deleted} snapshot rows older than {$keepDays} days.");
$this->info(sprintf(
'[nova:prune-metric-snapshots] cutoff=%s keep_days=%d eligible=%d chunk=%d sleep_ms=%d max_batches=%s%s',
$cutoff->toDateTimeString(),
$keepDays,
$eligible,
$chunk,
$sleepMs,
$maxBatches === 0 ? 'unlimited' : (string) $maxBatches,
$dryRun ? ' (dry-run)' : ''
));
if ($dryRun) {
Log::info('[nova:prune-metric-snapshots] dry-run', [
'eligible' => $eligible,
'keep_days' => $keepDays,
'cutoff' => $cutoff->toDateTimeString(),
]);
return self::SUCCESS;
}
$deleted = 0;
$batches = 0;
while (true) {
if ($maxBatches > 0 && $batches >= $maxBatches) {
$this->warn("Stopped after max-batches={$maxBatches}.");
break;
}
$ids = DB::table('artwork_metric_snapshots_hourly')
->where('bucket_hour', '<', $cutoff)
->orderBy('id')
->limit($chunk)
->pluck('id');
if ($ids->isEmpty()) {
break;
}
$batchDeleted = DB::table('artwork_metric_snapshots_hourly')
->whereIn('id', $ids->all())
->delete();
$deleted += $batchDeleted;
$batches++;
Log::info('[nova:prune-metric-snapshots] batch', [
'batch' => $batches,
'deleted' => $batchDeleted,
'deleted_total' => $deleted,
]);
if ($sleepMs > 0) {
usleep($sleepMs * 1000);
}
}
$this->info("Pruned {$deleted} snapshot rows older than {$keepDays} days in {$batches} batch(es).");
Log::info('[nova:prune-metric-snapshots] completed', [
'deleted' => $deleted,
'deleted' => $deleted,
'batches' => $batches,
'keep_days' => $keepDays,
'cutoff' => $cutoff->toDateTimeString(),
'eligible_at_start' => $eligible,
]);
return self::SUCCESS;
}
private function resolveKeepDays(): int
{
$option = $this->option('keep-days');
if ($option === null || $option === '') {
return (int) config('metrics.hourly_snapshot_retention_days', 30);
}
return (int) $option;
}
private function resolvePositiveInt(string $option, int $default, int $minimum): int
{
$raw = $this->option($option);
$value = ($raw === null || $raw === '') ? $default : (int) $raw;
return max($minimum, $value);
}
private function resolveNonNegativeInt(string $option, int $default): int
{
$raw = $this->option($option);
$value = ($raw === null || $raw === '') ? $default : (int) $raw;
return max(0, $value);
}
}
@@ -0,0 +1,88 @@
<?php
declare(strict_types=1);
namespace App\Console\Commands;
use App\Services\Traffic\OnlineVisitorRepository;
use App\Services\Traffic\PresenceIndexPruner;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Redis;
class PruneOnlineVisitorIndexCommand extends Command
{
protected $signature = 'skinbase:prune-online-visitor-index
{--dry-run : Count stale members without SREM}
{--execute : Remove stale members}
{--scan-count= : SSCAN COUNT}
{--max-batches= : Batches this run}
{--max-members= : Members to inspect this run}
{--sleep-ms= : Pause between batches}
{--time-limit= : Stop after N seconds (0 = none)}';
protected $description = 'Remove stale members from the online-visitor Redis set (SSCAN, never SMEMBERS)';
public function handle(PresenceIndexPruner $pruner): int
{
$dryRun = (bool) $this->option('dry-run');
$execute = (bool) $this->option('execute');
if ($dryRun === $execute) {
$this->error('Pass exactly one of --dry-run or --execute.');
return self::FAILURE;
}
$scanCount = $this->intOption('scan-count', (int) config('traffic.online_visitors.prune_scan_count', 500));
$maxBatches = $this->intOption('max-batches', (int) config('traffic.online_visitors.prune_max_batches', 200));
$maxMembers = $this->intOption('max-members', (int) config('traffic.online_visitors.prune_max_members', 100000));
$sleepMs = $this->intOption('sleep-ms', (int) config('traffic.online_visitors.prune_sleep_ms', 25));
$timeLimit = $this->intOption('time-limit', 0);
$scard = (int) Redis::scard(OnlineVisitorRepository::INDEX_KEY);
$this->info(sprintf(
'[prune-online-visitor-index] key=%s prefix=%s scard=%d scan_count=%d max_batches=%d max_members=%d sleep_ms=%d %s',
OnlineVisitorRepository::INDEX_KEY,
(string) config('database.redis.options.prefix'),
$scard,
$scanCount,
$maxBatches,
$maxMembers,
$sleepMs,
$dryRun ? 'dry-run' : 'execute'
));
$result = $pruner->prune(
$scanCount,
$maxBatches,
$maxMembers,
$sleepMs,
$dryRun,
$timeLimit > 0 ? $timeLimit : null,
);
$this->info(sprintf(
'scanned=%d stale=%d live=%d removed=%d batches=%d',
$result['scanned'],
$result['stale'],
$result['live'],
$result['removed'],
$result['batches']
));
Log::info('[prune-online-visitor-index] completed', $result + ['scard_before' => $scard]);
return self::SUCCESS;
}
private function intOption(string $name, int $default): int
{
$raw = $this->option($name);
if ($raw === null || $raw === '') {
return $default;
}
return max(0, (int) $raw);
}
}
@@ -0,0 +1,94 @@
<?php
declare(strict_types=1);
namespace App\Console\Commands;
use App\Jobs\RecComputeSimilarByTagsJob;
use App\Support\Queues\QueuedJobClassMatcher;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Redis;
/**
* Deployment cleanup: remove waiting RecComputeSimilarByTagsJob payloads only.
*
* Does not run from the scheduler. Operator must pass --dry-run or --execute.
*
* Usage:
* php artisan skinbase:purge-queued-rec-tags --dry-run
* php artisan skinbase:purge-queued-rec-tags --execute
*/
class PurgeQueuedRecComputeTagsCommand extends Command
{
protected $signature = 'skinbase:purge-queued-rec-tags
{--queue=default : Redis queue name (waiting list only)}
{--dry-run : Count matches without removing}
{--execute : Remove matched waiting payloads}';
protected $description = 'Remove waiting RecComputeSimilarByTagsJob payloads from a Redis queue (opt-in)';
public function handle(): int
{
$dryRun = (bool) $this->option('dry-run');
$execute = (bool) $this->option('execute');
if ($dryRun === $execute) {
$this->error('Pass exactly one of --dry-run or --execute.');
return self::FAILURE;
}
$queue = (string) $this->option('queue');
if (! preg_match('/^[A-Za-z0-9_-]+$/', $queue)) {
$this->error('Invalid queue name.');
return self::FAILURE;
}
$key = 'queues:'.$queue;
$target = RecComputeSimilarByTagsJob::class;
$redis = Redis::connection();
$len = (int) $redis->llen($key);
$matchedPayloads = [];
$chunk = 200;
$this->info(sprintf(
'[purge-queued-rec-tags] queue=%s waiting=%d target=%s %s',
$queue,
$len,
$target,
$dryRun ? 'dry-run' : 'execute'
));
for ($start = 0; $start < $len; $start += $chunk) {
$rows = $redis->lrange($key, $start, min($start + $chunk - 1, $len - 1));
foreach ($rows as $raw) {
if (QueuedJobClassMatcher::isClass((string) $raw, $target)) {
$matchedPayloads[] = (string) $raw;
}
}
}
$matched = count($matchedPayloads);
$removed = 0;
if (! $dryRun) {
foreach ($matchedPayloads as $raw) {
$removed += (int) $redis->lrem($key, 1, $raw);
}
}
$this->info(sprintf('matched=%d removed=%d preserved_other=%s', $matched, $removed, $dryRun ? 'yes' : 'yes'));
Log::info('[purge-queued-rec-tags] completed', [
'queue' => $queue,
'waiting' => $len,
'matched' => $matched,
'removed' => $removed,
'dry_run' => $dryRun,
]);
return self::SUCCESS;
}
}
+27 -21
View File
@@ -57,7 +57,7 @@ class RecalculateHeatCommand extends Command
$updatedCount = 0;
$skippedCount = 0;
// Process in chunks using artwork IDs that have at least one snapshot in the smoothing window
// Distinct IDs only — do not hydrate the full 24h snapshot window in one query.
$artworkIds = DB::table('artwork_metric_snapshots_hourly')
->whereBetween('bucket_hour', [$lookbackStart, $currentHour])
->distinct()
@@ -68,27 +68,33 @@ class RecalculateHeatCommand extends Command
return self::SUCCESS;
}
// Load all snapshots for the lookback window in bulk
$snapshots = DB::table('artwork_metric_snapshots_hourly')
->whereBetween('bucket_hour', [$lookbackStart, $currentHour])
->whereIn('artwork_id', $artworkIds)
->orderBy('bucket_hour')
->get()
->groupBy('artwork_id');
// Load artwork published_at dates for age factor (use published_at, fall back to created_at)
$artworkDates = DB::table('artworks')
->whereIn('id', $artworkIds)
->whereNull('deleted_at')
->where('is_approved', true)
->select('id', 'published_at', 'created_at')
->get()
->mapWithKeys(fn ($row) => [
$row->id => \Carbon\Carbon::parse($row->published_at ?? $row->created_at),
]);
// Process in chunks
foreach ($artworkIds->chunk($chunk) as $chunkIds) {
$snapshots = DB::table('artwork_metric_snapshots_hourly')
->select([
'artwork_id',
'bucket_hour',
'views_count',
'downloads_count',
'favourites_count',
'comments_count',
'shares_count',
])
->whereBetween('bucket_hour', [$lookbackStart, $currentHour])
->whereIn('artwork_id', $chunkIds)
->orderBy('bucket_hour')
->get()
->groupBy('artwork_id');
$artworkDates = DB::table('artworks')
->whereIn('id', $chunkIds)
->whereNull('deleted_at')
->where('is_approved', true)
->select('id', 'published_at', 'created_at')
->get()
->mapWithKeys(fn ($row) => [
$row->id => \Carbon\Carbon::parse($row->published_at ?? $row->created_at),
]);
$upsertRows = [];
foreach ($chunkIds as $artworkId) {
@@ -0,0 +1,139 @@
<?php
declare(strict_types=1);
namespace App\Console\Commands;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Log;
use Predis\Client as PredisClient;
class RedisCleanupLegacyPrefixCommand extends Command
{
protected $signature = 'skinbase:redis-cleanup-legacy-prefix
{--dry-run : SCAN and report only}
{--execute : UNLINK matching obsolete keys}
{--max-keys=500 : Cap keys this run}';
protected $description = 'UNLINK obsolete skinbasenova-database-* and skinbasenova_horizon:* keys only';
private const LEGACY_PREFIXES = [
'skinbasenova-database-',
'skinbasenova_horizon:',
];
private const PROTECTED_PREFIXES = [
'skinbase-database-',
'skinbase_horizon:',
];
public function handle(): int
{
$dryRun = (bool) $this->option('dry-run');
$execute = (bool) $this->option('execute');
if ($dryRun === $execute) {
$this->error('Pass exactly one of --dry-run or --execute.');
return self::FAILURE;
}
$maxKeys = max(1, (int) $this->option('max-keys'));
$currentPrefix = (string) config('database.redis.options.prefix');
$horizonPrefix = (string) config('horizon.prefix');
$this->info(sprintf(
'[redis-cleanup-legacy-prefix] current_prefix=%s horizon_prefix=%s max_keys=%d %s',
$currentPrefix,
$horizonPrefix,
$maxKeys,
$dryRun ? 'dry-run' : 'execute'
));
if (in_array($currentPrefix, self::LEGACY_PREFIXES, true) || in_array($horizonPrefix, self::LEGACY_PREFIXES, true)) {
$this->error('Current process still uses a legacy prefix. Aborting.');
return self::FAILURE;
}
$client = $this->unprefixedClient();
$found = [];
$cursor = '0';
do {
[$cursor, $keys] = $client->scan($cursor, ['COUNT' => 200, 'MATCH' => '*']);
foreach ($keys as $key) {
$key = (string) $key;
if ($this->isProtected($key)) {
continue;
}
if (! $this->isLegacy($key)) {
continue;
}
$found[] = $key;
if (count($found) >= $maxKeys) {
$cursor = '0';
break;
}
}
} while ($cursor !== '0' && $cursor !== 0);
$this->info('matched='.count($found));
foreach (array_slice($found, 0, 20) as $key) {
$this->line('key='.$key);
}
if (count($found) > 20) {
$this->line('... truncated listing');
}
$unlinked = 0;
if (! $dryRun && $found !== []) {
foreach (array_chunk($found, 50) as $chunk) {
$unlinked += (int) $client->unlink(...$chunk);
}
}
$this->info('unlinked='.$unlinked);
Log::info('[redis-cleanup-legacy-prefix] completed', [
'matched' => count($found),
'unlinked' => $unlinked,
'dry_run' => $dryRun,
]);
return self::SUCCESS;
}
private function isLegacy(string $key): bool
{
foreach (self::LEGACY_PREFIXES as $prefix) {
if (str_starts_with($key, $prefix)) {
return true;
}
}
return false;
}
private function isProtected(string $key): bool
{
foreach (self::PROTECTED_PREFIXES as $prefix) {
if (str_starts_with($key, $prefix)) {
return true;
}
}
return false;
}
private function unprefixedClient(): PredisClient
{
$redis = config('database.redis.default', []);
return new PredisClient([
'scheme' => 'tcp',
'host' => $redis['host'] ?? '127.0.0.1',
'port' => (int) ($redis['port'] ?? 6379),
'password' => $redis['password'] ?? null,
'database' => (int) ($redis['database'] ?? 0),
]);
}
}
@@ -0,0 +1,79 @@
<?php
declare(strict_types=1);
namespace App\Console\Commands;
use App\Support\Redis\OrphanQueueCleanup;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Log;
use RuntimeException;
class RedisCleanupOrphansCommand extends Command
{
protected $signature = 'skinbase:redis-cleanup-orphans
{target : forum-moderation, forum-security, or collections}
{--dry-run : Inspect and report without UNLINK}
{--execute : UNLINK the waiting queue and notify list}
{--force : Allow unexpected sampled job classes}';
protected $description = 'UNLINK an orphan Redis queue after producer and class checks (operator only)';
public function handle(OrphanQueueCleanup $cleanup): int
{
$dryRun = (bool) $this->option('dry-run');
$execute = (bool) $this->option('execute');
if ($dryRun === $execute) {
$this->error('Pass exactly one of --dry-run or --execute.');
return self::FAILURE;
}
$target = (string) $this->argument('target');
$force = (bool) $this->option('force');
$this->info(sprintf(
'[redis-cleanup-orphans] prefix=%s connection=default target=%s force=%s mode=%s',
(string) config('database.redis.options.prefix'),
$target,
$force ? 'yes' : 'no',
$dryRun ? 'dry-run' : 'execute'
));
try {
$plan = $execute
? $cleanup->execute($target, $force)
: $cleanup->plan($target, $force);
} catch (RuntimeException $e) {
$this->error($e->getMessage());
Log::warning('[redis-cleanup-orphans] refused', ['target' => $target, 'error' => $e->getMessage()]);
return self::FAILURE;
}
$this->line('logical_key='.$plan['logical_key']);
$this->line('notify_key='.$plan['notify_key']);
$this->line('llen='.$plan['llen'].' reserved='.$plan['reserved'].' delayed='.$plan['delayed']);
$this->line('sampled='.$plan['sampled'].' est_bytes='.$plan['est_bytes']);
$this->line('classes='.json_encode($plan['classes']));
if ($plan['unexpected_classes'] !== []) {
$this->warn('unexpected_classes='.json_encode($plan['unexpected_classes']));
}
if ($plan['errors'] !== []) {
$this->error('errors='.implode(',', $plan['errors']));
}
if ($execute) {
$this->info('unlinked='.($plan['unlinked'] ?? 0));
}
Log::info('[redis-cleanup-orphans] completed', [
'target' => $target,
'dry_run' => $dryRun,
'llen' => $plan['llen'],
'errors' => $plan['errors'],
'unlinked' => $plan['unlinked'] ?? 0,
]);
return ($plan['errors'] === [] || $dryRun) ? self::SUCCESS : self::FAILURE;
}
}