Ship production optimization M1-M12.5A: queues, metrics, HTTP observability, and vector search reliability.

Keep similar-ai from tripping the global circuit on a lone URL 502, clamp Qdrant search to 100, and add Server-Timing plus slow-request logging. Studio shared props, Academy S3 exists caching, heat chunking, and Redis/scheduler hygiene stay in this rollout.
This commit is contained in:
2026-08-25 07:58:47 +02:00
parent f52879edbb
commit 8a80aae21e
114 changed files with 9293 additions and 284 deletions
@@ -0,0 +1,322 @@
<?php
declare(strict_types=1);
namespace App\Support\Http;
/**
* Read-only parser for nginx skinbase_perf JSON lines.
*
* Multi-value upstream timings (retries): use the **sum** of numeric parts
* as the upstream total for that request. "-" and empty → null.
* Percentiles use nearest-rank (ceil(p * n), 1-indexed).
*/
final class HttpPerformanceLogAnalyzer
{
public function __construct(private readonly HttpUriNormalizer $uris)
{
}
/**
* @param array{
* since?: ?string,
* min_requests?: int,
* top?: int,
* status?: ?string,
* uri?: ?string,
* host?: ?string,
* slow_ms?: ?float,
* now?: ?\DateTimeImmutable
* } $options
* @return array<string, mixed>
*/
public function analyze(string $file, array $options = []): array
{
$since = $this->sinceCutoff($options['since'] ?? null, $options['now'] ?? new \DateTimeImmutable('now'));
$minRequests = max(1, (int) ($options['min_requests'] ?? 1));
$top = max(1, (int) ($options['top'] ?? 20));
$statusFilter = isset($options['status']) ? strtolower((string) $options['status']) : null;
$uriFilter = $options['uri'] ?? null;
$hostFilter = isset($options['host']) ? (string) $options['host'] : '';
$slowMs = isset($options['slow_ms']) ? (float) $options['slow_ms'] / 1000 : null;
$skipped = 0;
$families = [];
$slowest = [];
$statusCounts = ['4xx' => 0, '5xx' => 0, '499' => 0, 'total' => 0];
$handle = fopen($file, 'r');
if ($handle === false) {
throw new \RuntimeException('Unable to open performance log: '.$file);
}
try {
while (($line = fgets($handle)) !== false) {
$line = trim($line);
if ($line === '') {
continue;
}
$row = json_decode($line, true);
if (! is_array($row)) {
$skipped++;
continue;
}
$time = $this->parseTime((string) ($row['time'] ?? ''));
if ($since !== null && ($time === null || $time < $since)) {
continue;
}
$requestTime = $this->parseFloat($row['request_time'] ?? null);
if ($requestTime === null) {
$skipped++;
continue;
}
$status = (int) ($row['status'] ?? 0);
if ($statusFilter === '5xx' && ($status < 500 || $status > 599)) {
continue;
}
if ($statusFilter === '4xx' && ($status < 400 || $status > 499)) {
continue;
}
if ($statusFilter === '499' && $status !== 499) {
continue;
}
$host = (string) ($row['host'] ?? '');
if ($hostFilter !== '' && $host !== $hostFilter) {
continue;
}
$uri = $this->uris->normalize((string) ($row['uri'] ?? '/'));
if (is_string($uriFilter) && $uriFilter !== '' && ! str_starts_with($uri, $uriFilter)) {
continue;
}
if ($slowMs !== null && $requestTime < $slowMs) {
continue;
}
$upstreamResponse = $this->parseUpstream($row['upstream_response_time'] ?? null);
$upstreamHeader = $this->parseUpstream($row['upstream_header_time'] ?? null);
$bytes = (int) ($row['bytes'] ?? 0);
$method = (string) ($row['method'] ?? '');
$statusCounts['total']++;
if ($status === 499) {
$statusCounts['499']++;
} elseif ($status >= 500) {
$statusCounts['5xx']++;
} elseif ($status >= 400) {
$statusCounts['4xx']++;
}
$familyKey = $host."\n".$uri;
$families[$familyKey] ??= [
'host' => $host,
'uri' => $uri,
'count' => 0,
'times' => [],
'upstream_sum' => 0.0,
'upstream_n' => 0,
'header_sum' => 0.0,
'header_n' => 0,
'bytes_sum' => 0,
'4xx' => 0,
'5xx' => 0,
'499' => 0,
'404' => 0,
];
$families[$familyKey]['count']++;
$families[$familyKey]['times'][] = $requestTime;
$families[$familyKey]['bytes_sum'] += $bytes;
if ($upstreamResponse !== null) {
$families[$familyKey]['upstream_sum'] += $upstreamResponse;
$families[$familyKey]['upstream_n']++;
}
if ($upstreamHeader !== null) {
$families[$familyKey]['header_sum'] += $upstreamHeader;
$families[$familyKey]['header_n']++;
}
if ($status === 499) {
$families[$familyKey]['499']++;
} elseif ($status >= 500) {
$families[$familyKey]['5xx']++;
} elseif ($status >= 400) {
$families[$familyKey]['4xx']++;
if ($status === 404) {
$families[$familyKey]['404']++;
}
}
$slowest[] = [
'time' => $row['time'] ?? null,
'host' => $host,
'uri' => $uri,
'method' => $method,
'status' => $status,
'request_time' => $requestTime,
'upstream_response_time' => $upstreamResponse,
];
}
} finally {
fclose($handle);
}
usort($slowest, static fn (array $a, array $b): int => $b['request_time'] <=> $a['request_time']);
$slowest = array_slice($slowest, 0, $top);
$summaries = [];
foreach ($families as $family) {
if ($family['count'] < $minRequests) {
continue;
}
$times = $family['times'];
sort($times, SORT_NUMERIC);
$avg = array_sum($times) / $family['count'];
$summaries[] = [
'host' => $family['host'],
'uri' => $family['uri'],
'count' => $family['count'],
'p50' => $this->percentile($times, 0.50),
'p95' => $this->percentile($times, 0.95),
'p99' => $this->percentile($times, 0.99),
'max' => $times[array_key_last($times)],
'avg' => round($avg, 4),
'total' => round(array_sum($times), 4),
'avg_upstream_response' => $family['upstream_n'] > 0 ? round($family['upstream_sum'] / $family['upstream_n'], 4) : null,
'avg_upstream_header' => $family['header_n'] > 0 ? round($family['header_sum'] / $family['header_n'], 4) : null,
'avg_bytes' => (int) round($family['bytes_sum'] / $family['count']),
'4xx' => $family['4xx'],
'4xx_rate' => round($family['4xx'] / $family['count'], 4),
'404' => $family['404'],
'5xx' => $family['5xx'],
'5xx_rate' => round($family['5xx'] / $family['count'], 4),
'499' => $family['499'],
];
}
$byP95 = $summaries;
usort($byP95, static fn (array $a, array $b): int => ($b['p95'] ?? 0) <=> ($a['p95'] ?? 0));
$byP99 = $summaries;
usort($byP99, static fn (array $a, array $b): int => ($b['p99'] ?? 0) <=> ($a['p99'] ?? 0));
$byTotal = $summaries;
usort($byTotal, static fn (array $a, array $b): int => $b['total'] <=> $a['total']);
$byCount = $summaries;
usort($byCount, static fn (array $a, array $b): int => $b['count'] <=> $a['count']);
$byCost = $summaries;
usort($byCost, static fn (array $a, array $b): int => ($b['count'] * $b['avg']) <=> ($a['count'] * $a['avg']));
$by5xx = array_values(array_filter($summaries, static fn (array $row): bool => $row['5xx'] > 0));
usort($by5xx, static fn (array $a, array $b): int => $b['5xx'] <=> $a['5xx']);
$by404 = array_values(array_filter($summaries, static fn (array $row): bool => ($row['404'] ?? 0) > 0));
usort($by404, static fn (array $a, array $b): int => $b['404'] <=> $a['404']);
return [
'skipped' => $skipped,
'totals' => $statusCounts,
'by_p95' => array_slice($byP95, 0, $top),
'by_p99' => array_slice($byP99, 0, $top),
'by_total_time' => array_slice($byTotal, 0, $top),
'by_count' => array_slice($byCount, 0, $top),
'by_cost' => array_slice($byCost, 0, $top),
'by_5xx' => array_slice($by5xx, 0, $top),
'by_4xx' => array_slice($by404, 0, $top),
'slowest' => $slowest,
'percentile_method' => 'nearest-rank (ceil(p * n), 1-indexed)',
'upstream_aggregation' => 'sum of numeric comma-separated upstream timings; "-" is null',
];
}
/**
* Nearest-rank: rank = ceil(p * n), 1-indexed.
*
* @param list<float> $sorted
*/
public function percentile(array $sorted, float $p): ?float
{
$n = count($sorted);
if ($n === 0) {
return null;
}
$rank = (int) ceil($p * $n);
$index = max(0, min($n - 1, $rank - 1));
return round($sorted[$index], 4);
}
public function parseUpstream(mixed $value): ?float
{
if ($value === null || $value === '' || $value === '-') {
return null;
}
if (is_int($value) || is_float($value)) {
return (float) $value;
}
$parts = preg_split('/\s*,\s*/', (string) $value) ?: [];
$sum = 0.0;
$found = false;
foreach ($parts as $part) {
if ($part === '' || $part === '-') {
continue;
}
if (! is_numeric($part)) {
continue;
}
$sum += (float) $part;
$found = true;
}
return $found ? $sum : null;
}
private function parseFloat(mixed $value): ?float
{
if ($value === null || $value === '' || $value === '-') {
return null;
}
if (! is_numeric($value)) {
return null;
}
return (float) $value;
}
private function parseTime(string $value): ?\DateTimeImmutable
{
if ($value === '') {
return null;
}
try {
return new \DateTimeImmutable($value);
} catch (\Exception) {
return null;
}
}
private function sinceCutoff(?string $since, \DateTimeImmutable $now): ?\DateTimeImmutable
{
if ($since === null || $since === '') {
return null;
}
if (preg_match('/^(\d+)(h|d|m)$/', strtolower($since), $matches) !== 1) {
return null;
}
$n = (int) $matches[1];
$unit = $matches[2];
$interval = match ($unit) {
'm' => new \DateInterval('PT'.$n.'M'),
'h' => new \DateInterval('PT'.$n.'H'),
'd' => new \DateInterval('P'.$n.'D'),
};
return $now->sub($interval);
}
}
+96
View File
@@ -0,0 +1,96 @@
<?php
declare(strict_types=1);
namespace App\Support\Http;
final class HttpUriNormalizer
{
public function normalize(string $uri): string
{
$path = parse_url($uri, PHP_URL_PATH);
if (! is_string($path) || $path === '') {
$path = $uri;
}
$path = '/'.ltrim($path, '/');
if ($path !== '/') {
$path = rtrim($path, '/');
}
if (preg_match('#^/@([^/]+)(/.*)?$#', $path, $matches) === 1) {
$rest = $matches[2] ?? '';
return '/@{user}'.$this->normalizeSegments($rest);
}
return $this->normalizeSegments($path);
}
private function normalizeSegments(string $path): string
{
if ($path === '' || $path === '/') {
return $path === '' ? '' : '/';
}
$segments = explode('/', $path);
$normalized = [];
foreach ($segments as $index => $segment) {
if ($segment === '') {
$normalized[] = '';
continue;
}
$normalized[] = $this->normalizeSegment($segment, $index, $segments);
}
$result = implode('/', $normalized);
return $result === '' ? '/' : $result;
}
/**
* @param array<int, string> $segments
*/
private function normalizeSegment(string $segment, int $index, array $segments): string
{
if (ctype_digit($segment)) {
return '{id}';
}
if (preg_match('/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i', $segment) === 1) {
return '{uuid}';
}
if (preg_match('/^[0-9a-f]{32,64}$/i', $segment) === 1) {
return '{hash}';
}
if ($this->isArtworkPublicSlug($segment, $index, $segments)) {
return '{slug}';
}
return $segment;
}
/**
* /art/{id}/{slug} stays distinct from /art/{id}/similar and other actions.
*
* @param array<int, string> $segments
*/
private function isArtworkPublicSlug(string $segment, int $index, array $segments): bool
{
if ($index !== 3 || ($segments[1] ?? '') !== 'art') {
return false;
}
if (! ctype_digit((string) ($segments[2] ?? ''))) {
return false;
}
$actions = ['similar', 'view', 'download', 'edit', 'comments', 'favourite', 'favorite'];
return ! in_array(strtolower($segment), $actions, true);
}
}
@@ -0,0 +1,34 @@
<?php
declare(strict_types=1);
namespace App\Support\Http;
use Illuminate\Http\Request;
use Inertia\Ssr\Gateway;
use Inertia\Ssr\HttpGateway;
use Inertia\Ssr\Response;
/**
* Times the existing Inertia SSR HTTP round-trip. Does not change SSR behavior.
*/
final class TimedInertiaSsrGateway implements Gateway
{
public function __construct(private readonly HttpGateway $inner)
{
}
public function dispatch(array $page): ?Response
{
$started = hrtime(true);
$response = $this->inner->dispatch($page);
$durationMs = (hrtime(true) - $started) / 1_000_000;
$request = request();
if ($request instanceof Request) {
$request->attributes->set('http.ssr_ms', round($durationMs, 1));
}
return $response;
}
}
@@ -0,0 +1,33 @@
<?php
declare(strict_types=1);
namespace App\Support\Queues;
final class QueuedJobClassMatcher
{
public static function className(string $rawPayload): ?string
{
$decoded = json_decode($rawPayload, true);
if (! is_array($decoded)) {
return null;
}
$display = $decoded['displayName'] ?? null;
if (is_string($display) && $display !== '') {
return $display;
}
$command = $decoded['data']['commandName'] ?? null;
if (is_string($command) && $command !== '') {
return $command;
}
return null;
}
public static function isClass(string $rawPayload, string $class): bool
{
return self::className($rawPayload) === $class;
}
}
+159
View File
@@ -0,0 +1,159 @@
<?php
declare(strict_types=1);
namespace App\Support\Redis;
use App\Jobs\RefreshCollectionHealthJob;
use App\Jobs\RefreshCollectionRecommendationJob;
use App\Jobs\ScanCollectionDuplicateCandidatesJob;
use App\Support\Queues\QueuedJobClassMatcher;
use Illuminate\Support\Facades\Redis;
use RuntimeException;
final class OrphanQueueCleanup
{
public const PROTECTED_QUEUES = ['default', 'search', 'mail', 'broadcasts', 'notifications'];
/**
* @var array<string, array{expected: list<string>, dispatch_config: string}>
*/
public const TARGETS = [
'forum-moderation' => [
'expected' => ['cPad\\Plugins\\Forum\\Jobs\\AnalyzeForumPostJob'],
'dispatch_config' => 'skinbase_ai_moderation.queue.dispatch_enabled',
],
'forum-security' => [
'expected' => [
'cPad\\Plugins\\Forum\\Jobs\\FirewallActivityMonitor',
'cPad\\Plugins\\Forum\\Jobs\\BotActivityMonitor',
],
'dispatch_config' => 'forum_security.queues.dispatch_enabled',
],
'collections' => [
'expected' => [
RefreshCollectionHealthJob::class,
RefreshCollectionRecommendationJob::class,
ScanCollectionDuplicateCandidatesJob::class,
],
'dispatch_config' => 'collections.v5.queue.dispatch_enabled',
],
];
/**
* @return array<string, mixed>
*/
public function inspect(string $queue, int $sampleSize = 400): array
{
$this->assertQueueName($queue);
$redis = Redis::connection();
$waitingKey = 'queues:'.$queue;
$llen = (int) $redis->llen($waitingKey);
$delayed = (int) $redis->zcard('queues:'.$queue.':delayed');
$reserved = (int) $redis->zcard('queues:'.$queue.':reserved');
$notify = (int) $redis->llen('queues:'.$queue.':notify');
$prefix = (string) config('database.redis.options.prefix', '');
$classes = [];
$bytes = [];
$sampled = 0;
if ($llen > 0) {
$step = max(1, (int) floor($llen / max(1, $sampleSize)));
for ($i = 0; $i < $llen && $sampled < $sampleSize; $i += $step) {
$raw = $redis->lindex($waitingKey, $i);
if (! is_string($raw) || $raw === '') {
continue;
}
$sampled++;
$class = QueuedJobClassMatcher::className($raw) ?? 'unknown';
$classes[$class] = ($classes[$class] ?? 0) + 1;
$bytes[] = strlen($raw);
}
}
arsort($classes);
$avg = $bytes !== [] ? (int) round(array_sum($bytes) / count($bytes)) : 0;
return [
'queue' => $queue,
'logical_key' => $waitingKey,
'notify_key' => 'queues:'.$queue.':notify',
'prefix' => $prefix,
'llen' => $llen,
'delayed' => $delayed,
'reserved' => $reserved,
'notify' => $notify,
'sampled' => $sampled,
'classes' => $classes,
'avg_bytes' => $avg,
'est_bytes' => $llen * $avg,
];
}
/**
* @return array<string, mixed>
*/
public function plan(string $queue, bool $force = false): array
{
if (in_array($queue, self::PROTECTED_QUEUES, true)) {
throw new RuntimeException("Refusing protected queue [{$queue}].");
}
if (! isset(self::TARGETS[$queue])) {
throw new RuntimeException("Unknown orphan target [{$queue}].");
}
$meta = self::TARGETS[$queue];
$dispatchEnabled = (bool) config($meta['dispatch_config'], false);
$inspect = $this->inspect($queue);
$unexpected = array_values(array_diff(array_keys($inspect['classes']), $meta['expected']));
$errors = [];
if ($dispatchEnabled) {
$errors[] = 'producer_dispatch_enabled';
}
if ($inspect['reserved'] > 0) {
$errors[] = 'reserved_jobs_present';
}
if ($inspect['delayed'] > 0) {
$errors[] = 'delayed_jobs_present';
}
if ($unexpected !== [] && ! $force) {
$errors[] = 'unexpected_job_classes';
}
return $inspect + [
'expected_classes' => $meta['expected'],
'unexpected_classes' => $unexpected,
'dispatch_enabled' => $dispatchEnabled,
'errors' => $errors,
'can_execute' => $errors === [],
];
}
/**
* @return array<string, mixed>
*/
public function execute(string $queue, bool $force = false): array
{
$plan = $this->plan($queue, $force);
if (! $plan['can_execute']) {
throw new RuntimeException('Cleanup refused: '.implode(',', $plan['errors']));
}
$redis = Redis::connection();
$keys = ['queues:'.$queue, 'queues:'.$queue.':notify'];
$unlinked = 0;
foreach ($keys as $key) {
$unlinked += (int) $redis->unlink($key);
}
return $plan + ['unlinked_keys' => $keys, 'unlinked' => $unlinked];
}
private function assertQueueName(string $queue): void
{
if (! preg_match('/^[A-Za-z0-9_-]+$/', $queue)) {
throw new RuntimeException('Invalid queue name.');
}
}
}