klevze/SkinbaseNova
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Add HTTP performance log analyzer (M12.6).

Browse Source 
Parse rotated nginx JSON access logs into host/route timings so production hotspots can be measured without ad-hoc one-off scripts.
This commit is contained in:
klevze
2026-08-29 12:25:27 +02:00
parent 4ab315e9d7
commit fb560fb7dd
10 changed files with 737 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Support/Http/HttpPerformanceLogAnalyzer.php
+363 -121
View File
@@ -10,16 +10,23 @@ namespace App\Support\Http;
* Multi-value upstream timings (retries): use the **sum** of numeric parts
* as the upstream total for that request. "-" and empty → null.
* Percentiles use nearest-rank (ceil(p * n), 1-indexed).
*
* Exact --status=404 means HTTP 404 only (not 4xx). Classes are 4xx/5xx.
*/
final class HttpPerformanceLogAnalyzer
{
/** Ignore gaps shorter than this when judging 24h source coverage. */
public const COVERAGE_TOLERANCE_SECONDS = 900;
public function __construct(private readonly HttpUriNormalizer $uris)
{
}
/**
* @param string|list<string> $input
* @param array{
* since?: ?string,
* until?: ?string,
* min_requests?: int,
* top?: int,
* status?: ?string,
@@ -30,139 +37,178 @@ final class HttpPerformanceLogAnalyzer
* } $options
* @return array<string, mixed>
*/
public function analyze(string $file, array $options = []): array
public function analyze(string|array $input, array $options = []): array
{
$since = $this->sinceCutoff($options['since'] ?? null, $options['now'] ?? new \DateTimeImmutable('now'));
$files = $this->normalizeInputList($input);
$now = $options['now'] ?? new \DateTimeImmutable('now');
$requestedStart = $this->sinceCutoff($options['since'] ?? null, $now);
$requestedEnd = $this->untilInstant($options['until'] ?? null, $now);
$minRequests = max(1, (int) ($options['min_requests'] ?? 1));
$top = max(1, (int) ($options['top'] ?? 20));
$statusFilter = isset($options['status']) ? strtolower((string) $options['status']) : null;
$statusFilter = $this->parseStatusFilter($options['status'] ?? null);
$uriFilter = $options['uri'] ?? null;
$hostFilter = isset($options['host']) ? (string) $options['host'] : '';
$slowMs = isset($options['slow_ms']) ? (float) $options['slow_ms'] / 1000 : null;
$slowMs = isset($options['slow_ms']) && $options['slow_ms'] !== null
? (float) $options['slow_ms'] / 1000
: null;
$skipped = 0;
$linesScanned = 0;
$parseable = 0;
$families = [];
$slowest = [];
$statusCounts = ['4xx' => 0, '5xx' => 0, '499' => 0, 'total' => 0];
$sources = [];
$earliestParseable = null;
$latestParseable = null;
$earliestMatching = null;
$latestMatching = null;
$handle = fopen($file, 'r');
if ($handle === false) {
throw new \RuntimeException('Unable to open performance log: '.$file);
}
foreach ($files as $file) {
$source = [
'path' => $file,
'gzip' => str_ends_with(strtolower($file), '.gz'),
'lines_scanned' => 0,
'parseable' => 0,
'in_window' => 0,
'error' => null,
];
try {
while (($line = fgets($handle)) !== false) {
$line = trim($line);
if ($line === '') {
continue;
}
$row = json_decode($line, true);
if (! is_array($row)) {
$skipped++;
continue;
}
$time = $this->parseTime((string) ($row['time'] ?? ''));
if ($since !== null && ($time === null || $time < $since)) {
continue;
}
$requestTime = $this->parseFloat($row['request_time'] ?? null);
if ($requestTime === null) {
$skipped++;
continue;
}
$status = (int) ($row['status'] ?? 0);
if ($statusFilter === '5xx' && ($status < 500 || $status > 599)) {
continue;
}
if ($statusFilter === '4xx' && ($status < 400 || $status > 499)) {
continue;
}
if ($statusFilter === '499' && $status !== 499) {
continue;
}
$host = (string) ($row['host'] ?? '');
if ($hostFilter !== '' && $host !== $hostFilter) {
continue;
}
$uri = $this->uris->normalize((string) ($row['uri'] ?? '/'));
if (is_string($uriFilter) && $uriFilter !== '' && ! str_starts_with($uri, $uriFilter)) {
continue;
}
if ($slowMs !== null && $requestTime < $slowMs) {
continue;
}
$upstreamResponse = $this->parseUpstream($row['upstream_response_time'] ?? null);
$upstreamHeader = $this->parseUpstream($row['upstream_header_time'] ?? null);
$bytes = (int) ($row['bytes'] ?? 0);
$method = (string) ($row['method'] ?? '');
$statusCounts['total']++;
if ($status === 499) {
$statusCounts['499']++;
} elseif ($status >= 500) {
$statusCounts['5xx']++;
} elseif ($status >= 400) {
$statusCounts['4xx']++;
}
$familyKey = $host."\n".$uri;
$families[$familyKey] ??= [
'host' => $host,
'uri' => $uri,
'count' => 0,
'times' => [],
'upstream_sum' => 0.0,
'upstream_n' => 0,
'header_sum' => 0.0,
'header_n' => 0,
'bytes_sum' => 0,
'4xx' => 0,
'5xx' => 0,
'499' => 0,
'404' => 0,
];
$families[$familyKey]['count']++;
$families[$familyKey]['times'][] = $requestTime;
$families[$familyKey]['bytes_sum'] += $bytes;
if ($upstreamResponse !== null) {
$families[$familyKey]['upstream_sum'] += $upstreamResponse;
$families[$familyKey]['upstream_n']++;
}
if ($upstreamHeader !== null) {
$families[$familyKey]['header_sum'] += $upstreamHeader;
$families[$familyKey]['header_n']++;
}
if ($status === 499) {
$families[$familyKey]['499']++;
} elseif ($status >= 500) {
$families[$familyKey]['5xx']++;
} elseif ($status >= 400) {
$families[$familyKey]['4xx']++;
if ($status === 404) {
$families[$familyKey]['404']++;
}
}
$slowest[] = [
'time' => $row['time'] ?? null,
'host' => $host,
'uri' => $uri,
'method' => $method,
'status' => $status,
'request_time' => $requestTime,
'upstream_response_time' => $upstreamResponse,
];
$handle = $this->openStream($file);
if ($handle === false) {
$source['error'] = 'unreadable';
$sources[] = $source;
continue;
}
} finally {
fclose($handle);
try {
while (($line = fgets($handle)) !== false) {
$linesScanned++;
$source['lines_scanned']++;
$line = trim($line);
if ($line === '') {
continue;
}
$row = json_decode($line, true);
if (! is_array($row)) {
$skipped++;
continue;
}
$time = $this->parseTime((string) ($row['time'] ?? ''));
$requestTime = $this->parseFloat($row['request_time'] ?? null);
if ($requestTime === null) {
$skipped++;
continue;
}
$parseable++;
$source['parseable']++;
if ($time instanceof \DateTimeImmutable) {
$earliestParseable = $this->minTime($earliestParseable, $time);
$latestParseable = $this->maxTime($latestParseable, $time);
}
if ($requestedStart !== null && ($time === null || $time < $requestedStart)) {
continue;
}
if ($requestedEnd !== null && ($time === null || $time > $requestedEnd)) {
continue;
}
$status = (int) ($row['status'] ?? 0);
if (! $this->statusMatches($status, $statusFilter)) {
continue;
}
$host = (string) ($row['host'] ?? '');
if ($hostFilter !== '' && $host !== $hostFilter) {
continue;
}
$uri = $this->uris->normalize((string) ($row['uri'] ?? '/'));
if (is_string($uriFilter) && $uriFilter !== '' && ! str_starts_with($uri, $uriFilter)) {
continue;
}
if ($slowMs !== null && $requestTime < $slowMs) {
continue;
}
$source['in_window']++;
if ($time instanceof \DateTimeImmutable) {
$earliestMatching = $this->minTime($earliestMatching, $time);
$latestMatching = $this->maxTime($latestMatching, $time);
}
$upstreamResponse = $this->parseUpstream($row['upstream_response_time'] ?? null);
$upstreamHeader = $this->parseUpstream($row['upstream_header_time'] ?? null);
$bytes = (int) ($row['bytes'] ?? 0);
$method = (string) ($row['method'] ?? '');
$statusCounts['total']++;
if ($status === 499) {
$statusCounts['499']++;
} elseif ($status >= 500) {
$statusCounts['5xx']++;
} elseif ($status >= 400) {
$statusCounts['4xx']++;
}
$familyKey = $host."\n".$uri;
$families[$familyKey] ??= [
'host' => $host,
'uri' => $uri,
'count' => 0,
'times' => [],
'upstream_sum' => 0.0,
'upstream_n' => 0,
'header_sum' => 0.0,
'header_n' => 0,
'bytes_sum' => 0,
'4xx' => 0,
'5xx' => 0,
'499' => 0,
'404' => 0,
];
$families[$familyKey]['count']++;
$families[$familyKey]['times'][] = $requestTime;
$families[$familyKey]['bytes_sum'] += $bytes;
if ($upstreamResponse !== null) {
$families[$familyKey]['upstream_sum'] += $upstreamResponse;
$families[$familyKey]['upstream_n']++;
}
if ($upstreamHeader !== null) {
$families[$familyKey]['header_sum'] += $upstreamHeader;
$families[$familyKey]['header_n']++;
}
if ($status === 499) {
$families[$familyKey]['499']++;
} elseif ($status >= 500) {
$families[$familyKey]['5xx']++;
} elseif ($status >= 400) {
$families[$familyKey]['4xx']++;
if ($status === 404) {
$families[$familyKey]['404']++;
}
}
$slowest[] = [
'time' => $row['time'] ?? null,
'host' => $host,
'uri' => $uri,
'method' => $method,
'status' => $status,
'request_time' => $requestTime,
];
$slowest[array_key_last($slowest)]['upstream_response_time'] = $upstreamResponse;
}
} finally {
fclose($handle);
}
$sources[] = $source;
}
usort($slowest, static fn (array $a, array $b): int => $b['request_time'] <=> $a['request_time']);
@@ -213,8 +259,24 @@ final class HttpPerformanceLogAnalyzer
$by404 = array_values(array_filter($summaries, static fn (array $row): bool => ($row['404'] ?? 0) > 0));
usort($by404, static fn (array $a, array $b): int => $b['404'] <=> $a['404']);
$coverage = $this->coverageReport(
$requestedStart,
$requestedEnd,
$earliestParseable,
$latestParseable,
$earliestMatching,
$latestMatching,
);
return [
'skipped' => $skipped,
'lines_scanned' => $linesScanned,
'parseable' => $parseable,
'source_files' => count($files),
'sources' => $sources,
'coverage' => $coverage,
'warnings' => $coverage['warnings'],
'sample_empty' => $statusCounts['total'] === 0,
'totals' => $statusCounts,
'by_p95' => array_slice($byP95, 0, $top),
'by_p99' => array_slice($byP99, 0, $top),
@@ -229,6 +291,48 @@ final class HttpPerformanceLogAnalyzer
];
}
/**
* @param list<string> $paths
* @return list<string>
*/
public function deduplicatePaths(array $paths): array
{
$seen = [];
$out = [];
foreach ($paths as $path) {
$path = str_replace('\\', '/', (string) $path);
if ($path === '') {
continue;
}
$resolved = realpath($path) ?: $path;
$key = strtolower($resolved);
if (isset($seen[$key])) {
continue;
}
$seen[$key] = true;
$out[] = $path;
}
return array_values($out);
}
/**
* @return list<string>
*/
public function expandGlob(string $pattern): array
{
$matches = glob($pattern, GLOB_NOSORT) ?: [];
$files = [];
foreach ($matches as $match) {
if (is_file($match)) {
$files[] = $match;
}
}
natsort($files);
return array_values($files);
}
/**
* Nearest-rank: rank = ceil(p * n), 1-indexed.
*
@@ -274,6 +378,135 @@ final class HttpPerformanceLogAnalyzer
return $found ? $sum : null;
}
/**
* @param string|list<string> $input
* @return list<string>
*/
private function normalizeInputList(string|array $input): array
{
$files = is_array($input) ? $input : [$input];
$files = $this->deduplicatePaths(array_map('strval', $files));
if ($files === []) {
throw new \InvalidArgumentException('No performance log files were provided.');
}
return $files;
}
/**
* @return resource|false
*/
private function openStream(string $file)
{
if (str_ends_with(strtolower($file), '.gz')) {
return @gzopen($file, 'rb');
}
return @fopen($file, 'r');
}
/**
* @return array{type: string, code?: int, class?: string}
*/
public function parseStatusFilter(?string $raw): array
{
if ($raw === null || $raw === '') {
return ['type' => 'none'];
}
$value = strtolower(trim($raw));
if ($value === '4xx') {
return ['type' => 'class', 'class' => '4xx'];
}
if ($value === '5xx') {
return ['type' => 'class', 'class' => '5xx'];
}
if (preg_match('/^[1-5][0-9]{2}$/', $value) === 1) {
return ['type' => 'exact', 'code' => (int) $value];
}
throw new \InvalidArgumentException('Invalid --status value: '.$raw);
}
/**
* @param array{type: string, code?: int, class?: string} $filter
*/
private function statusMatches(int $status, array $filter): bool
{
return match ($filter['type']) {
'none' => true,
'exact' => $status === (int) ($filter['code'] ?? 0),
'class' => ($filter['class'] ?? '') === '5xx'
? ($status >= 500 && $status <= 599)
: ($status >= 400 && $status <= 499),
default => true,
};
}
/**
* @return array<string, mixed>
*/
private function coverageReport(
?\DateTimeImmutable $requestedStart,
?\DateTimeImmutable $requestedEnd,
?\DateTimeImmutable $earliestParseable,
?\DateTimeImmutable $latestParseable,
?\DateTimeImmutable $earliestMatching,
?\DateTimeImmutable $latestMatching,
): array {
$warnings = [];
$complete = true;
if ($requestedStart instanceof \DateTimeImmutable && $requestedEnd instanceof \DateTimeImmutable) {
if (! $earliestParseable instanceof \DateTimeImmutable || ! $latestParseable instanceof \DateTimeImmutable) {
$complete = false;
$warnings[] = 'Observed records do not demonstrate full requested coverage.';
} else {
$startGap = $earliestParseable->getTimestamp() - $requestedStart->getTimestamp();
$endGap = $requestedEnd->getTimestamp() - $latestParseable->getTimestamp();
if ($startGap > self::COVERAGE_TOLERANCE_SECONDS || $endGap > self::COVERAGE_TOLERANCE_SECONDS) {
$complete = false;
$warnings[] = sprintf(
'WARNING: Requested window is not fully covered by supplied logs. Requested: %s → %s Observed: %s → %s',
$requestedStart->format(\DateTimeInterface::ATOM),
$requestedEnd->format(\DateTimeInterface::ATOM),
$earliestParseable->format(\DateTimeInterface::ATOM),
$latestParseable->format(\DateTimeInterface::ATOM),
);
}
}
}
return [
'requested_start' => $requestedStart?->format(\DateTimeInterface::ATOM),
'requested_end' => $requestedEnd?->format(\DateTimeInterface::ATOM),
'observed_earliest_parseable' => $earliestParseable?->format(\DateTimeInterface::ATOM),
'observed_latest_parseable' => $latestParseable?->format(\DateTimeInterface::ATOM),
'observed_earliest_matching' => $earliestMatching?->format(\DateTimeInterface::ATOM),
'observed_latest_matching' => $latestMatching?->format(\DateTimeInterface::ATOM),
'tolerance_seconds' => self::COVERAGE_TOLERANCE_SECONDS,
'complete' => $complete,
'warnings' => $warnings,
];
}
private function minTime(?\DateTimeImmutable $current, \DateTimeImmutable $candidate): \DateTimeImmutable
{
if ($current === null || $candidate < $current) {
return $candidate;
}
return $current;
}
private function maxTime(?\DateTimeImmutable $current, \DateTimeImmutable $candidate): \DateTimeImmutable
{
if ($current === null || $candidate > $current) {
return $candidate;
}
return $current;
}
private function parseFloat(mixed $value): ?float
{
if ($value === null || $value === '' || $value === '-') {
@@ -306,7 +539,7 @@ final class HttpPerformanceLogAnalyzer
}
if (preg_match('/^(\d+)(h|d|m)$/', strtolower($since), $matches) !== 1) {
return null;
return $this->parseTime($since);
}
$n = (int) $matches[1];
@@ -319,4 +552,13 @@ final class HttpPerformanceLogAnalyzer
return $now->sub($interval);
}
private function untilInstant(?string $until, \DateTimeImmutable $now): \DateTimeImmutable
{
if ($until === null || $until === '') {
return $now;
}
return $this->parseTime($until) ?? $now;
}
}