klevze/SkinbaseNova
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Wire admin studio SSR and search infrastructure

Browse Source
This commit is contained in:
Gregor Klevže
2026-05-01 11:46:06 +02:00
parent 257b0dbef6
commit 18cea8b0f0
329 changed files with 197465 additions and 2741 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
app/Jobs/AutoTagArtworkJob.php
View File

@@ -60,6 +60,16 @@ final class AutoTagArtworkJob implements ShouldQueue
return;
}
// Dedup: skip if this artwork+hash was already successfully tagged (7-day window).
// This prevents redundant processing when the job is enqueued multiple times.
try {
if (Redis::exists($this->processedKey($this->artworkId, $this->hash))) {
return;
}
} catch (\Throwable) {
// Redis unavailable — proceed without dedup guard.
}
$artwork = Artwork::query()->with(['categories.contentType'])->find($this->artworkId);
if (! $artwork) {
return;

17
app/Jobs/DeleteArtworkFromIndexJob.php
View File

@@ -6,6 +6,7 @@ namespace App\Jobs;
use App\Models\Artwork;
use Illuminate\Bus\Queueable;
use Meilisearch\Client as MeilisearchClient;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
@@ -22,14 +23,16 @@ class DeleteArtworkFromIndexJob implements ShouldQueue
public int $tries = 3;
public int $timeout = 30;
public function __construct(public readonly int $artworkId) {}
public function handle(): void
public function __construct(public readonly int $artworkId)
{
// Create a bare model instance just to call unsearchable() with the right ID.
$artwork = new Artwork();
$artwork->id = $this->artworkId;
$artwork->unsearchable();
$this->afterCommit = true;
}
public function handle(MeilisearchClient $client): void
{
// Delete directly from the Meilisearch index — no Scout after_commit hop.
$indexName = (new Artwork())->searchableAs();
$client->index($indexName)->deleteDocument($this->artworkId);
}
public function failed(\Throwable $e): void

2
app/Jobs/GenerateArtworkEmbeddingJob.php
View File

@@ -89,7 +89,7 @@ final class GenerateArtworkEmbeddingJob implements ShouldQueue
}
try {
$imageUrl = $imageUrlBuilder->fromHash($sourceHash, (string) ($artwork->thumb_ext ?: 'webp'));
$imageUrl = $imageUrlBuilder->fromArtwork($artwork);
if ($imageUrl === null) {
return;
}

23
app/Jobs/GenerateDerivativesJob.php
View File

@@ -4,6 +4,7 @@ declare(strict_types=1);
namespace App\Jobs;
use App\Services\Uploads\UploadQueueService;
use App\Services\Uploads\UploadPipelineService;
use App\Jobs\AnalyzeArtworkAiAssistJob;
use App\Jobs\AutoTagArtworkJob;
@@ -30,11 +31,12 @@ final class GenerateDerivativesJob implements ShouldQueue
private readonly ?string $archiveSessionId = null,
private readonly ?string $archiveHash = null,
private readonly ?string $archiveOriginalFileName = null,
private readonly array $additionalScreenshotSessions = []
private readonly array $additionalScreenshotSessions = [],
private readonly ?int $batchItemId = null,
) {
}
public function handle(UploadPipelineService $pipeline): void
public function handle(UploadPipelineService $pipeline, UploadQueueService $queue): void
{
$pipeline->processAndPublish(
$this->sessionId,
@@ -47,10 +49,27 @@ final class GenerateDerivativesJob implements ShouldQueue
$this->additionalScreenshotSessions
);
if ($this->batchItemId) {
$queue->markItemMediaProcessed($this->batchItemId);
}
// Auto-tagging is async and must never block publish.
AutoTagArtworkJob::dispatch($this->artworkId, $this->hash)->afterCommit();
DetectArtworkMaturityJob::dispatch($this->artworkId, $this->hash)->afterCommit();
GenerateArtworkEmbeddingJob::dispatch($this->artworkId, $this->hash)->afterCommit();
AnalyzeArtworkAiAssistJob::dispatch($this->artworkId)->afterCommit();
}
public function failed(\Throwable $exception): void
{
if (! $this->batchItemId) {
return;
}
app(UploadQueueService::class)->markItemFailed(
$this->batchItemId,
'derivatives_failed',
$exception->getMessage()
);
}
}

34
app/Jobs/IndexArtworkJob.php
View File

@@ -11,35 +11,51 @@ use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use Meilisearch\Client as MeilisearchClient;
/**
* Queued job: index (or re-index) a single Artwork in Meilisearch.
*
* Writes directly to the Meilisearch HTTP API instead of going through
* Scout's searchable() / MakeSearchable pipeline. This avoids the
* after_commit double-dispatch problem and ensures the document lands
* in the index within this job's execution, with no extra queue hop.
*/
class IndexArtworkJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public int $tries = 3;
public int $timeout = 30;
public int $timeout = 60;
public function __construct(public readonly int $artworkId) {}
public function handle(): void
public function __construct(public readonly int $artworkId)
{
$artwork = Artwork::with(['user', 'tags', 'categories', 'stats', 'awardStat'])
->find($this->artworkId);
$this->afterCommit = true;
}
public function handle(MeilisearchClient $client): void
{
$artwork = Artwork::with([
'user',
'group',
'tags',
'categories.contentType',
'stats',
'awardStat',
])->find($this->artworkId);
if (! $artwork) {
return;
}
if (! $artwork->is_public || ! $artwork->is_approved || ! $artwork->published_at) {
// Not public/approved — ensure it is removed from the index.
$artwork->unsearchable();
// Not eligible — remove from index if present.
$client->index($artwork->searchableAs())->deleteDocument($this->artworkId);
return;
}
$artwork->searchable();
$document = $artwork->toSearchableArray();
$client->index($artwork->searchableAs())->addDocuments([$document]);
}
public function failed(\Throwable $e): void

138
app/Jobs/RecBuildItemPairsFromFavouritesJob.php
View File

@@ -46,33 +46,22 @@ final class RecBuildItemPairsFromFavouritesJob implements ShouldQueue
->pluck('cnt', 'artwork_id')
->all();
// ── Accumulate co-occurrence counts across all users ──
$coOccurrenceCounts = [];
// ── Rebuild weights from scratch to avoid cross-run accumulation ──
DB::table('rec_item_pairs')->delete();
DB::table('artwork_favourites')
->select('user_id')
->groupBy('user_id')
->orderBy('user_id')
->chunk($this->userBatchSize, function ($userRows) use ($favCap, &$coOccurrenceCounts) {
->chunk($this->userBatchSize, function ($userRows) use ($favCap) {
$userIds = [];
foreach ($userRows as $row) {
$pairs = $this->pairsForUser((int) $row->user_id, $favCap);
foreach ($pairs as $pair) {
$key = $pair[0] . ':' . $pair[1];
$coOccurrenceCounts[$key] = ($coOccurrenceCounts[$key] ?? 0) + 1;
}
$userIds[] = (int) $row->user_id;
}
$this->flushPairCountChunk($this->pairCountsForUsers($userIds, $favCap));
});
// ── Normalize to cosine-like scores and flush ──
$normalized = [];
foreach ($coOccurrenceCounts as $key => $count) {
[$a, $b] = explode(':', $key);
$likesA = $this->artworkLikeCounts[(int) $a] ?? 1;
$likesB = $this->artworkLikeCounts[(int) $b] ?? 1;
$normalized[$key] = $count / sqrt($likesA * $likesB);
}
$this->flushPairs($normalized);
}
/** @var array<int, int> artwork_id => total favourite count */
@@ -93,6 +82,56 @@ final class RecBuildItemPairsFromFavouritesJob implements ShouldQueue
->map(fn ($id) => (int) $id)
->all();
return $this->pairsForArtworkIds($artworkIds);
}
/**
* Collect chunk-local pair counts using one capped favourites query for the chunk.
*
* @param list<int> $userIds
* @return array<string, int>
*/
private function pairCountsForUsers(array $userIds, int $cap): array
{
if ($userIds === []) {
return [];
}
$rankedFavourites = DB::query()
->fromSub(
DB::table('artwork_favourites')
->selectRaw('user_id, artwork_id, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY created_at DESC, artwork_id DESC) as favourite_rank')
->whereIn('user_id', $userIds),
'ranked_favourites'
)
->where('favourite_rank', '<=', $cap)
->orderBy('user_id')
->orderBy('favourite_rank')
->get(['user_id', 'artwork_id']);
$artworksByUser = [];
foreach ($rankedFavourites as $row) {
$artworksByUser[(int) $row->user_id][] = (int) $row->artwork_id;
}
$pairCounts = [];
foreach ($artworksByUser as $artworkIds) {
foreach ($this->pairsForArtworkIds($artworkIds) as [$a, $b]) {
$key = $this->pairKey($a, $b);
$pairCounts[$key] = ($pairCounts[$key] ?? 0) + 1;
}
}
return $pairCounts;
}
/**
* @param list<int> $artworkIds
* @return list<array{0: int, 1: int}>
*/
private function pairsForArtworkIds(array $artworkIds): array
{
$count = count($artworkIds);
if ($count < 2) {
return [];
@@ -112,28 +151,50 @@ final class RecBuildItemPairsFromFavouritesJob implements ShouldQueue
}
/**
* Upsert normalized pair weights into rec_item_pairs.
* Upsert one chunk of pair counts into rec_item_pairs.
*
* Uses Laravel's DB-agnostic upsert (works on MySQL, Postgres, SQLite).
*
* @param array<string, float> $upserts key = "a:b", value = cosine-normalized weight
* @param array<string, int> $pairCounts key = "a:b", value = chunk-local co-occurrence count
*/
private function flushPairs(array $upserts): void
private function flushPairCountChunk(array $pairCounts): void
{
if ($upserts === []) {
if ($pairCounts === []) {
return;
}
$now = now();
foreach (array_chunk($upserts, 500, preserve_keys: true) as $chunk) {
foreach (array_chunk($pairCounts, 500, preserve_keys: true) as $chunk) {
$pairIds = [];
$aIds = [];
$bIds = [];
foreach ($chunk as $key => $count) {
[$a, $b] = $this->pairIdsFromKey($key);
$pairIds[$key] = [$a, $b];
$aIds[] = $a;
$bIds[] = $b;
}
$existingWeights = DB::table('rec_item_pairs')
->whereIn('a_artwork_id', array_values(array_unique($aIds)))
->whereIn('b_artwork_id', array_values(array_unique($bIds)))
->get(['a_artwork_id', 'b_artwork_id', 'weight'])
->mapWithKeys(fn ($row): array => [
$this->pairKey((int) $row->a_artwork_id, (int) $row->b_artwork_id) => (float) $row->weight,
])
->all();
$rows = [];
foreach ($chunk as $key => $weight) {
[$a, $b] = explode(':', $key);
foreach ($chunk as $key => $count) {
[$a, $b] = $pairIds[$key];
$likesA = $this->artworkLikeCounts[$a] ?? 1;
$likesB = $this->artworkLikeCounts[$b] ?? 1;
$deltaWeight = $count / sqrt($likesA * $likesB);
$rows[] = [
'a_artwork_id' => (int) $a,
'b_artwork_id' => (int) $b,
'weight' => $weight,
'a_artwork_id' => $a,
'b_artwork_id' => $b,
'weight' => ($existingWeights[$key] ?? 0.0) + $deltaWeight,
'updated_at' => $now,
];
}
@@ -145,4 +206,19 @@ final class RecBuildItemPairsFromFavouritesJob implements ShouldQueue
);
}
}
private function pairKey(int $a, int $b): string
{
return $a . ':' . $b;
}
/**
* @return array{0: int, 1: int}
*/
private function pairIdsFromKey(string $key): array
{
[$a, $b] = explode(':', $key, 2);
return [(int) $a, (int) $b];
}
}