Harden vision vector search and embedding jobs.
Add point search, circuit-breaking, and safer gateway limits so artwork similarity lookups fail closed instead of hanging the request path.
This commit is contained in:
@@ -18,7 +18,9 @@ beforeEach(function (): void {
|
||||
config()->set('vision.vector_gateway.base_url', 'https://vision.klevze.net');
|
||||
config()->set('vision.vector_gateway.api_key', 'test-key');
|
||||
config()->set('vision.vector_gateway.upsert_endpoint', '/vectors/upsert');
|
||||
config()->set('vision.vector_gateway.upsert_file_endpoint', '/vectors/upsert/file');
|
||||
config()->set('vision.vector_gateway.search_endpoint', '/vectors/search');
|
||||
config()->set('vision.vector_gateway.search_point_endpoint', '/vectors/search/point');
|
||||
config()->set('vision.image_variant', 'md');
|
||||
config()->set('cdn.files_url', 'https://files.skinbase.org');
|
||||
});
|
||||
@@ -49,35 +51,110 @@ it('indexes artworks into the vector gateway with artwork metadata', function ()
|
||||
$artwork->tags()->attach($tag->id, ['source' => 'ai', 'confidence' => 0.88]);
|
||||
|
||||
Http::fake([
|
||||
'https://vision.klevze.net/vectors/upsert' => Http::response(['ok' => true], 200),
|
||||
'https://files.skinbase.org/*'
|
||||
=> Http::response(
|
||||
'fake-image-bytes',
|
||||
200,
|
||||
['Content-Type' => 'image/webp']
|
||||
),
|
||||
|
||||
'https://vision.klevze.net/vectors/upsert/file'
|
||||
=> Http::response([
|
||||
'id' => $artwork->id,
|
||||
'collection' => 'images',
|
||||
'dim' => 512,
|
||||
], 200),
|
||||
]);
|
||||
|
||||
artisan('artworks:vectors-index', ['--limit' => 1])
|
||||
->assertSuccessful();
|
||||
|
||||
$artwork->refresh();
|
||||
expect($artwork->last_vector_indexed_at)->not->toBeNull();
|
||||
|
||||
expect($artwork->last_vector_indexed_at)
|
||||
->not->toBeNull();
|
||||
|
||||
Http::assertSentCount(2);
|
||||
|
||||
Http::assertSent(function ($request): bool {
|
||||
return $request->method() === 'GET'
|
||||
&& $request->url()
|
||||
=== 'https://files.skinbase.org/artworks/md/aa/bb/aabbcc112233.webp';
|
||||
});
|
||||
|
||||
Http::assertSent(function ($request) use ($artwork): bool {
|
||||
if ($request->url() !== 'https://vision.klevze.net/vectors/upsert') {
|
||||
if (
|
||||
$request->url()
|
||||
!== 'https://vision.klevze.net/vectors/upsert/file'
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$payload = json_decode($request->body(), true);
|
||||
$parts = collect(
|
||||
$request->data()
|
||||
)->keyBy('name');
|
||||
|
||||
return $request->hasHeader('X-API-Key', 'test-key')
|
||||
&& is_array($payload)
|
||||
&& ($payload['id'] ?? null) === (string) $artwork->id
|
||||
&& ($payload['url'] ?? null) === 'https://files.skinbase.org/artworks/md/aa/bb/aabbcc112233.webp'
|
||||
&& ($payload['metadata']['content_type'] ?? null) === 'Photography'
|
||||
&& ($payload['metadata']['category'] ?? null) === 'Abstract'
|
||||
&& ($payload['metadata']['tags'] ?? null) === ['skyline']
|
||||
&& array_key_exists('is_public', $payload['metadata'])
|
||||
&& array_key_exists('is_deleted', $payload['metadata'])
|
||||
&& array_key_exists('is_nsfw', $payload['metadata'])
|
||||
&& array_key_exists('category_id', $payload['metadata'])
|
||||
&& array_key_exists('content_type_id', $payload['metadata'])
|
||||
&& array_key_exists('status', $payload['metadata']);
|
||||
$idPart = $parts->get('id', []);
|
||||
$metadataPart = $parts->get(
|
||||
'metadata_json',
|
||||
[]
|
||||
);
|
||||
$filePart = $parts->get('file', []);
|
||||
|
||||
$metadata = json_decode(
|
||||
(string) (
|
||||
$metadataPart['contents'] ?? ''
|
||||
),
|
||||
true
|
||||
);
|
||||
|
||||
return $request->hasHeader(
|
||||
'X-API-Key',
|
||||
'test-key'
|
||||
)
|
||||
&& (string) (
|
||||
$idPart['contents'] ?? ''
|
||||
) === (string) $artwork->id
|
||||
&& (string) (
|
||||
$filePart['contents'] ?? ''
|
||||
) === 'fake-image-bytes'
|
||||
&& (string) (
|
||||
$filePart['filename'] ?? ''
|
||||
) === sprintf(
|
||||
'artwork-%d.webp',
|
||||
$artwork->id
|
||||
)
|
||||
&& is_array($metadata)
|
||||
&& ($metadata['content_type'] ?? null)
|
||||
=== 'Photography'
|
||||
&& ($metadata['category'] ?? null)
|
||||
=== 'Abstract'
|
||||
&& ($metadata['tags'] ?? null)
|
||||
=== ['skyline']
|
||||
&& array_key_exists(
|
||||
'is_public',
|
||||
$metadata
|
||||
)
|
||||
&& array_key_exists(
|
||||
'is_deleted',
|
||||
$metadata
|
||||
)
|
||||
&& array_key_exists(
|
||||
'is_nsfw',
|
||||
$metadata
|
||||
)
|
||||
&& array_key_exists(
|
||||
'category_id',
|
||||
$metadata
|
||||
)
|
||||
&& array_key_exists(
|
||||
'content_type_id',
|
||||
$metadata
|
||||
)
|
||||
&& array_key_exists(
|
||||
'status',
|
||||
$metadata
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -113,12 +190,19 @@ it('searches similar artworks through the vector gateway', function (): void {
|
||||
$similar->categories()->attach($category->id);
|
||||
|
||||
Http::fake([
|
||||
'https://vision.klevze.net/vectors/search' => Http::response([
|
||||
'results' => [
|
||||
['id' => $source->id, 'score' => 1.0],
|
||||
['id' => $similar->id, 'score' => 0.9876],
|
||||
],
|
||||
], 200),
|
||||
'https://vision.klevze.net/vectors/search/point'
|
||||
=> Http::response([
|
||||
'results' => [
|
||||
[
|
||||
'id' => $source->id,
|
||||
'score' => 1.0,
|
||||
],
|
||||
[
|
||||
'id' => $similar->id,
|
||||
'score' => 0.9876,
|
||||
],
|
||||
],
|
||||
], 200),
|
||||
]);
|
||||
|
||||
artisan('artworks:vectors-search', [
|
||||
@@ -133,6 +217,25 @@ it('searches similar artworks through the vector gateway', function (): void {
|
||||
'category' => 'Nature',
|
||||
]])
|
||||
->assertSuccessful();
|
||||
|
||||
Http::assertSentCount(1);
|
||||
|
||||
Http::assertSent(
|
||||
function ($request) use ($source): bool {
|
||||
return $request->url()
|
||||
=== 'https://vision.klevze.net/vectors/search/point'
|
||||
&& ($request->data()['id'] ?? null)
|
||||
=== $source->id;
|
||||
}
|
||||
);
|
||||
|
||||
Http::assertNotSent(
|
||||
fn ($request): bool =>
|
||||
str_contains(
|
||||
$request->url(),
|
||||
'files.skinbase.org'
|
||||
)
|
||||
);
|
||||
});
|
||||
|
||||
it('can re-upsert only artworks that already have local embeddings', function (): void {
|
||||
@@ -184,28 +287,81 @@ it('can re-upsert only artworks that already have local embeddings', function ()
|
||||
$nonEmbeddedArtwork->categories()->attach($category->id);
|
||||
|
||||
Http::fake([
|
||||
'https://vision.klevze.net/vectors/upsert' => Http::response(['ok' => true], 200),
|
||||
'https://files.skinbase.org/*'
|
||||
=> Http::response(
|
||||
'fake-image-bytes',
|
||||
200,
|
||||
['Content-Type' => 'image/webp']
|
||||
),
|
||||
|
||||
'https://vision.klevze.net/vectors/upsert/file'
|
||||
=> Http::response([
|
||||
'id' => $embeddedArtwork->id,
|
||||
'collection' => 'images',
|
||||
'dim' => 512,
|
||||
], 200),
|
||||
]);
|
||||
|
||||
artisan('artworks:vectors-index', ['--embedded-only' => true, '--limit' => 10])
|
||||
->assertSuccessful();
|
||||
artisan('artworks:vectors-index', [
|
||||
'--embedded-only' => true,
|
||||
'--limit' => 10,
|
||||
])->assertSuccessful();
|
||||
|
||||
$embeddedArtwork->refresh();
|
||||
$nonEmbeddedArtwork->refresh();
|
||||
|
||||
expect($embeddedArtwork->last_vector_indexed_at)->not->toBeNull()
|
||||
->and($nonEmbeddedArtwork->last_vector_indexed_at)->toBeNull();
|
||||
expect($embeddedArtwork->last_vector_indexed_at)
|
||||
->not->toBeNull()
|
||||
->and(
|
||||
$nonEmbeddedArtwork->last_vector_indexed_at
|
||||
)
|
||||
->toBeNull();
|
||||
|
||||
Http::assertSentCount(1);
|
||||
Http::assertSent(function ($request) use ($embeddedArtwork): bool {
|
||||
if ($request->url() !== 'https://vision.klevze.net/vectors/upsert') {
|
||||
return false;
|
||||
/*
|
||||
* One indexed artwork:
|
||||
* 1x CDN image download
|
||||
* 1x /vectors/upsert/file
|
||||
*/
|
||||
Http::assertSentCount(2);
|
||||
|
||||
Http::assertSent(
|
||||
function ($request): bool {
|
||||
return $request->method() === 'GET'
|
||||
&& $request->url()
|
||||
=== 'https://files.skinbase.org/artworks/md/11/22/112233445566.webp';
|
||||
}
|
||||
);
|
||||
|
||||
$payload = json_decode($request->body(), true);
|
||||
Http::assertSent(
|
||||
function ($request) use (
|
||||
$embeddedArtwork
|
||||
): bool {
|
||||
if (
|
||||
$request->url()
|
||||
!== 'https://vision.klevze.net/vectors/upsert/file'
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return is_array($payload)
|
||||
&& ($payload['id'] ?? null) === (string) $embeddedArtwork->id
|
||||
&& ($payload['url'] ?? null) === 'https://files.skinbase.org/artworks/md/11/22/112233445566.webp';
|
||||
});
|
||||
$parts = collect(
|
||||
$request->data()
|
||||
)->keyBy('name');
|
||||
|
||||
$idPart = $parts->get('id', []);
|
||||
$filePart = $parts->get('file', []);
|
||||
|
||||
return (string) (
|
||||
$idPart['contents'] ?? ''
|
||||
) === (string) $embeddedArtwork->id
|
||||
&& (string) (
|
||||
$filePart['contents'] ?? ''
|
||||
) === 'fake-image-bytes'
|
||||
&& (string) (
|
||||
$filePart['filename'] ?? ''
|
||||
) === sprintf(
|
||||
'artwork-%d.webp',
|
||||
$embeddedArtwork->id
|
||||
);
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user