Implement creator studio and upload updates
This commit is contained in:
@@ -0,0 +1,43 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Moderation;
|
||||
|
||||
use App\Models\ContentModerationActionLog;
|
||||
use App\Models\ContentModerationFinding;
|
||||
use App\Models\User;
|
||||
|
||||
class ContentModerationActionLogService
|
||||
{
|
||||
/**
|
||||
* @param array<string, mixed>|null $meta
|
||||
*/
|
||||
public function log(
|
||||
?ContentModerationFinding $finding,
|
||||
string $targetType,
|
||||
?int $targetId,
|
||||
string $actionType,
|
||||
?User $actor = null,
|
||||
?string $oldStatus = null,
|
||||
?string $newStatus = null,
|
||||
?string $oldVisibility = null,
|
||||
?string $newVisibility = null,
|
||||
?string $notes = null,
|
||||
?array $meta = null,
|
||||
): ContentModerationActionLog {
|
||||
return ContentModerationActionLog::query()->create([
|
||||
'finding_id' => $finding?->id,
|
||||
'target_type' => $targetType,
|
||||
'target_id' => $targetId,
|
||||
'action_type' => $actionType,
|
||||
'actor_type' => $actor ? 'admin' : 'system',
|
||||
'actor_id' => $actor?->id,
|
||||
'old_status' => $oldStatus,
|
||||
'new_status' => $newStatus,
|
||||
'old_visibility' => $oldVisibility,
|
||||
'new_visibility' => $newVisibility,
|
||||
'notes' => $notes,
|
||||
'meta_json' => $meta,
|
||||
'created_at' => \now(),
|
||||
]);
|
||||
}
|
||||
}
|
||||
182
app/Services/Moderation/ContentModerationPersistenceService.php
Normal file
182
app/Services/Moderation/ContentModerationPersistenceService.php
Normal file
@@ -0,0 +1,182 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Moderation;
|
||||
|
||||
use App\Data\Moderation\ModerationResultData;
|
||||
use App\Enums\ModerationEscalationStatus;
|
||||
use App\Enums\ModerationContentType;
|
||||
use App\Enums\ModerationStatus;
|
||||
use App\Models\ContentModerationAiSuggestion;
|
||||
use App\Models\ContentModerationFinding;
|
||||
|
||||
class ContentModerationPersistenceService
|
||||
{
|
||||
public function __construct(
|
||||
private readonly ContentModerationReviewService $review,
|
||||
private readonly ContentModerationActionLogService $actionLogs,
|
||||
) {
|
||||
}
|
||||
|
||||
public function shouldQueue(ModerationResultData $result): bool
|
||||
{
|
||||
return $result->score >= (int) \app('config')->get('content_moderation.queue_threshold', 30);
|
||||
}
|
||||
|
||||
public function hasCurrentFinding(string $contentType, int $contentId, string $contentHash, string $scannerVersion): bool
|
||||
{
|
||||
return ContentModerationFinding::query()
|
||||
->where('content_type', $contentType)
|
||||
->where('content_id', $contentId)
|
||||
->where('content_hash', $contentHash)
|
||||
->where('scanner_version', $scannerVersion)
|
||||
->exists();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $context
|
||||
* @return array{finding:?ContentModerationFinding, created:bool, updated:bool}
|
||||
*/
|
||||
public function persist(ModerationResultData $result, array $context): array
|
||||
{
|
||||
$contentType = (string) ($context['content_type'] ?? '');
|
||||
$contentId = (int) ($context['content_id'] ?? 0);
|
||||
|
||||
if ($contentType === '' || $contentId <= 0) {
|
||||
return ['finding' => null, 'created' => false, 'updated' => false];
|
||||
}
|
||||
|
||||
$existing = ContentModerationFinding::query()
|
||||
->where('content_type', $contentType)
|
||||
->where('content_id', $contentId)
|
||||
->where('content_hash', $result->contentHash)
|
||||
->where('scanner_version', $result->scannerVersion)
|
||||
->first();
|
||||
|
||||
if (! $this->shouldQueue($result) && $existing === null) {
|
||||
return ['finding' => null, 'created' => false, 'updated' => false];
|
||||
}
|
||||
|
||||
$finding = $existing ?? new ContentModerationFinding();
|
||||
$isNew = ! $finding->exists;
|
||||
|
||||
$finding->fill([
|
||||
'content_type' => $contentType,
|
||||
'content_id' => $contentId,
|
||||
'content_target_type' => $result->contentTargetType,
|
||||
'content_target_id' => $result->contentTargetId,
|
||||
'artwork_id' => $context['artwork_id'] ?? null,
|
||||
'user_id' => $context['user_id'] ?? null,
|
||||
'severity' => $result->severity->value,
|
||||
'score' => $result->score,
|
||||
'content_hash' => $result->contentHash,
|
||||
'scanner_version' => $result->scannerVersion,
|
||||
'reasons_json' => $result->reasons,
|
||||
'matched_links_json' => $result->matchedLinks,
|
||||
'matched_domains_json' => $result->matchedDomains,
|
||||
'matched_keywords_json' => $result->matchedKeywords,
|
||||
'rule_hits_json' => $result->ruleHits,
|
||||
'score_breakdown_json' => $result->scoreBreakdown,
|
||||
'content_hash_normalized' => $result->contentHashNormalized,
|
||||
'group_key' => $result->groupKey,
|
||||
'campaign_key' => $result->campaignKey,
|
||||
'cluster_score' => $result->clusterScore,
|
||||
'cluster_reason' => $result->clusterReason,
|
||||
'priority_score' => $result->priorityScore,
|
||||
'policy_name' => $result->policyName,
|
||||
'review_bucket' => $result->reviewBucket,
|
||||
'escalation_status' => $result->escalationStatus ?? ModerationEscalationStatus::None->value,
|
||||
'ai_provider' => $result->aiProvider,
|
||||
'ai_label' => $result->aiLabel,
|
||||
'ai_suggested_action' => $result->aiSuggestedAction,
|
||||
'ai_confidence' => $result->aiConfidence,
|
||||
'ai_explanation' => $result->aiExplanation,
|
||||
'user_risk_score' => $result->userRiskScore,
|
||||
'content_snapshot' => (string) ($context['content_snapshot'] ?? ''),
|
||||
'auto_action_taken' => $result->autoHideRecommended ? 'recommended' : null,
|
||||
]);
|
||||
|
||||
if ($isNew) {
|
||||
$finding->status = $result->status;
|
||||
} elseif (! $this->shouldQueue($result) && $finding->isPending()) {
|
||||
$finding->status = ModerationStatus::Resolved;
|
||||
$finding->action_taken = 'rescanned_clean';
|
||||
$finding->resolved_at = \now();
|
||||
$finding->resolved_by = null;
|
||||
}
|
||||
|
||||
$finding->save();
|
||||
|
||||
if ($result->aiProvider !== null && ($result->aiLabel !== null || $result->aiExplanation !== null || $result->aiConfidence !== null)) {
|
||||
ContentModerationAiSuggestion::query()->create([
|
||||
'finding_id' => $finding->id,
|
||||
'provider' => $result->aiProvider,
|
||||
'suggested_label' => $result->aiLabel,
|
||||
'suggested_action' => $result->aiSuggestedAction,
|
||||
'confidence' => $result->aiConfidence,
|
||||
'explanation' => $result->aiExplanation,
|
||||
'campaign_tags_json' => $result->campaignKey ? [$result->campaignKey] : [],
|
||||
'raw_response_json' => $result->aiRawResponse,
|
||||
'created_at' => now(),
|
||||
]);
|
||||
}
|
||||
|
||||
if (! $isNew && ! $this->shouldQueue($result) && $finding->action_taken === 'rescanned_clean') {
|
||||
$this->actionLogs->log(
|
||||
$finding,
|
||||
'finding',
|
||||
$finding->id,
|
||||
'rescan',
|
||||
null,
|
||||
ModerationStatus::Pending->value,
|
||||
ModerationStatus::Resolved->value,
|
||||
null,
|
||||
null,
|
||||
'Finding resolved automatically after a clean rescan.',
|
||||
['scanner_version' => $result->scannerVersion],
|
||||
);
|
||||
}
|
||||
|
||||
return [
|
||||
'finding' => $finding,
|
||||
'created' => $isNew,
|
||||
'updated' => ! $isNew,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $context
|
||||
*/
|
||||
public function applyAutomatedActionIfNeeded(ContentModerationFinding $finding, ModerationResultData $result, array $context): bool
|
||||
{
|
||||
if (! $result->autoHideRecommended) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if ($finding->is_auto_hidden) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$supportedTypes = (array) \app('config')->get('content_moderation.auto_hide.supported_types', []);
|
||||
if (! in_array($finding->content_type->value, $supportedTypes, true)) {
|
||||
$finding->forceFill([
|
||||
'auto_action_taken' => 'recommended_review',
|
||||
])->save();
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
if (! in_array($finding->content_type, [ModerationContentType::ArtworkComment, ModerationContentType::ArtworkDescription], true)) {
|
||||
if ($finding->content_type !== ModerationContentType::ArtworkTitle) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (! in_array($finding->content_type, [ModerationContentType::ArtworkComment, ModerationContentType::ArtworkDescription, ModerationContentType::ArtworkTitle], true)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$this->review->autoHideContent($finding, 'Triggered by automated moderation threshold.');
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Moderation;
|
||||
|
||||
use App\Models\ContentModerationFinding;
|
||||
|
||||
class ContentModerationProcessingService
|
||||
{
|
||||
public function __construct(
|
||||
private readonly ContentModerationService $moderation,
|
||||
private readonly ContentModerationPersistenceService $persistence,
|
||||
private readonly DomainReputationService $domains,
|
||||
private readonly ModerationClusterService $clusters,
|
||||
private readonly DomainIntelligenceService $domainIntelligence,
|
||||
) {
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $context
|
||||
* @return array{result:\App\Data\Moderation\ModerationResultData,finding:?ContentModerationFinding,created:bool,updated:bool,auto_hidden:bool}
|
||||
*/
|
||||
public function process(string $content, array $context, bool $persist = true): array
|
||||
{
|
||||
$result = $this->moderation->analyze($content, $context);
|
||||
$this->domains->trackDomains(
|
||||
$result->matchedDomains,
|
||||
$this->persistence->shouldQueue($result),
|
||||
false,
|
||||
);
|
||||
|
||||
if (! $persist) {
|
||||
return [
|
||||
'result' => $result,
|
||||
'finding' => null,
|
||||
'created' => false,
|
||||
'updated' => false,
|
||||
'auto_hidden' => false,
|
||||
];
|
||||
}
|
||||
|
||||
$persisted = $this->persistence->persist($result, $context);
|
||||
$finding = $persisted['finding'];
|
||||
|
||||
if ($finding !== null) {
|
||||
$this->domains->attachDomainIds($finding);
|
||||
$autoHidden = $this->persistence->applyAutomatedActionIfNeeded($finding, $result, $context);
|
||||
$this->clusters->syncFinding($finding->fresh());
|
||||
foreach ($result->matchedDomains as $domain) {
|
||||
$this->domainIntelligence->refreshDomain($domain);
|
||||
}
|
||||
|
||||
return [
|
||||
'result' => $result,
|
||||
'finding' => $finding->fresh(),
|
||||
'created' => $persisted['created'],
|
||||
'updated' => $persisted['updated'],
|
||||
'auto_hidden' => $autoHidden,
|
||||
];
|
||||
}
|
||||
|
||||
return [
|
||||
'result' => $result,
|
||||
'finding' => null,
|
||||
'created' => false,
|
||||
'updated' => false,
|
||||
'auto_hidden' => false,
|
||||
];
|
||||
}
|
||||
|
||||
public function rescanFinding(ContentModerationFinding $finding, ContentModerationSourceService $sources): ?ContentModerationFinding
|
||||
{
|
||||
$resolved = $sources->contextForFinding($finding);
|
||||
if ($resolved['context'] === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$result = $this->process((string) $resolved['context']['content_snapshot'], $resolved['context'], true);
|
||||
|
||||
return $result['finding'];
|
||||
}
|
||||
}
|
||||
292
app/Services/Moderation/ContentModerationReviewService.php
Normal file
292
app/Services/Moderation/ContentModerationReviewService.php
Normal file
@@ -0,0 +1,292 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Moderation;
|
||||
|
||||
use App\Enums\ModerationActionType;
|
||||
use App\Enums\ModerationContentType;
|
||||
use App\Enums\ModerationStatus;
|
||||
use App\Models\Artwork;
|
||||
use App\Models\ArtworkComment;
|
||||
use App\Models\ContentModerationFinding;
|
||||
use App\Models\User;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
|
||||
class ContentModerationReviewService
|
||||
{
|
||||
public function __construct(
|
||||
private readonly ContentModerationActionLogService $actionLogs,
|
||||
private readonly DomainReputationService $domains,
|
||||
private readonly ModerationFeedbackService $feedback,
|
||||
) {
|
||||
}
|
||||
|
||||
public function markSafe(ContentModerationFinding $finding, User $reviewer, ?string $notes = null): void
|
||||
{
|
||||
$this->updateFinding($finding, ModerationStatus::ReviewedSafe, $reviewer, $notes, ModerationActionType::MarkSafe);
|
||||
$this->feedback->record($finding->fresh(), 'marked_safe', $reviewer, $notes);
|
||||
}
|
||||
|
||||
public function confirmSpam(ContentModerationFinding $finding, User $reviewer, ?string $notes = null): void
|
||||
{
|
||||
$this->updateFinding($finding, ModerationStatus::ConfirmedSpam, $reviewer, $notes, ModerationActionType::ConfirmSpam);
|
||||
$this->domains->trackDomains((array) $finding->matched_domains_json, true, true);
|
||||
$this->feedback->record($finding->fresh(), 'confirmed_spam', $reviewer, $notes);
|
||||
}
|
||||
|
||||
public function ignore(ContentModerationFinding $finding, User $reviewer, ?string $notes = null): void
|
||||
{
|
||||
$this->updateFinding($finding, ModerationStatus::Ignored, $reviewer, $notes, ModerationActionType::Ignore);
|
||||
}
|
||||
|
||||
public function resolve(ContentModerationFinding $finding, User $reviewer, ?string $notes = null): void
|
||||
{
|
||||
$this->updateFinding($finding, ModerationStatus::Resolved, $reviewer, $notes, ModerationActionType::Resolve);
|
||||
$this->feedback->record($finding->fresh(), 'resolved', $reviewer, $notes);
|
||||
}
|
||||
|
||||
public function markFalsePositive(ContentModerationFinding $finding, User $reviewer, ?string $notes = null): void
|
||||
{
|
||||
DB::transaction(function () use ($finding, $reviewer, $notes): void {
|
||||
$oldVisibility = null;
|
||||
$newVisibility = null;
|
||||
|
||||
if (in_array($finding->content_type, [ModerationContentType::ArtworkComment, ModerationContentType::ArtworkDescription, ModerationContentType::ArtworkTitle], true)) {
|
||||
[$action, $oldVisibility, $newVisibility] = match ($finding->content_type) {
|
||||
ModerationContentType::ArtworkComment => $this->restoreComment($finding),
|
||||
default => $this->restoreArtwork($finding),
|
||||
};
|
||||
|
||||
$actionType = $action;
|
||||
} else {
|
||||
$actionType = ModerationActionType::MarkFalsePositive;
|
||||
}
|
||||
|
||||
$oldStatus = $finding->status->value;
|
||||
$finding->forceFill([
|
||||
'status' => ModerationStatus::ReviewedSafe,
|
||||
'reviewed_by' => $reviewer->id,
|
||||
'reviewed_at' => now(),
|
||||
'resolved_by' => $reviewer->id,
|
||||
'resolved_at' => now(),
|
||||
'restored_by' => $oldVisibility !== null ? $reviewer->id : $finding->restored_by,
|
||||
'restored_at' => $oldVisibility !== null ? now() : $finding->restored_at,
|
||||
'is_auto_hidden' => false,
|
||||
'is_false_positive' => true,
|
||||
'false_positive_count' => ((int) $finding->false_positive_count) + 1,
|
||||
'action_taken' => ModerationActionType::MarkFalsePositive->value,
|
||||
'admin_notes' => $this->normalizeNotes($notes, $finding->admin_notes),
|
||||
])->save();
|
||||
|
||||
$this->actionLogs->log(
|
||||
$finding,
|
||||
$finding->content_type->value,
|
||||
$finding->content_id,
|
||||
ModerationActionType::MarkFalsePositive->value,
|
||||
$reviewer,
|
||||
$oldStatus,
|
||||
ModerationStatus::ReviewedSafe->value,
|
||||
$oldVisibility,
|
||||
$newVisibility,
|
||||
$notes,
|
||||
['restored_action' => $actionType->value],
|
||||
);
|
||||
|
||||
$this->feedback->record($finding->fresh(), 'false_positive', $reviewer, $notes, ['restored' => $oldVisibility !== null]);
|
||||
});
|
||||
}
|
||||
|
||||
public function hideContent(ContentModerationFinding $finding, User $reviewer, ?string $notes = null): ModerationActionType
|
||||
{
|
||||
return DB::transaction(function () use ($finding, $reviewer, $notes): ModerationActionType {
|
||||
[$action, $oldVisibility, $newVisibility] = match ($finding->content_type) {
|
||||
ModerationContentType::ArtworkComment => $this->hideComment($finding, false),
|
||||
ModerationContentType::ArtworkDescription, ModerationContentType::ArtworkTitle => $this->hideArtwork($finding, false),
|
||||
};
|
||||
|
||||
$this->updateFinding($finding, ModerationStatus::ConfirmedSpam, $reviewer, $notes, $action, $oldVisibility, $newVisibility);
|
||||
$this->domains->trackDomains((array) $finding->matched_domains_json, true, true);
|
||||
|
||||
return $action;
|
||||
});
|
||||
}
|
||||
|
||||
public function autoHideContent(ContentModerationFinding $finding, ?string $notes = null): ModerationActionType
|
||||
{
|
||||
return DB::transaction(function () use ($finding, $notes): ModerationActionType {
|
||||
[$action, $oldVisibility, $newVisibility] = match ($finding->content_type) {
|
||||
ModerationContentType::ArtworkComment => $this->hideComment($finding, true),
|
||||
ModerationContentType::ArtworkDescription, ModerationContentType::ArtworkTitle => $this->hideArtwork($finding, true),
|
||||
};
|
||||
|
||||
$finding->forceFill([
|
||||
'is_auto_hidden' => true,
|
||||
'auto_action_taken' => $action->value,
|
||||
'auto_hidden_at' => \now(),
|
||||
'action_taken' => $action->value,
|
||||
'admin_notes' => $this->normalizeNotes($notes, $finding->admin_notes),
|
||||
])->save();
|
||||
|
||||
$this->actionLogs->log(
|
||||
$finding,
|
||||
$finding->content_type->value,
|
||||
$finding->content_id,
|
||||
$action->value,
|
||||
null,
|
||||
$finding->status->value,
|
||||
$finding->status->value,
|
||||
$oldVisibility,
|
||||
$newVisibility,
|
||||
$notes,
|
||||
['automated' => true],
|
||||
);
|
||||
|
||||
$this->domains->trackDomains((array) $finding->matched_domains_json, true, false);
|
||||
|
||||
return $action;
|
||||
});
|
||||
}
|
||||
|
||||
public function restoreContent(ContentModerationFinding $finding, User $reviewer, ?string $notes = null): ModerationActionType
|
||||
{
|
||||
return DB::transaction(function () use ($finding, $reviewer, $notes): ModerationActionType {
|
||||
[$action, $oldVisibility, $newVisibility] = match ($finding->content_type) {
|
||||
ModerationContentType::ArtworkComment => $this->restoreComment($finding),
|
||||
ModerationContentType::ArtworkDescription, ModerationContentType::ArtworkTitle => $this->restoreArtwork($finding),
|
||||
};
|
||||
|
||||
$oldStatus = $finding->status->value;
|
||||
|
||||
$finding->forceFill([
|
||||
'status' => ModerationStatus::ReviewedSafe,
|
||||
'reviewed_by' => $reviewer->id,
|
||||
'reviewed_at' => \now(),
|
||||
'resolved_by' => $reviewer->id,
|
||||
'resolved_at' => \now(),
|
||||
'restored_by' => $reviewer->id,
|
||||
'restored_at' => \now(),
|
||||
'is_auto_hidden' => false,
|
||||
'action_taken' => $action->value,
|
||||
'admin_notes' => $this->normalizeNotes($notes, $finding->admin_notes),
|
||||
])->save();
|
||||
|
||||
$this->actionLogs->log(
|
||||
$finding,
|
||||
$finding->content_type->value,
|
||||
$finding->content_id,
|
||||
$action->value,
|
||||
$reviewer,
|
||||
$oldStatus,
|
||||
ModerationStatus::ReviewedSafe->value,
|
||||
$oldVisibility,
|
||||
$newVisibility,
|
||||
$notes,
|
||||
['restored' => true],
|
||||
);
|
||||
|
||||
return $action;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{0:ModerationActionType,1:string,2:string}
|
||||
*/
|
||||
private function hideComment(ContentModerationFinding $finding, bool $automated): array
|
||||
{
|
||||
$comment = ArtworkComment::query()->find($finding->content_id);
|
||||
$oldVisibility = $comment && $comment->is_approved ? 'visible' : 'hidden';
|
||||
|
||||
if ($comment) {
|
||||
$comment->forceFill(['is_approved' => false])->save();
|
||||
}
|
||||
|
||||
return [$automated ? ModerationActionType::AutoHideComment : ModerationActionType::HideComment, $oldVisibility, 'hidden'];
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{0:ModerationActionType,1:string,2:string}
|
||||
*/
|
||||
private function hideArtwork(ContentModerationFinding $finding, bool $automated): array
|
||||
{
|
||||
$artworkId = (int) ($finding->artwork_id ?? $finding->content_id);
|
||||
$artwork = Artwork::query()->find($artworkId);
|
||||
$oldVisibility = $artwork && $artwork->is_public ? 'visible' : 'hidden';
|
||||
|
||||
if ($artwork) {
|
||||
$artwork->forceFill(['is_public' => false])->save();
|
||||
}
|
||||
|
||||
return [$automated ? ModerationActionType::AutoHideArtwork : ModerationActionType::HideArtwork, $oldVisibility, 'hidden'];
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{0:ModerationActionType,1:string,2:string}
|
||||
*/
|
||||
private function restoreComment(ContentModerationFinding $finding): array
|
||||
{
|
||||
$comment = ArtworkComment::query()->find($finding->content_id);
|
||||
$oldVisibility = $comment && $comment->is_approved ? 'visible' : 'hidden';
|
||||
|
||||
if ($comment) {
|
||||
$comment->forceFill(['is_approved' => true])->save();
|
||||
}
|
||||
|
||||
return [ModerationActionType::RestoreComment, $oldVisibility, 'visible'];
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{0:ModerationActionType,1:string,2:string}
|
||||
*/
|
||||
private function restoreArtwork(ContentModerationFinding $finding): array
|
||||
{
|
||||
$artworkId = (int) ($finding->artwork_id ?? $finding->content_id);
|
||||
$artwork = Artwork::query()->find($artworkId);
|
||||
$oldVisibility = $artwork && $artwork->is_public ? 'visible' : 'hidden';
|
||||
|
||||
if ($artwork) {
|
||||
$artwork->forceFill(['is_public' => true])->save();
|
||||
}
|
||||
|
||||
return [ModerationActionType::RestoreArtwork, $oldVisibility, 'visible'];
|
||||
}
|
||||
|
||||
private function updateFinding(
|
||||
ContentModerationFinding $finding,
|
||||
ModerationStatus $status,
|
||||
User $reviewer,
|
||||
?string $notes,
|
||||
ModerationActionType $action,
|
||||
?string $oldVisibility = null,
|
||||
?string $newVisibility = null,
|
||||
): void {
|
||||
$oldStatus = $finding->status->value;
|
||||
$finding->forceFill([
|
||||
'status' => $status,
|
||||
'reviewed_by' => $reviewer->id,
|
||||
'reviewed_at' => \now(),
|
||||
'resolved_by' => $reviewer->id,
|
||||
'resolved_at' => \now(),
|
||||
'action_taken' => $action->value,
|
||||
'admin_notes' => $this->normalizeNotes($notes, $finding->admin_notes),
|
||||
])->save();
|
||||
|
||||
$this->actionLogs->log(
|
||||
$finding,
|
||||
$finding->content_type->value,
|
||||
$finding->content_id,
|
||||
$action->value,
|
||||
$reviewer,
|
||||
$oldStatus,
|
||||
$status->value,
|
||||
$oldVisibility,
|
||||
$newVisibility,
|
||||
$notes,
|
||||
);
|
||||
}
|
||||
|
||||
private function normalizeNotes(?string $incoming, ?string $existing): ?string
|
||||
{
|
||||
$normalized = is_string($incoming) ? trim($incoming) : '';
|
||||
|
||||
return $normalized !== '' ? $normalized : $existing;
|
||||
}
|
||||
}
|
||||
203
app/Services/Moderation/ContentModerationService.php
Normal file
203
app/Services/Moderation/ContentModerationService.php
Normal file
@@ -0,0 +1,203 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Moderation;
|
||||
|
||||
use App\Contracts\Moderation\ModerationRuleInterface;
|
||||
use App\Data\Moderation\ModerationResultData;
|
||||
use App\Enums\ModerationSeverity;
|
||||
use App\Enums\ModerationStatus;
|
||||
use App\Services\Moderation\DuplicateDetectionService;
|
||||
use App\Services\Moderation\Rules\LinkPresenceRule;
|
||||
|
||||
class ContentModerationService
|
||||
{
|
||||
public function __construct(
|
||||
private readonly ModerationPolicyEngineService $policies,
|
||||
private readonly ModerationSuggestionService $suggestions,
|
||||
private readonly ModerationClusterService $clusters,
|
||||
private readonly ModerationPriorityService $priorities,
|
||||
) {
|
||||
}
|
||||
|
||||
public function analyze(string $content, array $context = []): ModerationResultData
|
||||
{
|
||||
$normalized = $this->normalize($content);
|
||||
$campaignNormalized = app(DuplicateDetectionService::class)->campaignText($content);
|
||||
$linkRule = app(LinkPresenceRule::class);
|
||||
$extractedUrls = $linkRule->extractUrls($content);
|
||||
$extractedDomains = array_values(array_unique(array_filter(array_map(
|
||||
static fn (string $url): ?string => $linkRule->extractHost($url),
|
||||
$extractedUrls
|
||||
))));
|
||||
|
||||
$riskAssessment = app(UserRiskScoreService::class)->assess(
|
||||
isset($context['user_id']) ? (int) $context['user_id'] : null,
|
||||
$extractedDomains,
|
||||
);
|
||||
|
||||
$context['extracted_urls'] = $extractedUrls;
|
||||
$context['extracted_domains'] = $extractedDomains;
|
||||
$context['user_risk_assessment'] = $riskAssessment;
|
||||
|
||||
$score = 0;
|
||||
$reasons = [];
|
||||
$matchedLinks = [];
|
||||
$matchedDomains = [];
|
||||
$matchedKeywords = [];
|
||||
$ruleHits = [];
|
||||
$scoreBreakdown = [];
|
||||
|
||||
foreach ($this->rules() as $rule) {
|
||||
foreach ($rule->analyze($content, $normalized, $context) as $finding) {
|
||||
$ruleScore = (int) ($finding['score'] ?? 0);
|
||||
$score += $ruleScore;
|
||||
$reason = (string) ($finding['reason'] ?? 'Flagged by moderation rule');
|
||||
$reasons[] = $reason;
|
||||
$matchedLinks = array_merge($matchedLinks, (array) ($finding['links'] ?? []));
|
||||
$matchedDomains = array_merge($matchedDomains, array_filter((array) ($finding['domains'] ?? [])));
|
||||
$matchedKeywords = array_merge($matchedKeywords, array_filter((array) ($finding['keywords'] ?? [])));
|
||||
$ruleKey = (string) ($finding['rule'] ?? 'unknown');
|
||||
$ruleHits[$ruleKey] = ($ruleHits[$ruleKey] ?? 0) + 1;
|
||||
$scoreBreakdown[] = [
|
||||
'rule' => $ruleKey,
|
||||
'score' => $ruleScore,
|
||||
'reason' => $reason,
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
$modifier = (int) ($riskAssessment['score_modifier'] ?? 0);
|
||||
if ($modifier !== 0) {
|
||||
$score += $modifier;
|
||||
$reasons[] = $modifier > 0
|
||||
? 'User risk profile increased moderation score by ' . $modifier
|
||||
: 'User trust profile reduced moderation score by ' . abs($modifier);
|
||||
$ruleHits['user_risk_modifier'] = 1;
|
||||
$scoreBreakdown[] = [
|
||||
'rule' => 'user_risk_modifier',
|
||||
'score' => $modifier,
|
||||
'reason' => $modifier > 0
|
||||
? 'User risk profile increased moderation score by ' . $modifier
|
||||
: 'User trust profile reduced moderation score by ' . abs($modifier),
|
||||
];
|
||||
}
|
||||
|
||||
$score = max(0, $score);
|
||||
$severity = ModerationSeverity::fromScore($score);
|
||||
$policy = $this->policies->resolve($context, $riskAssessment);
|
||||
$autoHideRecommended = $this->shouldAutoHide($score, $ruleHits, $matchedDomains ?: $extractedDomains, $policy);
|
||||
$groupKey = app(DuplicateDetectionService::class)->buildGroupKey($content, $matchedDomains ?: $extractedDomains);
|
||||
|
||||
$draft = new ModerationResultData(
|
||||
score: $score,
|
||||
severity: $severity,
|
||||
status: $score >= (int) ($policy['queue_threshold'] ?? app('config')->get('content_moderation.queue_threshold', 30))
|
||||
? ModerationStatus::Pending
|
||||
: ModerationStatus::ReviewedSafe,
|
||||
reasons: array_values(array_unique(array_filter($reasons))),
|
||||
matchedLinks: array_values(array_unique(array_filter($matchedLinks))),
|
||||
matchedDomains: array_values(array_unique(array_filter($matchedDomains))),
|
||||
matchedKeywords: array_values(array_unique(array_filter($matchedKeywords))),
|
||||
contentHash: hash('sha256', $normalized),
|
||||
scannerVersion: (string) app('config')->get('content_moderation.scanner_version', '1.0'),
|
||||
ruleHits: $ruleHits,
|
||||
contentHashNormalized: hash('sha256', $campaignNormalized),
|
||||
groupKey: $groupKey,
|
||||
userRiskScore: (int) ($riskAssessment['risk_score'] ?? 0),
|
||||
autoHideRecommended: $autoHideRecommended,
|
||||
contentTargetType: isset($context['content_target_type']) ? (string) $context['content_target_type'] : null,
|
||||
contentTargetId: isset($context['content_target_id']) ? (int) $context['content_target_id'] : null,
|
||||
policyName: (string) ($policy['name'] ?? 'default'),
|
||||
scoreBreakdown: $scoreBreakdown,
|
||||
);
|
||||
|
||||
$suggestion = $this->suggestions->suggest($content, $draft, $context);
|
||||
$cluster = $this->clusters->classify($content, $draft, $context, [
|
||||
'campaign_tags' => $suggestion->campaignTags,
|
||||
'confidence' => $suggestion->confidence,
|
||||
]);
|
||||
$priority = $this->priorities->score($draft, $context, $policy, [
|
||||
'confidence' => $suggestion->confidence,
|
||||
'campaign_tags' => $suggestion->campaignTags,
|
||||
]);
|
||||
|
||||
return new ModerationResultData(
|
||||
score: $score,
|
||||
severity: $severity,
|
||||
status: $score >= (int) ($policy['queue_threshold'] ?? app('config')->get('content_moderation.queue_threshold', 30))
|
||||
? ModerationStatus::Pending
|
||||
: ModerationStatus::ReviewedSafe,
|
||||
reasons: array_values(array_unique(array_filter($reasons))),
|
||||
matchedLinks: array_values(array_unique(array_filter($matchedLinks))),
|
||||
matchedDomains: array_values(array_unique(array_filter($matchedDomains))),
|
||||
matchedKeywords: array_values(array_unique(array_filter($matchedKeywords))),
|
||||
contentHash: hash('sha256', $normalized),
|
||||
scannerVersion: (string) app('config')->get('content_moderation.scanner_version', '1.0'),
|
||||
ruleHits: $ruleHits,
|
||||
contentHashNormalized: hash('sha256', $campaignNormalized),
|
||||
groupKey: $groupKey,
|
||||
userRiskScore: (int) ($riskAssessment['risk_score'] ?? 0),
|
||||
autoHideRecommended: $autoHideRecommended,
|
||||
contentTargetType: isset($context['content_target_type']) ? (string) $context['content_target_type'] : null,
|
||||
contentTargetId: isset($context['content_target_id']) ? (int) $context['content_target_id'] : null,
|
||||
campaignKey: $cluster['campaign_key'],
|
||||
clusterScore: $cluster['cluster_score'],
|
||||
clusterReason: $cluster['cluster_reason'],
|
||||
policyName: (string) ($policy['name'] ?? 'default'),
|
||||
priorityScore: (int) ($priority['priority_score'] ?? $score),
|
||||
reviewBucket: (string) ($priority['review_bucket'] ?? ($policy['review_bucket'] ?? 'standard')),
|
||||
escalationStatus: (string) ($priority['escalation_status'] ?? 'none'),
|
||||
aiProvider: $suggestion->provider,
|
||||
aiLabel: $suggestion->suggestedLabel,
|
||||
aiSuggestedAction: $suggestion->suggestedAction,
|
||||
aiConfidence: $suggestion->confidence,
|
||||
aiExplanation: $suggestion->explanation,
|
||||
aiRawResponse: $suggestion->rawResponse,
|
||||
scoreBreakdown: $scoreBreakdown,
|
||||
);
|
||||
}
|
||||
|
||||
public function normalize(string $content): string
|
||||
{
|
||||
$normalized = preg_replace('/\s+/u', ' ', trim($content));
|
||||
|
||||
return mb_strtolower((string) $normalized);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<int, ModerationRuleInterface>
|
||||
*/
|
||||
private function rules(): array
|
||||
{
|
||||
$classes = app('config')->get('content_moderation.rules.enabled', []);
|
||||
|
||||
return array_values(array_filter(array_map(function (string $class): ?ModerationRuleInterface {
|
||||
$rule = app($class);
|
||||
|
||||
return $rule instanceof ModerationRuleInterface ? $rule : null;
|
||||
}, $classes)));
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, int> $ruleHits
|
||||
* @param array<int, string> $matchedDomains
|
||||
*/
|
||||
private function shouldAutoHide(int $score, array $ruleHits, array $matchedDomains, array $policy = []): bool
|
||||
{
|
||||
if (! app('config')->get('content_moderation.auto_hide.enabled', true)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$threshold = (int) ($policy['auto_hide_threshold'] ?? app('config')->get('content_moderation.auto_hide.threshold', 95));
|
||||
if ($score >= $threshold) {
|
||||
return true;
|
||||
}
|
||||
|
||||
$blockedHit = isset($ruleHits['blacklisted_domain']) || isset($ruleHits['blocked_domain']);
|
||||
$severeHitCount = collect($ruleHits)
|
||||
->only(['blacklisted_domain', 'blocked_domain', 'high_risk_keyword', 'near_duplicate_campaign', 'duplicate_comment'])
|
||||
->sum();
|
||||
|
||||
return $blockedHit && $severeHitCount >= 2 && count($matchedDomains) >= 1;
|
||||
}
|
||||
}
|
||||
352
app/Services/Moderation/ContentModerationSourceService.php
Normal file
352
app/Services/Moderation/ContentModerationSourceService.php
Normal file
@@ -0,0 +1,352 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Moderation;
|
||||
|
||||
use App\Enums\ModerationContentType;
|
||||
use App\Models\Collection;
|
||||
use App\Models\NovaCard;
|
||||
use App\Models\Story;
|
||||
use App\Models\Artwork;
|
||||
use App\Models\ArtworkComment;
|
||||
use App\Models\ContentModerationFinding;
|
||||
use App\Models\UserProfile;
|
||||
use App\Models\UserSocialLink;
|
||||
use Illuminate\Database\Eloquent\Builder as EloquentBuilder;
|
||||
|
||||
class ContentModerationSourceService
|
||||
{
|
||||
public function queryForType(ModerationContentType $type): EloquentBuilder
|
||||
{
|
||||
return match ($type) {
|
||||
ModerationContentType::ArtworkComment => ArtworkComment::query()
|
||||
->with('artwork:id,title,slug,user_id')
|
||||
->whereNull('deleted_at')
|
||||
->where(function (EloquentBuilder $query): void {
|
||||
$query->whereNotNull('raw_content')->where('raw_content', '!=', '')
|
||||
->orWhere(function (EloquentBuilder $fallback): void {
|
||||
$fallback->whereNotNull('content')->where('content', '!=', '');
|
||||
});
|
||||
})
|
||||
->orderBy('id'),
|
||||
ModerationContentType::ArtworkDescription => Artwork::query()
|
||||
->whereNotNull('description')
|
||||
->where('description', '!=', '')
|
||||
->orderBy('id'),
|
||||
ModerationContentType::ArtworkTitle => Artwork::query()
|
||||
->whereNotNull('title')
|
||||
->where('title', '!=', '')
|
||||
->orderBy('id'),
|
||||
ModerationContentType::UserBio => UserProfile::query()
|
||||
->with('user:id,username,name')
|
||||
->where(function (EloquentBuilder $query): void {
|
||||
$query->whereNotNull('about')->where('about', '!=', '')
|
||||
->orWhere(function (EloquentBuilder $fallback): void {
|
||||
$fallback->whereNotNull('description')->where('description', '!=', '');
|
||||
});
|
||||
})
|
||||
->orderBy('user_id'),
|
||||
ModerationContentType::UserProfileLink => UserSocialLink::query()
|
||||
->with('user:id,username,name')
|
||||
->whereNotNull('url')
|
||||
->where('url', '!=', '')
|
||||
->orderBy('id'),
|
||||
ModerationContentType::CollectionTitle => Collection::query()
|
||||
->with('user:id,username,name')
|
||||
->whereNotNull('title')
|
||||
->where('title', '!=', '')
|
||||
->orderBy('id'),
|
||||
ModerationContentType::CollectionDescription => Collection::query()
|
||||
->with('user:id,username,name')
|
||||
->whereNotNull('description')
|
||||
->where('description', '!=', '')
|
||||
->orderBy('id'),
|
||||
ModerationContentType::StoryTitle => Story::query()
|
||||
->with('creator:id,username,name')
|
||||
->whereNotNull('title')
|
||||
->where('title', '!=', '')
|
||||
->orderBy('id'),
|
||||
ModerationContentType::StoryContent => Story::query()
|
||||
->with('creator:id,username,name')
|
||||
->whereNotNull('content')
|
||||
->where('content', '!=', '')
|
||||
->orderBy('id'),
|
||||
ModerationContentType::CardTitle => NovaCard::query()
|
||||
->with('user:id,username,name')
|
||||
->whereNotNull('title')
|
||||
->where('title', '!=', '')
|
||||
->orderBy('id'),
|
||||
ModerationContentType::CardText => NovaCard::query()
|
||||
->with('user:id,username,name')
|
||||
->where(function (EloquentBuilder $query): void {
|
||||
$query->whereNotNull('quote_text')->where('quote_text', '!=', '')
|
||||
->orWhere(function (EloquentBuilder $description): void {
|
||||
$description->whereNotNull('description')->where('description', '!=', '');
|
||||
});
|
||||
})
|
||||
->orderBy('id'),
|
||||
default => throw new \InvalidArgumentException('Unsupported moderation content type: ' . $type->value),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param Artwork|ArtworkComment|Collection|Story|NovaCard|UserProfile|UserSocialLink $row
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
public function buildContext(ModerationContentType $type, object $row): array
|
||||
{
|
||||
return match ($type) {
|
||||
ModerationContentType::ArtworkComment => [
|
||||
'content_type' => $type->value,
|
||||
'content_id' => (int) $row->id,
|
||||
'content_target_type' => 'artwork_comment',
|
||||
'content_target_id' => (int) $row->id,
|
||||
'artwork_id' => (int) $row->artwork_id,
|
||||
'user_id' => $row->user_id ? (int) $row->user_id : null,
|
||||
'content_snapshot' => (string) ($row->raw_content ?: $row->content),
|
||||
'is_publicly_exposed' => true,
|
||||
],
|
||||
ModerationContentType::ArtworkDescription => [
|
||||
'content_type' => $type->value,
|
||||
'content_id' => (int) $row->id,
|
||||
'content_target_type' => 'artwork',
|
||||
'content_target_id' => (int) $row->id,
|
||||
'artwork_id' => (int) $row->id,
|
||||
'user_id' => $row->user_id ? (int) $row->user_id : null,
|
||||
'content_snapshot' => (string) ($row->description ?? ''),
|
||||
'is_publicly_exposed' => (bool) ($row->is_public ?? false),
|
||||
],
|
||||
ModerationContentType::ArtworkTitle => [
|
||||
'content_type' => $type->value,
|
||||
'content_id' => (int) $row->id,
|
||||
'content_target_type' => 'artwork',
|
||||
'content_target_id' => (int) $row->id,
|
||||
'artwork_id' => (int) $row->id,
|
||||
'user_id' => $row->user_id ? (int) $row->user_id : null,
|
||||
'content_snapshot' => (string) ($row->title ?? ''),
|
||||
'is_publicly_exposed' => (bool) ($row->is_public ?? false),
|
||||
],
|
||||
ModerationContentType::UserBio => [
|
||||
'content_type' => $type->value,
|
||||
'content_id' => (int) $row->user_id,
|
||||
'content_target_type' => 'user_profile',
|
||||
'content_target_id' => (int) $row->user_id,
|
||||
'user_id' => (int) $row->user_id,
|
||||
'content_snapshot' => trim((string) ($row->about ?: $row->description ?: '')),
|
||||
'is_publicly_exposed' => true,
|
||||
],
|
||||
ModerationContentType::UserProfileLink => [
|
||||
'content_type' => $type->value,
|
||||
'content_id' => (int) $row->id,
|
||||
'content_target_type' => 'user_social_link',
|
||||
'content_target_id' => (int) $row->id,
|
||||
'user_id' => (int) $row->user_id,
|
||||
'content_snapshot' => trim((string) ($row->url ?? '')),
|
||||
'is_publicly_exposed' => true,
|
||||
],
|
||||
ModerationContentType::CollectionTitle => [
|
||||
'content_type' => $type->value,
|
||||
'content_id' => (int) $row->id,
|
||||
'content_target_type' => 'collection',
|
||||
'content_target_id' => (int) $row->id,
|
||||
'user_id' => $row->user_id ? (int) $row->user_id : null,
|
||||
'content_snapshot' => (string) ($row->title ?? ''),
|
||||
'is_publicly_exposed' => in_array((string) ($row->visibility ?? ''), ['public', 'unlisted'], true),
|
||||
],
|
||||
ModerationContentType::CollectionDescription => [
|
||||
'content_type' => $type->value,
|
||||
'content_id' => (int) $row->id,
|
||||
'content_target_type' => 'collection',
|
||||
'content_target_id' => (int) $row->id,
|
||||
'user_id' => $row->user_id ? (int) $row->user_id : null,
|
||||
'content_snapshot' => (string) ($row->description ?? ''),
|
||||
'is_publicly_exposed' => in_array((string) ($row->visibility ?? ''), ['public', 'unlisted'], true),
|
||||
],
|
||||
ModerationContentType::StoryTitle => [
|
||||
'content_type' => $type->value,
|
||||
'content_id' => (int) $row->id,
|
||||
'content_target_type' => 'story',
|
||||
'content_target_id' => (int) $row->id,
|
||||
'user_id' => $row->creator_id ? (int) $row->creator_id : null,
|
||||
'content_snapshot' => (string) ($row->title ?? ''),
|
||||
'is_publicly_exposed' => in_array((string) ($row->status ?? ''), ['published', 'scheduled'], true),
|
||||
],
|
||||
ModerationContentType::StoryContent => [
|
||||
'content_type' => $type->value,
|
||||
'content_id' => (int) $row->id,
|
||||
'content_target_type' => 'story',
|
||||
'content_target_id' => (int) $row->id,
|
||||
'user_id' => $row->creator_id ? (int) $row->creator_id : null,
|
||||
'content_snapshot' => (string) ($row->content ?? ''),
|
||||
'is_publicly_exposed' => in_array((string) ($row->status ?? ''), ['published', 'scheduled'], true),
|
||||
],
|
||||
ModerationContentType::CardTitle => [
|
||||
'content_type' => $type->value,
|
||||
'content_id' => (int) $row->id,
|
||||
'content_target_type' => 'nova_card',
|
||||
'content_target_id' => (int) $row->id,
|
||||
'user_id' => $row->user_id ? (int) $row->user_id : null,
|
||||
'content_snapshot' => (string) ($row->title ?? ''),
|
||||
'is_publicly_exposed' => in_array((string) ($row->visibility ?? ''), ['public', 'unlisted'], true),
|
||||
],
|
||||
ModerationContentType::CardText => [
|
||||
'content_type' => $type->value,
|
||||
'content_id' => (int) $row->id,
|
||||
'content_target_type' => 'nova_card',
|
||||
'content_target_id' => (int) $row->id,
|
||||
'user_id' => $row->user_id ? (int) $row->user_id : null,
|
||||
'content_snapshot' => trim(implode("\n", array_filter([
|
||||
(string) ($row->quote_text ?? ''),
|
||||
(string) ($row->description ?? ''),
|
||||
(string) ($row->quote_author ?? ''),
|
||||
(string) ($row->quote_source ?? ''),
|
||||
]))),
|
||||
'is_publicly_exposed' => in_array((string) ($row->visibility ?? ''), ['public', 'unlisted'], true),
|
||||
],
|
||||
default => throw new \InvalidArgumentException('Unsupported moderation content type: ' . $type->value),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{context:array<string, mixed>|null, type:ModerationContentType|null}
|
||||
*/
|
||||
public function contextForFinding(ContentModerationFinding $finding): array
|
||||
{
|
||||
return match ($finding->content_type) {
|
||||
ModerationContentType::ArtworkComment => $this->commentContextForFinding($finding),
|
||||
ModerationContentType::ArtworkDescription => $this->descriptionContextForFinding($finding),
|
||||
ModerationContentType::ArtworkTitle => $this->artworkTitleContextForFinding($finding),
|
||||
ModerationContentType::UserBio => $this->userBioContextForFinding($finding),
|
||||
ModerationContentType::UserProfileLink => $this->userProfileLinkContextForFinding($finding),
|
||||
ModerationContentType::CollectionTitle => $this->collectionTitleContextForFinding($finding),
|
||||
ModerationContentType::CollectionDescription => $this->collectionDescriptionContextForFinding($finding),
|
||||
ModerationContentType::StoryTitle => $this->storyTitleContextForFinding($finding),
|
||||
ModerationContentType::StoryContent => $this->storyContentContextForFinding($finding),
|
||||
ModerationContentType::CardTitle => $this->cardTitleContextForFinding($finding),
|
||||
ModerationContentType::CardText => $this->cardTextContextForFinding($finding),
|
||||
default => ['context' => null, 'type' => null],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{context:array<string, mixed>|null, type:ModerationContentType|null}
|
||||
*/
|
||||
private function commentContextForFinding(ContentModerationFinding $finding): array
|
||||
{
|
||||
$comment = ArtworkComment::query()->find($finding->content_id);
|
||||
if (! $comment) {
|
||||
return ['context' => null, 'type' => null];
|
||||
}
|
||||
|
||||
return [
|
||||
'context' => $this->buildContext(ModerationContentType::ArtworkComment, $comment),
|
||||
'type' => ModerationContentType::ArtworkComment,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{context:array<string, mixed>|null, type:ModerationContentType|null}
|
||||
*/
|
||||
private function descriptionContextForFinding(ContentModerationFinding $finding): array
|
||||
{
|
||||
$artworkId = (int) ($finding->artwork_id ?? $finding->content_id);
|
||||
$artwork = Artwork::query()->find($artworkId);
|
||||
if (! $artwork) {
|
||||
return ['context' => null, 'type' => null];
|
||||
}
|
||||
|
||||
return [
|
||||
'context' => $this->buildContext(ModerationContentType::ArtworkDescription, $artwork),
|
||||
'type' => ModerationContentType::ArtworkDescription,
|
||||
];
|
||||
}
|
||||
|
||||
private function artworkTitleContextForFinding(ContentModerationFinding $finding): array
|
||||
{
|
||||
$artwork = Artwork::query()->find((int) ($finding->artwork_id ?? $finding->content_id));
|
||||
if (! $artwork) {
|
||||
return ['context' => null, 'type' => null];
|
||||
}
|
||||
|
||||
return ['context' => $this->buildContext(ModerationContentType::ArtworkTitle, $artwork), 'type' => ModerationContentType::ArtworkTitle];
|
||||
}
|
||||
|
||||
private function userBioContextForFinding(ContentModerationFinding $finding): array
|
||||
{
|
||||
$profile = UserProfile::query()->find($finding->content_id);
|
||||
if (! $profile) {
|
||||
return ['context' => null, 'type' => null];
|
||||
}
|
||||
|
||||
return ['context' => $this->buildContext(ModerationContentType::UserBio, $profile), 'type' => ModerationContentType::UserBio];
|
||||
}
|
||||
|
||||
private function userProfileLinkContextForFinding(ContentModerationFinding $finding): array
|
||||
{
|
||||
$link = UserSocialLink::query()->find($finding->content_id);
|
||||
if (! $link) {
|
||||
return ['context' => null, 'type' => null];
|
||||
}
|
||||
|
||||
return ['context' => $this->buildContext(ModerationContentType::UserProfileLink, $link), 'type' => ModerationContentType::UserProfileLink];
|
||||
}
|
||||
|
||||
private function collectionTitleContextForFinding(ContentModerationFinding $finding): array
|
||||
{
|
||||
$collection = Collection::query()->find($finding->content_id);
|
||||
if (! $collection) {
|
||||
return ['context' => null, 'type' => null];
|
||||
}
|
||||
|
||||
return ['context' => $this->buildContext(ModerationContentType::CollectionTitle, $collection), 'type' => ModerationContentType::CollectionTitle];
|
||||
}
|
||||
|
||||
private function collectionDescriptionContextForFinding(ContentModerationFinding $finding): array
|
||||
{
|
||||
$collection = Collection::query()->find($finding->content_id);
|
||||
if (! $collection) {
|
||||
return ['context' => null, 'type' => null];
|
||||
}
|
||||
|
||||
return ['context' => $this->buildContext(ModerationContentType::CollectionDescription, $collection), 'type' => ModerationContentType::CollectionDescription];
|
||||
}
|
||||
|
||||
private function storyTitleContextForFinding(ContentModerationFinding $finding): array
|
||||
{
|
||||
$story = Story::query()->find($finding->content_id);
|
||||
if (! $story) {
|
||||
return ['context' => null, 'type' => null];
|
||||
}
|
||||
|
||||
return ['context' => $this->buildContext(ModerationContentType::StoryTitle, $story), 'type' => ModerationContentType::StoryTitle];
|
||||
}
|
||||
|
||||
private function storyContentContextForFinding(ContentModerationFinding $finding): array
|
||||
{
|
||||
$story = Story::query()->find($finding->content_id);
|
||||
if (! $story) {
|
||||
return ['context' => null, 'type' => null];
|
||||
}
|
||||
|
||||
return ['context' => $this->buildContext(ModerationContentType::StoryContent, $story), 'type' => ModerationContentType::StoryContent];
|
||||
}
|
||||
|
||||
private function cardTitleContextForFinding(ContentModerationFinding $finding): array
|
||||
{
|
||||
$card = NovaCard::query()->find($finding->content_id);
|
||||
if (! $card) {
|
||||
return ['context' => null, 'type' => null];
|
||||
}
|
||||
|
||||
return ['context' => $this->buildContext(ModerationContentType::CardTitle, $card), 'type' => ModerationContentType::CardTitle];
|
||||
}
|
||||
|
||||
private function cardTextContextForFinding(ContentModerationFinding $finding): array
|
||||
{
|
||||
$card = NovaCard::query()->find($finding->content_id);
|
||||
if (! $card) {
|
||||
return ['context' => null, 'type' => null];
|
||||
}
|
||||
|
||||
return ['context' => $this->buildContext(ModerationContentType::CardText, $card), 'type' => ModerationContentType::CardText];
|
||||
}
|
||||
}
|
||||
52
app/Services/Moderation/DomainIntelligenceService.php
Normal file
52
app/Services/Moderation/DomainIntelligenceService.php
Normal file
@@ -0,0 +1,52 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Moderation;
|
||||
|
||||
use App\Models\ContentModerationDomain;
|
||||
use App\Models\ContentModerationFinding;
|
||||
|
||||
class DomainIntelligenceService
|
||||
{
|
||||
public function refreshDomain(string $domain): ?ContentModerationDomain
|
||||
{
|
||||
$record = ContentModerationDomain::query()->where('domain', $domain)->first();
|
||||
if (! $record) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$findings = ContentModerationFinding::query()
|
||||
->whereJsonContains('matched_domains_json', $domain)
|
||||
->get(['id', 'user_id', 'campaign_key', 'matched_keywords_json', 'content_type', 'is_false_positive']);
|
||||
|
||||
$topKeywords = $findings
|
||||
->flatMap(static fn (ContentModerationFinding $finding): array => (array) $finding->matched_keywords_json)
|
||||
->filter()
|
||||
->countBy()
|
||||
->sortDesc()
|
||||
->take(8)
|
||||
->keys()
|
||||
->values()
|
||||
->all();
|
||||
|
||||
$topContentTypes = $findings
|
||||
->pluck('content_type')
|
||||
->filter()
|
||||
->countBy()
|
||||
->sortDesc()
|
||||
->take(8)
|
||||
->map(static fn (int $count, string $type): array => ['type' => $type, 'count' => $count])
|
||||
->values()
|
||||
->all();
|
||||
|
||||
$record->forceFill([
|
||||
'linked_users_count' => $findings->pluck('user_id')->filter()->unique()->count(),
|
||||
'linked_findings_count' => $findings->count(),
|
||||
'linked_clusters_count' => $findings->pluck('campaign_key')->filter()->unique()->count(),
|
||||
'top_keywords_json' => $topKeywords,
|
||||
'top_content_types_json' => $topContentTypes,
|
||||
'false_positive_count' => $findings->where('is_false_positive', true)->count(),
|
||||
])->save();
|
||||
|
||||
return $record->fresh();
|
||||
}
|
||||
}
|
||||
200
app/Services/Moderation/DomainReputationService.php
Normal file
200
app/Services/Moderation/DomainReputationService.php
Normal file
@@ -0,0 +1,200 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Moderation;
|
||||
|
||||
use App\Enums\ModerationActionType;
|
||||
use App\Enums\ModerationDomainStatus;
|
||||
use App\Models\ContentModerationActionLog;
|
||||
use App\Models\ContentModerationDomain;
|
||||
use App\Models\ContentModerationFinding;
|
||||
use App\Models\User;
|
||||
|
||||
class DomainReputationService
|
||||
{
|
||||
public function __construct(
|
||||
private readonly DomainIntelligenceService $intelligence,
|
||||
) {
|
||||
}
|
||||
|
||||
public function normalizeDomain(?string $domain): ?string
|
||||
{
|
||||
if (! is_string($domain)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$normalized = trim(mb_strtolower($domain));
|
||||
$normalized = preg_replace('/^www\./', '', $normalized);
|
||||
|
||||
return $normalized !== '' ? $normalized : null;
|
||||
}
|
||||
|
||||
public function statusForDomain(string $domain): ModerationDomainStatus
|
||||
{
|
||||
$normalized = $this->normalizeDomain($domain);
|
||||
if ($normalized === null) {
|
||||
return ModerationDomainStatus::Neutral;
|
||||
}
|
||||
|
||||
$record = ContentModerationDomain::query()->where('domain', $normalized)->first();
|
||||
if ($record !== null) {
|
||||
return $record->status;
|
||||
}
|
||||
|
||||
if ($this->matchesAnyPattern($normalized, (array) \app('config')->get('content_moderation.allowed_domains', []))) {
|
||||
return ModerationDomainStatus::Allowed;
|
||||
}
|
||||
|
||||
if ($this->matchesAnyPattern($normalized, (array) \app('config')->get('content_moderation.blacklisted_domains', []))) {
|
||||
return ModerationDomainStatus::Blocked;
|
||||
}
|
||||
|
||||
if ($this->matchesAnyPattern($normalized, (array) \app('config')->get('content_moderation.suspicious_domains', []))) {
|
||||
return ModerationDomainStatus::Suspicious;
|
||||
}
|
||||
|
||||
return ModerationDomainStatus::Neutral;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<int, string> $domains
|
||||
* @return array<int, ContentModerationDomain>
|
||||
*/
|
||||
public function trackDomains(array $domains, bool $flagged = false, bool $confirmedSpam = false): array
|
||||
{
|
||||
$normalized = \collect($domains)
|
||||
->map(fn (?string $domain): ?string => $this->normalizeDomain($domain))
|
||||
->filter()
|
||||
->unique()
|
||||
->values();
|
||||
|
||||
if ($normalized->isEmpty()) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$existing = ContentModerationDomain::query()
|
||||
->whereIn('domain', $normalized->all())
|
||||
->get()
|
||||
->keyBy('domain');
|
||||
|
||||
$records = [];
|
||||
$now = \now();
|
||||
|
||||
foreach ($normalized as $domain) {
|
||||
$defaultStatus = $this->statusForDomain($domain);
|
||||
$record = $existing[$domain] ?? new ContentModerationDomain([
|
||||
'domain' => $domain,
|
||||
'status' => $defaultStatus,
|
||||
'first_seen_at' => $now,
|
||||
]);
|
||||
|
||||
$record->forceFill([
|
||||
'status' => $record->status ?? $defaultStatus,
|
||||
'times_seen' => ((int) $record->times_seen) + 1,
|
||||
'times_flagged' => ((int) $record->times_flagged) + ($flagged ? 1 : 0),
|
||||
'times_confirmed_spam' => ((int) $record->times_confirmed_spam) + ($confirmedSpam ? 1 : 0),
|
||||
'first_seen_at' => $record->first_seen_at ?? $now,
|
||||
'last_seen_at' => $now,
|
||||
])->save();
|
||||
|
||||
$records[] = $record->fresh();
|
||||
}
|
||||
|
||||
return $records;
|
||||
}
|
||||
|
||||
public function updateStatus(string $domain, ModerationDomainStatus $status, ?User $actor = null, ?string $notes = null): ContentModerationDomain
|
||||
{
|
||||
$normalized = $this->normalizeDomain($domain);
|
||||
\abort_unless($normalized !== null, 422, 'Invalid domain.');
|
||||
|
||||
$record = ContentModerationDomain::query()->firstOrNew(['domain' => $normalized]);
|
||||
$previous = $record->status?->value;
|
||||
|
||||
$record->forceFill([
|
||||
'status' => $status,
|
||||
'first_seen_at' => $record->first_seen_at ?? \now(),
|
||||
'last_seen_at' => \now(),
|
||||
'notes' => $notes !== null && trim($notes) !== '' ? trim($notes) : $record->notes,
|
||||
])->save();
|
||||
|
||||
ContentModerationActionLog::query()->create([
|
||||
'target_type' => 'domain',
|
||||
'target_id' => $record->id,
|
||||
'action_type' => match ($status) {
|
||||
ModerationDomainStatus::Blocked => ModerationActionType::BlockDomain->value,
|
||||
ModerationDomainStatus::Suspicious => ModerationActionType::MarkDomainSuspicious->value,
|
||||
ModerationDomainStatus::Escalated => ModerationActionType::Escalate->value,
|
||||
ModerationDomainStatus::ReviewRequired => ModerationActionType::MarkDomainSuspicious->value,
|
||||
ModerationDomainStatus::Allowed, ModerationDomainStatus::Neutral => ModerationActionType::AllowDomain->value,
|
||||
},
|
||||
'actor_type' => $actor ? 'admin' : 'system',
|
||||
'actor_id' => $actor?->id,
|
||||
'notes' => $notes,
|
||||
'old_status' => $previous,
|
||||
'new_status' => $status->value,
|
||||
'meta_json' => ['domain' => $normalized],
|
||||
'created_at' => \now(),
|
||||
]);
|
||||
|
||||
$this->intelligence->refreshDomain($normalized);
|
||||
|
||||
return $record->fresh();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<int, string>
|
||||
*/
|
||||
public function shortenerDomains(): array
|
||||
{
|
||||
return \collect((array) \app('config')->get('content_moderation.shortener_domains', []))
|
||||
->map(fn (string $domain): ?string => $this->normalizeDomain($domain))
|
||||
->filter()
|
||||
->values()
|
||||
->all();
|
||||
}
|
||||
|
||||
public function attachDomainIds(ContentModerationFinding $finding): void
|
||||
{
|
||||
$domains = \collect((array) $finding->matched_domains_json)
|
||||
->map(fn (?string $domain): ?string => $this->normalizeDomain($domain))
|
||||
->filter()
|
||||
->unique()
|
||||
->values();
|
||||
|
||||
if ($domains->isEmpty()) {
|
||||
$finding->forceFill(['domain_ids_json' => []])->save();
|
||||
return;
|
||||
}
|
||||
|
||||
$ids = ContentModerationDomain::query()
|
||||
->whereIn('domain', $domains->all())
|
||||
->pluck('id')
|
||||
->map(static fn (int $id): int => $id)
|
||||
->values()
|
||||
->all();
|
||||
|
||||
$finding->forceFill(['domain_ids_json' => $ids])->save();
|
||||
|
||||
foreach ($domains as $domain) {
|
||||
$this->intelligence->refreshDomain((string) $domain);
|
||||
}
|
||||
}
|
||||
|
||||
private function matchesAnyPattern(string $domain, array $patterns): bool
|
||||
{
|
||||
foreach ($patterns as $pattern) {
|
||||
$pattern = trim(mb_strtolower((string) $pattern));
|
||||
if ($pattern === '') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$quoted = preg_quote($pattern, '/');
|
||||
$regex = '/^' . str_replace('\\*', '.*', $quoted) . '$/i';
|
||||
if (preg_match($regex, $domain) === 1) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
106
app/Services/Moderation/DuplicateDetectionService.php
Normal file
106
app/Services/Moderation/DuplicateDetectionService.php
Normal file
@@ -0,0 +1,106 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Moderation;
|
||||
|
||||
use App\Enums\ModerationContentType;
|
||||
use App\Models\Artwork;
|
||||
use App\Models\ArtworkComment;
|
||||
|
||||
class DuplicateDetectionService
|
||||
{
|
||||
public function campaignText(string $content): string
|
||||
{
|
||||
$text = mb_strtolower($content);
|
||||
$text = preg_replace('/https?:\/\/\S+/iu', ' [link] ', $text);
|
||||
$text = preg_replace('/www\.\S+/iu', ' [link] ', (string) $text);
|
||||
$text = preg_replace('/[^\p{L}\p{N}\s\[\]]+/u', ' ', (string) $text);
|
||||
$text = preg_replace('/\s+/u', ' ', trim((string) $text));
|
||||
|
||||
return (string) $text;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<int, string> $domains
|
||||
*/
|
||||
public function buildGroupKey(string $content, array $domains = []): string
|
||||
{
|
||||
$template = $this->campaignText($content);
|
||||
$tokens = preg_split('/\s+/u', $template, -1, PREG_SPLIT_NO_EMPTY) ?: [];
|
||||
$signature = implode(' ', array_slice($tokens, 0, 12));
|
||||
$domainPart = implode('|', array_slice(array_values(array_unique($domains)), 0, 2));
|
||||
|
||||
return hash('sha256', $domainPart . '::' . $signature);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $context
|
||||
* @param array<int, string> $domains
|
||||
*/
|
||||
public function nearDuplicateCount(string $content, array $context = [], array $domains = []): int
|
||||
{
|
||||
$type = (string) ($context['content_type'] ?? '');
|
||||
$contentId = (int) ($context['content_id'] ?? 0);
|
||||
$artworkId = (int) ($context['artwork_id'] ?? 0);
|
||||
$signature = $this->campaignText($content);
|
||||
if ($signature === '') {
|
||||
return 0;
|
||||
}
|
||||
|
||||
$candidates = match ($type) {
|
||||
ModerationContentType::ArtworkComment->value => ArtworkComment::query()
|
||||
->where('id', '!=', $contentId)
|
||||
->whereNull('deleted_at')
|
||||
->latest('id')
|
||||
->limit(80)
|
||||
->get(['id', 'artwork_id', 'raw_content', 'content']),
|
||||
ModerationContentType::ArtworkDescription->value => Artwork::query()
|
||||
->where('id', '!=', $contentId)
|
||||
->whereNotNull('description')
|
||||
->latest('id')
|
||||
->limit(80)
|
||||
->get(['id', 'description']),
|
||||
default => \collect(),
|
||||
};
|
||||
|
||||
$matches = 0;
|
||||
|
||||
foreach ($candidates as $candidate) {
|
||||
$candidateText = match ($type) {
|
||||
ModerationContentType::ArtworkComment->value => (string) ($candidate->raw_content ?: $candidate->content),
|
||||
ModerationContentType::ArtworkDescription->value => (string) ($candidate->description ?? ''),
|
||||
default => '',
|
||||
};
|
||||
|
||||
if ($candidateText === '') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$candidateSignature = $this->campaignText($candidateText);
|
||||
similar_text($signature, $candidateSignature, $similarity);
|
||||
|
||||
$sameArtworkPenalty = $artworkId > 0 && (int) ($candidate->artwork_id ?? $candidate->id ?? 0) === $artworkId ? 4 : 0;
|
||||
|
||||
if ($similarity >= (float) \app('config')->get('content_moderation.duplicate_detection.near_duplicate_similarity', 84) - $sameArtworkPenalty) {
|
||||
$matches++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($domains !== []) {
|
||||
$topDomain = $domains[0] ?? null;
|
||||
if ($topDomain !== null && str_contains(mb_strtolower($candidateText), mb_strtolower($topDomain))) {
|
||||
similar_text($this->stripLinks($signature), $this->stripLinks($candidateSignature), $linklessSimilarity);
|
||||
if ($linklessSimilarity >= 72) {
|
||||
$matches++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $matches;
|
||||
}
|
||||
|
||||
private function stripLinks(string $text): string
|
||||
{
|
||||
return trim(str_replace('[link]', '', $text));
|
||||
}
|
||||
}
|
||||
89
app/Services/Moderation/ModerationClusterService.php
Normal file
89
app/Services/Moderation/ModerationClusterService.php
Normal file
@@ -0,0 +1,89 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Moderation;
|
||||
|
||||
use App\Data\Moderation\ModerationResultData;
|
||||
use App\Models\ContentModerationCluster;
|
||||
use App\Models\ContentModerationFinding;
|
||||
|
||||
class ModerationClusterService
|
||||
{
|
||||
/**
|
||||
* @param array<string, mixed> $context
|
||||
* @param array<string, mixed> $suggestion
|
||||
* @return array{campaign_key:string,cluster_score:int,cluster_reason:string}
|
||||
*/
|
||||
public function classify(string $content, ModerationResultData $result, array $context = [], array $suggestion = []): array
|
||||
{
|
||||
$domains = array_values(array_filter($result->matchedDomains));
|
||||
$keywords = array_values(array_filter($result->matchedKeywords));
|
||||
$reason = 'normalized_content';
|
||||
|
||||
if ($domains !== [] && $keywords !== []) {
|
||||
$reason = 'domain_keyword_cta';
|
||||
$key = 'campaign:' . sha1(implode('|', [implode(',', array_slice($domains, 0, 3)), implode(',', array_slice($keywords, 0, 3))]));
|
||||
} elseif ($domains !== []) {
|
||||
$reason = 'domain_fingerprint';
|
||||
$key = 'campaign:' . sha1(implode(',', array_slice($domains, 0, 3)) . '|' . ($result->contentHashNormalized ?? $result->contentHash));
|
||||
} elseif (! empty($suggestion['campaign_tags'])) {
|
||||
$reason = 'suggested_cluster';
|
||||
$key = 'campaign:' . sha1(implode('|', (array) $suggestion['campaign_tags']));
|
||||
} else {
|
||||
$key = 'campaign:' . sha1((string) ($result->groupKey ?? $result->contentHashNormalized ?? $result->contentHash));
|
||||
}
|
||||
|
||||
$clusterScore = min(100, $result->score + (count($domains) * 8) + (count($keywords) * 4));
|
||||
|
||||
return [
|
||||
'campaign_key' => $key,
|
||||
'cluster_score' => $clusterScore,
|
||||
'cluster_reason' => $reason,
|
||||
];
|
||||
}
|
||||
|
||||
public function syncFinding(ContentModerationFinding $finding): void
|
||||
{
|
||||
if (! $finding->campaign_key) {
|
||||
return;
|
||||
}
|
||||
|
||||
$query = ContentModerationFinding::query()->where('campaign_key', $finding->campaign_key);
|
||||
$findings = $query->get(['id', 'user_id', 'matched_domains_json', 'matched_keywords_json', 'review_bucket', 'cluster_score', 'created_at']);
|
||||
|
||||
$domains = $findings
|
||||
->flatMap(static fn (ContentModerationFinding $item): array => (array) $item->matched_domains_json)
|
||||
->filter()
|
||||
->unique()
|
||||
->values();
|
||||
|
||||
$keywords = $findings
|
||||
->flatMap(static fn (ContentModerationFinding $item): array => (array) $item->matched_keywords_json)
|
||||
->filter()
|
||||
->unique()
|
||||
->take(8)
|
||||
->values();
|
||||
|
||||
ContentModerationCluster::query()->updateOrCreate(
|
||||
['campaign_key' => $finding->campaign_key],
|
||||
[
|
||||
'cluster_reason' => $finding->cluster_reason,
|
||||
'review_bucket' => $finding->review_bucket,
|
||||
'escalation_status' => $finding->escalation_status?->value ?? (string) $finding->escalation_status,
|
||||
'cluster_score' => (int) ($findings->max('cluster_score') ?? $finding->cluster_score ?? 0),
|
||||
'findings_count' => $findings->count(),
|
||||
'unique_users_count' => $findings->pluck('user_id')->filter()->unique()->count(),
|
||||
'unique_domains_count' => $domains->count(),
|
||||
'latest_finding_at' => $findings->max('created_at') ?: now(),
|
||||
'summary_json' => [
|
||||
'domains' => $domains->take(8)->all(),
|
||||
'keywords' => $keywords->all(),
|
||||
],
|
||||
],
|
||||
);
|
||||
|
||||
$clusterSize = $findings->count();
|
||||
if ($clusterSize > 1) {
|
||||
$query->update(['priority_score' => $finding->priority_score + min(25, ($clusterSize - 1) * 3)]);
|
||||
}
|
||||
}
|
||||
}
|
||||
25
app/Services/Moderation/ModerationFeedbackService.php
Normal file
25
app/Services/Moderation/ModerationFeedbackService.php
Normal file
@@ -0,0 +1,25 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Moderation;
|
||||
|
||||
use App\Models\ContentModerationFeedback;
|
||||
use App\Models\ContentModerationFinding;
|
||||
use App\Models\User;
|
||||
|
||||
class ModerationFeedbackService
|
||||
{
|
||||
/**
|
||||
* @param array<string, mixed> $meta
|
||||
*/
|
||||
public function record(ContentModerationFinding $finding, string $feedbackType, ?User $actor = null, ?string $notes = null, array $meta = []): ContentModerationFeedback
|
||||
{
|
||||
return ContentModerationFeedback::query()->create([
|
||||
'finding_id' => $finding->id,
|
||||
'feedback_type' => $feedbackType,
|
||||
'actor_id' => $actor?->id,
|
||||
'notes' => $notes,
|
||||
'meta_json' => $meta,
|
||||
'created_at' => now(),
|
||||
]);
|
||||
}
|
||||
}
|
||||
51
app/Services/Moderation/ModerationPolicyEngineService.php
Normal file
51
app/Services/Moderation/ModerationPolicyEngineService.php
Normal file
@@ -0,0 +1,51 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Moderation;
|
||||
|
||||
use App\Enums\ModerationContentType;
|
||||
|
||||
class ModerationPolicyEngineService
|
||||
{
|
||||
/**
|
||||
* @param array<string, mixed> $context
|
||||
* @param array<string, mixed> $riskAssessment
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
public function resolve(array $context, array $riskAssessment = []): array
|
||||
{
|
||||
$policies = (array) app('config')->get('content_moderation.policies', []);
|
||||
$contentType = ModerationContentType::tryFrom((string) ($context['content_type'] ?? ''));
|
||||
$accountAgeDays = (int) data_get($riskAssessment, 'signals.account_age_days', 0);
|
||||
$riskScore = (int) ($riskAssessment['risk_score'] ?? 0);
|
||||
$hasLinks = ! empty($context['extracted_urls'] ?? []) || ! empty($context['extracted_domains'] ?? []);
|
||||
|
||||
$name = 'default';
|
||||
|
||||
if ($riskScore >= 70 || ($accountAgeDays > 0 && $accountAgeDays < 14)) {
|
||||
$name = 'new_user_strict_mode';
|
||||
} elseif ($riskScore <= 8 && $accountAgeDays >= 180) {
|
||||
$name = 'trusted_user_relaxed_mode';
|
||||
}
|
||||
|
||||
if ($contentType === ModerationContentType::ArtworkComment && $riskScore >= 45) {
|
||||
$name = 'comments_high_volume_antispam';
|
||||
}
|
||||
|
||||
if ($hasLinks && in_array($contentType, [
|
||||
ModerationContentType::UserProfileLink,
|
||||
ModerationContentType::CollectionDescription,
|
||||
ModerationContentType::CollectionTitle,
|
||||
ModerationContentType::StoryContent,
|
||||
ModerationContentType::StoryTitle,
|
||||
ModerationContentType::CardText,
|
||||
ModerationContentType::CardTitle,
|
||||
], true)) {
|
||||
$name = 'strict_seo_protection';
|
||||
}
|
||||
|
||||
$policy = $policies[$name] ?? ($policies['default'] ?? []);
|
||||
$policy['name'] = $name;
|
||||
|
||||
return $policy;
|
||||
}
|
||||
}
|
||||
47
app/Services/Moderation/ModerationPriorityService.php
Normal file
47
app/Services/Moderation/ModerationPriorityService.php
Normal file
@@ -0,0 +1,47 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Moderation;
|
||||
|
||||
use App\Data\Moderation\ModerationResultData;
|
||||
|
||||
class ModerationPriorityService
|
||||
{
|
||||
/**
|
||||
* @param array<string, mixed> $context
|
||||
* @param array<string, mixed> $policy
|
||||
* @param array<string, mixed> $suggestion
|
||||
*/
|
||||
public function score(ModerationResultData $result, array $context = [], array $policy = [], array $suggestion = []): array
|
||||
{
|
||||
$score = $result->score;
|
||||
$score += $result->isSuspicious() ? 10 : 0;
|
||||
$score += $result->autoHideRecommended ? 25 : 0;
|
||||
$score += max(0, (int) ($result->userRiskScore ?? 0) / 2);
|
||||
$score += (int) ($policy['priority_bonus'] ?? 0);
|
||||
$score += max(0, (int) (($suggestion['confidence'] ?? 0) / 5));
|
||||
$score += ! empty($context['is_publicly_exposed']) ? 12 : 0;
|
||||
$score += ! empty($result->matchedDomains) ? 10 : 0;
|
||||
$score += isset($result->ruleHits['blocked_domain']) ? 18 : 0;
|
||||
$score += isset($result->ruleHits['near_duplicate_campaign']) ? 14 : 0;
|
||||
|
||||
$bucket = match (true) {
|
||||
$score >= 140 => 'urgent',
|
||||
$score >= 95 => 'high',
|
||||
$score >= 50 => 'priority',
|
||||
default => 'standard',
|
||||
};
|
||||
|
||||
$escalation = match ($bucket) {
|
||||
'urgent' => 'urgent',
|
||||
'high' => 'escalated',
|
||||
'priority' => 'review_required',
|
||||
default => 'none',
|
||||
};
|
||||
|
||||
return [
|
||||
'priority_score' => $score,
|
||||
'review_bucket' => $bucket,
|
||||
'escalation_status' => $escalation,
|
||||
];
|
||||
}
|
||||
}
|
||||
81
app/Services/Moderation/ModerationRuleRegistryService.php
Normal file
81
app/Services/Moderation/ModerationRuleRegistryService.php
Normal file
@@ -0,0 +1,81 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Moderation;
|
||||
|
||||
use App\Enums\ModerationRuleType;
|
||||
use App\Models\ContentModerationRule;
|
||||
|
||||
class ModerationRuleRegistryService
|
||||
{
|
||||
/**
|
||||
* @return array<int, string>
|
||||
*/
|
||||
public function suspiciousKeywords(): array
|
||||
{
|
||||
return $this->mergeValues(
|
||||
(array) \app('config')->get('content_moderation.keywords.suspicious', []),
|
||||
$this->rulesByType(ModerationRuleType::SuspiciousKeyword)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<int, string>
|
||||
*/
|
||||
public function highRiskKeywords(): array
|
||||
{
|
||||
return $this->mergeValues(
|
||||
(array) \app('config')->get('content_moderation.keywords.high_risk', []),
|
||||
$this->rulesByType(ModerationRuleType::HighRiskKeyword)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<int, array{pattern:string,weight:?int,id:int|null}>
|
||||
*/
|
||||
public function regexRules(): array
|
||||
{
|
||||
return ContentModerationRule::query()
|
||||
->where('enabled', true)
|
||||
->where('type', ModerationRuleType::Regex->value)
|
||||
->orderByDesc('id')
|
||||
->get(['id', 'value', 'weight'])
|
||||
->map(static fn (ContentModerationRule $rule): array => [
|
||||
'pattern' => (string) $rule->value,
|
||||
'weight' => $rule->weight,
|
||||
'id' => $rule->id,
|
||||
])
|
||||
->values()
|
||||
->all();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<int, string>
|
||||
*/
|
||||
private function rulesByType(ModerationRuleType $type): array
|
||||
{
|
||||
return ContentModerationRule::query()
|
||||
->where('enabled', true)
|
||||
->where('type', $type->value)
|
||||
->orderByDesc('id')
|
||||
->pluck('value')
|
||||
->map(static fn (string $value): string => trim($value))
|
||||
->filter()
|
||||
->values()
|
||||
->all();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<int, string> $configValues
|
||||
* @param array<int, string> $dbValues
|
||||
* @return array<int, string>
|
||||
*/
|
||||
private function mergeValues(array $configValues, array $dbValues): array
|
||||
{
|
||||
return \collect(array_merge($configValues, $dbValues))
|
||||
->map(static fn (string $value): string => trim($value))
|
||||
->filter()
|
||||
->unique(static fn (string $value): string => mb_strtolower($value))
|
||||
->values()
|
||||
->all();
|
||||
}
|
||||
}
|
||||
30
app/Services/Moderation/ModerationSuggestionService.php
Normal file
30
app/Services/Moderation/ModerationSuggestionService.php
Normal file
@@ -0,0 +1,30 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Moderation;
|
||||
|
||||
use App\Contracts\Moderation\ModerationSuggestionProviderInterface;
|
||||
use App\Data\Moderation\ModerationResultData;
|
||||
use App\Data\Moderation\ModerationSuggestionData;
|
||||
use App\Services\Moderation\Providers\HeuristicModerationSuggestionProvider;
|
||||
use App\Services\Moderation\Providers\NullModerationSuggestionProvider;
|
||||
|
||||
class ModerationSuggestionService
|
||||
{
|
||||
public function provider(): ModerationSuggestionProviderInterface
|
||||
{
|
||||
$provider = (string) app('config')->get('content_moderation.suggestions.provider', 'heuristic');
|
||||
|
||||
return match ($provider) {
|
||||
'null' => app(NullModerationSuggestionProvider::class),
|
||||
default => app(HeuristicModerationSuggestionProvider::class),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $context
|
||||
*/
|
||||
public function suggest(string $content, ModerationResultData $result, array $context = []): ModerationSuggestionData
|
||||
{
|
||||
return $this->provider()->suggest($content, $result, $context);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Moderation\Providers;
|
||||
|
||||
use App\Contracts\Moderation\ModerationSuggestionProviderInterface;
|
||||
use App\Data\Moderation\ModerationResultData;
|
||||
use App\Data\Moderation\ModerationSuggestionData;
|
||||
|
||||
class HeuristicModerationSuggestionProvider implements ModerationSuggestionProviderInterface
|
||||
{
|
||||
public function suggest(string $content, ModerationResultData $result, array $context = []): ModerationSuggestionData
|
||||
{
|
||||
$label = null;
|
||||
$action = null;
|
||||
$confidence = null;
|
||||
$reason = null;
|
||||
$campaignTags = [];
|
||||
|
||||
if ($result->score === 0) {
|
||||
return new ModerationSuggestionData(
|
||||
provider: 'heuristic_assist',
|
||||
suggestedLabel: 'likely_safe',
|
||||
suggestedAction: 'mark_safe',
|
||||
confidence: 82,
|
||||
explanation: 'No suspicious signals were detected by the deterministic moderation rules.',
|
||||
);
|
||||
}
|
||||
|
||||
if (isset($result->ruleHits['blocked_domain']) || isset($result->ruleHits['blacklisted_domain'])) {
|
||||
$label = 'seo_spam';
|
||||
$action = $result->autoHideRecommended ? 'auto_hide_review' : 'confirm_spam';
|
||||
$confidence = 94;
|
||||
$reason = 'Blocked-domain activity was detected and strongly correlates with outbound spam campaigns.';
|
||||
$campaignTags[] = 'blocked-domain';
|
||||
} elseif (isset($result->ruleHits['high_risk_keyword'])) {
|
||||
$label = $this->labelFromKeywords($result->matchedKeywords);
|
||||
$action = 'confirm_spam';
|
||||
$confidence = 88;
|
||||
$reason = 'High-risk spam keywords were matched across the content snapshot.';
|
||||
$campaignTags[] = 'high-risk-keywords';
|
||||
} elseif (isset($result->ruleHits['near_duplicate_campaign']) || isset($result->ruleHits['duplicate_comment'])) {
|
||||
$label = 'campaign_spam';
|
||||
$action = 'review_cluster';
|
||||
$confidence = 86;
|
||||
$reason = 'The content appears linked to a repeated spam template or campaign cluster.';
|
||||
$campaignTags[] = 'duplicate-campaign';
|
||||
} else {
|
||||
$label = 'needs_review';
|
||||
$action = 'review';
|
||||
$confidence = max(55, min(84, $result->score));
|
||||
$reason = 'Multiple suspicious signals were detected, but the content should remain human-reviewed.';
|
||||
}
|
||||
|
||||
if ($result->matchedDomains !== []) {
|
||||
$campaignTags[] = 'domains:' . implode(',', array_slice($result->matchedDomains, 0, 3));
|
||||
}
|
||||
|
||||
return new ModerationSuggestionData(
|
||||
provider: 'heuristic_assist',
|
||||
suggestedLabel: $label,
|
||||
suggestedAction: $action,
|
||||
confidence: $confidence,
|
||||
explanation: $reason,
|
||||
campaignTags: array_values(array_unique($campaignTags)),
|
||||
rawResponse: [
|
||||
'rule_hits' => $result->ruleHits,
|
||||
'matched_domains' => $result->matchedDomains,
|
||||
'matched_keywords' => $result->matchedKeywords,
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<int, string> $keywords
|
||||
*/
|
||||
private function labelFromKeywords(array $keywords): string
|
||||
{
|
||||
$joined = mb_strtolower(implode(' ', $keywords));
|
||||
|
||||
return match (true) {
|
||||
str_contains($joined, 'casino'), str_contains($joined, 'bet') => 'casino_spam',
|
||||
str_contains($joined, 'adult'), str_contains($joined, 'webcam') => 'adult_spam',
|
||||
str_contains($joined, 'bitcoin'), str_contains($joined, 'crypto') => 'crypto_spam',
|
||||
str_contains($joined, 'pharma'), str_contains($joined, 'viagra') => 'pharma_spam',
|
||||
default => 'spam',
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Moderation\Providers;
|
||||
|
||||
use App\Contracts\Moderation\ModerationSuggestionProviderInterface;
|
||||
use App\Data\Moderation\ModerationResultData;
|
||||
use App\Data\Moderation\ModerationSuggestionData;
|
||||
|
||||
class NullModerationSuggestionProvider implements ModerationSuggestionProviderInterface
|
||||
{
|
||||
public function suggest(string $content, ModerationResultData $result, array $context = []): ModerationSuggestionData
|
||||
{
|
||||
return new ModerationSuggestionData(provider: 'null');
|
||||
}
|
||||
}
|
||||
68
app/Services/Moderation/Rules/DomainBlacklistRule.php
Normal file
68
app/Services/Moderation/Rules/DomainBlacklistRule.php
Normal file
@@ -0,0 +1,68 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Moderation\Rules;
|
||||
|
||||
use App\Contracts\Moderation\ModerationRuleInterface;
|
||||
use App\Enums\ModerationDomainStatus;
|
||||
use App\Services\Moderation\DomainReputationService;
|
||||
|
||||
class DomainBlacklistRule implements ModerationRuleInterface
|
||||
{
|
||||
public function analyze(string $content, string $normalized, array $context = []): array
|
||||
{
|
||||
$linkRule = app(LinkPresenceRule::class);
|
||||
$urls = (array) ($context['extracted_urls'] ?? $linkRule->extractUrls($content));
|
||||
|
||||
if (empty($urls)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$weights = app('config')->get('content_moderation.weights', []);
|
||||
$domainService = app(DomainReputationService::class);
|
||||
|
||||
$findings = [];
|
||||
$blockedMatches = [];
|
||||
$suspiciousMatches = [];
|
||||
|
||||
foreach ($urls as $url) {
|
||||
$host = $linkRule->extractHost($url);
|
||||
if ($host === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$status = $domainService->statusForDomain($host);
|
||||
if ($status === ModerationDomainStatus::Blocked) {
|
||||
$blockedMatches[] = $host;
|
||||
} elseif ($status === ModerationDomainStatus::Suspicious) {
|
||||
$suspiciousMatches[] = $host;
|
||||
}
|
||||
}
|
||||
|
||||
$blockedMatches = array_values(array_unique($blockedMatches));
|
||||
$suspiciousMatches = array_values(array_unique($suspiciousMatches));
|
||||
|
||||
if (!empty($blockedMatches)) {
|
||||
$findings[] = [
|
||||
'rule' => 'blocked_domain',
|
||||
'score' => ($weights['blacklisted_domain'] ?? 70) * count($blockedMatches),
|
||||
'reason' => 'Contains blocked domain(s): ' . implode(', ', $blockedMatches),
|
||||
'links' => $urls,
|
||||
'domains' => $blockedMatches,
|
||||
'keywords' => [],
|
||||
];
|
||||
}
|
||||
|
||||
if (!empty($suspiciousMatches)) {
|
||||
$findings[] = [
|
||||
'rule' => 'suspicious_domain',
|
||||
'score' => ($weights['suspicious_domain'] ?? 40) * count($suspiciousMatches),
|
||||
'reason' => 'Contains suspicious TLD domain(s): ' . implode(', ', $suspiciousMatches),
|
||||
'links' => $urls,
|
||||
'domains' => $suspiciousMatches,
|
||||
'keywords' => [],
|
||||
];
|
||||
}
|
||||
|
||||
return $findings;
|
||||
}
|
||||
}
|
||||
41
app/Services/Moderation/Rules/DuplicateCommentRule.php
Normal file
41
app/Services/Moderation/Rules/DuplicateCommentRule.php
Normal file
@@ -0,0 +1,41 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Moderation\Rules;
|
||||
|
||||
use App\Contracts\Moderation\ModerationRuleInterface;
|
||||
use App\Enums\ModerationContentType;
|
||||
use App\Models\ArtworkComment;
|
||||
|
||||
class DuplicateCommentRule implements ModerationRuleInterface
|
||||
{
|
||||
public function analyze(string $content, string $normalized, array $context = []): array
|
||||
{
|
||||
if (($context['content_type'] ?? null) !== ModerationContentType::ArtworkComment->value) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$contentId = (int) ($context['content_id'] ?? 0);
|
||||
if ($contentId <= 0 || $normalized === '') {
|
||||
return [];
|
||||
}
|
||||
|
||||
$duplicates = ArtworkComment::query()
|
||||
->where('id', '!=', $contentId)
|
||||
->whereNull('deleted_at')
|
||||
->whereRaw('LOWER(TRIM(COALESCE(raw_content, content))) = ?', [$normalized])
|
||||
->count();
|
||||
|
||||
if ($duplicates < 1) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return [[
|
||||
'rule' => 'duplicate_comment',
|
||||
'score' => app('config')->get('content_moderation.weights.duplicate_comment', 35),
|
||||
'reason' => 'Matches ' . $duplicates . ' existing comment(s) exactly',
|
||||
'links' => [],
|
||||
'domains' => [],
|
||||
'keywords' => [],
|
||||
]];
|
||||
}
|
||||
}
|
||||
54
app/Services/Moderation/Rules/ExcessivePunctuationRule.php
Normal file
54
app/Services/Moderation/Rules/ExcessivePunctuationRule.php
Normal file
@@ -0,0 +1,54 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Moderation\Rules;
|
||||
|
||||
use App\Contracts\Moderation\ModerationRuleInterface;
|
||||
|
||||
class ExcessivePunctuationRule implements ModerationRuleInterface
|
||||
{
|
||||
public function analyze(string $content, string $normalized, array $context = []): array
|
||||
{
|
||||
$config = app('config')->get('content_moderation.excessive_punctuation', []);
|
||||
$length = mb_strlen($content);
|
||||
|
||||
if ($length < (int) ($config['min_length'] ?? 20)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$exclamationRatio = substr_count($content, '!') / max($length, 1);
|
||||
$questionRatio = substr_count($content, '?') / max($length, 1);
|
||||
$capsRatio = $this->capsRatio($content);
|
||||
$symbolBurst = preg_match('/[!?$%*@#._\-]{6,}/', $content) === 1;
|
||||
|
||||
if (
|
||||
$exclamationRatio <= (float) ($config['max_exclamation_ratio'] ?? 0.1)
|
||||
&& $questionRatio <= (float) ($config['max_question_ratio'] ?? 0.1)
|
||||
&& $capsRatio <= (float) ($config['max_caps_ratio'] ?? 0.7)
|
||||
&& ! $symbolBurst
|
||||
) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return [[
|
||||
'rule' => 'excessive_punctuation',
|
||||
'score' => app('config')->get('content_moderation.weights.excessive_punctuation', 15),
|
||||
'reason' => 'Contains excessive punctuation, all-caps patterns, or symbol spam',
|
||||
'links' => [],
|
||||
'domains' => [],
|
||||
'keywords' => [],
|
||||
]];
|
||||
}
|
||||
|
||||
private function capsRatio(string $content): float
|
||||
{
|
||||
preg_match_all('/\p{Lu}/u', $content, $upperMatches);
|
||||
preg_match_all('/\p{L}/u', $content, $letterMatches);
|
||||
|
||||
$letters = count($letterMatches[0] ?? []);
|
||||
if ($letters === 0) {
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
return count($upperMatches[0] ?? []) / $letters;
|
||||
}
|
||||
}
|
||||
49
app/Services/Moderation/Rules/KeywordStuffingRule.php
Normal file
49
app/Services/Moderation/Rules/KeywordStuffingRule.php
Normal file
@@ -0,0 +1,49 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Moderation\Rules;
|
||||
|
||||
use App\Contracts\Moderation\ModerationRuleInterface;
|
||||
|
||||
class KeywordStuffingRule implements ModerationRuleInterface
|
||||
{
|
||||
public function analyze(string $content, string $normalized, array $context = []): array
|
||||
{
|
||||
preg_match_all('/[\p{L}\p{N}]+/u', $normalized, $matches);
|
||||
|
||||
$words = array_values(array_filter($matches[0] ?? [], static fn (string $word): bool => mb_strlen($word) > 1));
|
||||
$totalWords = count($words);
|
||||
$config = app('config')->get('content_moderation.keyword_stuffing', []);
|
||||
|
||||
if ($totalWords < (int) ($config['min_word_count'] ?? 20)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$frequencies = array_count_values($words);
|
||||
$uniqueRatio = count($frequencies) / max($totalWords, 1);
|
||||
$topFrequency = max($frequencies);
|
||||
$topWordRatio = $topFrequency / max($totalWords, 1);
|
||||
|
||||
$maxUniqueRatio = (float) ($config['max_unique_ratio'] ?? 0.3);
|
||||
$maxSingleWordFrequency = (float) ($config['max_single_word_frequency'] ?? 0.25);
|
||||
|
||||
if ($uniqueRatio >= $maxUniqueRatio && $topWordRatio <= $maxSingleWordFrequency) {
|
||||
return [];
|
||||
}
|
||||
|
||||
arsort($frequencies);
|
||||
$keywords = array_slice(array_keys($frequencies), 0, 5);
|
||||
|
||||
return [[
|
||||
'rule' => 'keyword_stuffing',
|
||||
'score' => app('config')->get('content_moderation.weights.keyword_stuffing', 20),
|
||||
'reason' => sprintf(
|
||||
'Likely keyword stuffing (unique ratio %.2f, top word ratio %.2f)',
|
||||
$uniqueRatio,
|
||||
$topWordRatio
|
||||
),
|
||||
'links' => [],
|
||||
'domains' => [],
|
||||
'keywords' => $keywords,
|
||||
]];
|
||||
}
|
||||
}
|
||||
118
app/Services/Moderation/Rules/LinkPresenceRule.php
Normal file
118
app/Services/Moderation/Rules/LinkPresenceRule.php
Normal file
@@ -0,0 +1,118 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Moderation\Rules;
|
||||
|
||||
use App\Contracts\Moderation\ModerationRuleInterface;
|
||||
use App\Enums\ModerationDomainStatus;
|
||||
use App\Services\Moderation\DomainReputationService;
|
||||
|
||||
class LinkPresenceRule implements ModerationRuleInterface
|
||||
{
|
||||
public function analyze(string $content, string $normalized, array $context = []): array
|
||||
{
|
||||
$urls = (array) ($context['extracted_urls'] ?? $this->extractUrls($content));
|
||||
|
||||
if (empty($urls)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$domainService = app(DomainReputationService::class);
|
||||
$shortenerDomains = $domainService->shortenerDomains();
|
||||
|
||||
$externalUrls = [];
|
||||
$shortenerUrls = [];
|
||||
|
||||
foreach ($urls as $url) {
|
||||
$host = $this->extractHost($url);
|
||||
if ($host === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($domainService->statusForDomain($host) === ModerationDomainStatus::Allowed) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($this->isDomainInList($host, $shortenerDomains)) {
|
||||
$shortenerUrls[] = $url;
|
||||
}
|
||||
|
||||
$externalUrls[] = $url;
|
||||
}
|
||||
|
||||
$findings = [];
|
||||
$weights = app('config')->get('content_moderation.weights', []);
|
||||
|
||||
if (count($shortenerUrls) > 0) {
|
||||
$findings[] = [
|
||||
'rule' => 'shortened_link',
|
||||
'score' => $weights['shortened_link'] ?? 30,
|
||||
'reason' => 'Contains ' . count($shortenerUrls) . ' shortened URL(s)',
|
||||
'links' => $shortenerUrls,
|
||||
'domains' => array_map(fn ($u) => $this->extractHost($u), $shortenerUrls),
|
||||
'keywords' => [],
|
||||
];
|
||||
}
|
||||
|
||||
if (count($externalUrls) > 1) {
|
||||
$findings[] = [
|
||||
'rule' => 'multiple_links',
|
||||
'score' => $weights['multiple_links'] ?? 40,
|
||||
'reason' => 'Contains ' . count($externalUrls) . ' external links',
|
||||
'links' => $externalUrls,
|
||||
'domains' => array_values(array_unique(array_filter(array_map(fn ($u) => $this->extractHost($u), $externalUrls)))),
|
||||
'keywords' => [],
|
||||
];
|
||||
} elseif (count($externalUrls) === 1) {
|
||||
$findings[] = [
|
||||
'rule' => 'single_external_link',
|
||||
'score' => $weights['single_external_link'] ?? 20,
|
||||
'reason' => 'Contains an external link',
|
||||
'links' => $externalUrls,
|
||||
'domains' => array_values(array_unique(array_filter(array_map(fn ($u) => $this->extractHost($u), $externalUrls)))),
|
||||
'keywords' => [],
|
||||
];
|
||||
}
|
||||
|
||||
return $findings;
|
||||
}
|
||||
|
||||
/** @return string[] */
|
||||
public function extractUrls(string $text): array
|
||||
{
|
||||
$matches = [];
|
||||
|
||||
preg_match_all("#https?://[^\\s<>\\[\\]\"'`\\)]+#iu", $text, $httpMatches);
|
||||
preg_match_all("#\\bwww\.[^\\s<>\\[\\]\"'`\\)]+#iu", $text, $wwwMatches);
|
||||
|
||||
$matches = array_merge($httpMatches[0] ?? [], $wwwMatches[0] ?? []);
|
||||
|
||||
return array_values(array_unique($matches));
|
||||
}
|
||||
|
||||
public function extractHost(string $url): ?string
|
||||
{
|
||||
$normalizedUrl = preg_match('#^https?://#i', $url) ? $url : 'https://' . ltrim($url, '/');
|
||||
$host = parse_url($normalizedUrl, PHP_URL_HOST);
|
||||
if (!is_string($host)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return app(DomainReputationService::class)->normalizeDomain($host);
|
||||
}
|
||||
|
||||
private function isDomainInList(string $host, array $list): bool
|
||||
{
|
||||
foreach ($list as $entry) {
|
||||
$entry = strtolower($entry);
|
||||
if ($host === $entry) {
|
||||
return true;
|
||||
}
|
||||
// Check if host is a subdomain of the entry
|
||||
if (str_ends_with($host, '.' . $entry)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
28
app/Services/Moderation/Rules/NearDuplicateCampaignRule.php
Normal file
28
app/Services/Moderation/Rules/NearDuplicateCampaignRule.php
Normal file
@@ -0,0 +1,28 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Moderation\Rules;
|
||||
|
||||
use App\Contracts\Moderation\ModerationRuleInterface;
|
||||
use App\Services\Moderation\DuplicateDetectionService;
|
||||
|
||||
class NearDuplicateCampaignRule implements ModerationRuleInterface
|
||||
{
|
||||
public function analyze(string $content, string $normalized, array $context = []): array
|
||||
{
|
||||
$domains = (array) ($context['extracted_domains'] ?? []);
|
||||
$duplicates = app(DuplicateDetectionService::class)->nearDuplicateCount($content, $context, $domains);
|
||||
|
||||
if ($duplicates < 2) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return [[
|
||||
'rule' => 'near_duplicate_campaign',
|
||||
'score' => app('config')->get('content_moderation.weights.near_duplicate_campaign', 30),
|
||||
'reason' => 'Appears to match an existing spam campaign template (' . $duplicates . ' similar item(s))',
|
||||
'links' => (array) ($context['extracted_urls'] ?? []),
|
||||
'domains' => $domains,
|
||||
'keywords' => [],
|
||||
]];
|
||||
}
|
||||
}
|
||||
38
app/Services/Moderation/Rules/RegexPatternRule.php
Normal file
38
app/Services/Moderation/Rules/RegexPatternRule.php
Normal file
@@ -0,0 +1,38 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Moderation\Rules;
|
||||
|
||||
use App\Contracts\Moderation\ModerationRuleInterface;
|
||||
use App\Services\Moderation\ModerationRuleRegistryService;
|
||||
|
||||
class RegexPatternRule implements ModerationRuleInterface
|
||||
{
|
||||
public function analyze(string $content, string $normalized, array $context = []): array
|
||||
{
|
||||
$registry = \app(ModerationRuleRegistryService::class);
|
||||
$findings = [];
|
||||
|
||||
foreach ($registry->regexRules() as $rule) {
|
||||
$pattern = (string) ($rule['pattern'] ?? '');
|
||||
if ($pattern === '') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$matched = @preg_match($pattern, $content) === 1 || @preg_match($pattern, $normalized) === 1;
|
||||
if (! $matched) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$findings[] = [
|
||||
'rule' => 'regex_pattern',
|
||||
'score' => (int) ($rule['weight'] ?? \app('config')->get('content_moderation.weights.regex_pattern', 30)),
|
||||
'reason' => 'Matched custom moderation regex rule',
|
||||
'links' => [],
|
||||
'domains' => [],
|
||||
'keywords' => [$pattern],
|
||||
];
|
||||
}
|
||||
|
||||
return $findings;
|
||||
}
|
||||
}
|
||||
56
app/Services/Moderation/Rules/RepeatedPhraseRule.php
Normal file
56
app/Services/Moderation/Rules/RepeatedPhraseRule.php
Normal file
@@ -0,0 +1,56 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Moderation\Rules;
|
||||
|
||||
use App\Contracts\Moderation\ModerationRuleInterface;
|
||||
|
||||
class RepeatedPhraseRule implements ModerationRuleInterface
|
||||
{
|
||||
public function analyze(string $content, string $normalized, array $context = []): array
|
||||
{
|
||||
$config = app('config')->get('content_moderation.repeated_phrase', []);
|
||||
$minPhraseLength = $config['min_phrase_length'] ?? 4;
|
||||
$minRepetitions = $config['min_repetitions'] ?? 3;
|
||||
$weights = app('config')->get('content_moderation.weights', []);
|
||||
|
||||
$words = preg_split('/\s+/', $normalized);
|
||||
if (count($words) < $minPhraseLength * $minRepetitions) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$findings = [];
|
||||
$repeatedPhrases = [];
|
||||
|
||||
// Check for repeated n-grams of various lengths
|
||||
for ($phraseLen = $minPhraseLength; $phraseLen <= min(8, intdiv(count($words), 2)); $phraseLen++) {
|
||||
$ngrams = [];
|
||||
for ($i = 0; $i <= count($words) - $phraseLen; $i++) {
|
||||
$ngram = implode(' ', array_slice($words, $i, $phraseLen));
|
||||
$ngrams[$ngram] = ($ngrams[$ngram] ?? 0) + 1;
|
||||
}
|
||||
|
||||
foreach ($ngrams as $phrase => $count) {
|
||||
if ($count >= $minRepetitions) {
|
||||
$repeatedPhrases[$phrase] = $count;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!empty($repeatedPhrases)) {
|
||||
$findings[] = [
|
||||
'rule' => 'repeated_phrase',
|
||||
'score' => $weights['repeated_phrase'] ?? 25,
|
||||
'reason' => 'Contains repeated phrases: ' . implode(', ', array_map(
|
||||
fn ($phrase, $count) => "\"{$phrase}\" ({$count}x)",
|
||||
array_keys($repeatedPhrases),
|
||||
array_values($repeatedPhrases)
|
||||
)),
|
||||
'links' => [],
|
||||
'domains' => [],
|
||||
'keywords' => array_keys($repeatedPhrases),
|
||||
];
|
||||
}
|
||||
|
||||
return $findings;
|
||||
}
|
||||
}
|
||||
55
app/Services/Moderation/Rules/SuspiciousKeywordRule.php
Normal file
55
app/Services/Moderation/Rules/SuspiciousKeywordRule.php
Normal file
@@ -0,0 +1,55 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Moderation\Rules;
|
||||
|
||||
use App\Contracts\Moderation\ModerationRuleInterface;
|
||||
use App\Services\Moderation\ModerationRuleRegistryService;
|
||||
|
||||
class SuspiciousKeywordRule implements ModerationRuleInterface
|
||||
{
|
||||
public function analyze(string $content, string $normalized, array $context = []): array
|
||||
{
|
||||
$registry = app(ModerationRuleRegistryService::class);
|
||||
$weights = app('config')->get('content_moderation.weights', []);
|
||||
$findings = [];
|
||||
|
||||
$highRiskMatched = [];
|
||||
$suspiciousMatched = [];
|
||||
|
||||
foreach ($registry->highRiskKeywords() as $phrase) {
|
||||
if (str_contains($normalized, strtolower($phrase))) {
|
||||
$highRiskMatched[] = $phrase;
|
||||
}
|
||||
}
|
||||
|
||||
foreach ($registry->suspiciousKeywords() as $phrase) {
|
||||
if (str_contains($normalized, strtolower($phrase))) {
|
||||
$suspiciousMatched[] = $phrase;
|
||||
}
|
||||
}
|
||||
|
||||
if (!empty($highRiskMatched)) {
|
||||
$findings[] = [
|
||||
'rule' => 'high_risk_keyword',
|
||||
'score' => ($weights['high_risk_keyword'] ?? 40) * count($highRiskMatched),
|
||||
'reason' => 'Contains high-risk keyword(s): ' . implode(', ', $highRiskMatched),
|
||||
'links' => [],
|
||||
'domains' => [],
|
||||
'keywords' => $highRiskMatched,
|
||||
];
|
||||
}
|
||||
|
||||
if (!empty($suspiciousMatched)) {
|
||||
$findings[] = [
|
||||
'rule' => 'suspicious_keyword',
|
||||
'score' => ($weights['suspicious_keyword'] ?? 25) * count($suspiciousMatched),
|
||||
'reason' => 'Contains suspicious keyword(s): ' . implode(', ', $suspiciousMatched),
|
||||
'links' => [],
|
||||
'domains' => [],
|
||||
'keywords' => $suspiciousMatched,
|
||||
];
|
||||
}
|
||||
|
||||
return $findings;
|
||||
}
|
||||
}
|
||||
49
app/Services/Moderation/Rules/UnicodeObfuscationRule.php
Normal file
49
app/Services/Moderation/Rules/UnicodeObfuscationRule.php
Normal file
@@ -0,0 +1,49 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Moderation\Rules;
|
||||
|
||||
use App\Contracts\Moderation\ModerationRuleInterface;
|
||||
|
||||
class UnicodeObfuscationRule implements ModerationRuleInterface
|
||||
{
|
||||
public function analyze(string $content, string $normalized, array $context = []): array
|
||||
{
|
||||
$findings = [];
|
||||
$weights = app('config')->get('content_moderation.weights', []);
|
||||
|
||||
// Detect homoglyph / lookalike characters
|
||||
// Common spam tactic: replace Latin chars with Cyrillic, Greek, or special Unicode
|
||||
$suspiciousPatterns = [
|
||||
// Mixed script detection: Latin + Cyrillic in same word
|
||||
'/\b(?=\S*[\x{0400}-\x{04FF}])(?=\S*[a-zA-Z])\S+\b/u',
|
||||
// Zero-width characters
|
||||
'/[\x{200B}\x{200C}\x{200D}\x{FEFF}\x{00AD}]/u',
|
||||
// Invisible formatting characters
|
||||
'/[\x{2060}\x{2061}\x{2062}\x{2063}\x{2064}]/u',
|
||||
// Fullwidth Latin letters (used to bypass filters)
|
||||
'/[\x{FF01}-\x{FF5E}]/u',
|
||||
// Mathematical alphanumeric symbols used as text
|
||||
'/[\x{1D400}-\x{1D7FF}]/u',
|
||||
];
|
||||
|
||||
$matchCount = 0;
|
||||
foreach ($suspiciousPatterns as $pattern) {
|
||||
if (preg_match($pattern, $content)) {
|
||||
$matchCount++;
|
||||
}
|
||||
}
|
||||
|
||||
if ($matchCount > 0) {
|
||||
$findings[] = [
|
||||
'rule' => 'unicode_obfuscation',
|
||||
'score' => ($weights['unicode_obfuscation'] ?? 30) * $matchCount,
|
||||
'reason' => 'Contains suspicious Unicode characters/obfuscation (' . $matchCount . ' pattern(s) matched)',
|
||||
'links' => [],
|
||||
'domains' => [],
|
||||
'keywords' => [],
|
||||
];
|
||||
}
|
||||
|
||||
return $findings;
|
||||
}
|
||||
}
|
||||
66
app/Services/Moderation/UserModerationProfileService.php
Normal file
66
app/Services/Moderation/UserModerationProfileService.php
Normal file
@@ -0,0 +1,66 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Moderation;
|
||||
|
||||
use App\Enums\ModerationStatus;
|
||||
use App\Models\ContentModerationFinding;
|
||||
use App\Models\User;
|
||||
|
||||
class UserModerationProfileService
|
||||
{
|
||||
/**
|
||||
* @return array<string, mixed>|null
|
||||
*/
|
||||
public function profile(?int $userId): ?array
|
||||
{
|
||||
if (! $userId) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$user = User::query()->with('statistics:user_id,uploads_count')->find($userId);
|
||||
if (! $user) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$findings = ContentModerationFinding::query()->where('user_id', $userId)->get([
|
||||
'id', 'status', 'is_auto_hidden', 'is_false_positive', 'matched_domains_json', 'campaign_key', 'priority_score', 'created_at',
|
||||
]);
|
||||
|
||||
$confirmedSpam = $findings->where('status', ModerationStatus::ConfirmedSpam)->count();
|
||||
$safeCount = $findings->where('status', ModerationStatus::ReviewedSafe)->count();
|
||||
$autoHidden = $findings->where('is_auto_hidden', true)->count();
|
||||
$falsePositives = $findings->where('is_false_positive', true)->count();
|
||||
$clusters = $findings->pluck('campaign_key')->filter()->unique()->count();
|
||||
$domains = $findings->flatMap(static fn (ContentModerationFinding $finding): array => (array) $finding->matched_domains_json)->filter()->unique()->values();
|
||||
$risk = app(UserRiskScoreService::class)->assess($userId, $domains->all());
|
||||
|
||||
$tier = match (true) {
|
||||
$risk['risk_score'] >= 75 => 'high_risk',
|
||||
$risk['risk_score'] >= 45 => 'monitored',
|
||||
$risk['risk_score'] <= 8 && $safeCount >= 2 => 'trusted',
|
||||
default => 'normal',
|
||||
};
|
||||
|
||||
$recommendedPolicy = match ($tier) {
|
||||
'high_risk' => 'new_user_strict_mode',
|
||||
'monitored' => 'comments_high_volume_antispam',
|
||||
'trusted' => 'trusted_user_relaxed_mode',
|
||||
default => 'default',
|
||||
};
|
||||
|
||||
return [
|
||||
'user' => $user,
|
||||
'risk_score' => $risk['risk_score'],
|
||||
'trust_score' => max(0, 100 - $risk['risk_score'] + ($safeCount * 3) - ($falsePositives * 2)),
|
||||
'tier' => $tier,
|
||||
'recommended_policy' => $recommendedPolicy,
|
||||
'confirmed_spam_count' => $confirmedSpam,
|
||||
'safe_count' => $safeCount,
|
||||
'auto_hidden_count' => $autoHidden,
|
||||
'false_positive_count' => $falsePositives,
|
||||
'cluster_memberships' => $clusters,
|
||||
'promoted_domains' => $domains->take(12)->all(),
|
||||
'signals' => $risk['signals'],
|
||||
];
|
||||
}
|
||||
}
|
||||
83
app/Services/Moderation/UserRiskScoreService.php
Normal file
83
app/Services/Moderation/UserRiskScoreService.php
Normal file
@@ -0,0 +1,83 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Moderation;
|
||||
|
||||
use App\Enums\ModerationStatus;
|
||||
use App\Models\ContentModerationFinding;
|
||||
use App\Models\User;
|
||||
|
||||
class UserRiskScoreService
|
||||
{
|
||||
/**
|
||||
* @param array<int, string> $domains
|
||||
* @return array{risk_score:int,score_modifier:int,signals:array<string, int>}
|
||||
*/
|
||||
public function assess(?int $userId, array $domains = []): array
|
||||
{
|
||||
if (! $userId) {
|
||||
return ['risk_score' => 0, 'score_modifier' => 0, 'signals' => []];
|
||||
}
|
||||
|
||||
$user = User::query()->with('statistics:user_id,uploads_count')->find($userId);
|
||||
if (! $user) {
|
||||
return ['risk_score' => 0, 'score_modifier' => 0, 'signals' => []];
|
||||
}
|
||||
|
||||
$summary = ContentModerationFinding::query()
|
||||
->where('user_id', $userId)
|
||||
->selectRaw('COUNT(*) as total_findings')
|
||||
->selectRaw("SUM(CASE WHEN status = ? THEN 1 ELSE 0 END) as confirmed_spam_count", [ModerationStatus::ConfirmedSpam->value])
|
||||
->selectRaw("SUM(CASE WHEN status = ? THEN 1 ELSE 0 END) as safe_count", [ModerationStatus::ReviewedSafe->value])
|
||||
->selectRaw("SUM(CASE WHEN status = ? THEN 1 ELSE 0 END) as pending_count", [ModerationStatus::Pending->value])
|
||||
->first();
|
||||
|
||||
$confirmedSpam = (int) ($summary?->confirmed_spam_count ?? 0);
|
||||
$safeCount = (int) ($summary?->safe_count ?? 0);
|
||||
$pendingCount = (int) ($summary?->pending_count ?? 0);
|
||||
$domainRepeatCount = ContentModerationFinding::query()
|
||||
->where('user_id', $userId)
|
||||
->whereNotNull('matched_domains_json')
|
||||
->get(['matched_domains_json'])
|
||||
->reduce(function (int $carry, ContentModerationFinding $finding) use ($domains): int {
|
||||
return $carry + (empty(array_intersect((array) $finding->matched_domains_json, $domains)) ? 0 : 1);
|
||||
}, 0);
|
||||
|
||||
$accountAgeDays = max(0, \now()->diffInDays($user->created_at));
|
||||
$uploadsCount = (int) ($user->statistics?->uploads_count ?? 0);
|
||||
|
||||
$riskScore = 0;
|
||||
$riskScore += $confirmedSpam * 18;
|
||||
$riskScore += $pendingCount * 5;
|
||||
$riskScore += min(20, $domainRepeatCount * 6);
|
||||
$riskScore -= min(18, $safeCount * 4);
|
||||
$riskScore -= $accountAgeDays >= 365 ? 8 : ($accountAgeDays >= 90 ? 4 : 0);
|
||||
$riskScore -= $uploadsCount >= 25 ? 6 : ($uploadsCount >= 10 ? 3 : 0);
|
||||
$riskScore = max(0, min(100, $riskScore));
|
||||
|
||||
$modifier = 0;
|
||||
if ($riskScore >= 75) {
|
||||
$modifier = (int) \app('config')->get('content_moderation.user_risk.high_modifier', 18);
|
||||
} elseif ($riskScore >= 50) {
|
||||
$modifier = (int) \app('config')->get('content_moderation.user_risk.medium_modifier', 10);
|
||||
} elseif ($riskScore >= 25) {
|
||||
$modifier = (int) \app('config')->get('content_moderation.user_risk.low_modifier', 4);
|
||||
} elseif ($riskScore <= 5 && $accountAgeDays >= 180 && $uploadsCount >= 10) {
|
||||
$modifier = (int) \app('config')->get('content_moderation.user_risk.trusted_modifier', -6);
|
||||
} elseif ($riskScore <= 12 && $safeCount >= 2) {
|
||||
$modifier = -3;
|
||||
}
|
||||
|
||||
return [
|
||||
'risk_score' => $riskScore,
|
||||
'score_modifier' => $modifier,
|
||||
'signals' => [
|
||||
'confirmed_spam_count' => $confirmedSpam,
|
||||
'safe_count' => $safeCount,
|
||||
'pending_count' => $pendingCount,
|
||||
'domain_repeat_count' => $domainRepeatCount,
|
||||
'account_age_days' => $accountAgeDays,
|
||||
'uploads_count' => $uploadsCount,
|
||||
],
|
||||
];
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user