203 lines
9.3 KiB
PHP
203 lines
9.3 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Moderation;
|
|
|
|
use App\Contracts\Moderation\ModerationRuleInterface;
|
|
use App\Data\Moderation\ModerationResultData;
|
|
use App\Enums\ModerationSeverity;
|
|
use App\Enums\ModerationStatus;
|
|
use App\Services\Moderation\DuplicateDetectionService;
|
|
use App\Services\Moderation\Rules\LinkPresenceRule;
|
|
|
|
class ContentModerationService
|
|
{
|
|
public function __construct(
|
|
private readonly ModerationPolicyEngineService $policies,
|
|
private readonly ModerationSuggestionService $suggestions,
|
|
private readonly ModerationClusterService $clusters,
|
|
private readonly ModerationPriorityService $priorities,
|
|
) {
|
|
}
|
|
|
|
public function analyze(string $content, array $context = []): ModerationResultData
|
|
{
|
|
$normalized = $this->normalize($content);
|
|
$campaignNormalized = app(DuplicateDetectionService::class)->campaignText($content);
|
|
$linkRule = app(LinkPresenceRule::class);
|
|
$extractedUrls = $linkRule->extractUrls($content);
|
|
$extractedDomains = array_values(array_unique(array_filter(array_map(
|
|
static fn (string $url): ?string => $linkRule->extractHost($url),
|
|
$extractedUrls
|
|
))));
|
|
|
|
$riskAssessment = app(UserRiskScoreService::class)->assess(
|
|
isset($context['user_id']) ? (int) $context['user_id'] : null,
|
|
$extractedDomains,
|
|
);
|
|
|
|
$context['extracted_urls'] = $extractedUrls;
|
|
$context['extracted_domains'] = $extractedDomains;
|
|
$context['user_risk_assessment'] = $riskAssessment;
|
|
|
|
$score = 0;
|
|
$reasons = [];
|
|
$matchedLinks = [];
|
|
$matchedDomains = [];
|
|
$matchedKeywords = [];
|
|
$ruleHits = [];
|
|
$scoreBreakdown = [];
|
|
|
|
foreach ($this->rules() as $rule) {
|
|
foreach ($rule->analyze($content, $normalized, $context) as $finding) {
|
|
$ruleScore = (int) ($finding['score'] ?? 0);
|
|
$score += $ruleScore;
|
|
$reason = (string) ($finding['reason'] ?? 'Flagged by moderation rule');
|
|
$reasons[] = $reason;
|
|
$matchedLinks = array_merge($matchedLinks, (array) ($finding['links'] ?? []));
|
|
$matchedDomains = array_merge($matchedDomains, array_filter((array) ($finding['domains'] ?? [])));
|
|
$matchedKeywords = array_merge($matchedKeywords, array_filter((array) ($finding['keywords'] ?? [])));
|
|
$ruleKey = (string) ($finding['rule'] ?? 'unknown');
|
|
$ruleHits[$ruleKey] = ($ruleHits[$ruleKey] ?? 0) + 1;
|
|
$scoreBreakdown[] = [
|
|
'rule' => $ruleKey,
|
|
'score' => $ruleScore,
|
|
'reason' => $reason,
|
|
];
|
|
}
|
|
}
|
|
|
|
$modifier = (int) ($riskAssessment['score_modifier'] ?? 0);
|
|
if ($modifier !== 0) {
|
|
$score += $modifier;
|
|
$reasons[] = $modifier > 0
|
|
? 'User risk profile increased moderation score by ' . $modifier
|
|
: 'User trust profile reduced moderation score by ' . abs($modifier);
|
|
$ruleHits['user_risk_modifier'] = 1;
|
|
$scoreBreakdown[] = [
|
|
'rule' => 'user_risk_modifier',
|
|
'score' => $modifier,
|
|
'reason' => $modifier > 0
|
|
? 'User risk profile increased moderation score by ' . $modifier
|
|
: 'User trust profile reduced moderation score by ' . abs($modifier),
|
|
];
|
|
}
|
|
|
|
$score = max(0, $score);
|
|
$severity = ModerationSeverity::fromScore($score);
|
|
$policy = $this->policies->resolve($context, $riskAssessment);
|
|
$autoHideRecommended = $this->shouldAutoHide($score, $ruleHits, $matchedDomains ?: $extractedDomains, $policy);
|
|
$groupKey = app(DuplicateDetectionService::class)->buildGroupKey($content, $matchedDomains ?: $extractedDomains);
|
|
|
|
$draft = new ModerationResultData(
|
|
score: $score,
|
|
severity: $severity,
|
|
status: $score >= (int) ($policy['queue_threshold'] ?? app('config')->get('content_moderation.queue_threshold', 30))
|
|
? ModerationStatus::Pending
|
|
: ModerationStatus::ReviewedSafe,
|
|
reasons: array_values(array_unique(array_filter($reasons))),
|
|
matchedLinks: array_values(array_unique(array_filter($matchedLinks))),
|
|
matchedDomains: array_values(array_unique(array_filter($matchedDomains))),
|
|
matchedKeywords: array_values(array_unique(array_filter($matchedKeywords))),
|
|
contentHash: hash('sha256', $normalized),
|
|
scannerVersion: (string) app('config')->get('content_moderation.scanner_version', '1.0'),
|
|
ruleHits: $ruleHits,
|
|
contentHashNormalized: hash('sha256', $campaignNormalized),
|
|
groupKey: $groupKey,
|
|
userRiskScore: (int) ($riskAssessment['risk_score'] ?? 0),
|
|
autoHideRecommended: $autoHideRecommended,
|
|
contentTargetType: isset($context['content_target_type']) ? (string) $context['content_target_type'] : null,
|
|
contentTargetId: isset($context['content_target_id']) ? (int) $context['content_target_id'] : null,
|
|
policyName: (string) ($policy['name'] ?? 'default'),
|
|
scoreBreakdown: $scoreBreakdown,
|
|
);
|
|
|
|
$suggestion = $this->suggestions->suggest($content, $draft, $context);
|
|
$cluster = $this->clusters->classify($content, $draft, $context, [
|
|
'campaign_tags' => $suggestion->campaignTags,
|
|
'confidence' => $suggestion->confidence,
|
|
]);
|
|
$priority = $this->priorities->score($draft, $context, $policy, [
|
|
'confidence' => $suggestion->confidence,
|
|
'campaign_tags' => $suggestion->campaignTags,
|
|
]);
|
|
|
|
return new ModerationResultData(
|
|
score: $score,
|
|
severity: $severity,
|
|
status: $score >= (int) ($policy['queue_threshold'] ?? app('config')->get('content_moderation.queue_threshold', 30))
|
|
? ModerationStatus::Pending
|
|
: ModerationStatus::ReviewedSafe,
|
|
reasons: array_values(array_unique(array_filter($reasons))),
|
|
matchedLinks: array_values(array_unique(array_filter($matchedLinks))),
|
|
matchedDomains: array_values(array_unique(array_filter($matchedDomains))),
|
|
matchedKeywords: array_values(array_unique(array_filter($matchedKeywords))),
|
|
contentHash: hash('sha256', $normalized),
|
|
scannerVersion: (string) app('config')->get('content_moderation.scanner_version', '1.0'),
|
|
ruleHits: $ruleHits,
|
|
contentHashNormalized: hash('sha256', $campaignNormalized),
|
|
groupKey: $groupKey,
|
|
userRiskScore: (int) ($riskAssessment['risk_score'] ?? 0),
|
|
autoHideRecommended: $autoHideRecommended,
|
|
contentTargetType: isset($context['content_target_type']) ? (string) $context['content_target_type'] : null,
|
|
contentTargetId: isset($context['content_target_id']) ? (int) $context['content_target_id'] : null,
|
|
campaignKey: $cluster['campaign_key'],
|
|
clusterScore: $cluster['cluster_score'],
|
|
clusterReason: $cluster['cluster_reason'],
|
|
policyName: (string) ($policy['name'] ?? 'default'),
|
|
priorityScore: (int) ($priority['priority_score'] ?? $score),
|
|
reviewBucket: (string) ($priority['review_bucket'] ?? ($policy['review_bucket'] ?? 'standard')),
|
|
escalationStatus: (string) ($priority['escalation_status'] ?? 'none'),
|
|
aiProvider: $suggestion->provider,
|
|
aiLabel: $suggestion->suggestedLabel,
|
|
aiSuggestedAction: $suggestion->suggestedAction,
|
|
aiConfidence: $suggestion->confidence,
|
|
aiExplanation: $suggestion->explanation,
|
|
aiRawResponse: $suggestion->rawResponse,
|
|
scoreBreakdown: $scoreBreakdown,
|
|
);
|
|
}
|
|
|
|
public function normalize(string $content): string
|
|
{
|
|
$normalized = preg_replace('/\s+/u', ' ', trim($content));
|
|
|
|
return mb_strtolower((string) $normalized);
|
|
}
|
|
|
|
/**
|
|
* @return array<int, ModerationRuleInterface>
|
|
*/
|
|
private function rules(): array
|
|
{
|
|
$classes = app('config')->get('content_moderation.rules.enabled', []);
|
|
|
|
return array_values(array_filter(array_map(function (string $class): ?ModerationRuleInterface {
|
|
$rule = app($class);
|
|
|
|
return $rule instanceof ModerationRuleInterface ? $rule : null;
|
|
}, $classes)));
|
|
}
|
|
|
|
/**
|
|
* @param array<string, int> $ruleHits
|
|
* @param array<int, string> $matchedDomains
|
|
*/
|
|
private function shouldAutoHide(int $score, array $ruleHits, array $matchedDomains, array $policy = []): bool
|
|
{
|
|
if (! app('config')->get('content_moderation.auto_hide.enabled', true)) {
|
|
return false;
|
|
}
|
|
|
|
$threshold = (int) ($policy['auto_hide_threshold'] ?? app('config')->get('content_moderation.auto_hide.threshold', 95));
|
|
if ($score >= $threshold) {
|
|
return true;
|
|
}
|
|
|
|
$blockedHit = isset($ruleHits['blacklisted_domain']) || isset($ruleHits['blocked_domain']);
|
|
$severeHitCount = collect($ruleHits)
|
|
->only(['blacklisted_domain', 'blocked_domain', 'high_risk_keyword', 'near_duplicate_campaign', 'duplicate_comment'])
|
|
->sum();
|
|
|
|
return $blockedHit && $severeHitCount >= 2 && count($matchedDomains) >= 1;
|
|
}
|
|
} |