Implement creator studio and upload updates
This commit is contained in:
203
app/Services/Moderation/ContentModerationService.php
Normal file
203
app/Services/Moderation/ContentModerationService.php
Normal file
@@ -0,0 +1,203 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Moderation;
|
||||
|
||||
use App\Contracts\Moderation\ModerationRuleInterface;
|
||||
use App\Data\Moderation\ModerationResultData;
|
||||
use App\Enums\ModerationSeverity;
|
||||
use App\Enums\ModerationStatus;
|
||||
use App\Services\Moderation\DuplicateDetectionService;
|
||||
use App\Services\Moderation\Rules\LinkPresenceRule;
|
||||
|
||||
class ContentModerationService
|
||||
{
|
||||
public function __construct(
|
||||
private readonly ModerationPolicyEngineService $policies,
|
||||
private readonly ModerationSuggestionService $suggestions,
|
||||
private readonly ModerationClusterService $clusters,
|
||||
private readonly ModerationPriorityService $priorities,
|
||||
) {
|
||||
}
|
||||
|
||||
public function analyze(string $content, array $context = []): ModerationResultData
|
||||
{
|
||||
$normalized = $this->normalize($content);
|
||||
$campaignNormalized = app(DuplicateDetectionService::class)->campaignText($content);
|
||||
$linkRule = app(LinkPresenceRule::class);
|
||||
$extractedUrls = $linkRule->extractUrls($content);
|
||||
$extractedDomains = array_values(array_unique(array_filter(array_map(
|
||||
static fn (string $url): ?string => $linkRule->extractHost($url),
|
||||
$extractedUrls
|
||||
))));
|
||||
|
||||
$riskAssessment = app(UserRiskScoreService::class)->assess(
|
||||
isset($context['user_id']) ? (int) $context['user_id'] : null,
|
||||
$extractedDomains,
|
||||
);
|
||||
|
||||
$context['extracted_urls'] = $extractedUrls;
|
||||
$context['extracted_domains'] = $extractedDomains;
|
||||
$context['user_risk_assessment'] = $riskAssessment;
|
||||
|
||||
$score = 0;
|
||||
$reasons = [];
|
||||
$matchedLinks = [];
|
||||
$matchedDomains = [];
|
||||
$matchedKeywords = [];
|
||||
$ruleHits = [];
|
||||
$scoreBreakdown = [];
|
||||
|
||||
foreach ($this->rules() as $rule) {
|
||||
foreach ($rule->analyze($content, $normalized, $context) as $finding) {
|
||||
$ruleScore = (int) ($finding['score'] ?? 0);
|
||||
$score += $ruleScore;
|
||||
$reason = (string) ($finding['reason'] ?? 'Flagged by moderation rule');
|
||||
$reasons[] = $reason;
|
||||
$matchedLinks = array_merge($matchedLinks, (array) ($finding['links'] ?? []));
|
||||
$matchedDomains = array_merge($matchedDomains, array_filter((array) ($finding['domains'] ?? [])));
|
||||
$matchedKeywords = array_merge($matchedKeywords, array_filter((array) ($finding['keywords'] ?? [])));
|
||||
$ruleKey = (string) ($finding['rule'] ?? 'unknown');
|
||||
$ruleHits[$ruleKey] = ($ruleHits[$ruleKey] ?? 0) + 1;
|
||||
$scoreBreakdown[] = [
|
||||
'rule' => $ruleKey,
|
||||
'score' => $ruleScore,
|
||||
'reason' => $reason,
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
$modifier = (int) ($riskAssessment['score_modifier'] ?? 0);
|
||||
if ($modifier !== 0) {
|
||||
$score += $modifier;
|
||||
$reasons[] = $modifier > 0
|
||||
? 'User risk profile increased moderation score by ' . $modifier
|
||||
: 'User trust profile reduced moderation score by ' . abs($modifier);
|
||||
$ruleHits['user_risk_modifier'] = 1;
|
||||
$scoreBreakdown[] = [
|
||||
'rule' => 'user_risk_modifier',
|
||||
'score' => $modifier,
|
||||
'reason' => $modifier > 0
|
||||
? 'User risk profile increased moderation score by ' . $modifier
|
||||
: 'User trust profile reduced moderation score by ' . abs($modifier),
|
||||
];
|
||||
}
|
||||
|
||||
$score = max(0, $score);
|
||||
$severity = ModerationSeverity::fromScore($score);
|
||||
$policy = $this->policies->resolve($context, $riskAssessment);
|
||||
$autoHideRecommended = $this->shouldAutoHide($score, $ruleHits, $matchedDomains ?: $extractedDomains, $policy);
|
||||
$groupKey = app(DuplicateDetectionService::class)->buildGroupKey($content, $matchedDomains ?: $extractedDomains);
|
||||
|
||||
$draft = new ModerationResultData(
|
||||
score: $score,
|
||||
severity: $severity,
|
||||
status: $score >= (int) ($policy['queue_threshold'] ?? app('config')->get('content_moderation.queue_threshold', 30))
|
||||
? ModerationStatus::Pending
|
||||
: ModerationStatus::ReviewedSafe,
|
||||
reasons: array_values(array_unique(array_filter($reasons))),
|
||||
matchedLinks: array_values(array_unique(array_filter($matchedLinks))),
|
||||
matchedDomains: array_values(array_unique(array_filter($matchedDomains))),
|
||||
matchedKeywords: array_values(array_unique(array_filter($matchedKeywords))),
|
||||
contentHash: hash('sha256', $normalized),
|
||||
scannerVersion: (string) app('config')->get('content_moderation.scanner_version', '1.0'),
|
||||
ruleHits: $ruleHits,
|
||||
contentHashNormalized: hash('sha256', $campaignNormalized),
|
||||
groupKey: $groupKey,
|
||||
userRiskScore: (int) ($riskAssessment['risk_score'] ?? 0),
|
||||
autoHideRecommended: $autoHideRecommended,
|
||||
contentTargetType: isset($context['content_target_type']) ? (string) $context['content_target_type'] : null,
|
||||
contentTargetId: isset($context['content_target_id']) ? (int) $context['content_target_id'] : null,
|
||||
policyName: (string) ($policy['name'] ?? 'default'),
|
||||
scoreBreakdown: $scoreBreakdown,
|
||||
);
|
||||
|
||||
$suggestion = $this->suggestions->suggest($content, $draft, $context);
|
||||
$cluster = $this->clusters->classify($content, $draft, $context, [
|
||||
'campaign_tags' => $suggestion->campaignTags,
|
||||
'confidence' => $suggestion->confidence,
|
||||
]);
|
||||
$priority = $this->priorities->score($draft, $context, $policy, [
|
||||
'confidence' => $suggestion->confidence,
|
||||
'campaign_tags' => $suggestion->campaignTags,
|
||||
]);
|
||||
|
||||
return new ModerationResultData(
|
||||
score: $score,
|
||||
severity: $severity,
|
||||
status: $score >= (int) ($policy['queue_threshold'] ?? app('config')->get('content_moderation.queue_threshold', 30))
|
||||
? ModerationStatus::Pending
|
||||
: ModerationStatus::ReviewedSafe,
|
||||
reasons: array_values(array_unique(array_filter($reasons))),
|
||||
matchedLinks: array_values(array_unique(array_filter($matchedLinks))),
|
||||
matchedDomains: array_values(array_unique(array_filter($matchedDomains))),
|
||||
matchedKeywords: array_values(array_unique(array_filter($matchedKeywords))),
|
||||
contentHash: hash('sha256', $normalized),
|
||||
scannerVersion: (string) app('config')->get('content_moderation.scanner_version', '1.0'),
|
||||
ruleHits: $ruleHits,
|
||||
contentHashNormalized: hash('sha256', $campaignNormalized),
|
||||
groupKey: $groupKey,
|
||||
userRiskScore: (int) ($riskAssessment['risk_score'] ?? 0),
|
||||
autoHideRecommended: $autoHideRecommended,
|
||||
contentTargetType: isset($context['content_target_type']) ? (string) $context['content_target_type'] : null,
|
||||
contentTargetId: isset($context['content_target_id']) ? (int) $context['content_target_id'] : null,
|
||||
campaignKey: $cluster['campaign_key'],
|
||||
clusterScore: $cluster['cluster_score'],
|
||||
clusterReason: $cluster['cluster_reason'],
|
||||
policyName: (string) ($policy['name'] ?? 'default'),
|
||||
priorityScore: (int) ($priority['priority_score'] ?? $score),
|
||||
reviewBucket: (string) ($priority['review_bucket'] ?? ($policy['review_bucket'] ?? 'standard')),
|
||||
escalationStatus: (string) ($priority['escalation_status'] ?? 'none'),
|
||||
aiProvider: $suggestion->provider,
|
||||
aiLabel: $suggestion->suggestedLabel,
|
||||
aiSuggestedAction: $suggestion->suggestedAction,
|
||||
aiConfidence: $suggestion->confidence,
|
||||
aiExplanation: $suggestion->explanation,
|
||||
aiRawResponse: $suggestion->rawResponse,
|
||||
scoreBreakdown: $scoreBreakdown,
|
||||
);
|
||||
}
|
||||
|
||||
public function normalize(string $content): string
|
||||
{
|
||||
$normalized = preg_replace('/\s+/u', ' ', trim($content));
|
||||
|
||||
return mb_strtolower((string) $normalized);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<int, ModerationRuleInterface>
|
||||
*/
|
||||
private function rules(): array
|
||||
{
|
||||
$classes = app('config')->get('content_moderation.rules.enabled', []);
|
||||
|
||||
return array_values(array_filter(array_map(function (string $class): ?ModerationRuleInterface {
|
||||
$rule = app($class);
|
||||
|
||||
return $rule instanceof ModerationRuleInterface ? $rule : null;
|
||||
}, $classes)));
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, int> $ruleHits
|
||||
* @param array<int, string> $matchedDomains
|
||||
*/
|
||||
private function shouldAutoHide(int $score, array $ruleHits, array $matchedDomains, array $policy = []): bool
|
||||
{
|
||||
if (! app('config')->get('content_moderation.auto_hide.enabled', true)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$threshold = (int) ($policy['auto_hide_threshold'] ?? app('config')->get('content_moderation.auto_hide.threshold', 95));
|
||||
if ($score >= $threshold) {
|
||||
return true;
|
||||
}
|
||||
|
||||
$blockedHit = isset($ruleHits['blacklisted_domain']) || isset($ruleHits['blocked_domain']);
|
||||
$severeHitCount = collect($ruleHits)
|
||||
->only(['blacklisted_domain', 'blocked_domain', 'high_risk_keyword', 'near_duplicate_campaign', 'duplicate_comment'])
|
||||
->sum();
|
||||
|
||||
return $blockedHit && $severeHitCount >= 2 && count($matchedDomains) >= 1;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user