From 437d742ac4763dedcc3b32c3600edc68636d0308 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 8 Apr 2025 05:58:06 -0600 Subject: [PATCH] Add custom filters Add custom filters, compatible with Mastodon `/api/v2/filters` Todo: - [ ] fix routes - [ ] finish other context filtering --- app/Http/Controllers/Api/ApiV1Controller.php | 24 ++ .../Controllers/CustomFilterController.php | 225 +++++++++++++++ .../CustomFilterKeywordController.php | 10 + .../CustomFilterStatusController.php | 10 + app/Models/CustomFilter.php | 265 ++++++++++++++++++ app/Models/CustomFilterKeyword.php | 32 +++ app/Models/CustomFilterStatus.php | 23 ++ app/Policies/CustomFilterPolicy.php | 27 ++ app/Providers/AuthServiceProvider.php | 4 +- ..._08_102711_create_custom_filters_table.php | 32 +++ ...25_create_custom_filter_keywords_table.php | 30 ++ ...33_create_custom_filter_statuses_table.php | 29 ++ routes/api.php | 1 + 13 files changed, 711 insertions(+), 1 deletion(-) create mode 100644 app/Http/Controllers/CustomFilterController.php create mode 100644 app/Http/Controllers/CustomFilterKeywordController.php create mode 100644 app/Http/Controllers/CustomFilterStatusController.php create mode 100644 app/Models/CustomFilter.php create mode 100644 app/Models/CustomFilterKeyword.php create mode 100644 app/Models/CustomFilterStatus.php create mode 100644 app/Policies/CustomFilterPolicy.php create mode 100644 database/migrations/2025_04_08_102711_create_custom_filters_table.php create mode 100644 database/migrations/2025_04_08_103425_create_custom_filter_keywords_table.php create mode 100644 database/migrations/2025_04_08_103433_create_custom_filter_statuses_table.php diff --git a/app/Http/Controllers/Api/ApiV1Controller.php b/app/Http/Controllers/Api/ApiV1Controller.php index 9151107c4..2d25f236c 100644 --- a/app/Http/Controllers/Api/ApiV1Controller.php +++ b/app/Http/Controllers/Api/ApiV1Controller.php @@ -35,6 +35,7 @@ use App\Jobs\VideoPipeline\VideoThumbnail; use App\Like; use App\Media; use App\Models\Conversation; +use App\Models\CustomFilter; use App\Notification; use App\Profile; use App\Services\AccountService; @@ -2514,6 +2515,14 @@ class ApiV1Controller extends Controller ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album']; AccountService::setLastActive($request->user()->id); + $cachedFilters = CustomFilter::getCachedFiltersForAccount($pid); + + $homeFilters = array_filter($cachedFilters, function ($item) { + [$filter, $rules] = $item; + + return in_array('home', $filter->context); + }); + if (config('exp.cached_home_timeline')) { $paddedLimit = $includeReblogs ? $limit + 10 : $limit + 50; if ($min || $max) { @@ -2550,6 +2559,21 @@ class ApiV1Controller extends Controller ->filter(function ($s) use ($includeReblogs) { return $includeReblogs ? true : $s['reblog'] == null; }) + ->filter(function ($s) use ($homeFilters) { + $filterResults = CustomFilter::applyCachedFilters($homeFilters, $s); + + if (! empty($filterResults)) { + $shouldHide = collect($filterResults)->contains(function ($result) { + return $result['filter']->action === CustomFilter::ACTION_HIDE; + }); + + if ($shouldHide) { + return false; + } + } + + return true; + }) ->take($limit) ->map(function ($status) use ($pid) { if ($pid) { diff --git a/app/Http/Controllers/CustomFilterController.php b/app/Http/Controllers/CustomFilterController.php new file mode 100644 index 000000000..cf473e16c --- /dev/null +++ b/app/Http/Controllers/CustomFilterController.php @@ -0,0 +1,225 @@ +middleware('auth'); + } + + public function index(Request $request) + { + $filters = CustomFilter::where('profile_id', $request->user()->profile_id) + ->unexpired() + ->with(['keywords', 'statuses']) + ->get(); + + return response()->json([ + 'filters' => $filters, + ]); + } + + public function show(CustomFilter $filter) + { + $this->authorize('view', $filter); + + $filter->load(['keywords', 'statuses']); + + return response()->json([ + 'filter' => $filter, + ]); + } + + public function store(Request $request) + { + $validatedData = $request->validate([ + 'title' => 'required|string', + 'context' => 'required|array', + 'context.*' => 'string|in:'.implode(',', CustomFilter::VALID_CONTEXTS), + 'filter_action' => 'integer|in:0,1,2', + 'expires_in' => 'nullable|integer|min:0', + 'irreversible' => 'boolean', + 'keywords' => 'array', + 'keywords.*.keyword' => 'required|string', + 'keywords.*.whole_word' => 'boolean', + 'status_ids' => 'array', + 'status_ids.*' => 'integer|exists:statuses,id', + ]); + + DB::beginTransaction(); + + try { + $expiresAt = null; + if (isset($validatedData['expires_in']) && $validatedData['expires_in'] > 0) { + $expiresAt = now()->addSeconds($validatedData['expires_in']); + } + + $filter = CustomFilter::create([ + 'phrase' => $validatedData['title'], + 'context' => $validatedData['context'], + 'action' => $validatedData['filter_action'] ?? + (isset($validatedData['irreversible']) && $validatedData['irreversible'] ? + CustomFilter::ACTION_HIDE : CustomFilter::ACTION_WARN), + 'expires_at' => $expiresAt, + 'profile_id' => $request->user()->profile_id, + ]); + + if (isset($validatedData['keywords'])) { + foreach ($validatedData['keywords'] as $keywordData) { + $filter->keywords()->create([ + 'keyword' => $keywordData['keyword'], + 'whole_word' => $keywordData['whole_word'] ?? true, + ]); + } + } + + if (isset($validatedData['status_ids'])) { + foreach ($validatedData['status_ids'] as $statusId) { + $filter->statuses()->create([ + 'status_id' => $statusId, + ]); + } + } + + DB::commit(); + + $filter->load(['keywords', 'statuses']); + + return response()->json([ + 'filter' => $filter, + ], 201); + + } catch (\Exception $e) { + DB::rollBack(); + + return response()->json([ + 'error' => 'Failed to create filter', + 'message' => $e->getMessage(), + ], 500); + } + } + + public function update(Request $request, CustomFilter $filter) + { + $this->authorize('update', $filter); + + $validatedData = $request->validate([ + 'title' => 'string', + 'context' => 'array', + 'context.*' => 'string|in:'.implode(',', CustomFilter::VALID_CONTEXTS), + 'filter_action' => 'integer|in:0,1,2', + 'expires_in' => 'nullable|integer|min:0', + 'irreversible' => 'boolean', + 'keywords' => 'array', + 'keywords.*.id' => 'nullable|exists:custom_filter_keywords,id', + 'keywords.*.keyword' => 'required|string', + 'keywords.*.whole_word' => 'boolean', + 'keywords.*._destroy' => 'boolean', + 'status_ids' => 'array', + 'status_ids.*' => 'integer|exists:statuses,id', + ]); + + DB::beginTransaction(); + + try { + if (isset($validatedData['expires_in'])) { + if ($validatedData['expires_in'] > 0) { + $filter->expires_at = now()->addSeconds($validatedData['expires_in']); + } else { + $filter->expires_at = null; + } + } + + if (isset($validatedData['title'])) { + $filter->phrase = $validatedData['title']; + } + + if (isset($validatedData['context'])) { + $filter->context = $validatedData['context']; + } + + if (isset($validatedData['filter_action'])) { + $filter->action = $validatedData['filter_action']; + } elseif (isset($validatedData['irreversible'])) { + $filter->irreversible = $validatedData['irreversible']; + } + + $filter->save(); + + if (isset($validatedData['keywords'])) { + $existingKeywordIds = $filter->keywords->pluck('id')->toArray(); + $processedIds = []; + + foreach ($validatedData['keywords'] as $keywordData) { + if (isset($keywordData['id']) && isset($keywordData['_destroy']) && $keywordData['_destroy']) { + CustomFilterKeyword::destroy($keywordData['id']); + + continue; + } + + if (isset($keywordData['id']) && in_array($keywordData['id'], $existingKeywordIds)) { + $keyword = CustomFilterKeyword::find($keywordData['id']); + $keyword->update([ + 'keyword' => $keywordData['keyword'], + 'whole_word' => $keywordData['whole_word'] ?? $keyword->whole_word, + ]); + $processedIds[] = $keywordData['id']; + } else { + $newKeyword = $filter->keywords()->create([ + 'keyword' => $keywordData['keyword'], + 'whole_word' => $keywordData['whole_word'] ?? true, + ]); + $processedIds[] = $newKeyword->id; + } + } + + $keywordsToDelete = array_diff($existingKeywordIds, $processedIds); + if (! empty($keywordsToDelete)) { + CustomFilterKeyword::destroy($keywordsToDelete); + } + } + + if (isset($validatedData['status_ids'])) { + $filter->statuses()->delete(); + + foreach ($validatedData['status_ids'] as $statusId) { + $filter->statuses()->create([ + 'status_id' => $statusId, + ]); + } + } + + DB::commit(); + + $filter->load(['keywords', 'statuses']); + + return response()->json([ + 'filter' => $filter, + ]); + + } catch (\Exception $e) { + DB::rollBack(); + + return response()->json([ + 'error' => 'Failed to update filter', + 'message' => $e->getMessage(), + ], 500); + } + } + + public function destroy(CustomFilter $filter) + { + $this->authorize('delete', $filter); + + $filter->delete(); + + return response()->json(null, 204); + } +} diff --git a/app/Http/Controllers/CustomFilterKeywordController.php b/app/Http/Controllers/CustomFilterKeywordController.php new file mode 100644 index 000000000..4f251a198 --- /dev/null +++ b/app/Http/Controllers/CustomFilterKeywordController.php @@ -0,0 +1,10 @@ + 'array', + 'expires_at' => 'datetime', + 'action' => 'integer', + ]; + + protected $guarded = ['shouldInvalidateCache']; + + const VALID_CONTEXTS = [ + 'home', + 'notifications', + 'public', + 'thread', + 'account', + ]; + + const EXPIRATION_DURATIONS = [ + 1800, // 30 minutes + 3600, // 1 hour + 21600, // 6 hours + 43200, // 12 hours + 86400, // 1 day + 604800, // 1 week + ]; + + const ACTION_WARN = 0; + + const ACTION_HIDE = 1; + + const ACTION_BLUR = 2; + + public function account() + { + return $this->belongsTo(Profile::class, 'profile_id'); + } + + public function keywords() + { + return $this->hasMany(CustomFilterKeyword::class); + } + + public function statuses() + { + return $this->hasMany(CustomFilterStatus::class); + } + + public function getTitleAttribute() + { + return $this->phrase; + } + + public function setTitleAttribute($value) + { + $this->attributes['phrase'] = $value; + } + + public function getFilterActionAttribute() + { + return $this->action; + } + + public function setFilterActionAttribute($value) + { + $this->attributes['action'] = $value; + } + + public function setIrreversibleAttribute($value) + { + $this->attributes['action'] = $value ? self::ACTION_HIDE : self::ACTION_WARN; + } + + public function getIrreversibleAttribute() + { + return $this->action === self::ACTION_HIDE; + } + + public function getExpiresInAttribute() + { + if ($this->expires_at === null) { + return null; + } + + $now = now(); + foreach (self::EXPIRATION_DURATIONS as $duration) { + if ($now->addSeconds($duration)->gte($this->expires_at)) { + return $duration; + } + } + + return null; + } + + public function scopeUnexpired($query) + { + return $query->where(function ($q) { + $q->whereNull('expires_at') + ->orWhere('expires_at', '>', now()); + }); + } + + public function isExpired() + { + return $this->expires_at !== null && $this->expires_at->isPast(); + } + + protected static function boot() + { + parent::boot(); + + static::saving(function ($model) { + $model->prepareContextForStorage(); + $model->shouldInvalidateCache = true; + }); + + static::deleting(function ($model) { + $model->shouldInvalidateCache = true; + }); + + static::saved(function ($model) { + $model->invalidateCache(); + }); + + static::deleted(function ($model) { + $model->invalidateCache(); + }); + } + + protected function prepareContextForStorage() + { + if (is_array($this->context)) { + $this->context = array_values(array_filter(array_map('trim', $this->context))); + } + } + + protected function invalidateCache() + { + if (! isset($this->shouldInvalidateCache) || ! $this->shouldInvalidateCache) { + return; + } + + $this->shouldInvalidateCache = false; + + Cache::forget("filters:v3:{$this->profile_id}"); + } + + public static function getCachedFiltersForAccount($profileId) + { + $activeFilters = Cache::remember("filters:v3:{$profileId}", 3600, function () use ($profileId) { + $filtersHash = []; + + // Get keyword filters + $keywordFilters = CustomFilterKeyword::with(['customFilter' => function ($query) use ($profileId) { + $query->unexpired()->where('profile_id', $profileId); + }])->get(); + + $keywordFilters->groupBy('custom_filter_id')->each(function ($keywords, $filterId) use (&$filtersHash) { + $filter = $keywords->first()->customFilter; + + if (! $filter) { + return; + } + + $regexPatterns = $keywords->map(function ($keyword) { + $pattern = preg_quote($keyword->keyword, '/'); + if ($keyword->whole_word) { + $pattern = '\b'.$pattern.'\b'; + } + + return $pattern; + })->toArray(); + + $filtersHash[$filterId] = [ + 'keywords' => '/'.implode('|', $regexPatterns).'/i', + 'filter' => $filter, + ]; + }); + + $statusFilters = CustomFilterStatus::with(['customFilter' => function ($query) use ($profileId) { + $query->unexpired()->where('profile_id', $profileId); + }])->get(); + + $statusFilters->groupBy('custom_filter_id')->each(function ($statuses, $filterId) use (&$filtersHash) { + $filter = $statuses->first()->customFilter; + + if (! $filter) { + return; + } + + if (! isset($filtersHash[$filterId])) { + $filtersHash[$filterId] = ['filter' => $filter]; + } + + $filtersHash[$filterId]['status_ids'] = $statuses->pluck('status_id')->toArray(); + }); + + return array_map(function ($item) { + $filter = $item['filter']; + unset($item['filter']); + + return [$filter, $item]; + }, $filtersHash); + }); + + return collect($activeFilters)->reject(function ($item) { + [$filter, $rules] = $item; + + return $filter->isExpired(); + })->toArray(); + } + + public static function applyCachedFilters($cachedFilters, $status) + { + $results = []; + + foreach ($cachedFilters as [$filter, $rules]) { + $keywordMatches = []; + $statusMatches = []; + + if (isset($rules['keywords'])) { + $text = $status['content']; + preg_match_all($rules['keywords'], $text, $matches); + if (! empty($matches[0])) { + $keywordMatches = $matches[0]; + } + } + + // Check for status matches + if (isset($rules['status_ids'])) { + $statusId = $status->id; + $reblogId = $status->reblog_of_id ?? null; + + $matchingIds = array_intersect($rules['status_ids'], array_filter([$statusId, $reblogId])); + if (! empty($matchingIds)) { + $statusMatches = $matchingIds; + } + } + + if (! empty($keywordMatches) || ! empty($statusMatches)) { + $results[] = [ + 'filter' => $filter, + 'keyword_matches' => $keywordMatches, + 'status_matches' => $statusMatches, + ]; + } + } + + return $results; + } +} diff --git a/app/Models/CustomFilterKeyword.php b/app/Models/CustomFilterKeyword.php new file mode 100644 index 000000000..9994d191a --- /dev/null +++ b/app/Models/CustomFilterKeyword.php @@ -0,0 +1,32 @@ + 'boolean', + ]; + + public function customFilter() + { + return $this->belongsTo(CustomFilter::class); + } + + public function toRegex() + { + $pattern = preg_quote($this->keyword, '/'); + + if ($this->whole_word) { + $pattern = '\b'.$pattern.'\b'; + } + + return '/'.$pattern.'/i'; + } +} diff --git a/app/Models/CustomFilterStatus.php b/app/Models/CustomFilterStatus.php new file mode 100644 index 000000000..44084a6e7 --- /dev/null +++ b/app/Models/CustomFilterStatus.php @@ -0,0 +1,23 @@ +belongsTo(CustomFilter::class); + } + + public function status() + { + return $this->belongsTo(Status::class); + } +} diff --git a/app/Policies/CustomFilterPolicy.php b/app/Policies/CustomFilterPolicy.php new file mode 100644 index 000000000..3168c4608 --- /dev/null +++ b/app/Policies/CustomFilterPolicy.php @@ -0,0 +1,27 @@ +profile_id === $filter->profile_id; + } + + public function update(User $user, CustomFilter $filter) + { + return $user->profile_id === $filter->profile_id; + } + + public function delete(User $user, CustomFilter $filter) + { + return $user->profile_id === $filter->profile_id; + } +} diff --git a/app/Providers/AuthServiceProvider.php b/app/Providers/AuthServiceProvider.php index 8bedbfd53..57f59bc51 100644 --- a/app/Providers/AuthServiceProvider.php +++ b/app/Providers/AuthServiceProvider.php @@ -5,6 +5,8 @@ namespace App\Providers; use Gate; use Illuminate\Foundation\Support\Providers\AuthServiceProvider as ServiceProvider; use Laravel\Passport\Passport; +use App\Models\CustomFilter; +use App\Policies\CustomFilterPolicy; class AuthServiceProvider extends ServiceProvider { @@ -14,7 +16,7 @@ class AuthServiceProvider extends ServiceProvider * @var array */ protected $policies = [ - // 'App\Model' => 'App\Policies\ModelPolicy', + CustomFilter::class => CustomFilterPolicy::class, ]; /** diff --git a/database/migrations/2025_04_08_102711_create_custom_filters_table.php b/database/migrations/2025_04_08_102711_create_custom_filters_table.php new file mode 100644 index 000000000..151f01628 --- /dev/null +++ b/database/migrations/2025_04_08_102711_create_custom_filters_table.php @@ -0,0 +1,32 @@ +id(); + $table->foreignId('profile_id')->constrained()->onDelete('cascade'); + $table->text('phrase')->default('')->nullable(false); + $table->integer('action')->default(0)->nullable(false); // 0=warn, 1=hide, 2=blur + $table->json('context')->nullable(true); + $table->timestamp('expires_at')->nullable(); + $table->timestamps(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('custom_filters'); + } +}; diff --git a/database/migrations/2025_04_08_103425_create_custom_filter_keywords_table.php b/database/migrations/2025_04_08_103425_create_custom_filter_keywords_table.php new file mode 100644 index 000000000..0e1e93c59 --- /dev/null +++ b/database/migrations/2025_04_08_103425_create_custom_filter_keywords_table.php @@ -0,0 +1,30 @@ +id(); + $table->foreignId('custom_filter_id')->constrained()->onDelete('cascade'); + $table->string('keyword', 255)->nullable(false); + $table->boolean('whole_word')->default(true); + $table->timestamps(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('custom_filter_keywords'); + } +}; diff --git a/database/migrations/2025_04_08_103433_create_custom_filter_statuses_table.php b/database/migrations/2025_04_08_103433_create_custom_filter_statuses_table.php new file mode 100644 index 000000000..99d0eec1e --- /dev/null +++ b/database/migrations/2025_04_08_103433_create_custom_filter_statuses_table.php @@ -0,0 +1,29 @@ +id(); + $table->foreignId('custom_filter_id')->constrained()->onDelete('cascade'); + $table->foreignId('status_id')->constrained()->onDelete('cascade'); + $table->timestamps(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('custom_filter_statuses'); + } +}; diff --git a/routes/api.php b/routes/api.php index d3c2355dd..87c2dd4c3 100644 --- a/routes/api.php +++ b/routes/api.php @@ -187,6 +187,7 @@ Route::group(['prefix' => 'api'], function () use ($middleware) { Route::post('media', 'Api\ApiV2Controller@mediaUploadV2')->middleware($middleware); Route::get('streaming/config', 'Api\ApiV2Controller@getWebsocketConfig'); Route::get('instance', 'Api\ApiV2Controller@instance'); + Route::apiResource('filters', 'CustomFilterController')->middleware($middleware); }); Route::group(['prefix' => 'v1.1'], function () use ($middleware) {