diff --git a/app/Console/Commands/Admin/RebuildHomeFeeds.php b/app/Console/Commands/Admin/RebuildHomeFeeds.php new file mode 100644 index 000000000..0a0a4cf6e --- /dev/null +++ b/app/Console/Commands/Admin/RebuildHomeFeeds.php @@ -0,0 +1,156 @@ +argument('id'); + $all = (bool) $this->option('all'); + $activeOpt = $this->option('active'); + $active = $activeOpt !== [] && $activeOpt !== null && $activeOpt !== false; + $activeDays = is_array($activeOpt) + ? (int) ($activeOpt[0] ?? self::DEFAULT_ACTIVE_DAYS) + : (int) ($activeOpt ?: self::DEFAULT_ACTIVE_DAYS); + $sync = (bool) $this->option('sync'); + $dryRun = (bool) $this->option('dry-run'); + $local = (bool) $this->option('local'); + $network = (bool) $this->option('network'); + + $hasHomeTarget = $id || $all || $active; + + if (! $hasHomeTarget && ! $local && ! $network) { + $this->error('Specify a profile id/username, --active, --all, --local, or --network.'); + + return 1; + } + + if ($local) { + $this->rebuildPublic($dryRun); + } + + if ($network) { + $this->rebuildNetwork($dryRun); + } + + if (! $hasHomeTarget) { + return 0; + } + + if ($id) { + $profile = ctype_digit((string) $id) + ? Profile::find($id) + : Profile::whereUsername($id)->first(); + + if (! $profile) { + $this->error('Profile not found.'); + + return 1; + } + + if ($profile->domain !== null) { + $this->error('Home feeds exist only for local accounts.'); + + return 1; + } + + $this->rebuildOne($profile->id, $sync, $dryRun); + $this->info('Done.'); + + return 0; + } + + $scopeLabel = $active + ? 'local accounts active in the last '.$activeDays.' days' + : 'all local accounts'; + + if (! $dryRun && ! $this->option('force') && ! $this->confirm('Rebuild cached home feeds for '.$scopeLabel.'?', true)) { + $this->comment('Aborted.'); + + return 0; + } + + $query = Profile::whereNull('deleted_at')->whereNull('domain')->whereNotNull('user_id'); + + if ($active) { + $cutoff = now()->subDays($activeDays); + $query->whereHas('user', function ($q) use ($cutoff) { + $q->whereNotNull('last_active_at') + ->where('last_active_at', '>=', $cutoff); + }); + } + + $count = 0; + $query->lazyById(500)->each(function ($profile) use (&$count, $sync, $dryRun) { + $this->rebuildOne($profile->id, $sync, $dryRun); + $count++; + }); + + $this->info(($dryRun ? 'Would rebuild ' : 'Rebuilt ').$count.' home feed(s).'); + + return 0; + } + + protected function rebuildOne($pid, bool $sync, bool $dryRun): void + { + if ($dryRun) { + $this->line('pid '.$pid.': '.HomeTimelineService::count($pid).' cached entries'); + + return; + } + + if ($sync) { + HomeTimelineService::warmCache($pid, true, 400, true); + + return; + } + + FeedWarmCachePipeline::dispatch($pid)->onQueue('feed'); + } + + protected function rebuildPublic(bool $dryRun): void + { + if ($dryRun) { + $this->line('local timeline: '.PublicTimelineService::count().' cached entries'); + + return; + } + + PublicTimelineService::warmCache(true, 400); + $this->info('Rebuilt the local timeline.'); + } + + protected function rebuildNetwork(bool $dryRun): void + { + if ($dryRun) { + $this->line('network timeline: '.NetworkTimelineService::count().' cached entries'); + + return; + } + + NetworkTimelineService::warmCache(true, config('instance.timeline.network.cache_dropoff')); + $this->info('Rebuilt the network timeline.'); + } +} diff --git a/app/Services/HomeTimelineService.php b/app/Services/HomeTimelineService.php index 4968260d4..60efb2afd 100644 --- a/app/Services/HomeTimelineService.php +++ b/app/Services/HomeTimelineService.php @@ -6,6 +6,7 @@ use App\Models\Follower; use App\Models\Status; use App\Models\UserDomainBlock; use Illuminate\Support\Facades\Cache; +use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Redis; use Illuminate\Support\Str; @@ -112,14 +113,36 @@ class HomeTimelineService $domainBlocks = UserDomainBlock::whereProfileId($id)->pluck('domain')->toArray(); - $ids = Status::where('id', '>', $minId) - ->whereIn('profile_id', $following) - ->whereNull(['in_reply_to_id', 'reblog_of_id']) - ->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album']) - ->whereIn('visibility', ['public', 'unlisted', 'private']) - ->orderByDesc('id') - ->limit($limit) - ->pluck('id'); + $baseQuery = function () use ($following, $minId) { + return Status::where('id', '>', $minId) + ->whereIn('profile_id', $following) + ->whereNull(['in_reply_to_id', 'reblog_of_id']) + ->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album']) + ->whereIn('visibility', ['public', 'unlisted', 'private']); + }; + + $latestPerAccount = $baseQuery() + ->select(DB::raw('max(id) as id')) + ->groupBy('profile_id') + ->pluck('id') + ->sortDesc() + ->take($limit) + ->values(); + + $remaining = $limit - $latestPerAccount->count(); + + $recentIds = $remaining > 0 + ? $baseQuery() + ->whereNotIn('id', $latestPerAccount->all()) + ->orderByDesc('id') + ->limit($remaining) + ->pluck('id') + : collect(); + + $ids = $latestPerAccount + ->merge($recentIds) + ->unique() + ->values(); $tmpKey = self::CACHE_KEY.$id.':rebuild:'.Str::random(8); $added = 0; diff --git a/tests/Feature/Timeline/HomeWarmPerAccountTest.php b/tests/Feature/Timeline/HomeWarmPerAccountTest.php new file mode 100644 index 000000000..29bac5a3b --- /dev/null +++ b/tests/Feature/Timeline/HomeWarmPerAccountTest.php @@ -0,0 +1,80 @@ +create([ + 'profile_id' => $profileId, + 'type' => 'photo', + 'scope' => 'public', + 'visibility' => 'public', + 'in_reply_to_id' => null, + 'reblog_of_id' => null, + 'uri' => null, + ]); +} + +it('represents every followed account even when one posts far more', function () { + $viewer = User::factory()->create(); + $viewer->refresh(); + $pid = $viewer->profile_id; + + $prolific = User::factory()->create(); + $prolific->refresh(); + $quiet = User::factory()->create(); + $quiet->refresh(); + + foreach ([$prolific, $quiet] as $u) { + Follower::create(['profile_id' => $pid, 'following_id' => $u->profile_id]); + } + + // Quiet account posts first (older ids), prolific account floods afterwards. + $quietStatus = warmHomeStatus($quiet->profile_id); + for ($i = 0; $i < 10; $i++) { + warmHomeStatus($prolific->profile_id); + } + + // Warm with a cap smaller than the prolific account's output so the flat + // "newest N" query alone would never include the quiet account. + HomeTimelineService::warmCache($pid, true, 5); + + $cached = collect(HomeTimelineService::get($pid, 0, -1)) + ->map(fn ($v) => (int) $v) + ->all(); + + expect($cached)->toContain((int) $quietStatus->id); +}); + +it('keeps the newest posts as well as the per-account latest', function () { + $viewer = User::factory()->create(); + $viewer->refresh(); + $pid = $viewer->profile_id; + + $author = User::factory()->create(); + $author->refresh(); + Follower::create(['profile_id' => $pid, 'following_id' => $author->profile_id]); + + $ids = collect(range(1, 6)) + ->map(fn () => (int) warmHomeStatus($author->profile_id)->id) + ->sortDesc() + ->values(); + + HomeTimelineService::warmCache($pid, true, 4); + + $cached = collect(HomeTimelineService::get($pid, 0, -1))->map(fn ($v) => (int) $v)->sortDesc()->values()->all(); + + // The four newest are present. + expect(array_slice($cached, 0, 4))->toBe($ids->take(4)->all()); +}); diff --git a/tests/Feature/Timeline/RebuildHomeFeedsCommandTest.php b/tests/Feature/Timeline/RebuildHomeFeedsCommandTest.php new file mode 100644 index 000000000..6c95e128d --- /dev/null +++ b/tests/Feature/Timeline/RebuildHomeFeedsCommandTest.php @@ -0,0 +1,139 @@ +create([ + 'profile_id' => $profileId, + 'type' => 'photo', + 'scope' => 'public', + 'visibility' => 'public', + 'in_reply_to_id' => null, + 'reblog_of_id' => null, + 'uri' => null, + ]); +} + +it('force-rebuilds a stale home feed from the database', function () { + $viewer = User::factory()->create(); + $viewer->refresh(); + $pid = $viewer->profile_id; + + $author = User::factory()->create(); + $author->refresh(); + Follower::create(['profile_id' => $pid, 'following_id' => $author->profile_id]); + + $real = rebuildHomeStatus($author->profile_id); + + // Seed the cache with a stale id that no longer reflects the DB. + HomeTimelineService::add($pid, 999999999); + expect(collect(HomeTimelineService::get($pid, 0, -1))->map(fn ($v) => (int) $v)->all()) + ->toContain(999999999) + ->and(collect(HomeTimelineService::get($pid, 0, -1))->map(fn ($v) => (int) $v)->all()) + ->not->toContain((int) $real->id); + + $this->artisan('app:home-feed-rebuild', ['id' => $pid, '--sync' => true, '--force' => true]) + ->assertExitCode(0); + + $cached = collect(HomeTimelineService::get($pid, 0, -1))->map(fn ($v) => (int) $v)->all(); + + expect($cached)->toContain((int) $real->id) + ->and($cached)->not->toContain(999999999); +}); + +it('reports counts without changing anything in dry-run', function () { + $viewer = User::factory()->create(); + $viewer->refresh(); + $pid = $viewer->profile_id; + + HomeTimelineService::add($pid, 123); + + $this->artisan('app:home-feed-rebuild', ['id' => $pid, '--dry-run' => true]) + ->assertExitCode(0); + + expect(collect(HomeTimelineService::get($pid, 0, -1))->map(fn ($v) => (int) $v)->all()) + ->toBe([123]); +}); + +function rebuildLocalStatus(): Status +{ + return Status::factory()->create([ + 'type' => 'photo', + 'scope' => 'public', + 'visibility' => 'public', + 'local' => true, + 'in_reply_to_id' => null, + 'reblog_of_id' => null, + 'uri' => null, + ]); +} + +it('rebuilds the instance-wide local timeline with --local', function () { + config(['instance.hide_nsfw_on_public_feeds' => false]); + Redis::del(PublicTimelineService::CACHE_KEY); + + $real = rebuildLocalStatus(); + StatusService::get($real->id); + + PublicTimelineService::add(999999999); + + $this->artisan('app:home-feed-rebuild', ['--local' => true, '--force' => true]) + ->assertExitCode(0); + + $cached = collect(PublicTimelineService::get(0, -1))->map(fn ($v) => (int) $v)->all(); + + expect($cached)->toContain((int) $real->id) + ->and($cached)->not->toContain(999999999); +}); + +it('rebuilds the instance-wide network timeline with --network', function () { + config(['instance.hide_nsfw_on_public_feeds' => false]); + config(['instance.timeline.network.max_hours_old' => 2160]); + Redis::del(NetworkTimelineService::CACHE_KEY); + + $author = Profile::factory()->create(['user_id' => null, 'domain' => 'remote.example']); + $real = Status::factory()->create([ + 'profile_id' => $author->id, + 'type' => 'photo', + 'scope' => 'public', + 'visibility' => 'public', + 'local' => false, + 'in_reply_to_id' => null, + 'reblog_of_id' => null, + 'is_nsfw' => false, + 'uri' => 'https://remote.example/p/'.uniqid(), + 'created_at' => now()->subMinutes(5), + ]); + StatusService::get($real->id); + + NetworkTimelineService::add(999999999); + + $this->artisan('app:home-feed-rebuild', ['--network' => true, '--force' => true]) + ->assertExitCode(0); + + $cached = collect(NetworkTimelineService::get(0, -1))->map(fn ($v) => (int) $v)->all(); + + expect($cached)->toContain((int) $real->id) + ->and($cached)->not->toContain(999999999); +}); + +it('errors when no target is given', function () { + $this->artisan('app:home-feed-rebuild') + ->assertExitCode(1); +});