Represent every followed account when warming the home feed

The warm query took the newest 400 posts across all follows combined, so a few prolific accounts could consume the whole cache and quieter accounts were missing entirely, even though the uncached timeline showed them. Reserve the latest post from each followed account, then fill the rest with the newest posts overall, so every account appears within the cached window. Add app:home-feed-rebuild to force a rebuild of existing cached feeds: home feeds for a single account, --active, or --all, and the instance-wide local and network timelines via --local and --network.
2 weeks ago · d0122acd1c
parent 1562bc48b2
commit d0122acd1c
4 changed files with 406 additions and 8 deletions
--- a/app/Console/Commands/Admin/RebuildHomeFeeds.php
+++ b/app/Console/Commands/Admin/RebuildHomeFeeds.php
@ -0,0 +1,156 @@
+<?php
+
+namespace App\Console\Commands\Admin;
+
+use App\Jobs\HomeFeedPipeline\FeedWarmCachePipeline;
+use App\Models\Profile;
+use App\Services\HomeTimelineService;
+use App\Services\NetworkTimelineService;
+use App\Services\PublicTimelineService;
+use Illuminate\Console\Command;
+
+class RebuildHomeFeeds extends Command
+{
+    protected $signature = 'app:home-feed-rebuild
+        {id? : Profile id or username to rebuild (omit with --all or --active)}
+        {--all : Rebuild every local account}
+        {--active=* : Rebuild only local accounts active within N days (default 30)}
+        {--local : Also rebuild the instance-wide local timeline}
+        {--network : Also rebuild the instance-wide network timeline}
+        {--sync : Rebuild home feeds inline instead of queueing FeedWarmCachePipeline}
+        {--dry-run : Report what would be rebuilt without changing anything}
+        {--force : Skip the confirmation prompt}';
+
+    protected const DEFAULT_ACTIVE_DAYS = 30;
+
+    protected $description = 'Clear and rebuild cached timelines from the database. Rebuild home feeds for a single account, --active, or --all, and optionally the instance-wide --local and --network timelines.';
+
+    public function handle()
+    {
+        $id = $this->argument('id');
+        $all = (bool) $this->option('all');
+        $activeOpt = $this->option('active');
+        $active = $activeOpt !== [] && $activeOpt !== null && $activeOpt !== false;
+        $activeDays = is_array($activeOpt)
+            ? (int) ($activeOpt[0] ?? self::DEFAULT_ACTIVE_DAYS)
+            : (int) ($activeOpt ?: self::DEFAULT_ACTIVE_DAYS);
+        $sync = (bool) $this->option('sync');
+        $dryRun = (bool) $this->option('dry-run');
+        $local = (bool) $this->option('local');
+        $network = (bool) $this->option('network');
+
+        $hasHomeTarget = $id || $all || $active;
+
+        if (! $hasHomeTarget && ! $local && ! $network) {
+            $this->error('Specify a profile id/username, --active, --all, --local, or --network.');
+
+            return 1;
+        }
+
+        if ($local) {
+            $this->rebuildPublic($dryRun);
+        }
+
+        if ($network) {
+            $this->rebuildNetwork($dryRun);
+        }
+
+        if (! $hasHomeTarget) {
+            return 0;
+        }
+
+        if ($id) {
+            $profile = ctype_digit((string) $id)
+                ? Profile::find($id)
+                : Profile::whereUsername($id)->first();
+
+            if (! $profile) {
+                $this->error('Profile not found.');
+
+                return 1;
+            }
+
+            if ($profile->domain !== null) {
+                $this->error('Home feeds exist only for local accounts.');
+
+                return 1;
+            }
+
+            $this->rebuildOne($profile->id, $sync, $dryRun);
+            $this->info('Done.');
+
+            return 0;
+        }
+
+        $scopeLabel = $active
+            ? 'local accounts active in the last '.$activeDays.' days'
+            : 'all local accounts';
+
+        if (! $dryRun && ! $this->option('force') && ! $this->confirm('Rebuild cached home feeds for '.$scopeLabel.'?', true)) {
+            $this->comment('Aborted.');
+
+            return 0;
+        }
+
+        $query = Profile::whereNull('deleted_at')->whereNull('domain')->whereNotNull('user_id');
+
+        if ($active) {
+            $cutoff = now()->subDays($activeDays);
+            $query->whereHas('user', function ($q) use ($cutoff) {
+                $q->whereNotNull('last_active_at')
+                    ->where('last_active_at', '>=', $cutoff);
+            });
+        }
+
+        $count = 0;
+        $query->lazyById(500)->each(function ($profile) use (&$count, $sync, $dryRun) {
+            $this->rebuildOne($profile->id, $sync, $dryRun);
+            $count++;
+        });
+
+        $this->info(($dryRun ? 'Would rebuild ' : 'Rebuilt ').$count.' home feed(s).');
+
+        return 0;
+    }
+
+    protected function rebuildOne($pid, bool $sync, bool $dryRun): void
+    {
+        if ($dryRun) {
+            $this->line('pid '.$pid.': '.HomeTimelineService::count($pid).' cached entries');
+
+            return;
+        }
+
+        if ($sync) {
+            HomeTimelineService::warmCache($pid, true, 400, true);
+
+            return;
+        }
+
+        FeedWarmCachePipeline::dispatch($pid)->onQueue('feed');
+    }
+
+    protected function rebuildPublic(bool $dryRun): void
+    {
+        if ($dryRun) {
+            $this->line('local timeline: '.PublicTimelineService::count().' cached entries');
+
+            return;
+        }
+
+        PublicTimelineService::warmCache(true, 400);
+        $this->info('Rebuilt the local timeline.');
+    }
+
+    protected function rebuildNetwork(bool $dryRun): void
+    {
+        if ($dryRun) {
+            $this->line('network timeline: '.NetworkTimelineService::count().' cached entries');
+
+            return;
+        }
+
+        NetworkTimelineService::warmCache(true, config('instance.timeline.network.cache_dropoff'));
+        $this->info('Rebuilt the network timeline.');
+    }
+}
--- a/app/Services/HomeTimelineService.php
+++ b/app/Services/HomeTimelineService.php
@ -6,6 +6,7 @@ use App\Models\Follower;
 use App\Models\Status;
 use App\Models\UserDomainBlock;
 use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Redis;
 use Illuminate\Support\Str;

@ -112,14 +113,36 @@ class HomeTimelineService

            $domainBlocks = UserDomainBlock::whereProfileId($id)->pluck('domain')->toArray();

-            $ids = Status::where('id', '>', $minId)
-                ->whereIn('profile_id', $following)
-                ->whereNull(['in_reply_to_id', 'reblog_of_id'])
-                ->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
-                ->whereIn('visibility', ['public', 'unlisted', 'private'])
-                ->orderByDesc('id')
-                ->limit($limit)
-                ->pluck('id');
+            $baseQuery = function () use ($following, $minId) {
+                return Status::where('id', '>', $minId)
+                    ->whereIn('profile_id', $following)
+                    ->whereNull(['in_reply_to_id', 'reblog_of_id'])
+                    ->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
+                    ->whereIn('visibility', ['public', 'unlisted', 'private']);
+            };
+
+            $latestPerAccount = $baseQuery()
+                ->select(DB::raw('max(id) as id'))
+                ->groupBy('profile_id')
+                ->pluck('id')
+                ->sortDesc()
+                ->take($limit)
+                ->values();
+
+            $remaining = $limit - $latestPerAccount->count();
+
+            $recentIds = $remaining > 0
+                ? $baseQuery()
+                    ->whereNotIn('id', $latestPerAccount->all())
+                    ->orderByDesc('id')
+                    ->limit($remaining)
+                    ->pluck('id')
+                : collect();
+
+            $ids = $latestPerAccount
+                ->merge($recentIds)
+                ->unique()
+                ->values();

            $tmpKey = self::CACHE_KEY.$id.':rebuild:'.Str::random(8);
            $added = 0;
--- a/tests/Feature/Timeline/HomeWarmPerAccountTest.php
+++ b/tests/Feature/Timeline/HomeWarmPerAccountTest.php
@ -0,0 +1,80 @@
+<?php
+
+use App\Models\Follower;
+use App\Models\Status;
+use App\Models\User;
+use App\Services\HomeTimelineService;
+use Illuminate\Foundation\Testing\LazilyRefreshDatabase;
+use Illuminate\Support\Facades\Redis;
+
+uses(LazilyRefreshDatabase::class);
+
+afterEach(function () {
+    Redis::flushdb();
+});
+
+function warmHomeStatus(int $profileId): Status
+{
+    return Status::factory()->create([
+        'profile_id' => $profileId,
+        'type' => 'photo',
+        'scope' => 'public',
+        'visibility' => 'public',
+        'in_reply_to_id' => null,
+        'reblog_of_id' => null,
+        'uri' => null,
+    ]);
+}
+
+it('represents every followed account even when one posts far more', function () {
+    $viewer = User::factory()->create();
+    $viewer->refresh();
+    $pid = $viewer->profile_id;
+
+    $prolific = User::factory()->create();
+    $prolific->refresh();
+    $quiet = User::factory()->create();
+    $quiet->refresh();
+
+    foreach ([$prolific, $quiet] as $u) {
+        Follower::create(['profile_id' => $pid, 'following_id' => $u->profile_id]);
+    }
+
+    // Quiet account posts first (older ids), prolific account floods afterwards.
+    $quietStatus = warmHomeStatus($quiet->profile_id);
+    for ($i = 0; $i < 10; $i++) {
+        warmHomeStatus($prolific->profile_id);
+    }
+
+    // Warm with a cap smaller than the prolific account's output so the flat
+    // "newest N" query alone would never include the quiet account.
+    HomeTimelineService::warmCache($pid, true, 5);
+
+    $cached = collect(HomeTimelineService::get($pid, 0, -1))
+        ->map(fn ($v) => (int) $v)
+        ->all();
+
+    expect($cached)->toContain((int) $quietStatus->id);
+});
+
+it('keeps the newest posts as well as the per-account latest', function () {
+    $viewer = User::factory()->create();
+    $viewer->refresh();
+    $pid = $viewer->profile_id;
+
+    $author = User::factory()->create();
+    $author->refresh();
+    Follower::create(['profile_id' => $pid, 'following_id' => $author->profile_id]);
+
+    $ids = collect(range(1, 6))
+        ->map(fn () => (int) warmHomeStatus($author->profile_id)->id)
+        ->sortDesc()
+        ->values();
+
+    HomeTimelineService::warmCache($pid, true, 4);
+
+    $cached = collect(HomeTimelineService::get($pid, 0, -1))->map(fn ($v) => (int) $v)->sortDesc()->values()->all();
+
+    // The four newest are present.
+    expect(array_slice($cached, 0, 4))->toBe($ids->take(4)->all());
+});
--- a/tests/Feature/Timeline/RebuildHomeFeedsCommandTest.php
+++ b/tests/Feature/Timeline/RebuildHomeFeedsCommandTest.php
@ -0,0 +1,139 @@
+<?php
+
+use App\Models\Follower;
+use App\Models\Profile;
+use App\Models\Status;
+use App\Models\User;
+use App\Services\HomeTimelineService;
+use App\Services\NetworkTimelineService;
+use App\Services\PublicTimelineService;
+use App\Services\StatusService;
+use Illuminate\Foundation\Testing\LazilyRefreshDatabase;
+use Illuminate\Support\Facades\Redis;
+
+uses(LazilyRefreshDatabase::class);
+
+afterEach(function () {
+    Redis::flushdb();
+});
+
+function rebuildHomeStatus(int $profileId): Status
+{
+    return Status::factory()->create([
+        'profile_id' => $profileId,
+        'type' => 'photo',
+        'scope' => 'public',
+        'visibility' => 'public',
+        'in_reply_to_id' => null,
+        'reblog_of_id' => null,
+        'uri' => null,
+    ]);
+}
+
+it('force-rebuilds a stale home feed from the database', function () {
+    $viewer = User::factory()->create();
+    $viewer->refresh();
+    $pid = $viewer->profile_id;
+
+    $author = User::factory()->create();
+    $author->refresh();
+    Follower::create(['profile_id' => $pid, 'following_id' => $author->profile_id]);
+
+    $real = rebuildHomeStatus($author->profile_id);
+
+    // Seed the cache with a stale id that no longer reflects the DB.
+    HomeTimelineService::add($pid, 999999999);
+    expect(collect(HomeTimelineService::get($pid, 0, -1))->map(fn ($v) => (int) $v)->all())
+        ->toContain(999999999)
+        ->and(collect(HomeTimelineService::get($pid, 0, -1))->map(fn ($v) => (int) $v)->all())
+        ->not->toContain((int) $real->id);
+
+    $this->artisan('app:home-feed-rebuild', ['id' => $pid, '--sync' => true, '--force' => true])
+        ->assertExitCode(0);
+
+    $cached = collect(HomeTimelineService::get($pid, 0, -1))->map(fn ($v) => (int) $v)->all();
+
+    expect($cached)->toContain((int) $real->id)
+        ->and($cached)->not->toContain(999999999);
+});
+
+it('reports counts without changing anything in dry-run', function () {
+    $viewer = User::factory()->create();
+    $viewer->refresh();
+    $pid = $viewer->profile_id;
+
+    HomeTimelineService::add($pid, 123);
+
+    $this->artisan('app:home-feed-rebuild', ['id' => $pid, '--dry-run' => true])
+        ->assertExitCode(0);
+
+    expect(collect(HomeTimelineService::get($pid, 0, -1))->map(fn ($v) => (int) $v)->all())
+        ->toBe([123]);
+});
+
+function rebuildLocalStatus(): Status
+{
+    return Status::factory()->create([
+        'type' => 'photo',
+        'scope' => 'public',
+        'visibility' => 'public',
+        'local' => true,
+        'in_reply_to_id' => null,
+        'reblog_of_id' => null,
+        'uri' => null,
+    ]);
+}
+
+it('rebuilds the instance-wide local timeline with --local', function () {
+    config(['instance.hide_nsfw_on_public_feeds' => false]);
+    Redis::del(PublicTimelineService::CACHE_KEY);
+
+    $real = rebuildLocalStatus();
+    StatusService::get($real->id);
+
+    PublicTimelineService::add(999999999);
+
+    $this->artisan('app:home-feed-rebuild', ['--local' => true, '--force' => true])
+        ->assertExitCode(0);
+
+    $cached = collect(PublicTimelineService::get(0, -1))->map(fn ($v) => (int) $v)->all();
+
+    expect($cached)->toContain((int) $real->id)
+        ->and($cached)->not->toContain(999999999);
+});
+
+it('rebuilds the instance-wide network timeline with --network', function () {
+    config(['instance.hide_nsfw_on_public_feeds' => false]);
+    config(['instance.timeline.network.max_hours_old' => 2160]);
+    Redis::del(NetworkTimelineService::CACHE_KEY);
+
+    $author = Profile::factory()->create(['user_id' => null, 'domain' => 'remote.example']);
+    $real = Status::factory()->create([
+        'profile_id' => $author->id,
+        'type' => 'photo',
+        'scope' => 'public',
+        'visibility' => 'public',
+        'local' => false,
+        'in_reply_to_id' => null,
+        'reblog_of_id' => null,
+        'is_nsfw' => false,
+        'uri' => 'https://remote.example/p/'.uniqid(),
+        'created_at' => now()->subMinutes(5),
+    ]);
+    StatusService::get($real->id);
+
+    NetworkTimelineService::add(999999999);
+
+    $this->artisan('app:home-feed-rebuild', ['--network' => true, '--force' => true])
+        ->assertExitCode(0);
+
+    $cached = collect(NetworkTimelineService::get(0, -1))->map(fn ($v) => (int) $v)->all();
+
+    expect($cached)->toContain((int) $real->id)
+        ->and($cached)->not->toContain(999999999);
+});
+
+it('errors when no target is given', function () {
+    $this->artisan('app:home-feed-rebuild')
+        ->assertExitCode(1);
+});