Walk on when a home window renders nothing

With boosts switched off, which is the default, a window of 51 members
that happens to be all boosts produced an empty page and stopped. One
followed account that mostly boosts is enough to do that indefinitely.
The same shape happens behind a muted account's run of posts or a block
of members that were just pruned.

The read now takes up to three further windows while the page is empty
and at least one member was dropped as unrenderable, as a boost, or by a
viewer filter. A page filtered out by the viewer's own rules does not
continue: they asked for that. Direction follows the request: a min_id
read deepens the same bounded window, because Redis enforces the floor
inside the query and walking older would hand a polling client posts it
already has, while any other read walks older from the last member it saw.

The Link cursor is derived from every member the request fetched rather
than the last pass alone, so it cannot point back into a block the client
has already walked.
pull/7871/head
Ashley 7 days ago
parent 24e278759a
commit 266cbe9869

@ -2835,6 +2835,26 @@ class ApiV1Controller extends Controller
return $this->fetchHomeWindow($pid, $cursor, $paddedLimit);
}
/**
* The next window to try when the current one rendered nothing.
*
* A min_id request deepens the same bounded window rather than walking
* older: Redis enforces the min_id floor inside the query, so a member at
* or below the cursor can never be returned, whereas paging older would
* serve the client posts it has already displayed. Anything else walks
* older from the last member seen.
*/
protected function continueHomeWindow($pid, array $cursor, array $prevIds, int $pass, int $paddedLimit): array
{
if ($cursor['min']) {
$all = HomeTimelineService::getRankedMinId($pid, $cursor['min'], $paddedLimit * ($pass + 1));
return array_slice($all, $paddedLimit * $pass);
}
return HomeTimelineService::getRankedMaxId($pid, end($prevIds), $paddedLimit);
}
protected function fetchHomeWindow($pid, array $cursor, int $paddedLimit): array
{
if ($cursor['min']) {
@ -3065,6 +3085,35 @@ class ApiV1Controller extends Controller
$res = $hydrate($ids);
// Every member this request fetched, across continuation passes. The
// Link cursor is derived from it rather than from the last pass alone,
// which would point the client back inside a block it has already
// walked.
$cursorIds = $ids;
$pass = 0;
// A window can hold plenty of members and render none of them: boosts
// for a viewer who has boosts off, a muted account's run of posts, the
// members that were just pruned. Walking on is the difference between
// a short page and a timeline that reports the end of the feed. An
// explicit filter is not a reason to walk on: the viewer asked for it.
while (
$res->isEmpty()
&& $pass < 3
&& $ids
&& ($drops['unrenderable'] || $drops['reblog'] || $drops['viewer'])
) {
$pass++;
$ids = $this->continueHomeWindow($pid, $cursor, $ids, $pass, $paddedLimit);
if (! $ids) {
break;
}
$cursorIds = array_merge($cursorIds, $ids);
$res = $hydrate($ids);
}
if ($pruned) {
Log::info('timelineHome: pruned unrenderable home feed members', [
'pid' => $pid,
@ -3078,6 +3127,11 @@ class ApiV1Controller extends Controller
// time within the request.
if ($res->isEmpty() && $drops['unrenderable'] > 0 && ! HomeTimelineService::cacheKeyExists($pid)) {
$ids = $this->homeColdBoot($pid, $cursor, $paddedLimit);
if ($ids) {
$cursorIds = $ids;
}
$res = $hydrate($ids);
}
@ -3094,7 +3148,7 @@ class ApiV1Controller extends Controller
$headers['Link'] = '<'.$baseUrl.'max_id='.$oldest.'>; rel="next",<'.$baseUrl.'min_id='.$newest.'>; rel="prev"';
} else {
$headers['Link'] = '<'.$baseUrl.'max_id='.collect($ids)->map(fn ($id) => (int) $id)->min().'>; rel="next"';
$headers['Link'] = '<'.$baseUrl.'max_id='.collect($cursorIds)->map(fn ($id) => (int) $id)->min().'>; rel="next"';
}
return $this->json($res->toArray(), 200, $headers);

@ -1,14 +1,11 @@
<?php
use App\Models\CustomFilter;
use App\Models\Follower;
use App\Models\Status;
use App\Models\User;
use App\Services\HomeTimelineService;
use App\Services\StatusService;
use Illuminate\Foundation\Testing\LazilyRefreshDatabase;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Redis;
use Laravel\Passport\Passport;
@ -16,15 +13,14 @@ uses(LazilyRefreshDatabase::class);
/*
|--------------------------------------------------------------------------
| A home feed that holds members but can render none of them
| Walking on when a home window renders nothing
|--------------------------------------------------------------------------
|
| Members whose statuses have gone away produce an empty page from a feed
| that reports plenty of members, and nothing in the system ever removed
| them. The hydration pass prunes such a member as it finds it and rebuilds
| in the same request when the prune empties the set. A page the viewer
| explicitly filtered out is a different thing: it prunes nothing and runs
| no candidate query.
| A window of members that are all boosts for a viewer with boosts off, or
| a run behind a muted account, produces an empty page from a feed that
| reports plenty of members, and that is not end-of-feed. The read takes up
| to three further windows while the page is empty and at least one member
| was dropped as unrenderable, as a boost, or by a viewer filter.
|
*/
@ -65,71 +61,38 @@ function unrenderablePhoto(int $profileId, int $minutesAgo, array $overrides = [
return $status;
}
/** Remove a status so its feed member can never hydrate again. */
function unrenderableDestroy(Status $status): void
{
DB::table('statuses')->where('id', $status->id)->delete();
Cache::forget(StatusService::key($status->id, true));
Cache::forget(StatusService::key($status->id, false));
}
function unrenderableCandidateQueries(callable $fn): int
{
$count = 0;
DB::listen(function ($query) use (&$count) {
if (preg_match('/max\(id\)/i', $query->sql)) {
$count++;
}
});
$fn();
return $count;
}
it('prunes members whose statuses have gone away and serves what remains', function () {
it('pages past a window of boosts for a viewer with boosts switched off', function () {
[$viewer, $author] = unrenderableViewer();
$pid = $viewer->profile_id;
$stale = collect(range(1, 5))->map(fn ($i) => unrenderablePhoto($author->profile_id, 100 + $i));
foreach ($stale as $status) {
HomeTimelineService::add($pid, $status->id);
}
foreach ($stale as $status) {
unrenderableDestroy($status);
}
// Eligible, recent, and not in the feed: the repair the request can make.
$fresh = unrenderablePhoto($author->profile_id, 5);
Passport::actingAs($viewer, ['read']);
$ids = collect($this->getJson('/api/v1/timelines/home?limit=10')->assertOk()->json())
->pluck('id')->map(fn ($id) => (int) $id)->all();
expect($ids)->toContain((int) $fresh->id);
foreach ($stale as $status) {
expect(Redis::zscore(HomeTimelineService::CACHE_KEY.$pid, $status->id))->toBeNull();
}
});
$stranger = User::factory()->create();
$stranger->refresh();
$original = unrenderablePhoto($stranger->profile_id, 500);
$photo = unrenderablePhoto($author->profile_id, 400);
HomeTimelineService::add($pid, $photo->id);
// The whole first window (limit 1 plus the 50-member reblog pad, and the
// inclusive stop index that makes it one wider) is boosts.
$shares = collect(range(1, 60))->map(function ($i) use ($author, $original) {
$share = Status::factory()->create([
'profile_id' => $author->profile_id,
'type' => 'share',
'scope' => 'public',
'visibility' => 'public',
'local' => true,
'in_reply_to_id' => null,
'reblog_of_id' => $original->id,
'uri' => null,
'created_at' => now()->subMinutes(120 - $i),
]);
StatusService::get($share->id, false);
StatusService::getMastodon($share->id, false);
it('leaves a page the viewer filtered out alone', function () {
[$viewer, $author] = unrenderableViewer();
$pid = $viewer->profile_id;
$statuses = collect(range(1, 3))->map(fn ($i) => unrenderablePhoto($author->profile_id, 10 + $i, [
'caption' => 'hideme sunset photo',
'rendered' => 'hideme sunset photo',
]));
foreach ($statuses as $status) {
HomeTimelineService::add($pid, $status->id);
return $share;
});
foreach ($shares as $share) {
HomeTimelineService::add($pid, $share->id);
}
$filter = CustomFilter::create([
'profile_id' => $pid,
'title' => 'hide sunsets',
'context' => ['home'],
'action' => 1,
]);
$filter->keywords()->create(['keyword' => 'hideme', 'whole_word' => false]);
Cache::forget('filters:v3:'.$pid);
Passport::actingAs($viewer, ['read']);
$candidateQueries = unrenderableCandidateQueries(function () {
$res = $this->getJson('/api/v1/timelines/home?limit=5')->assertOk();
expect($res->json())->toBe([]);
});
expect($candidateQueries)->toBe(0)
->and(HomeTimelineService::count($pid))->toBe(3);
$ids = collect($this->getJson('/api/v1/timelines/home?limit=1')->assertOk()->json())
->pluck('id')->map(fn ($id) => (int) $id)->all();
expect($ids)->toContain((int) $photo->id);
});

Loading…
Cancel
Save