Merge pull request #7871 from pixelfed/fix/timeline-walk-empty-window

Walk on when a home window renders nothing
pull/7875/head
Shlee 6 days ago committed by GitHub
commit 87c2d6bd9e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -2835,6 +2835,26 @@ class ApiV1Controller extends Controller
return $this->fetchHomeWindow($pid, $cursor, $paddedLimit);
}
/**
* The next window to try when the current one rendered nothing.
*
* A min_id request deepens the same bounded window rather than walking
* older: Redis enforces the min_id floor inside the query, so a member at
* or below the cursor can never be returned, whereas paging older would
* serve the client posts it has already displayed. Anything else walks
* older from the last member seen.
*/
protected function continueHomeWindow($pid, array $cursor, array $prevIds, int $pass, int $paddedLimit): array
{
if ($cursor['min']) {
$all = HomeTimelineService::getRankedMinId($pid, $cursor['min'], $paddedLimit * ($pass + 1));
return array_slice($all, $paddedLimit * $pass);
}
return HomeTimelineService::getRankedMaxId($pid, end($prevIds), $paddedLimit);
}
protected function fetchHomeWindow($pid, array $cursor, int $paddedLimit): array
{
if ($cursor['min']) {
@ -3065,6 +3085,35 @@ class ApiV1Controller extends Controller
$res = $hydrate($ids);
// Every member this request fetched, across continuation passes. The
// Link cursor is derived from it rather than from the last pass alone,
// which would point the client back inside a block it has already
// walked.
$cursorIds = $ids;
$pass = 0;
// A window can hold plenty of members and render none of them: boosts
// for a viewer who has boosts off, a muted account's run of posts, the
// members that were just pruned. Walking on is the difference between
// a short page and a timeline that reports the end of the feed. An
// explicit filter is not a reason to walk on: the viewer asked for it.
while (
$res->isEmpty()
&& $pass < 3
&& $ids
&& ($drops['unrenderable'] || $drops['reblog'] || $drops['viewer'])
) {
$pass++;
$ids = $this->continueHomeWindow($pid, $cursor, $ids, $pass, $paddedLimit);
if (! $ids) {
break;
}
$cursorIds = array_merge($cursorIds, $ids);
$res = $hydrate($ids);
}
if ($pruned) {
Log::info('timelineHome: pruned unrenderable home feed members', [
'pid' => $pid,
@ -3078,6 +3127,11 @@ class ApiV1Controller extends Controller
// time within the request.
if ($res->isEmpty() && $drops['unrenderable'] > 0 && ! HomeTimelineService::cacheKeyExists($pid)) {
$ids = $this->homeColdBoot($pid, $cursor, $paddedLimit);
if ($ids) {
$cursorIds = $ids;
}
$res = $hydrate($ids);
}
@ -3094,7 +3148,7 @@ class ApiV1Controller extends Controller
$headers['Link'] = '<'.$baseUrl.'max_id='.$oldest.'>; rel="next",<'.$baseUrl.'min_id='.$newest.'>; rel="prev"';
} else {
$headers['Link'] = '<'.$baseUrl.'max_id='.collect($ids)->map(fn ($id) => (int) $id)->min().'>; rel="next"';
$headers['Link'] = '<'.$baseUrl.'max_id='.collect($cursorIds)->map(fn ($id) => (int) $id)->min().'>; rel="next"';
}
return $this->json($res->toArray(), 200, $headers);

@ -1,14 +1,11 @@
<?php
use App\Models\CustomFilter;
use App\Models\Follower;
use App\Models\Status;
use App\Models\User;
use App\Services\HomeTimelineService;
use App\Services\StatusService;
use Illuminate\Foundation\Testing\LazilyRefreshDatabase;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Redis;
use Laravel\Passport\Passport;
@ -16,15 +13,14 @@ uses(LazilyRefreshDatabase::class);
/*
|--------------------------------------------------------------------------
| A home feed that holds members but can render none of them
| Walking on when a home window renders nothing
|--------------------------------------------------------------------------
|
| Members whose statuses have gone away produce an empty page from a feed
| that reports plenty of members, and nothing in the system ever removed
| them. The hydration pass prunes such a member as it finds it and rebuilds
| in the same request when the prune empties the set. A page the viewer
| explicitly filtered out is a different thing: it prunes nothing and runs
| no candidate query.
| A window of members that are all boosts for a viewer with boosts off, or
| a run behind a muted account, produces an empty page from a feed that
| reports plenty of members, and that is not end-of-feed. The read takes up
| to three further windows while the page is empty and at least one member
| was dropped as unrenderable, as a boost, or by a viewer filter.
|
*/
@ -65,71 +61,38 @@ function unrenderablePhoto(int $profileId, int $minutesAgo, array $overrides = [
return $status;
}
/** Remove a status so its feed member can never hydrate again. */
function unrenderableDestroy(Status $status): void
{
DB::table('statuses')->where('id', $status->id)->delete();
Cache::forget(StatusService::key($status->id, true));
Cache::forget(StatusService::key($status->id, false));
}
function unrenderableCandidateQueries(callable $fn): int
{
$count = 0;
DB::listen(function ($query) use (&$count) {
if (preg_match('/max\(id\)/i', $query->sql)) {
$count++;
}
});
$fn();
return $count;
}
it('prunes members whose statuses have gone away and serves what remains', function () {
it('pages past a window of boosts for a viewer with boosts switched off', function () {
[$viewer, $author] = unrenderableViewer();
$pid = $viewer->profile_id;
$stale = collect(range(1, 5))->map(fn ($i) => unrenderablePhoto($author->profile_id, 100 + $i));
foreach ($stale as $status) {
HomeTimelineService::add($pid, $status->id);
}
foreach ($stale as $status) {
unrenderableDestroy($status);
}
// Eligible, recent, and not in the feed: the repair the request can make.
$fresh = unrenderablePhoto($author->profile_id, 5);
Passport::actingAs($viewer, ['read']);
$ids = collect($this->getJson('/api/v1/timelines/home?limit=10')->assertOk()->json())
->pluck('id')->map(fn ($id) => (int) $id)->all();
expect($ids)->toContain((int) $fresh->id);
foreach ($stale as $status) {
expect(Redis::zscore(HomeTimelineService::CACHE_KEY.$pid, $status->id))->toBeNull();
}
});
$stranger = User::factory()->create();
$stranger->refresh();
$original = unrenderablePhoto($stranger->profile_id, 500);
$photo = unrenderablePhoto($author->profile_id, 400);
HomeTimelineService::add($pid, $photo->id);
// The whole first window (limit 1 plus the 50-member reblog pad, and the
// inclusive stop index that makes it one wider) is boosts.
$shares = collect(range(1, 60))->map(function ($i) use ($author, $original) {
$share = Status::factory()->create([
'profile_id' => $author->profile_id,
'type' => 'share',
'scope' => 'public',
'visibility' => 'public',
'local' => true,
'in_reply_to_id' => null,
'reblog_of_id' => $original->id,
'uri' => null,
'created_at' => now()->subMinutes(120 - $i),
]);
StatusService::get($share->id, false);
StatusService::getMastodon($share->id, false);
it('leaves a page the viewer filtered out alone', function () {
[$viewer, $author] = unrenderableViewer();
$pid = $viewer->profile_id;
$statuses = collect(range(1, 3))->map(fn ($i) => unrenderablePhoto($author->profile_id, 10 + $i, [
'caption' => 'hideme sunset photo',
'rendered' => 'hideme sunset photo',
]));
foreach ($statuses as $status) {
HomeTimelineService::add($pid, $status->id);
return $share;
});
foreach ($shares as $share) {
HomeTimelineService::add($pid, $share->id);
}
$filter = CustomFilter::create([
'profile_id' => $pid,
'title' => 'hide sunsets',
'context' => ['home'],
'action' => 1,
]);
$filter->keywords()->create(['keyword' => 'hideme', 'whole_word' => false]);
Cache::forget('filters:v3:'.$pid);
Passport::actingAs($viewer, ['read']);
$candidateQueries = unrenderableCandidateQueries(function () {
$res = $this->getJson('/api/v1/timelines/home?limit=5')->assertOk();
expect($res->json())->toBe([]);
});
expect($candidateQueries)->toBe(0)
->and(HomeTimelineService::count($pid))->toBe(3);
$ids = collect($this->getJson('/api/v1/timelines/home?limit=1')->assertOk()->json())
->pluck('id')->map(fn ($id) => (int) $id)->all();
expect($ids)->toContain((int) $photo->id);
});

Loading…
Cancel
Save