Merge pull request #6923 from pixelfed/feature/user-status-command

Refactor: FixProfileCounts
pull/6927/head
Shlee 4 weeks ago committed by GitHub
commit 511527fc5a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -3,7 +3,6 @@
namespace App\Console\Commands;
use App\Models\Profile;
use App\Models\Status;
use App\Services\Account\AccountStatService;
use App\Services\AccountService;
use Illuminate\Console\Command;
@ -61,21 +60,18 @@ class AccountPostCountStatUpdate extends Command
return;
}
$statusCount = Status::whereProfileId($id)->count();
if ($statusCount != $acct['statuses_count']) {
$profile = Profile::find($id);
if (! $profile) {
AccountStatService::removeFromPostCount($id);
return;
}
$profile->status_count = $statusCount;
$profile->save();
$profile = Profile::find($id);
if (! $profile) {
AccountStatService::removeFromPostCount($id);
AccountService::del($id);
return;
}
// Reconcile only the status_count column (this queue is fed by
// status create/delete events). Shared recompute logic lives in
// AccountStatService so it stays consistent with fix:profilecounts.
AccountStatService::reconcileProfileCounts($profile, ['statuses']);
AccountStatService::removeFromPostCount($id);
}
}

@ -0,0 +1,235 @@
<?php
namespace App\Console\Commands;
use App\Jobs\FollowPipeline\FollowServiceWarmCache;
use App\Models\Profile;
use App\Services\Account\AccountStatService;
use App\Services\FollowerService;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Cache;
class FixProfileCounts extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'admin:fixProfileCounts
{id? : Profile id or username to resync (omit with --all)}
{--all : Scan all profiles and resync any with drifted counts}
{--active=* : Scan only local accounts active within N days (default 30). Bulk mode; mutually exclusive with --all}
{--type= : Restrict to a single metric: followers, following, or statuses (default: all three)}
{--dispatch : Queue FollowServiceWarmCache for follower/following instead of recomputing inline}
{--dry-run : Report drift without changing anything}
{--force : Skip the confirmation prompt (for scheduled/unattended runs)}';
/**
* Default active window (days) when --active is passed without a value.
*/
protected const DEFAULT_ACTIVE_DAYS = 30;
/**
* Metrics this command can reconcile.
*
* @var array<int, string>
*/
protected const METRICS = ['followers', 'following', 'statuses'];
/**
* The console command description.
*
* @var string
*/
protected $description = 'Resync a profile\'s cached counts (followers, following, statuses) from source-of-truth tables. Use --all or --active for bulk reconciliation.';
/**
* Execute the console command.
*
* @return int
*/
public function handle()
{
$id = $this->argument('id');
$all = $this->option('all');
$activeDays = $this->resolveActiveDays();
$active = $activeDays !== null;
// Exactly one mode: a single id, --all, or --active.
$modes = (int) (bool) $id + (int) $all + (int) $active;
if ($modes === 0) {
$this->error('Provide a profile id/username, or pass --all or --active.');
return 1;
}
if ($modes > 1) {
$this->error('Pass only one of: <id>, --all, or --active.');
return 1;
}
$type = $this->option('type');
if ($type !== null && ! in_array($type, self::METRICS, true)) {
$this->error('Invalid --type "'.$type.'". Use one of: '.implode(', ', self::METRICS).'.');
return 1;
}
if ($id) {
$profile = ctype_digit((string) $id)
? Profile::find($id)
: Profile::where('username', $id)->first();
if (! $profile) {
$this->error('No profile found for "'.$id.'".');
return 1;
}
$this->resyncOne($profile);
return 0;
}
// Bulk mode (--all or --active): scan and only touch drifted profiles.
$dryRun = $this->option('dry-run');
$scope = $active
? 'local accounts active in the last '.$activeDays.' days'
: 'all drifted profiles';
if (! $dryRun && ! $this->option('force') && ! $this->confirm('Resync cached counts for '.$scope.'?', true)) {
$this->comment('Aborted.');
return 0;
}
$query = Profile::whereNull('deleted_at');
if ($active) {
// Restrict to LOCAL profiles whose linked user logged in recently.
// Remote profiles have no user row, so they are excluded here.
$cutoff = now()->subDays($activeDays);
$query->whereNotNull('user_id')
->whereHas('user', function ($q) use ($cutoff) {
$q->whereNotNull('last_active_at')
->where('last_active_at', '>=', $cutoff);
});
}
$fixed = 0;
$scanned = 0;
$query->lazyById(500)->each(function ($profile) use (&$fixed, &$scanned) {
$scanned++;
if ($this->resyncOne($profile)) {
$fixed++;
}
});
$this->newLine();
$this->info('Scanned '.$scanned.' profiles ('.$scope.'); '.($this->option('dry-run') ? 'drifted' : 'resynced').': '.$fixed.'.');
return 0;
}
/**
* Resolve the --active window in days, or null if the flag was not passed.
*/
protected function resolveActiveDays(): ?int
{
$values = (array) $this->option('active');
if (empty($values)) {
return null;
}
$last = end($values);
// `--active` with no value arrives as an empty string / null.
if ($last === null || $last === '') {
return self::DEFAULT_ACTIVE_DAYS;
}
$days = (int) $last;
return $days > 0 ? $days : self::DEFAULT_ACTIVE_DAYS;
}
/**
* Recompute (or report) the cached counts for a single profile.
* Only emits output when drift is detected; silent otherwise.
*
* @return bool whether the profile was drifted
*/
protected function resyncOne(Profile $profile): bool
{
// Which metrics to consider: a single --type, or all three.
$type = $this->option('type');
$metrics = $type !== null ? [$type] : self::METRICS;
// Compute drift only for the selected metrics, using the canonical
// source-of-truth logic (shared with the scheduled updater).
$drift = [];
if (in_array('followers', $metrics, true)) {
$drift['followers'] = [
'cached' => (int) $profile->followers_count,
'live' => AccountStatService::recalculateFollowerCount($profile->id),
];
}
if (in_array('following', $metrics, true)) {
$drift['following'] = [
'cached' => (int) $profile->following_count,
'live' => AccountStatService::recalculateFollowingCount($profile->id),
];
}
if (in_array('statuses', $metrics, true)) {
$drift['statuses'] = [
'cached' => (int) $profile->status_count,
'live' => AccountStatService::recalculateStatusCount($profile->id),
];
}
$drifted = array_filter($drift, fn ($m) => $m['cached'] !== $m['live']);
if (empty($drifted)) {
// No drift on the selected metrics: stay silent.
return false;
}
// Drift detected: report exactly what drifted.
$this->warn($profile->username.' (id '.$profile->id.') drift detected:');
foreach ($drifted as $metric => $m) {
$this->line(' '.str_pad($metric.':', 11).'cached='.$m['cached'].' live='.$m['live']);
}
if ($this->option('dry-run')) {
return true;
}
$followOrFollowingDrift = isset($drifted['followers']) || isset($drifted['following']);
if ($this->option('dispatch') && $followOrFollowingDrift) {
// Fix status_count now (if in scope), then let the warm-cache job
// own the follower/following columns and rebuild the Redis sets.
if (isset($drift['statuses'])) {
AccountStatService::reconcileProfileCounts($profile, ['statuses']);
}
Cache::forget(FollowerService::FOLLOWERS_SYNC_KEY.$profile->id);
Cache::forget(FollowerService::FOLLOWING_SYNC_KEY.$profile->id);
FollowServiceWarmCache::dispatch($profile->id)->onQueue('low');
$this->info(' queued FollowServiceWarmCache for profile '.$profile->id.'.');
return true;
}
// Inline recompute of the selected metrics via the shared reconciler.
AccountStatService::reconcileProfileCounts($profile, $metrics);
$profile->refresh();
$this->info(' resynced to followers='.$profile->followers_count.', following='.$profile->following_count.', statuses='.$profile->status_count.'.');
return true;
}
}

@ -1,53 +0,0 @@
<?php
namespace App\Console\Commands;
use App\Models\Profile;
use App\Models\Status;
use Illuminate\Console\Command;
class FixRemotePostCount extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'fix:rpc';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Fix remote accounts post count';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return int
*/
public function handle()
{
Profile::whereNotNull('domain')->chunk(50, function ($profiles) {
foreach ($profiles as $profile) {
$count = Status::whereNull(['in_reply_to_id', 'reblog_of_id'])->whereProfileId($profile->id)->count();
$this->info("Checking {$profile->id} {$profile->username} - found {$count} statuses");
$profile->status_count = $count;
$profile->save();
}
});
return 0;
}
}

@ -1,137 +0,0 @@
<?php
namespace App\Console\Commands;
use App\Models\Profile;
use Illuminate\Console\Command;
class FixStatusCount extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'fix:statuscount {--remote} {--resync} {--remote-only} {--dlog}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'fix profile status count';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return int
*/
public function handle()
{
if (! $this->confirm('Are you sure you want to run the fix status command?')) {
return;
}
$this->line(' ');
$this->info('Running fix status command...');
$now = now();
$nulls = ['domain', 'status', 'last_fetched_at'];
$resync = $this->option('resync');
$resync24hours = false;
if ($resync) {
$resyncChoices = ['Only resync accounts that havent been synced in 24 hours', 'Resync all accounts'];
$rsc = $this->choice(
'Do you want to resync all accounts, or just accounts that havent been resynced for 24 hours?',
$resyncChoices,
0
);
$rsci = array_search($rsc, $resyncChoices);
if ($rsci === 0) {
$resync24hours = true;
$nulls = ['status', 'domain', 'last_fetched_at'];
} else {
$resync24hours = false;
$nulls = ['status', 'domain'];
}
}
$remote = $this->option('remote');
if ($remote) {
$ni = array_search('domain', $nulls);
unset($nulls[$ni]);
$ni = array_search('last_fetched_at', $nulls);
unset($nulls[$ni]);
}
$remoteOnly = $this->option('remote-only');
if ($remoteOnly) {
$ni = array_search('domain', $nulls);
unset($nulls[$ni]);
$ni = array_search('last_fetched_at', $nulls);
unset($nulls[$ni]);
$nulls[] = 'user_id';
}
$dlog = $this->option('dlog');
$nulls = array_values($nulls);
foreach (
Profile::when($resync24hours, function ($query, $resync24hours) use ($nulls) {
if (in_array('domain', $nulls)) {
return $query->whereNull('domain')
->whereNull('last_fetched_at')
->orWhere('last_fetched_at', '<', now()->subHours(24));
} else {
return $query->whereNull('last_fetched_at')
->orWhere('last_fetched_at', '<', now()->subHours(24));
}
})
->when($remoteOnly, function ($query, $remoteOnly) {
return $query->whereNull('last_fetched_at')
->orWhere('last_fetched_at', '<', now()->subHours(24));
})
->whereNull($nulls)
->lazyById(50, 'id') as $profile
) {
$ogc = $profile->status_count;
$upc = $profile->statuses()
->getQuery()
->whereIn('scope', ['public', 'private', 'unlisted'])
->count();
if ($ogc != $upc) {
$profile->status_count = $upc;
$profile->last_fetched_at = $now;
$profile->save();
if ($dlog) {
$this->info($profile->id.':'.$profile->username.' : '.$upc);
}
} else {
$profile->last_fetched_at = $now;
$profile->save();
if ($dlog) {
$this->info($profile->id.':'.$profile->username.' : '.$upc);
}
}
}
$this->line(' ');
$this->info('Finished fix status count command!');
return 0;
}
}

@ -2,12 +2,126 @@
namespace App\Services\Account;
use App\Models\Follower;
use App\Models\Profile;
use App\Models\Status;
use App\Services\AccountService;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Redis;
class AccountStatService
{
const REFRESH_CACHE_KEY = 'pf:services:accountstats:refresh:daily';
/**
* Status types that count toward a profile's status_count.
*
* Must mirror the increment/decrement logic in StatusEntityLexer and
* StatusDelete, which only adjust status_count for media post types.
*
* @var array<int, string>
*/
const COUNTABLE_STATUS_TYPES = [
'photo',
'photo:album',
'video',
'video:album',
'photo:video:album',
];
/**
* Canonical source-of-truth status_count for a profile.
*/
public static function recalculateStatusCount($pid): int
{
return (int) Status::whereProfileId($pid)
->whereIn('type', self::COUNTABLE_STATUS_TYPES)
->count();
}
/**
* Canonical source-of-truth follower count for a profile.
*/
public static function recalculateFollowerCount($pid): int
{
return (int) Follower::whereFollowingId($pid)->count();
}
/**
* Canonical source-of-truth following count for a profile.
*/
public static function recalculateFollowingCount($pid): int
{
return (int) Follower::whereProfileId($pid)->count();
}
/**
* Reconcile a profile's cached count columns against source-of-truth
* tables. Only writes/clears caches for columns that actually drifted.
*
* @param array<int, string> $only Restrict to a subset of
* ['statuses','followers','following'].
* @return array<string, array{cached:int,live:int,drifted:bool}>
* Per-metric before/after summary.
*/
public static function reconcileProfileCounts($profile, array $only = ['statuses', 'followers', 'following']): array
{
if (! $profile instanceof Profile) {
$profile = Profile::find($profile);
}
if (! $profile) {
return [];
}
$summary = [];
$changed = false;
if (in_array('statuses', $only, true)) {
$cached = (int) $profile->status_count;
$live = self::recalculateStatusCount($profile->id);
$drift = $cached !== $live;
if ($drift) {
$profile->status_count = $live;
$changed = true;
}
$summary['statuses'] = ['cached' => $cached, 'live' => $live, 'drifted' => $drift];
}
if (in_array('followers', $only, true)) {
$cached = (int) $profile->followers_count;
$live = self::recalculateFollowerCount($profile->id);
$drift = $cached !== $live;
if ($drift) {
$profile->followers_count = $live;
$changed = true;
}
$summary['followers'] = ['cached' => $cached, 'live' => $live, 'drifted' => $drift];
}
if (in_array('following', $only, true)) {
$cached = (int) $profile->following_count;
$live = self::recalculateFollowingCount($profile->id);
$drift = $cached !== $live;
if ($drift) {
$profile->following_count = $live;
$changed = true;
}
$summary['following'] = ['cached' => $cached, 'live' => $live, 'drifted' => $drift];
}
if ($changed) {
$profile->save();
Cache::forget('profile:status_count:'.$profile->id);
Cache::forget('profile:follower_count:'.$profile->id);
Cache::forget('profile:following_count:'.$profile->id);
AccountService::del($profile->id);
}
return $summary;
}
public static function incrementPostCount($pid)
{
return Redis::zadd(self::REFRESH_CACHE_KEY, $pid, $pid);

@ -0,0 +1,189 @@
<?php
use App\Models\Follower;
use App\Models\Status;
use App\Models\User;
use App\Services\Account\AccountStatService;
use Illuminate\Foundation\Testing\LazilyRefreshDatabase;
uses(LazilyRefreshDatabase::class);
/*
|--------------------------------------------------------------------------
| Profile Count Reconciliation
|--------------------------------------------------------------------------
|
| Tests for the shared AccountStatService recompute helpers used by both the
| scheduled app:account-post-count-stat-update command and admin:fixProfileCounts.
| status_count must mirror the increment logic (media post types only).
|
*/
describe('AccountStatService recompute helpers', function () {
it('counts only media post types toward status_count', function () {
$user = User::factory()->create();
$user->refresh();
$pid = $user->profile->id;
// 2 photos + 1 video = 3 countable; text + reply are NOT counted.
Status::factory()->count(2)->photo()->create(['profile_id' => $pid]);
Status::factory()->video()->create(['profile_id' => $pid]);
Status::factory()->create(['profile_id' => $pid, 'type' => 'text']);
Status::factory()->reply()->create(['profile_id' => $pid]);
expect(AccountStatService::recalculateStatusCount($pid))->toBe(3);
});
it('counts followers and following from the followers table', function () {
$a = User::factory()->create();
$a->refresh();
$b = User::factory()->create();
$b->refresh();
$c = User::factory()->create();
$c->refresh();
// b and c follow a; a follows c.
Follower::create(['profile_id' => $b->profile->id, 'following_id' => $a->profile->id]);
Follower::create(['profile_id' => $c->profile->id, 'following_id' => $a->profile->id]);
Follower::create(['profile_id' => $a->profile->id, 'following_id' => $c->profile->id]);
expect(AccountStatService::recalculateFollowerCount($a->profile->id))->toBe(2);
expect(AccountStatService::recalculateFollowingCount($a->profile->id))->toBe(1);
});
});
describe('AccountStatService::reconcileProfileCounts', function () {
it('fixes all drifted columns and reports the summary', function () {
$user = User::factory()->create();
$user->refresh();
$profile = $user->profile;
Status::factory()->count(2)->photo()->create(['profile_id' => $profile->id]);
// Deliberately set wrong cached values.
$profile->status_count = 99;
$profile->followers_count = 42;
$profile->following_count = 7;
$profile->save();
$summary = AccountStatService::reconcileProfileCounts($profile->fresh());
expect($summary['statuses']['drifted'])->toBeTrue();
expect($summary['statuses']['live'])->toBe(2);
expect($summary['followers']['live'])->toBe(0);
expect($summary['following']['live'])->toBe(0);
$profile->refresh();
expect((int) $profile->status_count)->toBe(2);
expect((int) $profile->followers_count)->toBe(0);
expect((int) $profile->following_count)->toBe(0);
});
it('reports no drift and writes nothing when counts are correct', function () {
$user = User::factory()->create();
$user->refresh();
$profile = $user->profile;
Status::factory()->photo()->create(['profile_id' => $profile->id]);
$profile->status_count = 1;
$profile->followers_count = 0;
$profile->following_count = 0;
$profile->save();
$updatedAt = $profile->fresh()->updated_at;
$summary = AccountStatService::reconcileProfileCounts($profile->fresh());
expect($summary['statuses']['drifted'])->toBeFalse();
expect($summary['followers']['drifted'])->toBeFalse();
expect($summary['following']['drifted'])->toBeFalse();
// No write should have occurred (updated_at unchanged).
expect($profile->fresh()->updated_at->eq($updatedAt))->toBeTrue();
});
it('restricts reconciliation to the requested metrics only', function () {
$user = User::factory()->create();
$user->refresh();
$profile = $user->profile;
Status::factory()->count(3)->photo()->create(['profile_id' => $profile->id]);
$profile->status_count = 0; // drifted, should be fixed
$profile->followers_count = 50; // drifted, but must be left alone
$profile->save();
$summary = AccountStatService::reconcileProfileCounts($profile->fresh(), ['statuses']);
expect($summary)->toHaveKey('statuses');
expect($summary)->not->toHaveKey('followers');
$profile->refresh();
expect((int) $profile->status_count)->toBe(3);
expect((int) $profile->followers_count)->toBe(50);
});
it('returns an empty summary for a missing profile id', function () {
expect(AccountStatService::reconcileProfileCounts(999999999999))->toBe([]);
});
});
describe('admin:fixProfileCounts command', function () {
it('is silent for an in-sync profile and reports drift otherwise', function () {
$user = User::factory()->create();
$user->refresh();
$profile = $user->profile;
Status::factory()->count(2)->photo()->create(['profile_id' => $profile->id]);
$profile->status_count = 5; // drift
$profile->save();
$this->artisan('admin:fixProfileCounts', ['id' => (string) $profile->id])
->expectsOutputToContain('drift detected')
->assertExitCode(0);
// Now in sync -> no drift output.
$this->artisan('admin:fixProfileCounts', ['id' => (string) $profile->id])
->doesntExpectOutputToContain('drift detected')
->assertExitCode(0);
});
it('does not modify anything in dry-run mode', function () {
$user = User::factory()->create();
$user->refresh();
$profile = $user->profile;
Status::factory()->photo()->create(['profile_id' => $profile->id]);
$profile->status_count = 88;
$profile->save();
$this->artisan('admin:fixProfileCounts', ['id' => (string) $profile->id, '--dry-run' => true])
->assertExitCode(0);
expect((int) $profile->fresh()->status_count)->toBe(88);
});
it('with --type restricts the fix to a single metric', function () {
$user = User::factory()->create();
$user->refresh();
$profile = $user->profile;
Status::factory()->count(2)->photo()->create(['profile_id' => $profile->id]);
// All three drifted.
$profile->status_count = 42;
$profile->followers_count = 100;
$profile->following_count = 50;
$profile->save();
$this->artisan('admin:fixProfileCounts', ['id' => (string) $profile->id, '--type' => 'statuses'])
->assertExitCode(0);
$profile->refresh();
// Only statuses reconciled; followers/following left untouched.
expect((int) $profile->status_count)->toBe(2);
expect((int) $profile->followers_count)->toBe(100);
expect((int) $profile->following_count)->toBe(50);
});
it('rejects an invalid --type', function () {
$this->artisan('admin:fixProfileCounts', ['id' => '1', '--type' => 'bogus'])
->assertExitCode(1);
});
});
Loading…
Cancel
Save