mirror of https://github.com/pixelfed/pixelfed
Backfill storage_used on upgrade via queued job + data migration
Repair accounts whose storage counter drifted before the self-heal logic existed (#7169). A data migration dispatches RecalculateAllUserStoragePipeline to the low queue so the deploy is not blocked while every user is recomputed from source. The job is unique and idempotent, so re-runs are harmless. - RecalculateAllUserStoragePipeline: chunked recalc of all active users - Migration dispatches the job (no inline heavy work during deploy) - Test covers bulk recalculation from actual mediapull/7175/head
parent
28573e863f
commit
5a9c235922
@ -0,0 +1,72 @@
|
||||
<?php
|
||||
|
||||
namespace App\Jobs\InternalPipeline;
|
||||
|
||||
use App\Models\User;
|
||||
use App\Services\UserStorageService;
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\Middleware\WithoutOverlapping;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
|
||||
/**
|
||||
* One-off backfill that recomputes users.storage_used from actual media.
|
||||
*
|
||||
* Repairs accounts whose cached storage counter drifted upward before the
|
||||
* upload/delete self-heal logic existed, unblocking users stuck at the
|
||||
* account size limit despite low real usage (#7169). Safe to run repeatedly:
|
||||
* each user is recomputed from source, so it is idempotent.
|
||||
*/
|
||||
class RecalculateAllUserStoragePipeline implements ShouldBeUniqueUntilProcessing, ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
public $timeout = 3600;
|
||||
|
||||
public $tries = 1;
|
||||
|
||||
public $maxExceptions = 1;
|
||||
|
||||
public $failOnTimeout = true;
|
||||
|
||||
/**
|
||||
* The number of seconds after which the job's unique lock will be released.
|
||||
*
|
||||
* @var int
|
||||
*/
|
||||
public $uniqueFor = 7200;
|
||||
|
||||
/**
|
||||
* Get the unique ID for the job.
|
||||
*/
|
||||
public function uniqueId(): string
|
||||
{
|
||||
return 'ip:recalculate-all-user-storage';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the middleware the job should pass through.
|
||||
*
|
||||
* @return array<int, object>
|
||||
*/
|
||||
public function middleware(): array
|
||||
{
|
||||
return [(new WithoutOverlapping('ip:recalculate-all-user-storage'))->shared()->dontRelease()];
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the job.
|
||||
*/
|
||||
public function handle(): void
|
||||
{
|
||||
User::whereNull('status')
|
||||
->chunkById(500, function ($users) {
|
||||
foreach ($users as $user) {
|
||||
UserStorageService::recalculateUpdateStorageUsed($user->id);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,25 @@
|
||||
<?php
|
||||
|
||||
use App\Jobs\InternalPipeline\RecalculateAllUserStoragePipeline;
|
||||
use Illuminate\Database\Migrations\Migration;
|
||||
|
||||
return new class extends Migration
|
||||
{
|
||||
/**
|
||||
* Repair storage_used counters that drifted before the self-heal logic
|
||||
* existed (#7169). The recalculation is dispatched to the queue rather than
|
||||
* run inline so the deploy is not blocked while every user is recomputed.
|
||||
*
|
||||
* The job is idempotent (recomputes each user from source) and unique, so a
|
||||
* re-run of this migration or a duplicate dispatch is harmless.
|
||||
*/
|
||||
public function up(): void
|
||||
{
|
||||
RecalculateAllUserStoragePipeline::dispatch()->onQueue('low');
|
||||
}
|
||||
|
||||
public function down(): void
|
||||
{
|
||||
// Data backfill only; nothing to reverse.
|
||||
}
|
||||
};
|
||||
@ -0,0 +1,51 @@
|
||||
<?php
|
||||
|
||||
use App\Jobs\InternalPipeline\RecalculateAllUserStoragePipeline;
|
||||
use App\Models\Media;
|
||||
use App\Models\User;
|
||||
use Illuminate\Foundation\Testing\LazilyRefreshDatabase;
|
||||
|
||||
uses(LazilyRefreshDatabase::class);
|
||||
|
||||
function makeUserMedia(User $user, int $bytes): Media
|
||||
{
|
||||
return Media::create([
|
||||
'status_id' => null,
|
||||
'profile_id' => $user->profile->id,
|
||||
'user_id' => $user->id,
|
||||
'media_path' => 'public/m/_v2/1/'.uniqid().'.jpeg',
|
||||
'mime' => 'image/jpeg',
|
||||
'size' => $bytes,
|
||||
'order' => 1,
|
||||
]);
|
||||
}
|
||||
|
||||
/*
|
||||
| The backfill job repairs storage_used counters that drifted before the
|
||||
| self-heal logic existed (#7169). It recomputes every active user from their
|
||||
| actual media, so stale inflated values are corrected in bulk.
|
||||
*/
|
||||
|
||||
it('recalculates storage_used from actual media for all active users', function () {
|
||||
$a = User::factory()->create();
|
||||
$b = User::factory()->create();
|
||||
$a->refresh();
|
||||
$b->refresh();
|
||||
|
||||
// Both start with wildly inflated stale counters.
|
||||
foreach ([$a, $b] as $u) {
|
||||
$u->storage_used = 999999;
|
||||
$u->storage_used_updated_at = now()->subYear();
|
||||
$u->save();
|
||||
}
|
||||
|
||||
makeUserMedia($a, 400000); // 400 KB real usage
|
||||
// $b has no media.
|
||||
|
||||
(new RecalculateAllUserStoragePipeline)->handle();
|
||||
|
||||
$a->refresh();
|
||||
$b->refresh();
|
||||
expect((int) $a->storage_used)->toBe(400);
|
||||
expect((int) $b->storage_used)->toBe(0);
|
||||
});
|
||||
Loading…
Reference in New Issue