refactor: run emoji cloud migration as a queued job, not inline in migrate

Running the ~25k-file S3 upload synchronously via Artisan::call inside the
migration blocked the upgrade with no visible output, and a mid-run failure
would leave the migration in a bad state.

Dispatch EmojiMigrateToCloudPipeline (ShouldQueue, ShouldBeUnique) onto the
mmo queue instead, so migrate returns immediately and the upload runs in the
background on Horizon. The job re-checks the cloud-storage guard at runtime,
runs the async command (--concurrency=100), has a 1h timeout and tries=1,
and is unique so it can't stack up.
feat/emoji-cloud-storage-v2
Your Name 3 weeks ago
parent 186f62fff9
commit 808a03117d

@ -0,0 +1,60 @@
<?php
namespace App\Jobs\MediaPipeline;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Artisan;
use Illuminate\Support\Facades\Log;
class EmojiMigrateToCloudPipeline implements ShouldBeUnique, ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
/**
* Bulk emoji upload can take a while; allow up to an hour.
*/
public $timeout = 3600;
/**
* Do not auto-retry a long bulk transfer. It is safe to re-dispatch
* manually, but automatic retries would re-run the whole upload.
*/
public $tries = 1;
/**
* Only one migration job should be queued/running at a time.
*/
public $uniqueFor = 3600;
public function uniqueId(): string
{
return 'emoji-migrate-to-cloud';
}
public function handle(): void
{
// Guard again at run time in case cloud storage was toggled off between
// dispatch and execution.
$cloudEnabled = (bool) config('pixelfed.cloud_storage') || (bool) config_cache('pixelfed.cloud_storage');
if (! $cloudEnabled) {
Log::info('EmojiMigrateToCloudPipeline: cloud storage not enabled, skipping.');
return;
}
Log::info('EmojiMigrateToCloudPipeline: starting emoji migration to cloud.');
Artisan::call('admin:EmojiMoveStorageLocalToCloud', [
'--force' => true,
'--concurrency' => 100,
]);
Log::info('EmojiMigrateToCloudPipeline: finished. '.trim(Artisan::output()));
}
}

@ -1,7 +1,7 @@
<?php
use App\Jobs\MediaPipeline\EmojiMigrateToCloudPipeline;
use Illuminate\Database\Migrations\Migration;
use Illuminate\Support\Facades\Artisan;
return new class extends Migration
{
@ -9,23 +9,24 @@ return new class extends Migration
* Run the migrations.
*
* When cloud storage is enabled, custom emoji URLs resolve to the cloud
* disk. Existing emoji may still only exist locally, so move them all to
* cloud now (in one pass) to avoid a window where their URLs 404. This is
* a no-op on local-only instances.
* disk. Existing emoji may still only exist locally, so migrate them to
* cloud to avoid a window where their URLs 404.
*
* The upload is potentially large and network-bound, so we dispatch it to
* the queue and return immediately rather than blocking the upgrade. The
* work runs in the background on Horizon. No-op on local-only instances.
*/
public function up(): void
{
// Consider cloud enabled if either the live config or the (possibly
// cached) config_cache value says so; the command guards again anyway.
// cached) config_cache value says so; the job guards again at runtime.
$cloudEnabled = (bool) config('pixelfed.cloud_storage') || (bool) config_cache('pixelfed.cloud_storage');
if (! $cloudEnabled) {
return;
}
Artisan::call('admin:EmojiMoveStorageLocalToCloud', [
'--force' => true,
]);
EmojiMigrateToCloudPipeline::dispatch()->onQueue('mmo');
}
/**

Loading…
Cancel
Save