diff --git a/app/Console/Commands/Admin/MediaMoveStorageLocalToCloud.php b/app/Console/Commands/Admin/MediaMoveStorageLocalToCloud.php
index 223c126bc..6338330d9 100644
--- a/app/Console/Commands/Admin/MediaMoveStorageLocalToCloud.php
+++ b/app/Console/Commands/Admin/MediaMoveStorageLocalToCloud.php
@@ -25,6 +25,7 @@ class MediaMoveStorageLocalToCloud extends Command
*/
protected $signature = 'admin:MediaMoveStorageLocalToCloud
{--limit=500 : Max media rows to process this run}
+ {--before-id= : Only process media with an ID lower than this value}
{--dry-run : Report what would happen without copying or writing}
{--keep-local : Do not delete local files after verifying the cloud copy}
{--debug : Print exactly what moves, from which local path to which cloud destination}
@@ -35,107 +36,258 @@ class MediaMoveStorageLocalToCloud extends Command
*
* @var string
*/
- protected $description = 'Migrate local media to cloud storage: copy up, verify (size/sha256), update URLs, then delete the local copy. Ensures new uploads go to cloud during the migration.';
+ protected $description = 'Migrate local media to cloud storage: copy up, verify, update URLs, then optionally delete the local copy.';
protected int $movedBytes = 0;
protected ?ProgressBar $bar = null;
- public function handle()
+ public function handle(): int
{
try {
$localDisk = Storage::disk('local');
$cloudDisk = Storage::disk(config('filesystems.cloud'));
} catch (\Throwable $e) {
- $this->error('Cloud disk ('.config('filesystems.cloud').') could not be resolved: '.$e->getMessage());
+ $this->error(
+ 'Cloud disk ('.config('filesystems.cloud').') could not be resolved: '.$e->getMessage()
+ );
- return 1;
+ return self::FAILURE;
}
if (! $this->cloudHost()) {
- $this->error('Cloud disk ('.config('filesystems.cloud').') is not configured (no resolvable URL).');
+ $this->error(
+ 'Cloud disk ('.config('filesystems.cloud').') is not configured (no resolvable URL).'
+ );
$this->line('Set AWS_URL / AWS_* in your environment before migrating to cloud.');
- return 1;
+ return self::FAILURE;
}
- // --- Ensure new uploads route to cloud during the migration --------
- // Read the effective, live setting the same way the rest of the app
- // does (config_cache is DB-backed and works with or without a .env
- // file, e.g. in containers that inject config via env vars).
- $cloudEnabled = (bool) config_cache('pixelfed.cloud_storage');
-
- if (! $cloudEnabled) {
- $this->warn('Cloud storage (pixelfed.cloud_storage) is currently disabled.');
- $this->line('New uploads would keep landing on LOCAL storage during this migration.');
- if ($this->option('dry-run')) {
- $this->line('[dry-run] Would enable cloud storage (runtime + config cache, and .env if writable).');
- } elseif ($this->option('force') || $this->confirm('Enable cloud storage now so new uploads go to cloud?', true)) {
- $this->setStorageEnv('PF_ENABLE_CLOUD', 'true', 'pixelfed.cloud_storage', true);
- $this->info('Cloud storage enabled (live runtime + config cache).');
- } else {
- $this->error('Aborting: refusing to migrate to cloud while new uploads stay local.');
-
- return 1;
- }
- } else {
- $this->info('Cloud storage is already enabled; new uploads route to cloud. ✓');
+ if (! $this->ensureCloudStorageEnabled()) {
+ return self::FAILURE;
}
if ($this->option('debug')) {
- $this->newLine();
- $this->line('[debug] Storage routing');
- $this->line(' local disk : '.config('filesystems.disks.local.root'));
- $this->line(' cloud disk : '.config('filesystems.cloud'));
- $this->line(' cloud driver : '.config('filesystems.disks.'.config('filesystems.cloud').'.driver'));
- $this->line(' cloud bucket : '.config('filesystems.disks.'.config('filesystems.cloud').'.bucket'));
- $this->line(' cloud host : '.$this->cloudHost());
+ $this->printStorageDebugInfo();
}
$this->newLine();
- if (! $this->option('dry-run') && ! $this->option('force')) {
- if (! $this->confirm('Begin migrating local media to cloud?', true)) {
- $this->comment('Aborted.');
- return 0;
- }
+ if (
+ ! $this->option('dry-run') &&
+ ! $this->option('force') &&
+ ! $this->confirm('Begin migrating local media to cloud?', true)
+ ) {
+ $this->comment('Aborted.');
+
+ return self::SUCCESS;
+ }
+
+ $limit = max(1, (int) $this->option('limit'));
+ $beforeId = $this->resolveBeforeId();
+
+ if ($beforeId === false) {
+ return self::FAILURE;
+ }
+
+ $query = $this->candidateQuery($beforeId, $limit);
+
+ /*
+ * Fetch candidates exactly once.
+ *
+ * Do NOT call $query->count() here. On very large media tables that can
+ * count the entire matching dataset before we immediately execute the
+ * SELECT again. We only need the count of this bounded result set.
+ */
+ $medias = $query->get();
+
+ if ($medias->isEmpty()) {
+ $this->info('No media candidates found.');
+
+ return self::SUCCESS;
}
- $limit = (int) $this->option('limit');
$moved = 0;
$skipped = 0;
$failed = 0;
- // Candidates: local, non-remote media not yet replicated to cloud.
- $query = Media::whereRemoteMedia(false)
- ->whereNotNull('media_path')
- ->where(function ($q) {
- $q->whereNull('cdn_url')->orWhereNull('replicated_at')->orWhereNot('version', '4');
- })
- ->orderByDesc('id')
- ->limit($limit);
-
- $this->bar = $this->output->createProgressBar($query->count());
+ $this->bar = $this->output->createProgressBar($medias->count());
$this->bar->start();
- foreach ($query->get() as $media) {
+ foreach ($medias as $media) {
$result = $this->migrateOne($media, $localDisk, $cloudDisk);
+
match ($result) {
'moved' => $moved++,
'skipped' => $skipped++,
default => $failed++,
};
+
$this->bar->advance();
}
$this->bar->finish();
$this->newLine(2);
- $this->info(($this->option('dry-run') ? '[dry-run] ' : '').'Done. moved='.$moved.' skipped='.$skipped.' failed='.$failed.'.');
+
+ $prefix = $this->option('dry-run') ? '[dry-run] ' : '';
+
+ $this->info(
+ $prefix.'Done. moved='.$moved.' skipped='.$skipped.' failed='.$failed.'.'
+ );
+
if ($this->movedBytes) {
- $this->info('Transferred '.PrettyNumber::size($this->movedBytes).' to cloud storage.');
+ $this->info(
+ 'Transferred '.PrettyNumber::size($this->movedBytes).' to cloud storage.'
+ );
}
- return 0;
+ /*
+ * The candidates are ordered newest -> oldest, so the final ID is a
+ * safe cursor for the next run. This is especially useful when some
+ * malformed/unrecoverable rows must remain untouched.
+ */
+ $lastMedia = $medias->last();
+
+ if ($lastMedia && $medias->count() === $limit) {
+ $this->newLine();
+ $this->comment('More media may remain.');
+
+ $nextCommand = sprintf(
+ 'php artisan admin:MediaMoveStorageLocalToCloud --limit=%d --before-id=%s',
+ $limit,
+ $lastMedia->id
+ );
+
+ if ($this->option('keep-local')) {
+ $nextCommand .= ' --keep-local';
+ }
+
+ if ($this->option('dry-run')) {
+ $nextCommand .= ' --dry-run';
+ }
+
+ if ($this->option('debug')) {
+ $nextCommand .= ' --debug';
+ }
+
+ if ($this->option('force')) {
+ $nextCommand .= ' --force';
+ }
+
+ $this->line('Next batch:');
+ $this->line($nextCommand);
+ }
+
+ return $failed > 0 ? self::FAILURE : self::SUCCESS;
+ }
+
+ protected function ensureCloudStorageEnabled(): bool
+ {
+ $cloudEnabled = (bool) config_cache('pixelfed.cloud_storage');
+
+ if ($cloudEnabled) {
+ $this->info('Cloud storage is already enabled; new uploads route to cloud. ✓');
+
+ return true;
+ }
+
+ $this->warn('Cloud storage (pixelfed.cloud_storage) is currently disabled.');
+ $this->line('New uploads would keep landing on LOCAL storage during this migration.');
+
+ if ($this->option('dry-run')) {
+ $this->line(
+ '[dry-run] Would enable cloud storage (runtime + config cache, and .env if writable).'
+ );
+
+ return true;
+ }
+
+ if (
+ ! $this->option('force') &&
+ ! $this->confirm(
+ 'Enable cloud storage now so new uploads go to cloud?',
+ true
+ )
+ ) {
+ $this->error(
+ 'Aborting: refusing to migrate to cloud while new uploads stay local.'
+ );
+
+ return false;
+ }
+
+ $this->setStorageEnv(
+ 'PF_ENABLE_CLOUD',
+ 'true',
+ 'pixelfed.cloud_storage',
+ true
+ );
+
+ $this->info('Cloud storage enabled (live runtime + config cache).');
+
+ return true;
+ }
+
+ protected function printStorageDebugInfo(): void
+ {
+ $this->newLine();
+ $this->line('[debug] Storage routing');
+ $this->line(' local disk : '.config('filesystems.disks.local.root'));
+ $this->line(' cloud disk : '.config('filesystems.cloud'));
+ $this->line(
+ ' cloud driver : '.config(
+ 'filesystems.disks.'.config('filesystems.cloud').'.driver'
+ )
+ );
+ $this->line(
+ ' cloud bucket : '.config(
+ 'filesystems.disks.'.config('filesystems.cloud').'.bucket'
+ )
+ );
+ $this->line(' cloud host : '.$this->cloudHost());
+
+ if ($this->option('before-id')) {
+ $this->line(' before id : '.$this->option('before-id'));
+ }
+ }
+
+ protected function resolveBeforeId(): string|null|false
+ {
+ $beforeId = $this->option('before-id');
+
+ if ($beforeId === null || $beforeId === '') {
+ return null;
+ }
+
+ $beforeId = trim((string) $beforeId);
+
+ if (! ctype_digit($beforeId) || $beforeId === '0') {
+ $this->error('--before-id must be a positive numeric media ID.');
+
+ return false;
+ }
+
+ return $beforeId;
+ }
+
+ protected function candidateQuery(?string $beforeId, int $limit)
+ {
+ return Media::query()
+ ->whereRemoteMedia(false)
+ ->whereNotNull('media_path')
+ ->where(function ($query) {
+ $query
+ ->whereNull('cdn_url')
+ ->orWhereNull('replicated_at')
+ ->orWhere('version', '!=', 4);
+ })
+ ->when(
+ $beforeId,
+ fn ($query) => $query->where('id', '<', $beforeId)
+ )
+ ->orderByDesc('id')
+ ->limit($limit);
}
/**
@@ -143,42 +295,80 @@ class MediaMoveStorageLocalToCloud extends Command
*/
protected function migrateOne(Media $media, $localDisk, $cloudDisk): string
{
- if (Str::startsWith((string) $media->media_path, 'http')) {
- $this->debugLine('media '.$media->id.' skipped: media_path is a remote URL ('.$media->media_path.')');
+ $mediaPath = (string) $media->media_path;
+
+ if (Str::startsWith($mediaPath, ['http://', 'https://'])) {
+ $this->debugLine(
+ 'media '.$media->id.' skipped: media_path is a remote URL ('.$mediaPath.')'
+ );
return 'skipped';
}
- // Nothing to do if the local file is gone.
- if (! $localDisk->exists($media->media_path)) {
- // Already on cloud only? mark version and move on.
- if ($cloudDisk->exists($media->media_path)) {
- $this->debugLine('media '.$media->id.' skipped: local file missing but already on cloud ('.$media->media_path.')');
- if (! $this->option('dry-run') && $media->version !== '4') {
- $media->version = 4;
- $media->save();
+ /*
+ * If the local file no longer exists, check whether migration had
+ * actually succeeded previously but the database state was only
+ * partially updated.
+ */
+ if (! $localDisk->exists($mediaPath)) {
+ if ($cloudDisk->exists($mediaPath)) {
+ $this->debugLine(
+ 'media '.$media->id.' recovered: local file missing but cloud copy exists ('.$mediaPath.')'
+ );
+
+ if (! $this->option('dry-run')) {
+ $this->markAsReplicated($media, $cloudDisk, false);
}
return 'skipped';
}
- $this->debugLine('media '.$media->id.' skipped: local file missing and not on cloud ('.$media->media_path.')');
+ $this->debugLine(
+ 'media '.$media->id.' skipped: local file missing and not on cloud ('.$mediaPath.')'
+ );
return 'skipped';
}
- // Basic per-item info (always on): what moves and where it lands.
- $this->basicLine('media '.$media->id.' ('.PrettyNumber::size((int) $media->size).'): '.$media->media_path.' → '.$this->cloudDestination($media->media_path, $cloudDisk));
+ $this->basicLine(
+ 'media '.$media->id.
+ ' ('.PrettyNumber::size((int) $media->size).'): '.
+ $mediaPath.
+ ' → '.
+ $this->cloudDestination($mediaPath, $cloudDisk)
+ );
if ($this->option('debug')) {
- $this->debugLine(' status_id : '.($media->status_id ?? 'null'));
- $this->debugLine(' primary from : '.$localDisk->path($media->media_path));
- $this->debugLine(' primary to : '.$cloudDisk->url($media->media_path));
- if ($media->thumbnail_path && $localDisk->exists($media->thumbnail_path)) {
- $this->debugLine(' thumbnail from: '.$localDisk->path($media->thumbnail_path));
- $this->debugLine(' thumbnail to : '.$cloudDisk->url($media->thumbnail_path));
+ $this->debugLine(
+ ' status_id : '.($media->status_id ?? 'null')
+ );
+
+ $this->debugLine(
+ ' primary from : '.$localDisk->path($mediaPath)
+ );
+
+ $this->debugLine(
+ ' primary to : '.$cloudDisk->url($mediaPath)
+ );
+
+ if (
+ $media->thumbnail_path &&
+ $localDisk->exists($media->thumbnail_path)
+ ) {
+ $this->debugLine(
+ ' thumbnail from: '.$localDisk->path($media->thumbnail_path)
+ );
+
+ $this->debugLine(
+ ' thumbnail to : '.$cloudDisk->url($media->thumbnail_path)
+ );
}
- $this->debugLine(' after copy : '.($this->option('keep-local') ? 'local kept' : 'local deleted').($this->option('dry-run') ? ' (dry-run: no changes)' : ''));
+
+ $this->debugLine(
+ ' after copy : '.
+ ($this->option('keep-local') ? 'local kept' : 'local deleted').
+ ($this->option('dry-run') ? ' (dry-run: no changes)' : '')
+ );
}
if ($this->option('dry-run')) {
@@ -186,37 +376,61 @@ class MediaMoveStorageLocalToCloud extends Command
}
try {
- // Copy the primary file (and thumbnail) to cloud.
- $this->copyToCloud($media->media_path, $localDisk, $cloudDisk);
- if ($media->thumbnail_path && $localDisk->exists($media->thumbnail_path)) {
- $this->copyToCloud($media->thumbnail_path, $localDisk, $cloudDisk);
+ $this->copyToCloud(
+ $mediaPath,
+ $localDisk
+ );
+
+ if (
+ $media->thumbnail_path &&
+ $localDisk->exists($media->thumbnail_path)
+ ) {
+ $this->copyToCloud(
+ $media->thumbnail_path,
+ $localDisk
+ );
}
- // Verify the primary file before touching anything else.
- if (! $this->verify($media->media_path, $localDisk, $cloudDisk, $media->original_sha256)) {
- $this->warn(PHP_EOL.'Verify failed for media '.$media->id.' ('.$media->media_path.'); left local copy intact.');
+ if (
+ ! $this->verify(
+ $mediaPath,
+ $localDisk,
+ $cloudDisk,
+ $media->original_sha256
+ )
+ ) {
+ $this->warn(
+ PHP_EOL.
+ 'Verify failed for media '.
+ $media->id.
+ ' ('.
+ $mediaPath.
+ '); left local copy intact.'
+ );
return 'failed';
}
- // Update URL fields to the cloud disk.
- $media->cdn_url = $cloudDisk->url($media->media_path);
- $media->optimized_url = $media->cdn_url;
- if ($media->thumbnail_path && $cloudDisk->exists($media->thumbnail_path)) {
- $media->thumbnail_url = $cloudDisk->url($media->thumbnail_path);
- }
- $media->replicated_at = now();
+ /*
+ * Mark the cloud copy as authoritative after verification.
+ *
+ * version=4 represents the migrated format/state regardless of
+ * whether --keep-local was requested. Keeping the local file
+ * should not cause this media row to be selected forever.
+ */
+ $this->markAsReplicated($media, $cloudDisk, true);
- // Integrated GC: delete the verified local copy unless --keep-local.
if (! $this->option('keep-local')) {
- $localDisk->delete($media->media_path);
- if ($media->thumbnail_path && $localDisk->exists($media->thumbnail_path)) {
+ $localDisk->delete($mediaPath);
+
+ if (
+ $media->thumbnail_path &&
+ $localDisk->exists($media->thumbnail_path)
+ ) {
$localDisk->delete($media->thumbnail_path);
}
- $media->version = 4;
}
- $media->save();
$this->movedBytes += (int) $media->size;
if ($media->status_id) {
@@ -226,18 +440,68 @@ class MediaMoveStorageLocalToCloud extends Command
return 'moved';
} catch (\Throwable $e) {
- Log::error('MediaMoveStorageLocalToCloud: failed to migrate media', [
- 'media_id' => $media->id,
- 'error' => $e->getMessage(),
- ]);
- $this->warn(PHP_EOL.'Error migrating media '.$media->id.': '.$e->getMessage());
+ Log::error(
+ 'MediaMoveStorageLocalToCloud: failed to migrate media',
+ [
+ 'media_id' => $media->id,
+ 'error' => $e->getMessage(),
+ ]
+ );
+
+ $this->warn(
+ PHP_EOL.
+ 'Error migrating media '.
+ $media->id.
+ ': '.
+ $e->getMessage()
+ );
return 'failed';
}
}
/**
- * Write a line that always shows, printed cleanly above an active progress bar.
+ * Normalize all DB state that indicates the media has been replicated.
+ *
+ * $freshReplication should be true when this command just copied and
+ * verified the object. For recovery of an already-existing cloud object,
+ * we preserve replicated_at if it already has a value.
+ */
+ protected function markAsReplicated(
+ Media $media,
+ $cloudDisk,
+ bool $freshReplication
+ ): void {
+ $media->cdn_url = $cloudDisk->url($media->media_path);
+ $media->optimized_url = $media->cdn_url;
+
+ if (
+ $media->thumbnail_path &&
+ $cloudDisk->exists($media->thumbnail_path)
+ ) {
+ $media->thumbnail_url = $cloudDisk->url(
+ $media->thumbnail_path
+ );
+ }
+
+ if ($freshReplication || ! $media->replicated_at) {
+ $media->replicated_at = now();
+ }
+
+ /*
+ * version=4 denotes completed cloud migration.
+ *
+ * This must be set even with --keep-local; otherwise those rows
+ * continue matching "version != 4" forever.
+ */
+ $media->version = 4;
+
+ $media->save();
+ }
+
+ /**
+ * Write a line that always shows, printed cleanly above an active
+ * progress bar.
*/
protected function basicLine(string $message): void
{
@@ -253,7 +517,8 @@ class MediaMoveStorageLocalToCloud extends Command
}
/**
- * Write a line only when --debug is set, cleanly above an active progress bar.
+ * Write a line only when --debug is set, cleanly above an active
+ * progress bar.
*/
protected function debugLine(string $message): void
{
@@ -284,40 +549,61 @@ class MediaMoveStorageLocalToCloud extends Command
}
}
- protected function copyToCloud(string $path, $localDisk, $cloudDisk): void
+ protected function copyToCloud(string $path, $localDisk): void
{
- $p = explode('/', $path);
- $name = array_pop($p);
- $storagePath = implode('/', $p);
-
- // Reuse the resilient uploader (handles alt disks + retries).
- ResilientMediaStorageService::store($storagePath, $localDisk->path($path), $name);
+ $parts = explode('/', $path);
+ $name = array_pop($parts);
+ $storagePath = implode('/', $parts);
+
+ /*
+ * Reuse the resilient uploader, which handles the configured cloud
+ * destination, alternate disks, and retries.
+ */
+ ResilientMediaStorageService::store(
+ $storagePath,
+ $localDisk->path($path),
+ $name
+ );
}
/**
- * Verify the cloud copy matches the local source by size, and by sha256
- * when a checksum is available/cheap. Fails closed.
+ * Verify the cloud copy matches the local source by size and, when an
+ * original checksum is available, verify the local source against it.
+ *
+ * Cloud content hashing would require downloading the object, so size
+ * parity plus a known-good local SHA-256 is used here.
*/
- protected function verify(string $path, $localDisk, $cloudDisk, ?string $expectedSha = null): bool
- {
+ protected function verify(
+ string $path,
+ $localDisk,
+ $cloudDisk,
+ ?string $expectedSha = null
+ ): bool {
if (! $cloudDisk->exists($path)) {
return false;
}
$localSize = $localDisk->size($path);
$cloudSize = $cloudDisk->size($path);
- if ($localSize === false || $cloudSize === false || $localSize !== $cloudSize) {
+
+ if (
+ $localSize === false ||
+ $cloudSize === false ||
+ $localSize !== $cloudSize
+ ) {
return false;
}
- // If we already have the original checksum, verify the local file still
- // matches it (so we never delete a locally-corrupted-but-uploaded file
- // without noticing). Cloud content hashing would require a full
- // download, which we avoid for large media; size parity + known sha
- // is a strong signal.
if ($expectedSha) {
- $localSha = @hash_file('sha256', $localDisk->path($path));
- if ($localSha && ! hash_equals($expectedSha, $localSha)) {
+ $localSha = @hash_file(
+ 'sha256',
+ $localDisk->path($path)
+ );
+
+ if (
+ $localSha &&
+ ! hash_equals($expectedSha, $localSha)
+ ) {
return false;
}
}