diff --git a/app/Console/Commands/MediaCloudUrlRewrite.php b/app/Console/Commands/MediaCloudUrlRewrite.php deleted file mode 100644 index 337fee178..000000000 --- a/app/Console/Commands/MediaCloudUrlRewrite.php +++ /dev/null @@ -1,209 +0,0 @@ - 'The old S3 domain', - 'newDomain' => 'The new S3 domain', - ]; - } - - /** - * The console command description. - * - * @var string - */ - protected $description = 'Rewrite S3 media urls from local users'; - - /** - * Execute the console command. - */ - public function handle() - { - $this->preflightCheck(); - $this->bootMessage(); - $this->confirmCloudUrl(); - $this->handleTask(); - } - - protected function preflightCheck() - { - if (! (bool) config_cache('pixelfed.cloud_storage')) { - $this->info('Error: Cloud storage is not enabled! Please enable before proceeding.'); - $this->error('Aborting...'); - exit; - } - } - - protected function bootMessage() - { - $this->info(' ____ _ ______ __ '); - $this->info(' / __ \(_) _____ / / __/__ ____/ / '); - $this->info(' / /_/ / / |/_/ _ \/ / /_/ _ \/ __ / '); - $this->info(' / ____/ /> info(' /_/ /_/_/|_|\___/_/_/ \___/\__,_/ '); - $this->info(' '); - $this->info(' Media Cloud Url Rewrite Tool'); - $this->info(' ==='); - $this->info(' Old S3: '.trim($this->argument('oldDomain'))); - $this->info(' New S3: '.trim($this->argument('newDomain'))); - $this->info(' '); - } - - protected function confirmCloudUrl() - { - $disk = Storage::disk(config('filesystems.cloud'))->url('test'); - $domain = parse_url($disk, PHP_URL_HOST); - if (trim($this->argument('newDomain')) !== $domain) { - $this->error('Error: The new S3 domain you entered is not currently configured'); - exit; - } - - if (! $this->confirm('Confirm this is correct')) { - $this->error('Aborting...'); - exit; - } - } - - protected function handleTask() - { - $task = select( - label: 'What action would you like to perform?', - options: ['Migrate All', 'Migrate Media', 'Migrate Avatars'] - ); - - switch ($task) { - case 'Migrate All': - $this->updateMediaUrls(); - $this->updateAvatarUrls(); - break; - case 'Migrate Media': - $this->updateMediaUrls(); - break; - case 'Migrate Avatars': - $this->updateAvatarUrls(); - break; - default: - $this->error('Invalid selection'); - - return; - } - } - - protected function updateMediaUrls() - { - $this->info('Updating media urls...'); - $oldDomain = trim($this->argument('oldDomain')); - $newDomain = trim($this->argument('newDomain')); - $disk = Storage::disk(config('filesystems.cloud')); - $count = Media::whereNotNull('cdn_url')->count(); - $bar = $this->output->createProgressBar($count); - $counter = 0; - $bar->start(); - foreach (Media::whereNotNull('cdn_url')->lazyById(1000, 'id') as $media) { - if (strncmp($media->media_path, 'http', 4) === 0) { - $bar->advance(); - - continue; - } - $cdnHost = parse_url($media->cdn_url, PHP_URL_HOST); - if ($oldDomain != $cdnHost || $newDomain == $cdnHost) { - $bar->advance(); - - continue; - } - - $media->cdn_url = str_replace($oldDomain, $newDomain, $media->cdn_url); - - if ($media->thumbnail_url != null) { - $thumbHost = parse_url($media->thumbnail_url, PHP_URL_HOST); - if ($thumbHost == $oldDomain) { - $thumbUrl = $disk->url($media->thumbnail_path); - $media->thumbnail_url = $thumbUrl; - } - } - - if ($media->optimized_url != null) { - $optiHost = parse_url($media->optimized_url, PHP_URL_HOST); - if ($optiHost == $oldDomain) { - $optiUrl = str_replace($oldDomain, $newDomain, $media->optimized_url); - $media->optimized_url = $optiUrl; - } - } - - $media->save(); - $counter++; - $bar->advance(); - } - - $bar->finish(); - - $this->line(' '); - $this->info('Finished! Updated '.$counter.' total records!'); - $this->line(' '); - $this->info('Tip: Run `php artisan cache:clear` to purge cached urls'); - } - - protected function updateAvatarUrls() - { - $this->info('Updating avatar urls...'); - $oldDomain = trim($this->argument('oldDomain')); - $newDomain = trim($this->argument('newDomain')); - $disk = Storage::disk(config('filesystems.cloud')); - $count = Avatar::count(); - $bar = $this->output->createProgressBar($count); - $counter = 0; - $bar->start(); - foreach (Avatar::lazyById(1000, 'id') as $avatar) { - if (! $avatar->cdn_url) { - $bar->advance(); - - continue; - } - - $cdnHost = parse_url($avatar->cdn_url, PHP_URL_HOST); - if (strcasecmp($oldDomain, $cdnHost) !== 0 || strcasecmp($newDomain, $cdnHost) === 0) { - $bar->advance(); - - continue; - } - - $avatar->cdn_url = str_replace($oldDomain, $newDomain, $avatar->cdn_url); - - $avatar->save(); - $counter++; - $bar->advance(); - } - - $bar->finish(); - - $this->line(' '); - $this->info('Finished! Updated '.$counter.' total records!'); - $this->line(' '); - $this->info('Tip: Run `php artisan cache:clear` to purge cached urls'); - } -} diff --git a/app/Console/Commands/MigrateLocalMediaURL.php b/app/Console/Commands/MigrateLocalMediaURL.php new file mode 100644 index 000000000..ae675f5f6 --- /dev/null +++ b/app/Console/Commands/MigrateLocalMediaURL.php @@ -0,0 +1,450 @@ +error('Cloud storage is not enabled (PF_ENABLE_CLOUD is false).'); + $this->line('This instance serves media from local storage; there are no cloud media URLs to migrate.'); + + return 1; + } + + // Safe default target = the currently configured cloud disk host, + // driven by AWS_URL in .env. Allow explicit override via --newDomain. + $configuredHost = $this->cloudHost(); + $override = $this->normalizeHost($this->option('newDomain')); + $this->newHost = $override ?: $configuredHost; + + if (! $this->newHost) { + $this->error('Could not resolve a target host.'); + $this->line('The cloud disk ('.config('filesystems.cloud').') did not return a usable URL.'); + $this->line('Set AWS_URL in your .env, or pass --newDomain explicitly.'); + + return 1; + } + + // Defensive guard: when auto-detecting the target (no --newDomain + // override), never rewrite media URLs to the app's own domain. That + // would indicate local storage or a misconfigured cloud disk URL + // (AWS_URL). An explicit --newDomain is treated as a deliberate choice. + if (! $override) { + $appHost = parse_url(config('app.url'), PHP_URL_HOST); + if ($appHost && strcasecmp($this->newHost, $appHost) === 0) { + $this->error('Refusing to run: auto-detected target host ('.$this->newHost.') is the app domain.'); + $this->line('That indicates local storage or a misconfigured cloud disk URL (AWS_URL).'); + $this->line('If you really intend this, pass an explicit --newDomain.'); + + return 1; + } + } + + $this->oldHost = $this->normalizeHost($this->option('oldDomain')); + + $id = $this->argument('id'); + $all = $this->option('all'); + $avatarsOnly = $this->option('avatars') && ! $all && ! $id; + + if (! $id && ! $all && ! $avatarsOnly) { + $this->error('Provide a status id/URL, or pass --all (optionally --avatars).'); + + return 1; + } + + if ($id && $all) { + $this->error('Pass either a status id or --all, not both.'); + + return 1; + } + + // Show the plan and require explicit approval of the target host. + $this->info('Target host (newDomain): '.$this->newHost.($override ? ' (override)' : ' (from configured cloud disk)')); + $this->info('Filter (oldDomain): '.($this->oldHost ?: 'none — rewriting all stale hosts')); + if ($this->newHost !== $configuredHost) { + $this->warn('Note: target host differs from the configured cloud disk host ('.($configuredHost ?: 'unresolved').').'); + } + + if (! $this->option('dry-run') && ! $this->option('force')) { + if (! $this->confirm('Rewrite media URLs to "'.$this->newHost.'"?', false)) { + $this->comment('Aborted.'); + + return 0; + } + } + $this->newLine(); + + if ($id) { + return $this->handleSingle($id); + } + + if ($avatarsOnly) { + $this->migrateAvatars(); + + return 0; + } + + return $this->handleAll(); + } + + /** + * Extract a bare host from a domain/URL option value. + */ + protected function normalizeHost(?string $value): ?string + { + $value = trim((string) $value); + if ($value === '') { + return null; + } + // Accept full URLs or bare hosts. + if (str_contains($value, '://')) { + $host = parse_url($value, PHP_URL_HOST); + + return $host ?: null; + } + + // Strip any accidental path/scheme fragments. + $host = parse_url('https://'.$value, PHP_URL_HOST); + + return $host ?: null; + } + + protected function handleSingle(string $id): int + { + $statusId = $this->resolveStatusId($id); + if (! $statusId) { + $this->error('Could not extract a status id from "'.$id.'".'); + + return 1; + } + + $status = Status::withTrashed()->find($statusId); + if (! $status) { + $this->error('No status found with id '.$statusId.'.'); + + return 1; + } + + $media = Media::whereStatusId($status->id)->get(); + if ($media->isEmpty()) { + $this->comment('Status '.$status->id.' has no media.'); + + return 0; + } + + $fixed = 0; + foreach ($media as $m) { + if ($this->migrateOne($m)) { + $fixed++; + } + } + + if ($fixed > 0 && ! $this->option('dry-run')) { + $this->bustCaches($status->id); + } + + $this->newLine(); + $this->info(($this->option('dry-run') ? 'Would fix ' : 'Fixed ').$fixed.' media row(s) for status '.$status->id.'.'); + if ($fixed > 0 && ! $this->option('dry-run')) { + $this->comment('Caches busted for this status.'); + } + + return 0; + } + + protected function handleAll(): int + { + if (! $this->option('dry-run') && ! $this->option('force')) { + if (! $this->confirm('Rebuild stale URLs for all local media rows?', true)) { + $this->comment('Aborted.'); + + return 0; + } + } + + $fixed = 0; + $scanned = 0; + $affectedStatusIds = []; + + Media::whereNull('remote_url') + ->where(function ($q) { + $q->whereNull('remote_media')->orWhere('remote_media', false); + }) + ->lazyById(1000, 'id') + ->each(function ($m) use (&$fixed, &$scanned, &$affectedStatusIds) { + $scanned++; + if ($this->migrateOne($m)) { + $fixed++; + if ($m->status_id) { + $affectedStatusIds[$m->status_id] = true; + } + } + }); + + if (! $this->option('dry-run')) { + foreach (array_keys($affectedStatusIds) as $sid) { + $this->bustCaches($sid); + } + } + + $this->newLine(); + $this->info('Scanned '.$scanned.' local media rows; '.($this->option('dry-run') ? 'would fix ' : 'fixed ').$fixed.'.'); + if ($fixed > 0 && ! $this->option('dry-run')) { + $this->comment('Caches busted for '.count($affectedStatusIds).' affected status(es).'); + } + + // --all always includes avatars (parity with the old command's + // "Migrate All"); --avatars can also be passed explicitly. + $this->migrateAvatars(); + + if (! $this->option('dry-run')) { + $this->comment('Tip: run `php artisan cache:clear` if any stale URLs remain cached elsewhere.'); + } + + return 0; + } + + /** + * Rebuild stale avatar cdn_urls from their media_path. + */ + protected function migrateAvatars(): void + { + $this->newLine(); + $this->info('Checking avatars...'); + + $fixed = 0; + $scanned = 0; + + Avatar::whereNotNull('cdn_url')->lazyById(1000, 'id')->each(function ($avatar) use (&$fixed, &$scanned) { + $scanned++; + + if (! $avatar->cdn_url || ! $avatar->media_path) { + return; + } + if (Str::startsWith((string) $avatar->media_path, 'http')) { + return; + } + $host = parse_url($avatar->cdn_url, PHP_URL_HOST); + if (! $this->shouldRewrite($host)) { + return; + } + + $rebuilt = $this->targetUrl($avatar->media_path); + if (! $rebuilt) { + return; + } + + $this->line(' avatar '.$avatar->id.' (profile '.$avatar->profile_id.'): '.$host.' -> '.$this->newHost); + + if (! $this->option('dry-run')) { + $avatar->cdn_url = $rebuilt; + $avatar->save(); + AccountService::del($avatar->profile_id); + } + $fixed++; + }); + + $this->info('Scanned '.$scanned.' avatars; '.($this->option('dry-run') ? 'would fix ' : 'fixed ').$fixed.'.'); + } + + /** + * Rebuild any stale URL field on a single media row from its storage path. + * Only writes when a field's host differs from the cloud host. + * + * @return bool whether the row was (or would be) changed + */ + protected function migrateOne(Media $media): bool + { + // Never touch remote media or rows whose media_path is an absolute URL. + if ($media->remote_media || Str::startsWith((string) $media->media_path, 'http')) { + return false; + } + + $changes = []; + + // cdn_url and optimized_url are both derived from media_path; + // thumbnail_url is derived from thumbnail_path. + $map = [ + 'cdn_url' => $media->media_path, + 'optimized_url' => $media->media_path, + 'thumbnail_url' => $media->thumbnail_path, + ]; + + foreach ($map as $field => $path) { + $current = $media->{$field}; + if (! $current) { + // Field not set; leave it as-is (nothing to migrate). + continue; + } + if (! $path) { + // No source path to rebuild from; skip. + continue; + } + $host = parse_url($current, PHP_URL_HOST); + if (! $this->shouldRewrite($host)) { + continue; + } + + $rebuilt = $this->targetUrl($path); + if (! $rebuilt) { + continue; + } + + $changes[$field] = ['from' => $current, 'to' => $rebuilt]; + } + + if (empty($changes)) { + return false; + } + + $this->warn('media '.$media->id.(($media->status_id) ? ' (status '.$media->status_id.')' : '').':'); + foreach ($changes as $field => $c) { + $fromHost = parse_url($c['from'], PHP_URL_HOST); + $this->line(' '.$field.': '.$fromHost.' -> '.$this->newHost); + } + + if ($this->option('dry-run')) { + return true; + } + + foreach ($changes as $field => $c) { + $media->{$field} = $c['to']; + } + $media->save(); + + return true; + } + + protected function bustCaches($statusId): void + { + MediaService::del($statusId); + StatusService::del($statusId, true); + } + + /** + * Decide whether a URL on $currentHost should be rewritten. + * Skips when already on the target host, and honours the optional + * --oldDomain filter. + */ + protected function shouldRewrite(?string $currentHost): bool + { + if (! $currentHost) { + return false; + } + // Already on the target host. + if (strcasecmp($currentHost, $this->newHost) === 0) { + return false; + } + // With --oldDomain, only rewrite that specific host. + if ($this->oldHost !== null && strcasecmp($currentHost, $this->oldHost) !== 0) { + return false; + } + + return true; + } + + /** + * Build the target URL for a storage path against the target host. + * Uses the configured cloud disk to produce the correct path, then + * swaps in --newDomain when it overrides the configured host. + */ + protected function targetUrl(string $path): ?string + { + $url = $this->diskUrl($path); + if (! $url) { + return null; + } + + $diskHost = parse_url($url, PHP_URL_HOST); + if ($diskHost && strcasecmp($diskHost, $this->newHost) !== 0) { + // Override host was requested; swap it into the disk-built URL. + $url = preg_replace('#^(https?://)'.preg_quote($diskHost, '#').'#i', '$1'.$this->newHost, $url); + } + + return $url; + } + + protected function diskUrl(string $path): ?string + { + try { + return (string) Storage::disk(config('filesystems.cloud'))->url($path); + } catch (\Throwable $e) { + return null; + } + } + + protected function cloudHost(): ?string + { + try { + $url = Storage::disk(config('filesystems.cloud'))->url('probe'); + $host = parse_url($url, PHP_URL_HOST); + + return $host ?: null; + } catch (\Throwable $e) { + return null; + } + } + + protected function resolveStatusId(string $input): ?string + { + $input = trim($input); + if (ctype_digit($input)) { + return $input; + } + if (preg_match('#/p/[^/]+/(\d+)#', $input, $m)) { + return $m[1]; + } + if (preg_match('#(\d{6,})#', $input, $m)) { + return $m[1]; + } + + return null; + } +} diff --git a/tests/Feature/Account/MigrateLocalMediaUrlTest.php b/tests/Feature/Account/MigrateLocalMediaUrlTest.php new file mode 100644 index 000000000..27027b085 --- /dev/null +++ b/tests/Feature/Account/MigrateLocalMediaUrlTest.php @@ -0,0 +1,186 @@ +url($path) == https://cdn.test/. + Config::set('filesystems.cloud', 's3'); + Config::set('filesystems.disks.s3', [ + 'driver' => 's3', + 'key' => 'test', + 'secret' => 'test', + 'region' => 'us-east-1', + 'bucket' => 'bucket', + 'url' => 'https://cdn.test', + 'endpoint' => 'https://cdn.test', + 'use_path_style_endpoint' => true, + 'visibility' => 'public', + ]); + // Enable cloud storage (config_cache reads from ConfigCacheService). + ConfigCacheService::put('pixelfed.cloud_storage', true); +}); + +function makeStatusWithStaleMedia(string $staleHost = 'https://s3.old.example'): Media +{ + $user = User::factory()->create(); + $user->refresh(); + $pid = $user->profile->id; + + $status = Status::factory()->create(['profile_id' => $pid, 'type' => 'video']); + + $path = 'public/m/_v2/'.$pid.'/aa/bb/file.mp4'; + $thumbPath = 'public/m/_v2/'.$pid.'/aa/bb/file_thumb.jpeg'; + + return Media::create([ + 'status_id' => $status->id, + 'profile_id' => $pid, + 'user_id' => $user->id, + 'media_path' => $path, + 'thumbnail_path' => $thumbPath, + 'cdn_url' => 'https://cdn.test/'.$path, // already correct + 'thumbnail_url' => $staleHost.'/'.$thumbPath, // stale + 'optimized_url' => $staleHost.'/'.$path, // stale + 'mime' => 'video/mp4', + 'remote_media' => false, + 'order' => 0, + ]); +} + +it('rebuilds stale thumbnail_url and optimized_url but leaves correct cdn_url', function () { + $media = makeStatusWithStaleMedia(); + + $this->artisan('admin:MigrateLocalMediaURL', ['id' => (string) $media->status_id, '--force' => true]) + ->assertExitCode(0); + + $media->refresh(); + expect($media->cdn_url)->toBe('https://cdn.test/'.$media->media_path); + expect($media->thumbnail_url)->toBe('https://cdn.test/'.$media->thumbnail_path); + expect($media->optimized_url)->toBe('https://cdn.test/'.$media->media_path); + expect(parse_url($media->thumbnail_url, PHP_URL_HOST))->toBe('cdn.test'); + expect(parse_url($media->optimized_url, PHP_URL_HOST))->toBe('cdn.test'); +}); + +it('does not change anything in dry-run mode', function () { + $media = makeStatusWithStaleMedia(); + $originalThumb = $media->thumbnail_url; + + $this->artisan('admin:MigrateLocalMediaURL', ['id' => (string) $media->status_id, '--dry-run' => true]) + ->assertExitCode(0); + + expect($media->fresh()->thumbnail_url)->toBe($originalThumb); +}); + +it('leaves already-correct media untouched', function () { + $user = User::factory()->create(); + $user->refresh(); + $pid = $user->profile->id; + $status = Status::factory()->create(['profile_id' => $pid, 'type' => 'photo']); + $path = 'public/m/_v2/'.$pid.'/aa/bb/ok.jpg'; + + $media = Media::create([ + 'status_id' => $status->id, + 'profile_id' => $pid, + 'media_path' => $path, + 'cdn_url' => 'https://cdn.test/'.$path, + 'thumbnail_url' => 'https://cdn.test/'.$path, + 'mime' => 'image/jpeg', + 'remote_media' => false, + 'order' => 0, + ]); + + $updatedAt = $media->fresh()->updated_at; + + $this->artisan('admin:MigrateLocalMediaURL', ['id' => (string) $status->id, '--force' => true]) + ->assertExitCode(0); + + expect($media->fresh()->updated_at->eq($updatedAt))->toBeTrue(); +}); + +it('never rewrites remote media', function () { + $user = User::factory()->create(); + $user->refresh(); + $pid = $user->profile->id; + $status = Status::factory()->create(['profile_id' => $pid, 'type' => 'photo']); + + $media = Media::create([ + 'status_id' => $status->id, + 'profile_id' => $pid, + 'media_path' => 'https://remote.example/image.jpg', + 'cdn_url' => 'https://s3.old.example/image.jpg', + 'remote_media' => true, + 'remote_url' => 'https://remote.example/image.jpg', + 'mime' => 'image/jpeg', + 'order' => 0, + ]); + + $this->artisan('admin:MigrateLocalMediaURL', ['id' => (string) $status->id, '--force' => true]) + ->assertExitCode(0); + + // Unchanged: remote media is skipped. + expect($media->fresh()->cdn_url)->toBe('https://s3.old.example/image.jpg'); +}); + +it('requires an id or --all', function () { + $this->artisan('admin:MigrateLocalMediaURL') + ->assertExitCode(1); +}); + +it('refuses to run on a local-storage instance', function () { + // Simulate local storage: cloud disabled. + ConfigCacheService::put('pixelfed.cloud_storage', false); + + $this->artisan('admin:MigrateLocalMediaURL', ['--all' => true, '--force' => true]) + ->expectsOutputToContain('Cloud storage is not enabled') + ->assertExitCode(1); +}); + +it('with --oldDomain only rewrites URLs on that host', function () { + // thumbnail_url on s3.old.example, optimized_url on other.example. + $media = makeStatusWithStaleMedia('https://s3.old.example'); + $media->optimized_url = 'https://other.example/'.$media->media_path; + $media->save(); + + $this->artisan('admin:MigrateLocalMediaURL', [ + 'id' => (string) $media->status_id, + '--oldDomain' => 's3.old.example', + '--force' => true, + ])->assertExitCode(0); + + $media->refresh(); + // Matched the filter -> rewritten. + expect(parse_url($media->thumbnail_url, PHP_URL_HOST))->toBe('cdn.test'); + // Did NOT match the filter -> left as-is. + expect(parse_url($media->optimized_url, PHP_URL_HOST))->toBe('other.example'); +}); + +it('with --newDomain override rewrites to the given host', function () { + $media = makeStatusWithStaleMedia('https://s3.old.example'); + + $this->artisan('admin:MigrateLocalMediaURL', [ + 'id' => (string) $media->status_id, + '--newDomain' => 'media.example', + '--force' => true, + ])->assertExitCode(0); + + $media->refresh(); + expect(parse_url($media->thumbnail_url, PHP_URL_HOST))->toBe('media.example'); + expect(parse_url($media->optimized_url, PHP_URL_HOST))->toBe('media.example'); +});