diff --git a/app/Console/Commands/Admin/StoryMoveStorageLocalToCloud.php b/app/Console/Commands/Admin/StoryMoveStorageLocalToCloud.php new file mode 100644 index 000000000..f7ad87085 --- /dev/null +++ b/app/Console/Commands/Admin/StoryMoveStorageLocalToCloud.php @@ -0,0 +1,249 @@ +error('Cloud storage is not enabled (pixelfed.cloud_storage is false).'); + + return self::FAILURE; + } + + try { + $localDisk = Storage::disk('local'); + $cloudDisk = Storage::disk(config('filesystems.cloud')); + } catch (\Throwable $e) { + $this->error('Cloud disk ('.config('filesystems.cloud').') could not be resolved: '.$e->getMessage()); + + return self::FAILURE; + } + + if (! $this->cloudHost()) { + $this->error('Cloud disk ('.config('filesystems.cloud').') is not configured (no resolvable URL).'); + $this->line('Set AWS_URL / AWS_* in your .env before migrating to cloud.'); + + return self::FAILURE; + } + + if (! $this->option('dry-run') && ! $this->option('force')) { + if (! $this->confirm('Begin migrating local story media to cloud?', true)) { + $this->comment('Aborted.'); + + return self::SUCCESS; + } + } + + $limit = (int) $this->option('limit'); + $moved = 0; + $skipped = 0; + $failed = 0; + + // Local stories whose media still lives on the local disk. Covers both + // active stories and archived ones (story_archives/*). Remote stories + // are excluded; their media is deleted on expiry, not archived. + $query = Story::whereLocal(true) + ->whereNotNull('path') + ->orderByDesc('id') + ->limit($limit); + + $bar = $this->output->createProgressBar($query->count()); + $bar->start(); + + foreach ($query->get() as $story) { + $result = $this->migrateOne($story, $localDisk, $cloudDisk); + match ($result) { + 'moved' => $moved++, + 'skipped' => $skipped++, + default => $failed++, + }; + $bar->advance(); + } + + $bar->finish(); + $this->newLine(2); + $this->info(($this->option('dry-run') ? '[dry-run] ' : '').'Tracked stories: moved='.$moved.' skipped='.$skipped.' failed='.$failed.'.'); + + if ($this->option('orphans')) { + $this->migrateOrphans($localDisk, $cloudDisk, $limit); + } + + if ($this->movedBytes) { + $this->info('Transferred '.PrettyNumber::size($this->movedBytes).' to cloud storage.'); + } + + return self::SUCCESS; + } + + /** + * Migrate files under story_archives/ that are not referenced by any story + * row (e.g. left behind after a story was deleted, or legacy files). These + * are relocated to cloud with the same copy -> verify -> delete flow, so no + * media is discarded, only moved. + */ + protected function migrateOrphans($localDisk, $cloudDisk, int $limit): void + { + if (! $localDisk->exists('story_archives')) { + $this->info('No local story_archives directory; nothing to scan for orphans.'); + + return; + } + + $moved = 0; + $skipped = 0; + $failed = 0; + + foreach ($localDisk->allFiles('story_archives') as $path) { + if ($moved + $failed >= $limit) { + break; + } + + // Referenced by a story row? Then it was handled above, not an orphan. + if (Story::wherePath($path)->exists()) { + $skipped++; + + continue; + } + + if ($this->option('dry-run')) { + $this->line('[dry-run] would migrate orphan: '.$path); + $moved++; + + continue; + } + + try { + $size = (int) $localDisk->size($path); + $this->copyToCloud($path, $localDisk); + + if (! $this->verify($path, $localDisk, $cloudDisk)) { + $this->warn(PHP_EOL.'Verify failed for orphan ('.$path.'); left local copy intact.'); + $failed++; + + continue; + } + + if (! $this->option('keep-local')) { + $localDisk->delete($path); + } + + $this->movedBytes += $size; + $moved++; + } catch (\Throwable $e) { + $this->warn(PHP_EOL.'Error migrating orphan '.$path.': '.$e->getMessage()); + $failed++; + } + } + + $this->info(($this->option('dry-run') ? '[dry-run] ' : '').'Orphan files: moved='.$moved.' skipped='.$skipped.' failed='.$failed.'.'); + } + + /** + * @return string one of moved|skipped|failed + */ + protected function migrateOne(Story $story, $localDisk, $cloudDisk): string + { + $path = $story->path; + + if (! $path || Str::startsWith($path, 'http')) { + return 'skipped'; + } + + // Nothing to do if the local file is gone (already on cloud only). + if (! $localDisk->exists($path)) { + return 'skipped'; + } + + if ($this->option('dry-run')) { + return 'moved'; + } + + try { + $this->copyToCloud($path, $localDisk); + + if (! $this->verify($path, $localDisk, $cloudDisk)) { + $this->warn(PHP_EOL.'Verify failed for story '.$story->id.' ('.$path.'); left local copy intact.'); + + return 'failed'; + } + + if (! $this->option('keep-local')) { + $localDisk->delete($path); + } + + $this->movedBytes += (int) $story->size; + + StoryService::delById($story->id); + StoryService::delLatest($story->profile_id); + + return 'moved'; + } catch (\Throwable $e) { + $this->warn(PHP_EOL.'Error migrating story '.$story->id.': '.$e->getMessage()); + + return 'failed'; + } + } + + protected function copyToCloud(string $path, $localDisk): void + { + $p = explode('/', $path); + $name = array_pop($p); + $storagePath = implode('/', $p); + + // Reuse the resilient uploader (handles alt disks + retries). + ResilientMediaStorageService::store($storagePath, $localDisk->path($path), $name); + } + + /** + * Verify the cloud copy matches the local source by size. Fails closed. + */ + protected function verify(string $path, $localDisk, $cloudDisk): bool + { + if (! $cloudDisk->exists($path)) { + return false; + } + + $localSize = $localDisk->size($path); + $cloudSize = $cloudDisk->size($path); + + if ($localSize === false || $cloudSize === false || $localSize !== $cloudSize) { + return false; + } + + return true; + } +} diff --git a/app/Jobs/StoryPipeline/StoryExpire.php b/app/Jobs/StoryPipeline/StoryExpire.php index 6c477f047..6249c8e65 100644 --- a/app/Jobs/StoryPipeline/StoryExpire.php +++ b/app/Jobs/StoryPipeline/StoryExpire.php @@ -14,6 +14,7 @@ use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; +use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Storage; class StoryExpire implements ShouldQueue @@ -85,17 +86,38 @@ class StoryExpire implements ShouldQueue $path = array_pop($paths); $newPath = $base.$path; - if (Storage::exists($old) == true) { - $dir = implode('/', $paths); - Storage::move($old, $newPath); - $story->bearcap_token = null; - $story->path = $newPath; - $story->save(); - - $remainingFiles = Storage::files($dir); - if (empty($remainingFiles)) { - Storage::deleteDirectory($dir); - } + // Archive on the same disk the story media lives on. When the instance + // is configured for cloud storage this is S3, and $disk->move() is + // performed by the Flysystem S3 adapter (server-side copy + delete). + $disk = config('filesystems.default') === 'local' + ? Storage::disk('local') + : Storage::disk(config('filesystems.default')); + + if (! $disk->exists($old)) { + return; + } + + try { + $disk->move($old, $newPath); + } catch (\Throwable $e) { + Log::error('StoryExpire: failed to archive story media', [ + 'story_id' => $story->id, + 'from' => $old, + 'to' => $newPath, + 'error' => $e->getMessage(), + ]); + + return; + } + + $story->bearcap_token = null; + $story->path = $newPath; + $story->save(); + + $dir = implode('/', $paths); + $remainingFiles = $disk->files($dir); + if (empty($remainingFiles)) { + $disk->deleteDirectory($dir); } } @@ -130,8 +152,12 @@ class StoryExpire implements ShouldQueue $path = $story->path; - if (Storage::exists($path) == true) { - Storage::delete($path); + $disk = config('filesystems.default') === 'local' + ? Storage::disk('local') + : Storage::disk(config('filesystems.default')); + + if ($disk->exists($path)) { + $disk->delete($path); } $story->views()->delete(); diff --git a/bootstrap/app.php b/bootstrap/app.php index dfeca931c..743a5fa02 100644 --- a/bootstrap/app.php +++ b/bootstrap/app.php @@ -146,6 +146,8 @@ return Application::configure(basePath: dirname(__DIR__)) if ((bool) config_cache('pixelfed.cloud_storage') && (bool) config_cache('media.delete_local_after_cloud')) { // Upload any local stragglers to cloud and GC verified local copies. $schedule->command('admin:MediaMoveStorageLocalToCloud --force --limit=500')->hourlyAt(15); + // Same for local story media and story_archives. + $schedule->command('admin:StoryMoveStorageLocalToCloud --force --limit=500')->hourlyAt(25); } if (config('import.instagram.enabled')) {