diff --git a/app/Console/Commands/Admin/EmojiMoveStorageLocalToCloud.php b/app/Console/Commands/Admin/EmojiMoveStorageLocalToCloud.php new file mode 100644 index 000000000..b22159ac7 --- /dev/null +++ b/app/Console/Commands/Admin/EmojiMoveStorageLocalToCloud.php @@ -0,0 +1,177 @@ +error('Cloud storage is not enabled (pixelfed.cloud_storage is false).'); + + return self::FAILURE; + } + + try { + $localDisk = Storage::disk('local'); + $cloudDisk = Storage::disk(config('filesystems.cloud')); + } catch (\Throwable $e) { + $this->error('Cloud disk ('.config('filesystems.cloud').') could not be resolved: '.$e->getMessage()); + + return self::FAILURE; + } + + if (! $this->cloudHost()) { + $this->error('Cloud disk ('.config('filesystems.cloud').') is not configured (no resolvable URL).'); + $this->line('Set AWS_URL / AWS_* in your .env before migrating to cloud.'); + + return self::FAILURE; + } + + if (! $this->option('dry-run') && ! $this->option('force')) { + if (! $this->confirm('Begin migrating local custom emoji to cloud?', true)) { + $this->comment('Aborted.'); + + return self::SUCCESS; + } + } + + $limit = (int) $this->option('limit'); + $moved = 0; + $skipped = 0; + $failed = 0; + + // Local (non-federated) emoji have no uri. Remote emoji already point + // at their origin server and are not stored on our disks. + $query = CustomEmoji::whereNull('uri') + ->whereNotNull('media_path') + ->orderByDesc('id') + ->limit($limit); + + $bar = $this->output->createProgressBar($query->count()); + $bar->start(); + + foreach ($query->get() as $emoji) { + $result = $this->migrateOne($emoji, $localDisk, $cloudDisk); + match ($result) { + 'moved' => $moved++, + 'skipped' => $skipped++, + default => $failed++, + }; + $bar->advance(); + } + + $bar->finish(); + $this->newLine(2); + + if ($moved > 0 && ! $this->option('dry-run')) { + Cache::forget('pf:custom_emoji'); + } + + $this->info(($this->option('dry-run') ? '[dry-run] ' : '').'Done. moved='.$moved.' skipped='.$skipped.' failed='.$failed.'.'); + if ($this->movedBytes) { + $this->info('Transferred '.PrettyNumber::size($this->movedBytes).' to cloud storage.'); + } + + return self::SUCCESS; + } + + /** + * @return string one of moved|skipped|failed + */ + protected function migrateOne(CustomEmoji $emoji, $localDisk, $cloudDisk): string + { + $mediaPath = $emoji->media_path; + + if (! $mediaPath || Str::startsWith($mediaPath, 'http')) { + return 'skipped'; + } + + // Local emoji live under the public/ disk prefix; cloud objects live at + // the bare media_path. + $localPath = 'public/'.$mediaPath; + + if (! $localDisk->exists($localPath)) { + // Already migrated (present on cloud, gone locally) or missing. + return 'skipped'; + } + + if ($this->option('dry-run')) { + return 'moved'; + } + + try { + $size = (int) $localDisk->size($localPath); + $cloudDisk->put($mediaPath, $localDisk->get($localPath), 'public'); + + if (! $this->verify($localPath, $mediaPath, $localDisk, $cloudDisk)) { + $this->warn(PHP_EOL.'Verify failed for emoji '.$emoji->id.' ('.$mediaPath.'); left local copy intact.'); + + return 'failed'; + } + + if (! $this->option('keep-local')) { + $localDisk->delete($localPath); + } + + $this->movedBytes += $size; + + Cache::forget('pf:custom_emoji:'.str_replace(':', '', (string) $emoji->shortcode)); + + return 'moved'; + } catch (\Throwable $e) { + $this->warn(PHP_EOL.'Error migrating emoji '.$emoji->id.': '.$e->getMessage()); + + return 'failed'; + } + } + + /** + * Verify the cloud copy matches the local source by size. Fails closed. + */ + protected function verify(string $localPath, string $cloudPath, $localDisk, $cloudDisk): bool + { + if (! $cloudDisk->exists($cloudPath)) { + return false; + } + + $localSize = $localDisk->size($localPath); + $cloudSize = $cloudDisk->size($cloudPath); + + if ($localSize === false || $cloudSize === false || $localSize !== $cloudSize) { + return false; + } + + return true; + } +} diff --git a/app/Console/Commands/Admin/ImportEmojis.php b/app/Console/Commands/Admin/ImportEmojis.php index 43500fbf2..d973e97b7 100644 --- a/app/Console/Commands/Admin/ImportEmojis.php +++ b/app/Console/Commands/Admin/ImportEmojis.php @@ -5,7 +5,6 @@ namespace App\Console\Commands\Admin; use App\Models\CustomEmoji; use Illuminate\Console\Command; use Illuminate\Support\Facades\Cache; -use Illuminate\Support\Facades\Storage; class ImportEmojis extends Command { @@ -87,8 +86,8 @@ class ImportEmojis extends Command $emoji->save(); $fileName = $emoji->id.'.'.$extension; - Storage::putFileAs('public/emoji', $entry->getPathname(), $fileName); $emoji->media_path = 'emoji/'.$fileName; + CustomEmoji::storeMediaFromFile($emoji->media_path, $entry->getPathname()); $emoji->save(); $imported++; Cache::forget('pf:custom_emoji'); diff --git a/app/Http/Controllers/AdminController.php b/app/Http/Controllers/AdminController.php index dee5bed73..94e6f71fd 100644 --- a/app/Http/Controllers/AdminController.php +++ b/app/Http/Controllers/AdminController.php @@ -35,7 +35,6 @@ use Illuminate\Http\Request; use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Mail; -use Illuminate\Support\Facades\Storage; use Illuminate\Support\Str; use Illuminate\Validation\Rule; @@ -652,8 +651,8 @@ class AdminController extends Controller $emoji->save(); $fileName = $emoji->id.'.'.$request->emoji->extension(); - $request->emoji->storePubliclyAs('public/emoji', $fileName); $emoji->media_path = 'emoji/'.$fileName; + CustomEmoji::storeMediaFromFile($emoji->media_path, $request->emoji->getPathname()); $emoji->save(); Cache::forget('pf:custom_emoji'); @@ -664,7 +663,7 @@ class AdminController extends Controller { abort_unless((bool) config_cache('federation.custom_emoji.enabled'), 404); $emoji = CustomEmoji::findOrFail($id); - Storage::delete("public/{$emoji->media_path}"); + CustomEmoji::deleteMedia($emoji->media_path); Cache::forget('pf:custom_emoji'); $emoji->delete(); diff --git a/app/Models/CustomEmoji.php b/app/Models/CustomEmoji.php index 12b62669d..7ea1d9f33 100644 --- a/app/Models/CustomEmoji.php +++ b/app/Models/CustomEmoji.php @@ -2,9 +2,11 @@ namespace App\Models; +use Illuminate\Contracts\Filesystem\Filesystem; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Model; use Illuminate\Support\Facades\Cache; +use Illuminate\Support\Facades\Storage; use Illuminate\Support\Str; class CustomEmoji extends Model @@ -17,6 +19,93 @@ class CustomEmoji extends Model protected $guarded = []; + /** + * Public URL for this emoji's media. + * + * When cloud storage is enabled the object is served from the cloud disk + * (emoji are stored at the same relative media_path on both disks), and + * falls back to the local /storage URL otherwise. + */ + public static function urlForPath(?string $mediaPath): ?string + { + if (! $mediaPath) { + return null; + } + + if ((bool) config_cache('pixelfed.cloud_storage')) { + return Storage::disk(config('filesystems.cloud'))->url($mediaPath); + } + + return url('/storage/'.$mediaPath); + } + + public function url(): ?string + { + return self::urlForPath($this->media_path); + } + + /** + * The disk emoji media is stored on, and the storage path prefix for it. + * + * On cloud storage, objects live at the bare media_path (emoji/{id}.ext). + * On local storage they live under the public/ disk prefix so they are + * served through the /storage symlink. + * + * @return array{disk: Filesystem, prefix: string} + */ + public static function storageTarget(): array + { + if ((bool) config_cache('pixelfed.cloud_storage')) { + return [ + 'disk' => Storage::disk(config('filesystems.cloud')), + 'prefix' => '', + ]; + } + + return [ + 'disk' => Storage::disk('local'), + 'prefix' => 'public/', + ]; + } + + /** + * Store emoji bytes for the given media_path on the active disk. + */ + public static function storeMedia(string $mediaPath, string $contents): void + { + $target = self::storageTarget(); + $target['disk']->put($target['prefix'].$mediaPath, $contents, 'public'); + } + + /** + * Store an emoji from a local source file for the given media_path on the + * active disk (used by uploads/imports that already have a file on disk). + */ + public static function storeMediaFromFile(string $mediaPath, string $sourcePath): void + { + $target = self::storageTarget(); + $target['disk']->put( + $target['prefix'].$mediaPath, + file_get_contents($sourcePath), + 'public' + ); + } + + /** + * Delete emoji media for the given media_path from the active disk. + */ + public static function deleteMedia(?string $mediaPath): void + { + if (! $mediaPath) { + return; + } + + $target = self::storageTarget(); + if ($target['disk']->exists($target['prefix'].$mediaPath)) { + $target['disk']->delete($target['prefix'].$mediaPath); + } + } + public static function scan($text, $activitypub = false) { if ((bool) config_cache('federation.custom_emoji.enabled') == false) { @@ -43,7 +132,7 @@ class CustomEmoji extends Model }); if ($tag) { - $url = url('/storage/'.$tag['media_path']); + $url = self::urlForPath($tag['media_path']); if ($activitypub == true) { $mediaType = Str::endsWith($url, '.png') ? 'image/png' : 'image/jpg'; diff --git a/app/Services/CustomEmojiService.php b/app/Services/CustomEmojiService.php index 438965def..a803acf80 100644 --- a/app/Services/CustomEmojiService.php +++ b/app/Services/CustomEmojiService.php @@ -7,7 +7,6 @@ use App\Util\ActivityPub\Helpers; use Illuminate\Http\Client\RequestException; use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Http; -use Illuminate\Support\Facades\Storage; class CustomEmojiService { @@ -118,9 +117,7 @@ class CustomEmojiService ]); if ($emoji->wasRecentlyCreated == false) { - if (Storage::exists('public/'.$emoji->media_path)) { - Storage::delete('public/'.$emoji->media_path); - } + CustomEmoji::deleteMedia($emoji->media_path); } $ext = '.'.last(explode('/', $json['icon']['mediaType'])); @@ -136,7 +133,7 @@ class CustomEmojiService return; } - Storage::put('public/'.$mediaPath, $body); + CustomEmoji::storeMedia($mediaPath, $body); $emoji->media_path = $mediaPath; $emoji->save(); @@ -191,7 +188,7 @@ class CustomEmojiService ->whereNull('uri') ->get() ->map(function ($emojo) { - $url = url('storage/'.$emojo->media_path); + $url = CustomEmoji::urlForPath($emojo->media_path); return [ 'shortcode' => str_replace(':', '', $emojo->shortcode), diff --git a/bootstrap/app.php b/bootstrap/app.php index 743a5fa02..cfc71ba4c 100644 --- a/bootstrap/app.php +++ b/bootstrap/app.php @@ -146,8 +146,8 @@ return Application::configure(basePath: dirname(__DIR__)) if ((bool) config_cache('pixelfed.cloud_storage') && (bool) config_cache('media.delete_local_after_cloud')) { // Upload any local stragglers to cloud and GC verified local copies. $schedule->command('admin:MediaMoveStorageLocalToCloud --force --limit=500')->hourlyAt(15); - // Same for local story media and story_archives. - $schedule->command('admin:StoryMoveStorageLocalToCloud --force --limit=500')->hourlyAt(25); + // Same for local custom emoji. + $schedule->command('admin:EmojiMoveStorageLocalToCloud --force --limit=1000')->dailyAt('04:35'); } if (config('import.instagram.enabled')) {