From 982cafd7f668c9c4f9a2e1f4a37c4cdb75246e25 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 30 Aug 2026 14:08:26 +0930 Subject: [PATCH] feat: emoji cloud storage with async S3 migration command Re-land the emoji cloud-storage work on a clean staging base, using the async AWS SDK upload path. - CustomEmoji model: cloud-aware URL + storage helpers (urlForPath, url, storageTarget, storeMedia, storeMediaFromFile, deleteMedia) - Route emoji writes/deletes/URLs through the model in ImportEmojis, CustomEmojiService and AdminController; admin views use $emoji->url() - admin:EmojiMoveStorageLocalToCloud: disk-driven migration using the AWS SDK CommandPool with --concurrency (default 100) for high throughput; skips missing.png; --dry-run/--keep-local/--offset/--limit/--no-acl/--debug - Deploy migration + daily schedule under the cloud-storage conditional --- .../Admin/EmojiMoveStorageLocalToCloud.php | 318 ++++++++++++++++++ app/Console/Commands/Admin/ImportEmojis.php | 3 +- app/Http/Controllers/AdminController.php | 5 +- app/Models/CustomEmoji.php | 91 ++++- app/Services/CustomEmojiService.php | 9 +- bootstrap/app.php | 2 + ...30_050000_migrate_local_emoji_to_cloud.php | 38 +++ .../admin/custom-emoji/duplicates.blade.php | 4 +- .../views/admin/custom-emoji/home.blade.php | 2 +- 9 files changed, 457 insertions(+), 15 deletions(-) create mode 100644 app/Console/Commands/Admin/EmojiMoveStorageLocalToCloud.php create mode 100644 database/migrations/2026_08_30_050000_migrate_local_emoji_to_cloud.php diff --git a/app/Console/Commands/Admin/EmojiMoveStorageLocalToCloud.php b/app/Console/Commands/Admin/EmojiMoveStorageLocalToCloud.php new file mode 100644 index 000000000..462c8e294 --- /dev/null +++ b/app/Console/Commands/Admin/EmojiMoveStorageLocalToCloud.php @@ -0,0 +1,318 @@ +error('Cloud storage is not enabled (pixelfed.cloud_storage is false).'); + + return self::FAILURE; + } + + try { + $localDisk = Storage::disk('local'); + } catch (\Throwable $e) { + $this->error('Local disk could not be resolved: '.$e->getMessage()); + + return self::FAILURE; + } + + if (! $this->cloudHost()) { + $this->error('Cloud disk ('.config('filesystems.cloud').') is not configured (no resolvable URL).'); + $this->line('Set AWS_URL / AWS_* in your .env before migrating to cloud.'); + + return self::FAILURE; + } + + if ($this->option('debug')) { + $this->printDebug($localDisk); + } + + // Build the list of local emoji files to migrate. Disk-driven: the file + // existing locally is the source of truth for "needs moving". We do NOT + // filter by DB columns — federated emoji have their media stored locally + // too (with a uri set), so a DB filter would wrongly exclude them. + $files = $this->collectFiles($localDisk); + $total = count($files); + + if ($total === 0) { + $this->info('No emoji files to migrate.'); + + return self::SUCCESS; + } + + $concurrency = max(0, (int) $this->option('concurrency')); + $mode = $concurrency > 0 ? "async S3 (concurrency={$concurrency})" : 'synchronous'; + + if ($this->option('dry-run')) { + $this->info("[dry-run] Would upload {$total} emoji to cloud using {$mode}."); + + return self::SUCCESS; + } + + if (! $this->option('force') && ! $this->confirm("Upload {$total} emoji to cloud using {$mode}?", true)) { + $this->comment('Aborted.'); + + return self::SUCCESS; + } + + return $concurrency > 0 + ? $this->uploadAsync($files, $concurrency, $localDisk) + : $this->uploadSync($files, $localDisk); + } + + /** + * Enumerate local emoji files, applying offset/limit and skipping the + * missing.png placeholder (hardcoded local /storage/emoji/missing.png + * onerror fallback) and dotfiles. + * + * @return list + */ + protected function collectFiles($localDisk): array + { + $files = $localDisk->exists('public/emoji') ? $localDisk->files('public/emoji') : []; + + $offset = max(0, (int) $this->option('offset')); + if ($offset > 0) { + $files = array_slice($files, $offset); + } + + $limit = (int) $this->option('limit'); + if ($limit > 0) { + $files = array_slice($files, 0, $limit); + } + + return array_values(array_filter($files, function ($p) { + $name = basename($p); + + return ! str_starts_with($name, '.') && $name !== 'missing.png'; + })); + } + + /** + * Upload via the async S3 SDK with a fixed number of concurrent in-flight + * PutObject requests. A successful PutObject response is the confirmation + * (no separate HEAD verify); the local copy is deleted on success. + * + * @param list $files + */ + protected function uploadAsync(array $files, int $concurrency, $localDisk): int + { + $conf = config('filesystems.disks.s3'); + $bucket = $conf['bucket'] ?? null; + + if (! $bucket) { + $this->error('S3 bucket is not configured (filesystems.disks.s3.bucket).'); + + return self::FAILURE; + } + + $args = [ + 'version' => 'latest', + 'region' => $conf['region'] ?? 'us-east-1', + 'credentials' => [ + 'key' => $conf['key'] ?? null, + 'secret' => $conf['secret'] ?? null, + ], + ]; + if (! empty($conf['endpoint'])) { + $args['endpoint'] = $conf['endpoint']; + } + if (! empty($conf['use_path_style_endpoint'])) { + $args['use_path_style_endpoint'] = true; + } + + try { + $client = new S3Client($args); + } catch (\Throwable $e) { + $this->error('Could not build S3 client: '.$e->getMessage()); + + return self::FAILURE; + } + + $keepLocal = (bool) $this->option('keep-local'); + $sendAcl = ! $this->option('no-acl'); + $visibility = ($conf['visibility'] ?? 'public') === 'public' ? 'public-read' : 'private'; + + $moved = 0; + $failed = 0; + $startedAt = microtime(true); + + $bar = $this->output->createProgressBar(count($files)); + $bar->setFormat(' %current%/%max% [%bar%] %percent:3s%% %rate% up/s'); + $bar->setMessage('0.0', 'rate'); + $bar->start(); + + // Lazily yield a PutObject command per file so CommandPool pulls work as + // concurrency slots free up (keeps memory flat over large runs). + $commands = function () use ($client, $files, $bucket, $localDisk, $visibility, $sendAcl) { + foreach ($files as $localPath) { + $params = [ + 'Bucket' => $bucket, + 'Key' => Str::after($localPath, 'public/'), + 'SourceFile' => $localDisk->path($localPath), + ]; + if ($sendAcl) { + $params['ACL'] = $visibility; + } + yield $client->getCommand('PutObject', $params); + } + }; + + $pool = new CommandPool($client, $commands(), [ + 'concurrency' => $concurrency, + 'fulfilled' => function ($result, $iterKey) use (&$moved, $files, $localDisk, $keepLocal, $bar, $startedAt) { + $moved++; + $localPath = $files[$iterKey] ?? null; + if ($localPath && ! $keepLocal) { + $localDisk->delete($localPath); + } + $elapsed = max(0.001, microtime(true) - $startedAt); + $bar->setMessage(sprintf('%.1f', $moved / $elapsed), 'rate'); + $bar->advance(); + }, + 'rejected' => function ($reason, $iterKey) use (&$failed, $files, $bar) { + $failed++; + $localPath = $files[$iterKey] ?? '?'; + $msg = $reason instanceof \Throwable ? $reason->getMessage() : (string) $reason; + $this->warn(PHP_EOL.'Upload failed for '.$localPath.': '.$msg); + $bar->advance(); + }, + ]); + + $pool->promise()->wait(); + + $bar->finish(); + $this->newLine(2); + + return $this->finish($moved, $failed, $startedAt, $concurrency); + } + + /** + * Simple synchronous fallback (concurrency=0): upload one file at a time via + * the cloud disk. Slower, but has no dependency on the S3 SDK internals. + * + * @param list $files + */ + protected function uploadSync(array $files, $localDisk): int + { + $cloudDisk = Storage::disk(config('filesystems.cloud')); + $keepLocal = (bool) $this->option('keep-local'); + + $moved = 0; + $failed = 0; + $startedAt = microtime(true); + + $bar = $this->output->createProgressBar(count($files)); + $bar->setFormat(' %current%/%max% [%bar%] %percent:3s%% %rate% up/s'); + $bar->setMessage('0.0', 'rate'); + $bar->start(); + + foreach ($files as $localPath) { + $mediaPath = Str::after($localPath, 'public/'); + + try { + $cloudDisk->put($mediaPath, $localDisk->get($localPath), 'public'); + if (! $keepLocal) { + $localDisk->delete($localPath); + } + $moved++; + } catch (\Throwable $e) { + $failed++; + $this->warn(PHP_EOL.'Upload failed for '.$localPath.': '.$e->getMessage()); + } + + $elapsed = max(0.001, microtime(true) - $startedAt); + $bar->setMessage(sprintf('%.1f', $moved / $elapsed), 'rate'); + $bar->advance(); + } + + $bar->finish(); + $this->newLine(2); + + return $this->finish($moved, $failed, $startedAt, 0); + } + + /** + * Bust the emoji cache and print the run summary. + */ + protected function finish(int $moved, int $failed, float $startedAt, int $concurrency): int + { + if ($moved > 0) { + Cache::forget('pf:custom_emoji'); + } + + $elapsed = max(0.001, microtime(true) - $startedAt); + $this->info(sprintf( + 'Done. moved=%d failed=%d in %.1fs, %.1f uploads/sec%s.', + $moved, + $failed, + $elapsed, + $moved / $elapsed, + $concurrency > 0 ? " (concurrency={$concurrency})" : '' + )); + + return $failed ? self::FAILURE : self::SUCCESS; + } + + protected function printDebug($localDisk): void + { + $this->line('--- debug: config ---'); + $this->line(' config(pixelfed.cloud_storage): '.var_export(config('pixelfed.cloud_storage'), true)); + $this->line(' config_cache(pixelfed.cloud_storage): '.var_export(config_cache('pixelfed.cloud_storage'), true)); + $this->line(' filesystems.cloud: '.config('filesystems.cloud')); + $this->line(' cloud host: '.($this->cloudHost() ?? 'null')); + $this->line(' local disk root: '.$localDisk->path('')); + + $this->line('--- debug: custom_emoji table ---'); + $this->line(' total rows: '.CustomEmoji::count()); + $this->line(' uri IS NULL: '.CustomEmoji::whereNull('uri')->count()); + $this->line(' uri NOT NULL: '.CustomEmoji::whereNotNull('uri')->count()); + $this->line(' media_path NOT NULL: '.CustomEmoji::whereNotNull('media_path')->count()); + + $this->line('--- debug: local emoji directory (public/emoji) ---'); + $this->line(' dir exists: '.var_export($localDisk->exists('public/emoji'), true)); + $this->line(' file count: '.count($localDisk->files('public/emoji'))); + } +} diff --git a/app/Console/Commands/Admin/ImportEmojis.php b/app/Console/Commands/Admin/ImportEmojis.php index 43500fbf2..d973e97b7 100644 --- a/app/Console/Commands/Admin/ImportEmojis.php +++ b/app/Console/Commands/Admin/ImportEmojis.php @@ -5,7 +5,6 @@ namespace App\Console\Commands\Admin; use App\Models\CustomEmoji; use Illuminate\Console\Command; use Illuminate\Support\Facades\Cache; -use Illuminate\Support\Facades\Storage; class ImportEmojis extends Command { @@ -87,8 +86,8 @@ class ImportEmojis extends Command $emoji->save(); $fileName = $emoji->id.'.'.$extension; - Storage::putFileAs('public/emoji', $entry->getPathname(), $fileName); $emoji->media_path = 'emoji/'.$fileName; + CustomEmoji::storeMediaFromFile($emoji->media_path, $entry->getPathname()); $emoji->save(); $imported++; Cache::forget('pf:custom_emoji'); diff --git a/app/Http/Controllers/AdminController.php b/app/Http/Controllers/AdminController.php index dee5bed73..94e6f71fd 100644 --- a/app/Http/Controllers/AdminController.php +++ b/app/Http/Controllers/AdminController.php @@ -35,7 +35,6 @@ use Illuminate\Http\Request; use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Mail; -use Illuminate\Support\Facades\Storage; use Illuminate\Support\Str; use Illuminate\Validation\Rule; @@ -652,8 +651,8 @@ class AdminController extends Controller $emoji->save(); $fileName = $emoji->id.'.'.$request->emoji->extension(); - $request->emoji->storePubliclyAs('public/emoji', $fileName); $emoji->media_path = 'emoji/'.$fileName; + CustomEmoji::storeMediaFromFile($emoji->media_path, $request->emoji->getPathname()); $emoji->save(); Cache::forget('pf:custom_emoji'); @@ -664,7 +663,7 @@ class AdminController extends Controller { abort_unless((bool) config_cache('federation.custom_emoji.enabled'), 404); $emoji = CustomEmoji::findOrFail($id); - Storage::delete("public/{$emoji->media_path}"); + CustomEmoji::deleteMedia($emoji->media_path); Cache::forget('pf:custom_emoji'); $emoji->delete(); diff --git a/app/Models/CustomEmoji.php b/app/Models/CustomEmoji.php index 12b62669d..7ea1d9f33 100644 --- a/app/Models/CustomEmoji.php +++ b/app/Models/CustomEmoji.php @@ -2,9 +2,11 @@ namespace App\Models; +use Illuminate\Contracts\Filesystem\Filesystem; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Model; use Illuminate\Support\Facades\Cache; +use Illuminate\Support\Facades\Storage; use Illuminate\Support\Str; class CustomEmoji extends Model @@ -17,6 +19,93 @@ class CustomEmoji extends Model protected $guarded = []; + /** + * Public URL for this emoji's media. + * + * When cloud storage is enabled the object is served from the cloud disk + * (emoji are stored at the same relative media_path on both disks), and + * falls back to the local /storage URL otherwise. + */ + public static function urlForPath(?string $mediaPath): ?string + { + if (! $mediaPath) { + return null; + } + + if ((bool) config_cache('pixelfed.cloud_storage')) { + return Storage::disk(config('filesystems.cloud'))->url($mediaPath); + } + + return url('/storage/'.$mediaPath); + } + + public function url(): ?string + { + return self::urlForPath($this->media_path); + } + + /** + * The disk emoji media is stored on, and the storage path prefix for it. + * + * On cloud storage, objects live at the bare media_path (emoji/{id}.ext). + * On local storage they live under the public/ disk prefix so they are + * served through the /storage symlink. + * + * @return array{disk: Filesystem, prefix: string} + */ + public static function storageTarget(): array + { + if ((bool) config_cache('pixelfed.cloud_storage')) { + return [ + 'disk' => Storage::disk(config('filesystems.cloud')), + 'prefix' => '', + ]; + } + + return [ + 'disk' => Storage::disk('local'), + 'prefix' => 'public/', + ]; + } + + /** + * Store emoji bytes for the given media_path on the active disk. + */ + public static function storeMedia(string $mediaPath, string $contents): void + { + $target = self::storageTarget(); + $target['disk']->put($target['prefix'].$mediaPath, $contents, 'public'); + } + + /** + * Store an emoji from a local source file for the given media_path on the + * active disk (used by uploads/imports that already have a file on disk). + */ + public static function storeMediaFromFile(string $mediaPath, string $sourcePath): void + { + $target = self::storageTarget(); + $target['disk']->put( + $target['prefix'].$mediaPath, + file_get_contents($sourcePath), + 'public' + ); + } + + /** + * Delete emoji media for the given media_path from the active disk. + */ + public static function deleteMedia(?string $mediaPath): void + { + if (! $mediaPath) { + return; + } + + $target = self::storageTarget(); + if ($target['disk']->exists($target['prefix'].$mediaPath)) { + $target['disk']->delete($target['prefix'].$mediaPath); + } + } + public static function scan($text, $activitypub = false) { if ((bool) config_cache('federation.custom_emoji.enabled') == false) { @@ -43,7 +132,7 @@ class CustomEmoji extends Model }); if ($tag) { - $url = url('/storage/'.$tag['media_path']); + $url = self::urlForPath($tag['media_path']); if ($activitypub == true) { $mediaType = Str::endsWith($url, '.png') ? 'image/png' : 'image/jpg'; diff --git a/app/Services/CustomEmojiService.php b/app/Services/CustomEmojiService.php index 438965def..a803acf80 100644 --- a/app/Services/CustomEmojiService.php +++ b/app/Services/CustomEmojiService.php @@ -7,7 +7,6 @@ use App\Util\ActivityPub\Helpers; use Illuminate\Http\Client\RequestException; use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Http; -use Illuminate\Support\Facades\Storage; class CustomEmojiService { @@ -118,9 +117,7 @@ class CustomEmojiService ]); if ($emoji->wasRecentlyCreated == false) { - if (Storage::exists('public/'.$emoji->media_path)) { - Storage::delete('public/'.$emoji->media_path); - } + CustomEmoji::deleteMedia($emoji->media_path); } $ext = '.'.last(explode('/', $json['icon']['mediaType'])); @@ -136,7 +133,7 @@ class CustomEmojiService return; } - Storage::put('public/'.$mediaPath, $body); + CustomEmoji::storeMedia($mediaPath, $body); $emoji->media_path = $mediaPath; $emoji->save(); @@ -191,7 +188,7 @@ class CustomEmojiService ->whereNull('uri') ->get() ->map(function ($emojo) { - $url = url('storage/'.$emojo->media_path); + $url = CustomEmoji::urlForPath($emojo->media_path); return [ 'shortcode' => str_replace(':', '', $emojo->shortcode), diff --git a/bootstrap/app.php b/bootstrap/app.php index dfeca931c..08965c52c 100644 --- a/bootstrap/app.php +++ b/bootstrap/app.php @@ -146,6 +146,8 @@ return Application::configure(basePath: dirname(__DIR__)) if ((bool) config_cache('pixelfed.cloud_storage') && (bool) config_cache('media.delete_local_after_cloud')) { // Upload any local stragglers to cloud and GC verified local copies. $schedule->command('admin:MediaMoveStorageLocalToCloud --force --limit=500')->hourlyAt(15); + // Same for local custom emoji (no limit: keep all emoji on cloud). + $schedule->command('admin:EmojiMoveStorageLocalToCloud --force')->dailyAt('04:35'); } if (config('import.instagram.enabled')) { diff --git a/database/migrations/2026_08_30_050000_migrate_local_emoji_to_cloud.php b/database/migrations/2026_08_30_050000_migrate_local_emoji_to_cloud.php new file mode 100644 index 000000000..8df4910e4 --- /dev/null +++ b/database/migrations/2026_08_30_050000_migrate_local_emoji_to_cloud.php @@ -0,0 +1,38 @@ + true, + ]); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + // + } +}; diff --git a/resources/views/admin/custom-emoji/duplicates.blade.php b/resources/views/admin/custom-emoji/duplicates.blade.php index 51ff3a714..678175e94 100644 --- a/resources/views/admin/custom-emoji/duplicates.blade.php +++ b/resources/views/admin/custom-emoji/duplicates.blade.php @@ -28,7 +28,7 @@
- +

{{ $emoji->shortcode }}

@@ -62,7 +62,7 @@ @foreach($emojis as $emoji)
- +

{{ $emoji->shortcode }}

diff --git a/resources/views/admin/custom-emoji/home.blade.php b/resources/views/admin/custom-emoji/home.blade.php index 54ac5886e..cfd746a35 100644 --- a/resources/views/admin/custom-emoji/home.blade.php +++ b/resources/views/admin/custom-emoji/home.blade.php @@ -105,7 +105,7 @@ @foreach($emojis as $emoji)
- +

{{ $emoji->shortcode }}