From 40b323bca4f76625c46a5147fcf703dc6c045408 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 30 Aug 2026 14:03:11 +0930 Subject: [PATCH] revert: remove emoji local-to-cloud storage changes Back out all emoji cloud-storage work from staging so it can be reworked and re-landed separately (the URL resolution flips to cloud on a global config flag, which created a broken-URL window, and the migration approach needs revisiting). Reverts to pre-emoji state: - CustomEmoji model URL/storage helpers (urlForPath, storageTarget, storeMedia, storeMediaFromFile, deleteMedia, url) and callers in ImportEmojis, CustomEmojiService, AdminController - admin custom-emoji blade views back to local /storage URLs - Remove admin:EmojiMoveStorageLocalToCloud command - Remove the deploy migration and its scheduler entry Media (and the already-reverted story) scheduler entries are untouched. --- .../Admin/EmojiMoveStorageLocalToCloud.php | 318 ------------------ app/Console/Commands/Admin/ImportEmojis.php | 3 +- app/Http/Controllers/AdminController.php | 5 +- app/Models/CustomEmoji.php | 91 +---- app/Services/CustomEmojiService.php | 9 +- bootstrap/app.php | 2 - ...30_050000_migrate_local_emoji_to_cloud.php | 38 --- .../admin/custom-emoji/duplicates.blade.php | 4 +- .../views/admin/custom-emoji/home.blade.php | 2 +- 9 files changed, 15 insertions(+), 457 deletions(-) delete mode 100644 app/Console/Commands/Admin/EmojiMoveStorageLocalToCloud.php delete mode 100644 database/migrations/2026_08_30_050000_migrate_local_emoji_to_cloud.php diff --git a/app/Console/Commands/Admin/EmojiMoveStorageLocalToCloud.php b/app/Console/Commands/Admin/EmojiMoveStorageLocalToCloud.php deleted file mode 100644 index 462c8e294..000000000 --- a/app/Console/Commands/Admin/EmojiMoveStorageLocalToCloud.php +++ /dev/null @@ -1,318 +0,0 @@ -error('Cloud storage is not enabled (pixelfed.cloud_storage is false).'); - - return self::FAILURE; - } - - try { - $localDisk = Storage::disk('local'); - } catch (\Throwable $e) { - $this->error('Local disk could not be resolved: '.$e->getMessage()); - - return self::FAILURE; - } - - if (! $this->cloudHost()) { - $this->error('Cloud disk ('.config('filesystems.cloud').') is not configured (no resolvable URL).'); - $this->line('Set AWS_URL / AWS_* in your .env before migrating to cloud.'); - - return self::FAILURE; - } - - if ($this->option('debug')) { - $this->printDebug($localDisk); - } - - // Build the list of local emoji files to migrate. Disk-driven: the file - // existing locally is the source of truth for "needs moving". We do NOT - // filter by DB columns — federated emoji have their media stored locally - // too (with a uri set), so a DB filter would wrongly exclude them. - $files = $this->collectFiles($localDisk); - $total = count($files); - - if ($total === 0) { - $this->info('No emoji files to migrate.'); - - return self::SUCCESS; - } - - $concurrency = max(0, (int) $this->option('concurrency')); - $mode = $concurrency > 0 ? "async S3 (concurrency={$concurrency})" : 'synchronous'; - - if ($this->option('dry-run')) { - $this->info("[dry-run] Would upload {$total} emoji to cloud using {$mode}."); - - return self::SUCCESS; - } - - if (! $this->option('force') && ! $this->confirm("Upload {$total} emoji to cloud using {$mode}?", true)) { - $this->comment('Aborted.'); - - return self::SUCCESS; - } - - return $concurrency > 0 - ? $this->uploadAsync($files, $concurrency, $localDisk) - : $this->uploadSync($files, $localDisk); - } - - /** - * Enumerate local emoji files, applying offset/limit and skipping the - * missing.png placeholder (hardcoded local /storage/emoji/missing.png - * onerror fallback) and dotfiles. - * - * @return list - */ - protected function collectFiles($localDisk): array - { - $files = $localDisk->exists('public/emoji') ? $localDisk->files('public/emoji') : []; - - $offset = max(0, (int) $this->option('offset')); - if ($offset > 0) { - $files = array_slice($files, $offset); - } - - $limit = (int) $this->option('limit'); - if ($limit > 0) { - $files = array_slice($files, 0, $limit); - } - - return array_values(array_filter($files, function ($p) { - $name = basename($p); - - return ! str_starts_with($name, '.') && $name !== 'missing.png'; - })); - } - - /** - * Upload via the async S3 SDK with a fixed number of concurrent in-flight - * PutObject requests. A successful PutObject response is the confirmation - * (no separate HEAD verify); the local copy is deleted on success. - * - * @param list $files - */ - protected function uploadAsync(array $files, int $concurrency, $localDisk): int - { - $conf = config('filesystems.disks.s3'); - $bucket = $conf['bucket'] ?? null; - - if (! $bucket) { - $this->error('S3 bucket is not configured (filesystems.disks.s3.bucket).'); - - return self::FAILURE; - } - - $args = [ - 'version' => 'latest', - 'region' => $conf['region'] ?? 'us-east-1', - 'credentials' => [ - 'key' => $conf['key'] ?? null, - 'secret' => $conf['secret'] ?? null, - ], - ]; - if (! empty($conf['endpoint'])) { - $args['endpoint'] = $conf['endpoint']; - } - if (! empty($conf['use_path_style_endpoint'])) { - $args['use_path_style_endpoint'] = true; - } - - try { - $client = new S3Client($args); - } catch (\Throwable $e) { - $this->error('Could not build S3 client: '.$e->getMessage()); - - return self::FAILURE; - } - - $keepLocal = (bool) $this->option('keep-local'); - $sendAcl = ! $this->option('no-acl'); - $visibility = ($conf['visibility'] ?? 'public') === 'public' ? 'public-read' : 'private'; - - $moved = 0; - $failed = 0; - $startedAt = microtime(true); - - $bar = $this->output->createProgressBar(count($files)); - $bar->setFormat(' %current%/%max% [%bar%] %percent:3s%% %rate% up/s'); - $bar->setMessage('0.0', 'rate'); - $bar->start(); - - // Lazily yield a PutObject command per file so CommandPool pulls work as - // concurrency slots free up (keeps memory flat over large runs). - $commands = function () use ($client, $files, $bucket, $localDisk, $visibility, $sendAcl) { - foreach ($files as $localPath) { - $params = [ - 'Bucket' => $bucket, - 'Key' => Str::after($localPath, 'public/'), - 'SourceFile' => $localDisk->path($localPath), - ]; - if ($sendAcl) { - $params['ACL'] = $visibility; - } - yield $client->getCommand('PutObject', $params); - } - }; - - $pool = new CommandPool($client, $commands(), [ - 'concurrency' => $concurrency, - 'fulfilled' => function ($result, $iterKey) use (&$moved, $files, $localDisk, $keepLocal, $bar, $startedAt) { - $moved++; - $localPath = $files[$iterKey] ?? null; - if ($localPath && ! $keepLocal) { - $localDisk->delete($localPath); - } - $elapsed = max(0.001, microtime(true) - $startedAt); - $bar->setMessage(sprintf('%.1f', $moved / $elapsed), 'rate'); - $bar->advance(); - }, - 'rejected' => function ($reason, $iterKey) use (&$failed, $files, $bar) { - $failed++; - $localPath = $files[$iterKey] ?? '?'; - $msg = $reason instanceof \Throwable ? $reason->getMessage() : (string) $reason; - $this->warn(PHP_EOL.'Upload failed for '.$localPath.': '.$msg); - $bar->advance(); - }, - ]); - - $pool->promise()->wait(); - - $bar->finish(); - $this->newLine(2); - - return $this->finish($moved, $failed, $startedAt, $concurrency); - } - - /** - * Simple synchronous fallback (concurrency=0): upload one file at a time via - * the cloud disk. Slower, but has no dependency on the S3 SDK internals. - * - * @param list $files - */ - protected function uploadSync(array $files, $localDisk): int - { - $cloudDisk = Storage::disk(config('filesystems.cloud')); - $keepLocal = (bool) $this->option('keep-local'); - - $moved = 0; - $failed = 0; - $startedAt = microtime(true); - - $bar = $this->output->createProgressBar(count($files)); - $bar->setFormat(' %current%/%max% [%bar%] %percent:3s%% %rate% up/s'); - $bar->setMessage('0.0', 'rate'); - $bar->start(); - - foreach ($files as $localPath) { - $mediaPath = Str::after($localPath, 'public/'); - - try { - $cloudDisk->put($mediaPath, $localDisk->get($localPath), 'public'); - if (! $keepLocal) { - $localDisk->delete($localPath); - } - $moved++; - } catch (\Throwable $e) { - $failed++; - $this->warn(PHP_EOL.'Upload failed for '.$localPath.': '.$e->getMessage()); - } - - $elapsed = max(0.001, microtime(true) - $startedAt); - $bar->setMessage(sprintf('%.1f', $moved / $elapsed), 'rate'); - $bar->advance(); - } - - $bar->finish(); - $this->newLine(2); - - return $this->finish($moved, $failed, $startedAt, 0); - } - - /** - * Bust the emoji cache and print the run summary. - */ - protected function finish(int $moved, int $failed, float $startedAt, int $concurrency): int - { - if ($moved > 0) { - Cache::forget('pf:custom_emoji'); - } - - $elapsed = max(0.001, microtime(true) - $startedAt); - $this->info(sprintf( - 'Done. moved=%d failed=%d in %.1fs, %.1f uploads/sec%s.', - $moved, - $failed, - $elapsed, - $moved / $elapsed, - $concurrency > 0 ? " (concurrency={$concurrency})" : '' - )); - - return $failed ? self::FAILURE : self::SUCCESS; - } - - protected function printDebug($localDisk): void - { - $this->line('--- debug: config ---'); - $this->line(' config(pixelfed.cloud_storage): '.var_export(config('pixelfed.cloud_storage'), true)); - $this->line(' config_cache(pixelfed.cloud_storage): '.var_export(config_cache('pixelfed.cloud_storage'), true)); - $this->line(' filesystems.cloud: '.config('filesystems.cloud')); - $this->line(' cloud host: '.($this->cloudHost() ?? 'null')); - $this->line(' local disk root: '.$localDisk->path('')); - - $this->line('--- debug: custom_emoji table ---'); - $this->line(' total rows: '.CustomEmoji::count()); - $this->line(' uri IS NULL: '.CustomEmoji::whereNull('uri')->count()); - $this->line(' uri NOT NULL: '.CustomEmoji::whereNotNull('uri')->count()); - $this->line(' media_path NOT NULL: '.CustomEmoji::whereNotNull('media_path')->count()); - - $this->line('--- debug: local emoji directory (public/emoji) ---'); - $this->line(' dir exists: '.var_export($localDisk->exists('public/emoji'), true)); - $this->line(' file count: '.count($localDisk->files('public/emoji'))); - } -} diff --git a/app/Console/Commands/Admin/ImportEmojis.php b/app/Console/Commands/Admin/ImportEmojis.php index d973e97b7..43500fbf2 100644 --- a/app/Console/Commands/Admin/ImportEmojis.php +++ b/app/Console/Commands/Admin/ImportEmojis.php @@ -5,6 +5,7 @@ namespace App\Console\Commands\Admin; use App\Models\CustomEmoji; use Illuminate\Console\Command; use Illuminate\Support\Facades\Cache; +use Illuminate\Support\Facades\Storage; class ImportEmojis extends Command { @@ -86,8 +87,8 @@ class ImportEmojis extends Command $emoji->save(); $fileName = $emoji->id.'.'.$extension; + Storage::putFileAs('public/emoji', $entry->getPathname(), $fileName); $emoji->media_path = 'emoji/'.$fileName; - CustomEmoji::storeMediaFromFile($emoji->media_path, $entry->getPathname()); $emoji->save(); $imported++; Cache::forget('pf:custom_emoji'); diff --git a/app/Http/Controllers/AdminController.php b/app/Http/Controllers/AdminController.php index 94e6f71fd..dee5bed73 100644 --- a/app/Http/Controllers/AdminController.php +++ b/app/Http/Controllers/AdminController.php @@ -35,6 +35,7 @@ use Illuminate\Http\Request; use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Mail; +use Illuminate\Support\Facades\Storage; use Illuminate\Support\Str; use Illuminate\Validation\Rule; @@ -651,8 +652,8 @@ class AdminController extends Controller $emoji->save(); $fileName = $emoji->id.'.'.$request->emoji->extension(); + $request->emoji->storePubliclyAs('public/emoji', $fileName); $emoji->media_path = 'emoji/'.$fileName; - CustomEmoji::storeMediaFromFile($emoji->media_path, $request->emoji->getPathname()); $emoji->save(); Cache::forget('pf:custom_emoji'); @@ -663,7 +664,7 @@ class AdminController extends Controller { abort_unless((bool) config_cache('federation.custom_emoji.enabled'), 404); $emoji = CustomEmoji::findOrFail($id); - CustomEmoji::deleteMedia($emoji->media_path); + Storage::delete("public/{$emoji->media_path}"); Cache::forget('pf:custom_emoji'); $emoji->delete(); diff --git a/app/Models/CustomEmoji.php b/app/Models/CustomEmoji.php index 7ea1d9f33..12b62669d 100644 --- a/app/Models/CustomEmoji.php +++ b/app/Models/CustomEmoji.php @@ -2,11 +2,9 @@ namespace App\Models; -use Illuminate\Contracts\Filesystem\Filesystem; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Model; use Illuminate\Support\Facades\Cache; -use Illuminate\Support\Facades\Storage; use Illuminate\Support\Str; class CustomEmoji extends Model @@ -19,93 +17,6 @@ class CustomEmoji extends Model protected $guarded = []; - /** - * Public URL for this emoji's media. - * - * When cloud storage is enabled the object is served from the cloud disk - * (emoji are stored at the same relative media_path on both disks), and - * falls back to the local /storage URL otherwise. - */ - public static function urlForPath(?string $mediaPath): ?string - { - if (! $mediaPath) { - return null; - } - - if ((bool) config_cache('pixelfed.cloud_storage')) { - return Storage::disk(config('filesystems.cloud'))->url($mediaPath); - } - - return url('/storage/'.$mediaPath); - } - - public function url(): ?string - { - return self::urlForPath($this->media_path); - } - - /** - * The disk emoji media is stored on, and the storage path prefix for it. - * - * On cloud storage, objects live at the bare media_path (emoji/{id}.ext). - * On local storage they live under the public/ disk prefix so they are - * served through the /storage symlink. - * - * @return array{disk: Filesystem, prefix: string} - */ - public static function storageTarget(): array - { - if ((bool) config_cache('pixelfed.cloud_storage')) { - return [ - 'disk' => Storage::disk(config('filesystems.cloud')), - 'prefix' => '', - ]; - } - - return [ - 'disk' => Storage::disk('local'), - 'prefix' => 'public/', - ]; - } - - /** - * Store emoji bytes for the given media_path on the active disk. - */ - public static function storeMedia(string $mediaPath, string $contents): void - { - $target = self::storageTarget(); - $target['disk']->put($target['prefix'].$mediaPath, $contents, 'public'); - } - - /** - * Store an emoji from a local source file for the given media_path on the - * active disk (used by uploads/imports that already have a file on disk). - */ - public static function storeMediaFromFile(string $mediaPath, string $sourcePath): void - { - $target = self::storageTarget(); - $target['disk']->put( - $target['prefix'].$mediaPath, - file_get_contents($sourcePath), - 'public' - ); - } - - /** - * Delete emoji media for the given media_path from the active disk. - */ - public static function deleteMedia(?string $mediaPath): void - { - if (! $mediaPath) { - return; - } - - $target = self::storageTarget(); - if ($target['disk']->exists($target['prefix'].$mediaPath)) { - $target['disk']->delete($target['prefix'].$mediaPath); - } - } - public static function scan($text, $activitypub = false) { if ((bool) config_cache('federation.custom_emoji.enabled') == false) { @@ -132,7 +43,7 @@ class CustomEmoji extends Model }); if ($tag) { - $url = self::urlForPath($tag['media_path']); + $url = url('/storage/'.$tag['media_path']); if ($activitypub == true) { $mediaType = Str::endsWith($url, '.png') ? 'image/png' : 'image/jpg'; diff --git a/app/Services/CustomEmojiService.php b/app/Services/CustomEmojiService.php index a803acf80..438965def 100644 --- a/app/Services/CustomEmojiService.php +++ b/app/Services/CustomEmojiService.php @@ -7,6 +7,7 @@ use App\Util\ActivityPub\Helpers; use Illuminate\Http\Client\RequestException; use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Http; +use Illuminate\Support\Facades\Storage; class CustomEmojiService { @@ -117,7 +118,9 @@ class CustomEmojiService ]); if ($emoji->wasRecentlyCreated == false) { - CustomEmoji::deleteMedia($emoji->media_path); + if (Storage::exists('public/'.$emoji->media_path)) { + Storage::delete('public/'.$emoji->media_path); + } } $ext = '.'.last(explode('/', $json['icon']['mediaType'])); @@ -133,7 +136,7 @@ class CustomEmojiService return; } - CustomEmoji::storeMedia($mediaPath, $body); + Storage::put('public/'.$mediaPath, $body); $emoji->media_path = $mediaPath; $emoji->save(); @@ -188,7 +191,7 @@ class CustomEmojiService ->whereNull('uri') ->get() ->map(function ($emojo) { - $url = CustomEmoji::urlForPath($emojo->media_path); + $url = url('storage/'.$emojo->media_path); return [ 'shortcode' => str_replace(':', '', $emojo->shortcode), diff --git a/bootstrap/app.php b/bootstrap/app.php index 08965c52c..dfeca931c 100644 --- a/bootstrap/app.php +++ b/bootstrap/app.php @@ -146,8 +146,6 @@ return Application::configure(basePath: dirname(__DIR__)) if ((bool) config_cache('pixelfed.cloud_storage') && (bool) config_cache('media.delete_local_after_cloud')) { // Upload any local stragglers to cloud and GC verified local copies. $schedule->command('admin:MediaMoveStorageLocalToCloud --force --limit=500')->hourlyAt(15); - // Same for local custom emoji (no limit: keep all emoji on cloud). - $schedule->command('admin:EmojiMoveStorageLocalToCloud --force')->dailyAt('04:35'); } if (config('import.instagram.enabled')) { diff --git a/database/migrations/2026_08_30_050000_migrate_local_emoji_to_cloud.php b/database/migrations/2026_08_30_050000_migrate_local_emoji_to_cloud.php deleted file mode 100644 index 8df4910e4..000000000 --- a/database/migrations/2026_08_30_050000_migrate_local_emoji_to_cloud.php +++ /dev/null @@ -1,38 +0,0 @@ - true, - ]); - } - - /** - * Reverse the migrations. - */ - public function down(): void - { - // - } -}; diff --git a/resources/views/admin/custom-emoji/duplicates.blade.php b/resources/views/admin/custom-emoji/duplicates.blade.php index 678175e94..51ff3a714 100644 --- a/resources/views/admin/custom-emoji/duplicates.blade.php +++ b/resources/views/admin/custom-emoji/duplicates.blade.php @@ -28,7 +28,7 @@
- +

{{ $emoji->shortcode }}

@@ -62,7 +62,7 @@ @foreach($emojis as $emoji)
- +

{{ $emoji->shortcode }}

diff --git a/resources/views/admin/custom-emoji/home.blade.php b/resources/views/admin/custom-emoji/home.blade.php index cfd746a35..54ac5886e 100644 --- a/resources/views/admin/custom-emoji/home.blade.php +++ b/resources/views/admin/custom-emoji/home.blade.php @@ -105,7 +105,7 @@ @foreach($emojis as $emoji)
- +

{{ $emoji->shortcode }}