Merge pull request #6968 from pixelfed/debug/emoji-cloud-migration

perf: async S3 SDK upload path for emoji migration

app/Console/Commands/Admin/EmojiMoveStorageLocalToCloud.php

@@ -5,6 +5,8 @@ namespace App\Console\Commands\Admin;
 use App\Console\Commands\Concerns\ManagesMediaStorageEnv;
 use App\Models\CustomEmoji;
 use App\Util\Lexer\PrettyNumber;
+use Aws\CommandPool;
+use Aws\S3\S3Client;
 use Illuminate\Console\Command;
 use Illuminate\Support\Facades\Cache;
 use Illuminate\Support\Facades\Storage;
@@ -24,6 +26,8 @@ class EmojiMoveStorageLocalToCloud extends Command
         {--limit=0 : Max files to process this run (0 = no limit, process all)}
         {--offset=0 : Skip this many files before processing (for manual chunking)}
         {--workers=1 : Spawn N parallel worker processes to upload concurrently}
+        {--concurrency=0 : Use the async S3 SDK with N concurrent in-flight uploads (0 = disabled, use sync path)}
+        {--no-acl : Do not send an ACL header on async uploads (some S3-compatible stores reject it)}
         {--stride=1 : Internal: total worker count for strided sharding}
         {--shard=0 : Internal: this worker index (0-based) for strided sharding}
         {--skip-verify : Do not re-check the cloud copy size after upload (faster)}
@@ -94,6 +98,16 @@ class EmojiMoveStorageLocalToCloud extends Command
             $this->line('  file count: '.count($localDisk->files('public/emoji')));
         }
 
+        $concurrency = max(0, (int) $this->option('concurrency'));
+
+        // Fastest path: async S3 SDK with many concurrent in-flight PutObject
+        // requests from a single process. This hides the high per-request
+        // latency of the object store far better than sequential uploads or
+        // process-level workers.
+        if ($concurrency > 0) {
+            return $this->runAsync($concurrency, $localDisk);
+        }
+
         $workers = max(1, (int) $this->option('workers'));
 
         // When asked to parallelise, spawn N child processes that each handle a
@@ -286,6 +300,153 @@ class EmojiMoveStorageLocalToCloud extends Command
         return $failed ? self::FAILURE : self::SUCCESS;
     }
 
+    /**
+     * Upload via the async S3 SDK with a fixed number of concurrent in-flight
+     * PutObject requests. A successful PutObject response is the confirmation
+     * (no separate HEAD verify), and the local copy is deleted on success.
+     */
+    protected function runAsync(int $concurrency, $localDisk): int
+    {
+        $conf = config('filesystems.disks.s3');
+        $bucket = $conf['bucket'] ?? null;
+
+        if (! $bucket) {
+            $this->error('S3 bucket is not configured (filesystems.disks.s3.bucket).');
+
+            return self::FAILURE;
+        }
+
+        // Build an S3 client from the same disk config Flysystem uses.
+        $args = [
+            'version' => 'latest',
+            'region' => $conf['region'] ?? 'us-east-1',
+            'credentials' => [
+                'key' => $conf['key'] ?? null,
+                'secret' => $conf['secret'] ?? null,
+            ],
+        ];
+        if (! empty($conf['endpoint'])) {
+            $args['endpoint'] = $conf['endpoint'];
+        }
+        if (! empty($conf['use_path_style_endpoint'])) {
+            $args['use_path_style_endpoint'] = true;
+        }
+
+        try {
+            $client = new S3Client($args);
+        } catch (\Throwable $e) {
+            $this->error('Could not build S3 client: '.$e->getMessage());
+
+            return self::FAILURE;
+        }
+
+        $limit = (int) $this->option('limit');
+        $offset = max(0, (int) $this->option('offset'));
+        $keepLocal = (bool) $this->option('keep-local');
+        $dryRun = (bool) $this->option('dry-run');
+
+        $files = $localDisk->exists('public/emoji') ? $localDisk->files('public/emoji') : [];
+        if ($offset > 0) {
+            $files = array_slice($files, $offset);
+        }
+        if ($limit > 0) {
+            $files = array_slice($files, 0, $limit);
+        }
+
+        // Filter out placeholders/dotfiles up front.
+        $files = array_values(array_filter($files, function ($p) {
+            $name = basename($p);
+
+            return ! str_starts_with($name, '.') && $name !== 'missing.png';
+        }));
+
+        $total = count($files);
+
+        if ($total === 0) {
+            $this->info('No emoji files to migrate.');
+
+            return self::SUCCESS;
+        }
+
+        if (! $dryRun && ! $this->option('force')) {
+            if (! $this->confirm("Upload {$total} emoji to cloud using async S3 (concurrency={$concurrency})?", true)) {
+                $this->comment('Aborted.');
+
+                return self::SUCCESS;
+            }
+        }
+
+        if ($dryRun) {
+            $this->info("[dry-run] Would upload {$total} files via async S3 (concurrency={$concurrency}).");
+
+            return self::SUCCESS;
+        }
+
+        $moved = 0;
+        $failed = 0;
+        $startedAt = microtime(true);
+        $visibility = ($conf['visibility'] ?? 'public') === 'public' ? 'public-read' : 'private';
+
+        $bar = $this->output->createProgressBar($total);
+        $bar->setFormat(' %current%/%max% [%bar%] %percent:3s%%  %rate% up/s');
+        $bar->setMessage('0.0', 'rate');
+        $bar->start();
+
+        // Lazily yield a PutObject command per file so CommandPool pulls work
+        // as concurrency slots free up (keeps memory flat over large runs).
+        $sendAcl = ! $this->option('no-acl');
+        $commands = function () use ($client, $files, $bucket, $localDisk, $visibility, $sendAcl) {
+            foreach ($files as $localPath) {
+                $mediaPath = Str::after($localPath, 'public/');
+                $params = [
+                    'Bucket' => $bucket,
+                    'Key' => $mediaPath,
+                    'SourceFile' => $localDisk->path($localPath),
+                ];
+                if ($sendAcl) {
+                    $params['ACL'] = $visibility;
+                }
+                yield $client->getCommand('PutObject', $params);
+            }
+        };
+
+        $pool = new CommandPool($client, $commands(), [
+            'concurrency' => $concurrency,
+            'fulfilled' => function ($result, $iterKey) use (&$moved, $files, $localDisk, $keepLocal, $bar, $startedAt) {
+                $moved++;
+                $localPath = $files[$iterKey] ?? null;
+                if ($localPath && ! $keepLocal) {
+                    $localDisk->delete($localPath);
+                }
+                $elapsed = max(0.001, microtime(true) - $startedAt);
+                $bar->setMessage(sprintf('%.1f', $moved / $elapsed), 'rate');
+                $bar->advance();
+            },
+            'rejected' => function ($reason, $iterKey) use (&$failed, $files, $bar) {
+                $failed++;
+                $localPath = $files[$iterKey] ?? '?';
+                $msg = $reason instanceof \Throwable ? $reason->getMessage() : (string) $reason;
+                $this->warn(PHP_EOL.'Upload failed for '.$localPath.': '.$msg);
+                $bar->advance();
+            },
+        ]);
+
+        // Block until all queued uploads settle.
+        $pool->promise()->wait();
+
+        $bar->finish();
+        $this->newLine(2);
+
+        if ($moved > 0) {
+            Cache::forget('pf:custom_emoji');
+        }
+
+        $elapsed = max(0.001, microtime(true) - $startedAt);
+        $this->info(sprintf('Done. moved=%d failed=%d in %.1fs, %.1f uploads/sec (concurrency=%d).', $moved, $failed, $elapsed, $moved / $elapsed, $concurrency));
+
+        return $failed ? self::FAILURE : self::SUCCESS;
+    }
+
     /**
      * @return string one of moved|skipped|failed
      */