S3 migration: copy existing objects from the old bucket (s3-old) | to the current cloud bucket (s3), verify, rewrite media URLs, GC the source. | */ beforeEach(function () { Config::set('filesystems.cloud', 's3'); // Destination (new) bucket. Storage::fake('s3', ['url' => 'https://cdneast.pixelfed.au']); // Source (old) bucket. Storage::fake('s3-old', ['url' => 'https://cdn.pixelfed.au']); }); function makeOldBucketMedia(string $oldHost = 'https://cdn.pixelfed.au', ?string $originalSha = null): Media { $user = User::factory()->create(); $user->refresh(); $pid = $user->profile->id; $status = Status::factory()->create(['profile_id' => $pid, 'type' => 'photo']); $path = 'public/m/_v2/'.$pid.'/aa/bb/file.jpg'; $thumb = 'public/m/_v2/'.$pid.'/aa/bb/file_thumb.jpeg'; // Files exist only on the OLD bucket. Storage::disk('s3-old')->put($path, 'PRIMARY-BYTES-1234567890'); Storage::disk('s3-old')->put($thumb, 'THUMB-BYTES'); return Media::create([ 'status_id' => $status->id, 'profile_id' => $pid, 'user_id' => $user->id, 'media_path' => $path, 'thumbnail_path' => $thumb, 'cdn_url' => $oldHost.'/'.$path, 'thumbnail_url' => $oldHost.'/'.$thumb, 'optimized_url' => $oldHost.'/'.$path, 'mime' => 'image/jpeg', 'size' => strlen('PRIMARY-BYTES-1234567890'), 'original_sha256' => $originalSha, 'remote_media' => false, 'version' => 4, 'replicated_at' => now(), 'order' => 0, ]); } it('copies old-bucket media to the new bucket, rewrites urls and GCs the source', function () { $media = makeOldBucketMedia(); $this->artisan('unstable:MediaMoveStorageCloudToCloud', ['--force' => true]) ->assertExitCode(0); // Copied to destination. expect(Storage::disk('s3')->exists($media->media_path))->toBeTrue(); // Removed from source (GC). expect(Storage::disk('s3-old')->exists($media->media_path))->toBeFalse(); $media->refresh(); expect(parse_url($media->cdn_url, PHP_URL_HOST))->toBe('cdneast.pixelfed.au'); expect(parse_url($media->thumbnail_url, PHP_URL_HOST))->toBe('cdneast.pixelfed.au'); expect(parse_url($media->optimized_url, PHP_URL_HOST))->toBe('cdneast.pixelfed.au'); }); it('keeps the source objects with --keep-source', function () { $media = makeOldBucketMedia(); $this->artisan('unstable:MediaMoveStorageCloudToCloud', ['--force' => true, '--keep-source' => true]) ->assertExitCode(0); expect(Storage::disk('s3')->exists($media->media_path))->toBeTrue(); expect(Storage::disk('s3-old')->exists($media->media_path))->toBeTrue(); }); it('makes no changes in dry-run', function () { $media = makeOldBucketMedia(); $this->artisan('unstable:MediaMoveStorageCloudToCloud', ['--force' => true, '--dry-run' => true]) ->assertExitCode(0); expect(Storage::disk('s3')->exists($media->media_path))->toBeFalse(); expect(parse_url($media->fresh()->cdn_url, PHP_URL_HOST))->toBe('cdn.pixelfed.au'); }); it('skips media already pointing at the destination host', function () { // cdn_url already on the destination host -> nothing to do. $media = makeOldBucketMedia('https://cdneast.pixelfed.au'); $before = $media->cdn_url; $this->artisan('unstable:MediaMoveStorageCloudToCloud', ['--force' => true]) ->assertExitCode(0); expect($media->fresh()->cdn_url)->toBe($before); // Not copied (was skipped). expect(Storage::disk('s3')->exists($media->media_path))->toBeFalse(); }); it('errors when source and destination are the same disk', function () { $this->artisan('unstable:MediaMoveStorageCloudToCloud', ['--sourceDisk' => 's3', '--force' => true]) ->assertExitCode(1); }); it('migrates media whose stored bytes no longer match original_sha256', function () { // Optimized images carry the pre-optimization upload hash in // original_sha256, while both buckets hold the optimized bytes. verify() // must not compare against original_sha256, or every optimized image is // left un-migrated on the old bucket. $media = makeOldBucketMedia('https://cdn.pixelfed.au', hash('sha256', 'ORIGINAL-UPLOAD-BYTES-BEFORE-OPTIMIZE')); $this->artisan('unstable:MediaMoveStorageCloudToCloud', ['--force' => true]) ->assertExitCode(0); expect(Storage::disk('s3')->exists($media->media_path))->toBeTrue(); expect(Storage::disk('s3-old')->exists($media->media_path))->toBeFalse(); expect(parse_url($media->fresh()->cdn_url, PHP_URL_HOST))->toBe('cdneast.pixelfed.au'); }); it('fails closed when a wrong object pre-exists at the destination key', function () { // A wrong object is already present on the destination bucket at the same // key (e.g. an external pre-seed), with different bytes than the source. $media = makeOldBucketMedia(); Storage::disk('s3')->put($media->media_path, 'WRONG-PRESEEDED-BYTES-of-different-length'); $this->artisan('unstable:MediaMoveStorageCloudToCloud', ['--force' => true]) ->assertExitCode(0); // Must fail closed: the good source is retained and the URL stays on the // old host, rather than adopting the wrong object and GCing the source. expect(Storage::disk('s3-old')->exists($media->media_path))->toBeTrue(); expect(parse_url($media->fresh()->cdn_url, PHP_URL_HOST))->toBe('cdn.pixelfed.au'); }); it('adopts an already-migrated destination object when the source is gone', function () { // Legitimate idempotent resume: pass 1 already copied + GC'd the source, // so only the destination object remains. The URL must still be rewritten. $media = makeOldBucketMedia(); Storage::disk('s3')->put($media->media_path, 'PRIMARY-BYTES-1234567890'); Storage::disk('s3-old')->delete($media->media_path); Storage::disk('s3-old')->delete($media->thumbnail_path); $this->artisan('unstable:MediaMoveStorageCloudToCloud', ['--force' => true]) ->assertExitCode(0); expect(parse_url($media->fresh()->cdn_url, PHP_URL_HOST))->toBe('cdneast.pixelfed.au'); });