mirror of https://github.com/pixelfed/pixelfed
commit
fe1037348b
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Admin;
|
||||
|
||||
use App\Mail\AdminInviteEmail;
|
||||
use App\Models\AdminInvite;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Admin;
|
||||
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Http\File;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Admin;
|
||||
|
||||
use App\Models\User;
|
||||
use App\Services\EmailService;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Admin;
|
||||
|
||||
use App\Services\ConfigCacheService;
|
||||
use Illuminate\Console\Command;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Admin;
|
||||
|
||||
use App\Mail\CuratedRegisterConfirmEmail;
|
||||
use App\Models\CuratedRegister;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Admin;
|
||||
|
||||
use App\Jobs\DeletePipeline\DeleteRemoteProfilePipeline;
|
||||
use App\Models\Profile;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Admin;
|
||||
|
||||
use App\Models\Place;
|
||||
use Illuminate\Console\Command;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Admin;
|
||||
|
||||
use App\Models\CustomEmoji;
|
||||
use Illuminate\Console\Command;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Admin;
|
||||
|
||||
use App\Jobs\InstancePipeline\FetchNodeinfoPipeline;
|
||||
use App\Models\Instance;
|
||||
@ -0,0 +1,267 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands\Admin;
|
||||
|
||||
use App\Models\Media;
|
||||
use App\Services\MediaService;
|
||||
use App\Services\StatusService;
|
||||
use App\Util\Lexer\PrettyNumber;
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Illuminate\Support\Str;
|
||||
|
||||
class MediaMoveStorageCloudToCloud extends Command
|
||||
{
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'admin:MediaMoveStorageCloudToCloud
|
||||
{--sourceDisk=s3-old : The disk holding the OLD bucket (default: s3-old, reads AWS_OLD_*)}
|
||||
{--limit=500 : Max media rows to process this run}
|
||||
{--dry-run : Report what would happen without copying or writing}
|
||||
{--keep-source : Do not delete objects from the source bucket after verifying the copy}
|
||||
{--force : Skip confirmation prompts}';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Cold-migrate existing media from an old S3 bucket (source disk) to the current cloud bucket, verifying each copy and rewriting media URLs one row at a time.';
|
||||
|
||||
protected int $movedBytes = 0;
|
||||
|
||||
public function handle()
|
||||
{
|
||||
$sourceName = (string) $this->option('sourceDisk');
|
||||
$destName = config('filesystems.cloud');
|
||||
|
||||
if ($sourceName === $destName) {
|
||||
$this->error('Source disk and destination (cloud) disk are the same ('.$sourceName.').');
|
||||
$this->line('Point AWS_* at the NEW bucket and keep the OLD bucket creds in the source disk.');
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
try {
|
||||
$sourceDisk = Storage::disk($sourceName);
|
||||
} catch (\Throwable $e) {
|
||||
$this->error('Source disk "'.$sourceName.'" could not be resolved: '.$e->getMessage());
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
try {
|
||||
$destDisk = Storage::disk($destName);
|
||||
} catch (\Throwable $e) {
|
||||
$this->error('Destination cloud disk "'.$destName.'" could not be resolved: '.$e->getMessage());
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
$sourceHost = $this->diskHost($sourceDisk);
|
||||
$destHost = $this->diskHost($destDisk);
|
||||
|
||||
if (! $sourceHost) {
|
||||
$this->error('Source disk "'.$sourceName.'" is not configured (no resolvable URL). Set AWS_OLD_* in your .env.');
|
||||
|
||||
return 1;
|
||||
}
|
||||
if (! $destHost) {
|
||||
$this->error('Destination cloud disk "'.$destName.'" is not configured (no resolvable URL). Set AWS_* in your .env.');
|
||||
|
||||
return 1;
|
||||
}
|
||||
if (strcasecmp($sourceHost, $destHost) === 0) {
|
||||
$this->error('Source and destination resolve to the same host ('.$sourceHost.'); nothing to migrate.');
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
$this->info('Source (old): '.$sourceName.' -> '.$sourceHost);
|
||||
$this->info('Destination (new): '.$destName.' -> '.$destHost);
|
||||
$this->newLine();
|
||||
|
||||
if (! $this->option('dry-run') && ! $this->option('force')) {
|
||||
if (! $this->confirm('Copy existing media from "'.$sourceHost.'" to "'.$destHost.'" and rewrite URLs?', true)) {
|
||||
$this->comment('Aborted.');
|
||||
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
$limit = (int) $this->option('limit');
|
||||
$moved = 0;
|
||||
$skipped = 0;
|
||||
$failed = 0;
|
||||
|
||||
// Candidates: non-remote media whose stored URL still points at the
|
||||
// source host (i.e. not yet migrated to the destination).
|
||||
$query = Media::whereRemoteMedia(false)
|
||||
->whereNotNull(['media_path', 'cdn_url'])
|
||||
->orderByDesc('id')
|
||||
->limit($limit);
|
||||
|
||||
$bar = $this->output->createProgressBar($query->count());
|
||||
$bar->start();
|
||||
|
||||
foreach ($query->get() as $media) {
|
||||
$result = $this->migrateOne($media, $sourceDisk, $destDisk, $sourceHost, $destHost);
|
||||
match ($result) {
|
||||
'moved' => $moved++,
|
||||
'skipped' => $skipped++,
|
||||
default => $failed++,
|
||||
};
|
||||
$bar->advance();
|
||||
}
|
||||
|
||||
$bar->finish();
|
||||
$this->newLine(2);
|
||||
$this->info(($this->option('dry-run') ? '[dry-run] ' : '').'Done. moved='.$moved.' skipped='.$skipped.' failed='.$failed.'.');
|
||||
if ($this->movedBytes) {
|
||||
$this->info('Transferred '.PrettyNumber::size($this->movedBytes).' to the new bucket.');
|
||||
}
|
||||
if ($moved > 0 && ! $this->option('dry-run')) {
|
||||
$this->comment('Tip: run `php artisan cache:clear` if any stale URLs remain cached elsewhere.');
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return string one of moved|skipped|failed
|
||||
*/
|
||||
protected function migrateOne(Media $media, $sourceDisk, $destDisk, string $sourceHost, string $destHost): string
|
||||
{
|
||||
if (Str::startsWith((string) $media->media_path, 'http')) {
|
||||
return 'skipped';
|
||||
}
|
||||
|
||||
// Only act on rows whose URL still references the source host.
|
||||
$currentHost = parse_url((string) $media->cdn_url, PHP_URL_HOST);
|
||||
if (! $currentHost || strcasecmp($currentHost, $sourceHost) !== 0) {
|
||||
return 'skipped';
|
||||
}
|
||||
|
||||
// If already present on the destination, just rewrite the URLs.
|
||||
$onDest = $destDisk->exists($media->media_path);
|
||||
$onSource = $sourceDisk->exists($media->media_path);
|
||||
|
||||
if (! $onDest && ! $onSource) {
|
||||
// File missing from both buckets; leave URLs untouched.
|
||||
return 'skipped';
|
||||
}
|
||||
|
||||
if ($this->option('dry-run')) {
|
||||
return 'moved';
|
||||
}
|
||||
|
||||
try {
|
||||
if (! $onDest) {
|
||||
// Copy primary + thumbnail source -> destination.
|
||||
$this->copy($media->media_path, $sourceDisk, $destDisk);
|
||||
if ($media->thumbnail_path && $sourceDisk->exists($media->thumbnail_path)) {
|
||||
$this->copy($media->thumbnail_path, $sourceDisk, $destDisk);
|
||||
}
|
||||
|
||||
if (! $this->verify($media->media_path, $sourceDisk, $destDisk, $media->original_sha256)) {
|
||||
$this->warn(PHP_EOL.'Verify failed for media '.$media->id.' ('.$media->media_path.'); left source intact, URLs unchanged.');
|
||||
|
||||
return 'failed';
|
||||
}
|
||||
}
|
||||
|
||||
// Rewrite URLs to the destination bucket.
|
||||
$media->cdn_url = $destDisk->url($media->media_path);
|
||||
$media->optimized_url = $media->cdn_url;
|
||||
if ($media->thumbnail_path && $destDisk->exists($media->thumbnail_path)) {
|
||||
$media->thumbnail_url = $destDisk->url($media->thumbnail_path);
|
||||
}
|
||||
$media->replicated_at = now();
|
||||
$media->save();
|
||||
|
||||
$this->movedBytes += (int) $media->size;
|
||||
|
||||
// Integrated GC on the OLD bucket, unless kept.
|
||||
if (! $this->option('keep-source') && $onSource) {
|
||||
$sourceDisk->delete($media->media_path);
|
||||
if ($media->thumbnail_path && $sourceDisk->exists($media->thumbnail_path)) {
|
||||
$sourceDisk->delete($media->thumbnail_path);
|
||||
}
|
||||
}
|
||||
|
||||
if ($media->status_id) {
|
||||
MediaService::del($media->status_id);
|
||||
StatusService::del($media->status_id, false);
|
||||
}
|
||||
|
||||
return 'moved';
|
||||
} catch (\Throwable $e) {
|
||||
$this->warn(PHP_EOL.'Error migrating media '.$media->id.': '.$e->getMessage());
|
||||
|
||||
return 'failed';
|
||||
}
|
||||
}
|
||||
|
||||
protected function copy(string $path, $sourceDisk, $destDisk): void
|
||||
{
|
||||
$stream = $sourceDisk->readStream($path);
|
||||
if ($stream === false || $stream === null) {
|
||||
throw new \RuntimeException('Could not open source stream for '.$path);
|
||||
}
|
||||
$destDisk->writeStream($path, $stream);
|
||||
if (is_resource($stream)) {
|
||||
fclose($stream);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify the destination copy matches the source by size, and by sha256
|
||||
* against the stored original checksum when available. Fails closed.
|
||||
*/
|
||||
protected function verify(string $path, $sourceDisk, $destDisk, ?string $expectedSha = null): bool
|
||||
{
|
||||
if (! $destDisk->exists($path)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$sourceSize = $sourceDisk->size($path);
|
||||
$destSize = $destDisk->size($path);
|
||||
if ($sourceSize === false || $destSize === false || $sourceSize !== $destSize) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if ($expectedSha) {
|
||||
// Hash the freshly written destination object to confirm integrity.
|
||||
$stream = $destDisk->readStream($path);
|
||||
if ($stream === false || $stream === null) {
|
||||
return false;
|
||||
}
|
||||
$ctx = hash_init('sha256');
|
||||
hash_update_stream($ctx, $stream);
|
||||
if (is_resource($stream)) {
|
||||
fclose($stream);
|
||||
}
|
||||
$destSha = hash_final($ctx);
|
||||
if (! hash_equals($expectedSha, $destSha)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
protected function diskHost($disk): ?string
|
||||
{
|
||||
try {
|
||||
$url = $disk->url('probe');
|
||||
$host = parse_url($url, PHP_URL_HOST);
|
||||
|
||||
return $host ?: null;
|
||||
} catch (\Throwable $e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,227 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands\Admin;
|
||||
|
||||
use App\Console\Commands\Concerns\ManagesMediaStorageEnv;
|
||||
use App\Models\Media;
|
||||
use App\Services\MediaService;
|
||||
use App\Services\StatusService;
|
||||
use App\Util\Lexer\PrettyNumber;
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Illuminate\Support\Str;
|
||||
|
||||
class MediaMoveStorageCloudToLocal extends Command
|
||||
{
|
||||
use ManagesMediaStorageEnv;
|
||||
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'admin:MediaMoveStorageCloudToLocal
|
||||
{--limit=500 : Max media rows to process this run}
|
||||
{--dry-run : Report what would happen without copying or writing}
|
||||
{--keep-cloud : Do not delete the cloud copy after verifying the local file}
|
||||
{--force : Skip confirmation prompts}';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Migrate cloud media back to local storage: download, verify (size/sha256), update URLs, then optionally delete the cloud copy. Ensures new uploads stay local during the migration.';
|
||||
|
||||
protected int $movedBytes = 0;
|
||||
|
||||
public function handle()
|
||||
{
|
||||
$localDisk = Storage::disk('local');
|
||||
$cloudDisk = Storage::disk(config('filesystems.cloud'));
|
||||
|
||||
// --- Ensure new uploads stay LOCAL during the migration -----------
|
||||
$envCloud = $this->readEnvValue('PF_ENABLE_CLOUD');
|
||||
$cloudEnabled = filter_var($envCloud, FILTER_VALIDATE_BOOLEAN);
|
||||
|
||||
if ($cloudEnabled) {
|
||||
$this->warn('PF_ENABLE_CLOUD is currently true.');
|
||||
$this->line('New uploads would keep landing on CLOUD storage during this migration.');
|
||||
if ($this->option('dry-run')) {
|
||||
$this->line('[dry-run] Would set PF_ENABLE_CLOUD=false (.env + runtime + config cache).');
|
||||
} elseif ($this->option('force') || $this->confirm('Set PF_ENABLE_CLOUD=false now so new uploads stay local?', true)) {
|
||||
$this->setStorageEnv('PF_ENABLE_CLOUD', 'false', 'pixelfed.cloud_storage', false);
|
||||
$this->info('PF_ENABLE_CLOUD set to false (.env + live runtime + config cache).');
|
||||
} else {
|
||||
$this->error('Aborting: refusing to migrate to local while new uploads go to cloud.');
|
||||
|
||||
return 1;
|
||||
}
|
||||
} else {
|
||||
$this->info('PF_ENABLE_CLOUD is already false; new uploads stay local. ✓');
|
||||
}
|
||||
|
||||
$this->newLine();
|
||||
if (! $this->option('dry-run') && ! $this->option('force')) {
|
||||
if (! $this->confirm('Begin migrating cloud media to local?', true)) {
|
||||
$this->comment('Aborted.');
|
||||
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
$limit = (int) $this->option('limit');
|
||||
$moved = 0;
|
||||
$skipped = 0;
|
||||
$failed = 0;
|
||||
|
||||
// Candidates: non-remote media that has a cloud copy (cdn_url set).
|
||||
$query = Media::whereRemoteMedia(false)
|
||||
->whereNotNull(['media_path', 'cdn_url'])
|
||||
->orderByDesc('id')
|
||||
->limit($limit);
|
||||
|
||||
$bar = $this->output->createProgressBar($query->count());
|
||||
$bar->start();
|
||||
|
||||
foreach ($query->get() as $media) {
|
||||
$result = $this->migrateOne($media, $localDisk, $cloudDisk);
|
||||
match ($result) {
|
||||
'moved' => $moved++,
|
||||
'skipped' => $skipped++,
|
||||
default => $failed++,
|
||||
};
|
||||
$bar->advance();
|
||||
}
|
||||
|
||||
$bar->finish();
|
||||
$this->newLine(2);
|
||||
$this->info(($this->option('dry-run') ? '[dry-run] ' : '').'Done. moved='.$moved.' skipped='.$skipped.' failed='.$failed.'.');
|
||||
if ($this->movedBytes) {
|
||||
$this->info('Transferred '.PrettyNumber::size($this->movedBytes).' back to local storage.');
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return string one of moved|skipped|failed
|
||||
*/
|
||||
protected function migrateOne(Media $media, $localDisk, $cloudDisk): string
|
||||
{
|
||||
if (Str::startsWith((string) $media->media_path, 'http')) {
|
||||
return 'skipped';
|
||||
}
|
||||
|
||||
// Must exist on cloud to pull down.
|
||||
if (! $cloudDisk->exists($media->media_path)) {
|
||||
// Already local-only? just clear the cloud url fields.
|
||||
if ($localDisk->exists($media->media_path)) {
|
||||
if (! $this->option('dry-run')) {
|
||||
$this->clearCloudFields($media);
|
||||
$media->save();
|
||||
}
|
||||
|
||||
return 'skipped';
|
||||
}
|
||||
|
||||
return 'failed';
|
||||
}
|
||||
|
||||
if ($this->option('dry-run')) {
|
||||
return 'moved';
|
||||
}
|
||||
|
||||
try {
|
||||
$this->copyToLocal($media->media_path, $localDisk, $cloudDisk);
|
||||
if ($media->thumbnail_path && $cloudDisk->exists($media->thumbnail_path)) {
|
||||
$this->copyToLocal($media->thumbnail_path, $localDisk, $cloudDisk);
|
||||
}
|
||||
|
||||
if (! $this->verify($media->media_path, $localDisk, $cloudDisk, $media->original_sha256)) {
|
||||
$this->warn(PHP_EOL.'Verify failed for media '.$media->id.' ('.$media->media_path.'); left cloud copy intact.');
|
||||
|
||||
return 'failed';
|
||||
}
|
||||
|
||||
// Point URLs back at local storage.
|
||||
$this->clearCloudFields($media);
|
||||
|
||||
// Integrated GC: delete the verified cloud copy unless --keep-cloud.
|
||||
if (! $this->option('keep-cloud')) {
|
||||
$cloudDisk->delete($media->media_path);
|
||||
if ($media->thumbnail_path && $cloudDisk->exists($media->thumbnail_path)) {
|
||||
$cloudDisk->delete($media->thumbnail_path);
|
||||
}
|
||||
}
|
||||
|
||||
$media->save();
|
||||
$this->movedBytes += (int) $media->size;
|
||||
|
||||
if ($media->status_id) {
|
||||
MediaService::del($media->status_id);
|
||||
StatusService::del($media->status_id, false);
|
||||
}
|
||||
|
||||
return 'moved';
|
||||
} catch (\Throwable $e) {
|
||||
$this->warn(PHP_EOL.'Error migrating media '.$media->id.': '.$e->getMessage());
|
||||
|
||||
return 'failed';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset a media row to local-served state.
|
||||
*/
|
||||
protected function clearCloudFields(Media $media): void
|
||||
{
|
||||
$media->cdn_url = null;
|
||||
$media->optimized_url = null;
|
||||
$media->thumbnail_url = null;
|
||||
$media->replicated_at = null;
|
||||
// version 4 meant "local deleted, cloud only"; reset so the file is
|
||||
// treated as locally present again.
|
||||
if ($media->version === '4' || $media->version === 4) {
|
||||
$media->version = 3;
|
||||
}
|
||||
}
|
||||
|
||||
protected function copyToLocal(string $path, $localDisk, $cloudDisk): void
|
||||
{
|
||||
$stream = $cloudDisk->readStream($path);
|
||||
if ($stream === false || $stream === null) {
|
||||
throw new \RuntimeException('Could not open cloud stream for '.$path);
|
||||
}
|
||||
$localDisk->writeStream($path, $stream);
|
||||
if (is_resource($stream)) {
|
||||
fclose($stream);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify the local copy matches the cloud source by size, and by sha256
|
||||
* against the stored original checksum when available. Fails closed.
|
||||
*/
|
||||
protected function verify(string $path, $localDisk, $cloudDisk, ?string $expectedSha = null): bool
|
||||
{
|
||||
if (! $localDisk->exists($path)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$localSize = $localDisk->size($path);
|
||||
$cloudSize = $cloudDisk->size($path);
|
||||
if ($localSize === false || $cloudSize === false || $localSize !== $cloudSize) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if ($expectedSha) {
|
||||
$localSha = @hash_file('sha256', $localDisk->path($path));
|
||||
if ($localSha && ! hash_equals($expectedSha, $localSha)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,239 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands\Admin;
|
||||
|
||||
use App\Console\Commands\Concerns\ManagesMediaStorageEnv;
|
||||
use App\Models\Media;
|
||||
use App\Services\MediaService;
|
||||
use App\Services\ResilientMediaStorageService;
|
||||
use App\Services\StatusService;
|
||||
use App\Util\Lexer\PrettyNumber;
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Illuminate\Support\Str;
|
||||
|
||||
class MediaMoveStorageLocalToCloud extends Command
|
||||
{
|
||||
use ManagesMediaStorageEnv;
|
||||
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'admin:MediaMoveStorageLocalToCloud
|
||||
{--limit=500 : Max media rows to process this run}
|
||||
{--dry-run : Report what would happen without copying or writing}
|
||||
{--keep-local : Do not delete local files after verifying the cloud copy}
|
||||
{--force : Skip confirmation prompts}';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Migrate local media to cloud storage: copy up, verify (size/sha256), update URLs, then delete the local copy. Ensures new uploads go to cloud during the migration.';
|
||||
|
||||
protected int $movedBytes = 0;
|
||||
|
||||
public function handle()
|
||||
{
|
||||
try {
|
||||
$localDisk = Storage::disk('local');
|
||||
$cloudDisk = Storage::disk(config('filesystems.cloud'));
|
||||
} catch (\Throwable $e) {
|
||||
$this->error('Cloud disk ('.config('filesystems.cloud').') could not be resolved: '.$e->getMessage());
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (! $this->cloudHost()) {
|
||||
$this->error('Cloud disk ('.config('filesystems.cloud').') is not configured (no resolvable URL).');
|
||||
$this->line('Set AWS_URL / AWS_* in your .env before migrating to cloud.');
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
// --- Ensure new uploads route to cloud during the migration --------
|
||||
$envCloud = $this->readEnvValue('PF_ENABLE_CLOUD');
|
||||
$cloudEnabled = filter_var($envCloud, FILTER_VALIDATE_BOOLEAN);
|
||||
|
||||
if (! $cloudEnabled) {
|
||||
$this->warn('PF_ENABLE_CLOUD is currently "'.($envCloud ?? 'unset').'".');
|
||||
$this->line('New uploads would keep landing on LOCAL storage during this migration.');
|
||||
if ($this->option('dry-run')) {
|
||||
$this->line('[dry-run] Would set PF_ENABLE_CLOUD=true (.env + runtime + config cache).');
|
||||
} elseif ($this->option('force') || $this->confirm('Set PF_ENABLE_CLOUD=true now so new uploads go to cloud?', true)) {
|
||||
$this->setStorageEnv('PF_ENABLE_CLOUD', 'true', 'pixelfed.cloud_storage', true);
|
||||
$this->info('PF_ENABLE_CLOUD set to true (.env + live runtime + config cache).');
|
||||
} else {
|
||||
$this->error('Aborting: refusing to migrate to cloud while new uploads stay local.');
|
||||
|
||||
return 1;
|
||||
}
|
||||
} else {
|
||||
$this->info('PF_ENABLE_CLOUD is already true; new uploads route to cloud. ✓');
|
||||
}
|
||||
|
||||
$this->newLine();
|
||||
if (! $this->option('dry-run') && ! $this->option('force')) {
|
||||
if (! $this->confirm('Begin migrating local media to cloud?', true)) {
|
||||
$this->comment('Aborted.');
|
||||
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
$limit = (int) $this->option('limit');
|
||||
$moved = 0;
|
||||
$skipped = 0;
|
||||
$failed = 0;
|
||||
|
||||
// Candidates: local, non-remote media not yet replicated to cloud.
|
||||
$query = Media::whereRemoteMedia(false)
|
||||
->whereNotNull('media_path')
|
||||
->where(function ($q) {
|
||||
$q->whereNull('cdn_url')->orWhereNull('replicated_at')->orWhereNot('version', '4');
|
||||
})
|
||||
->orderByDesc('id')
|
||||
->limit($limit);
|
||||
|
||||
$bar = $this->output->createProgressBar($query->count());
|
||||
$bar->start();
|
||||
|
||||
foreach ($query->get() as $media) {
|
||||
$result = $this->migrateOne($media, $localDisk, $cloudDisk);
|
||||
match ($result) {
|
||||
'moved' => $moved++,
|
||||
'skipped' => $skipped++,
|
||||
default => $failed++,
|
||||
};
|
||||
$bar->advance();
|
||||
}
|
||||
|
||||
$bar->finish();
|
||||
$this->newLine(2);
|
||||
$this->info(($this->option('dry-run') ? '[dry-run] ' : '').'Done. moved='.$moved.' skipped='.$skipped.' failed='.$failed.'.');
|
||||
if ($this->movedBytes) {
|
||||
$this->info('Transferred '.PrettyNumber::size($this->movedBytes).' to cloud storage.');
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return string one of moved|skipped|failed
|
||||
*/
|
||||
protected function migrateOne(Media $media, $localDisk, $cloudDisk): string
|
||||
{
|
||||
if (Str::startsWith((string) $media->media_path, 'http')) {
|
||||
return 'skipped';
|
||||
}
|
||||
|
||||
// Nothing to do if the local file is gone.
|
||||
if (! $localDisk->exists($media->media_path)) {
|
||||
// Already on cloud only? mark version and move on.
|
||||
if ($cloudDisk->exists($media->media_path)) {
|
||||
if (! $this->option('dry-run') && $media->version !== '4') {
|
||||
$media->version = 4;
|
||||
$media->save();
|
||||
}
|
||||
|
||||
return 'skipped';
|
||||
}
|
||||
|
||||
return 'skipped';
|
||||
}
|
||||
|
||||
if ($this->option('dry-run')) {
|
||||
return 'moved';
|
||||
}
|
||||
|
||||
try {
|
||||
// Copy the primary file (and thumbnail) to cloud.
|
||||
$this->copyToCloud($media->media_path, $localDisk, $cloudDisk);
|
||||
if ($media->thumbnail_path && $localDisk->exists($media->thumbnail_path)) {
|
||||
$this->copyToCloud($media->thumbnail_path, $localDisk, $cloudDisk);
|
||||
}
|
||||
|
||||
// Verify the primary file before touching anything else.
|
||||
if (! $this->verify($media->media_path, $localDisk, $cloudDisk, $media->original_sha256)) {
|
||||
$this->warn(PHP_EOL.'Verify failed for media '.$media->id.' ('.$media->media_path.'); left local copy intact.');
|
||||
|
||||
return 'failed';
|
||||
}
|
||||
|
||||
// Update URL fields to the cloud disk.
|
||||
$media->cdn_url = $cloudDisk->url($media->media_path);
|
||||
$media->optimized_url = $media->cdn_url;
|
||||
if ($media->thumbnail_path && $cloudDisk->exists($media->thumbnail_path)) {
|
||||
$media->thumbnail_url = $cloudDisk->url($media->thumbnail_path);
|
||||
}
|
||||
$media->replicated_at = now();
|
||||
|
||||
// Integrated GC: delete the verified local copy unless --keep-local.
|
||||
if (! $this->option('keep-local')) {
|
||||
$localDisk->delete($media->media_path);
|
||||
if ($media->thumbnail_path && $localDisk->exists($media->thumbnail_path)) {
|
||||
$localDisk->delete($media->thumbnail_path);
|
||||
}
|
||||
$media->version = 4;
|
||||
}
|
||||
|
||||
$media->save();
|
||||
$this->movedBytes += (int) $media->size;
|
||||
|
||||
if ($media->status_id) {
|
||||
MediaService::del($media->status_id);
|
||||
StatusService::del($media->status_id, false);
|
||||
}
|
||||
|
||||
return 'moved';
|
||||
} catch (\Throwable $e) {
|
||||
$this->warn(PHP_EOL.'Error migrating media '.$media->id.': '.$e->getMessage());
|
||||
|
||||
return 'failed';
|
||||
}
|
||||
}
|
||||
|
||||
protected function copyToCloud(string $path, $localDisk, $cloudDisk): void
|
||||
{
|
||||
$p = explode('/', $path);
|
||||
$name = array_pop($p);
|
||||
$storagePath = implode('/', $p);
|
||||
|
||||
// Reuse the resilient uploader (handles alt disks + retries).
|
||||
ResilientMediaStorageService::store($storagePath, $localDisk->path($path), $name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify the cloud copy matches the local source by size, and by sha256
|
||||
* when a checksum is available/cheap. Fails closed.
|
||||
*/
|
||||
protected function verify(string $path, $localDisk, $cloudDisk, ?string $expectedSha = null): bool
|
||||
{
|
||||
if (! $cloudDisk->exists($path)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$localSize = $localDisk->size($path);
|
||||
$cloudSize = $cloudDisk->size($path);
|
||||
if ($localSize === false || $cloudSize === false || $localSize !== $cloudSize) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// If we already have the original checksum, verify the local file still
|
||||
// matches it (so we never delete a locally-corrupted-but-uploaded file
|
||||
// without noticing). Cloud content hashing would require a full
|
||||
// download, which we avoid for large media; size parity + known sha
|
||||
// is a strong signal.
|
||||
if ($expectedSha) {
|
||||
$localSha = @hash_file('sha256', $localDisk->path($path));
|
||||
if ($localSha && ! hash_equals($expectedSha, $localSha)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,382 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands\Admin;
|
||||
|
||||
use App\Models\Media;
|
||||
use App\Models\Status;
|
||||
use App\Services\MediaService;
|
||||
use App\Services\StatusService;
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Illuminate\Support\Str;
|
||||
|
||||
class MigrateLocalS3MediaURL extends Command
|
||||
{
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'admin:MigrateLocalS3MediaURL
|
||||
{id? : A status id (or post URL) to fix; omit with --all}
|
||||
{--all : Scan every local media row and fix any with a stale host}
|
||||
{--oldDomain= : Only rewrite URLs whose host matches this old backend (default: rewrite all stale hosts)}
|
||||
{--newDomain= : Target host to rewrite to (default: the configured cloud disk host from .env)}
|
||||
{--dry-run : Report what would change without writing}
|
||||
{--force : Skip the confirmation prompt}';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Rewrite stale local media cloud URLs (cdn_url, thumbnail_url, optimized_url) from their storage paths to the configured S3/cloud host. Replaces media:cloud-url-rewrite.';
|
||||
|
||||
/**
|
||||
* The target host to rewrite URLs to.
|
||||
*/
|
||||
protected ?string $newHost = null;
|
||||
|
||||
/**
|
||||
* Optional old host filter; when set, only URLs on this host are rewritten.
|
||||
*/
|
||||
protected ?string $oldHost = null;
|
||||
|
||||
public function handle()
|
||||
{
|
||||
// This command only makes sense for instances serving media from a
|
||||
// cloud/object-storage backend. Local-storage instances (PF_ENABLE_CLOUD
|
||||
// unset/false) serve media from the app domain and have no cloud
|
||||
// cdn_url to migrate, so refuse rather than risk rewriting local URLs.
|
||||
if (! (bool) config_cache('pixelfed.cloud_storage')) {
|
||||
$this->error('Cloud storage is not enabled (PF_ENABLE_CLOUD is false).');
|
||||
$this->line('This instance serves media from local storage; there are no cloud media URLs to migrate.');
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Safe default target = the currently configured cloud disk host,
|
||||
// driven by AWS_URL in .env. Allow explicit override via --newDomain.
|
||||
$configuredHost = $this->cloudHost();
|
||||
$override = $this->normalizeHost($this->option('newDomain'));
|
||||
$this->newHost = $override ?: $configuredHost;
|
||||
|
||||
if (! $this->newHost) {
|
||||
$this->error('Could not resolve a target host.');
|
||||
$this->line('The cloud disk ('.config('filesystems.cloud').') did not return a usable URL.');
|
||||
$this->line('Set AWS_URL in your .env, or pass --newDomain explicitly.');
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Defensive guard: when auto-detecting the target (no --newDomain
|
||||
// override), never rewrite media URLs to the app's own domain. That
|
||||
// would indicate local storage or a misconfigured cloud disk URL
|
||||
// (AWS_URL). An explicit --newDomain is treated as a deliberate choice.
|
||||
if (! $override) {
|
||||
$appHost = parse_url(config('app.url'), PHP_URL_HOST);
|
||||
if ($appHost && strcasecmp($this->newHost, $appHost) === 0) {
|
||||
$this->error('Refusing to run: auto-detected target host ('.$this->newHost.') is the app domain.');
|
||||
$this->line('That indicates local storage or a misconfigured cloud disk URL (AWS_URL).');
|
||||
$this->line('If you really intend this, pass an explicit --newDomain.');
|
||||
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
$this->oldHost = $this->normalizeHost($this->option('oldDomain'));
|
||||
|
||||
$id = $this->argument('id');
|
||||
$all = $this->option('all');
|
||||
|
||||
if (! $id && ! $all) {
|
||||
$this->error('Provide a status id/URL, or pass --all.');
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
if ($id && $all) {
|
||||
$this->error('Pass either a status id or --all, not both.');
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Show the plan and require explicit approval of the target host.
|
||||
$this->info('Target host (newDomain): '.$this->newHost.($override ? ' (override)' : ' (from configured cloud disk)'));
|
||||
$this->info('Filter (oldDomain): '.($this->oldHost ?: 'none — rewriting all stale hosts'));
|
||||
if ($this->newHost !== $configuredHost) {
|
||||
$this->warn('Note: target host differs from the configured cloud disk host ('.($configuredHost ?: 'unresolved').').');
|
||||
}
|
||||
|
||||
if (! $this->option('dry-run') && ! $this->option('force')) {
|
||||
if (! $this->confirm('Rewrite media URLs to "'.$this->newHost.'"?', false)) {
|
||||
$this->comment('Aborted.');
|
||||
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
$this->newLine();
|
||||
|
||||
if ($id) {
|
||||
return $this->handleSingle($id);
|
||||
}
|
||||
|
||||
return $this->handleAll();
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract a bare host from a domain/URL option value.
|
||||
*/
|
||||
protected function normalizeHost(?string $value): ?string
|
||||
{
|
||||
$value = trim((string) $value);
|
||||
if ($value === '') {
|
||||
return null;
|
||||
}
|
||||
// Accept full URLs or bare hosts.
|
||||
if (str_contains($value, '://')) {
|
||||
$host = parse_url($value, PHP_URL_HOST);
|
||||
|
||||
return $host ?: null;
|
||||
}
|
||||
|
||||
// Strip any accidental path/scheme fragments.
|
||||
$host = parse_url('https://'.$value, PHP_URL_HOST);
|
||||
|
||||
return $host ?: null;
|
||||
}
|
||||
|
||||
protected function handleSingle(string $id): int
|
||||
{
|
||||
$statusId = $this->resolveStatusId($id);
|
||||
if (! $statusId) {
|
||||
$this->error('Could not extract a status id from "'.$id.'".');
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
$status = Status::withTrashed()->find($statusId);
|
||||
if (! $status) {
|
||||
$this->error('No status found with id '.$statusId.'.');
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
$media = Media::whereStatusId($status->id)->get();
|
||||
if ($media->isEmpty()) {
|
||||
$this->comment('Status '.$status->id.' has no media.');
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
$fixed = 0;
|
||||
foreach ($media as $m) {
|
||||
if ($this->migrateOne($m)) {
|
||||
$fixed++;
|
||||
}
|
||||
}
|
||||
|
||||
if ($fixed > 0 && ! $this->option('dry-run')) {
|
||||
$this->bustCaches($status->id);
|
||||
}
|
||||
|
||||
$this->newLine();
|
||||
$this->info(($this->option('dry-run') ? 'Would fix ' : 'Fixed ').$fixed.' media row(s) for status '.$status->id.'.');
|
||||
if ($fixed > 0 && ! $this->option('dry-run')) {
|
||||
$this->comment('Caches busted for this status.');
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
protected function handleAll(): int
|
||||
{
|
||||
$fixed = 0;
|
||||
$scanned = 0;
|
||||
$affectedStatusIds = [];
|
||||
|
||||
Media::whereNull('remote_url')
|
||||
->where(function ($q) {
|
||||
$q->whereNull('remote_media')->orWhere('remote_media', false);
|
||||
})
|
||||
->lazyById(1000, 'id')
|
||||
->each(function ($m) use (&$fixed, &$scanned, &$affectedStatusIds) {
|
||||
$scanned++;
|
||||
if ($this->migrateOne($m)) {
|
||||
$fixed++;
|
||||
if ($m->status_id) {
|
||||
$affectedStatusIds[$m->status_id] = true;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (! $this->option('dry-run')) {
|
||||
foreach (array_keys($affectedStatusIds) as $sid) {
|
||||
$this->bustCaches($sid);
|
||||
}
|
||||
}
|
||||
|
||||
$this->newLine();
|
||||
$this->info('Scanned '.$scanned.' local media rows; '.($this->option('dry-run') ? 'would fix ' : 'fixed ').$fixed.'.');
|
||||
if ($fixed > 0 && ! $this->option('dry-run')) {
|
||||
$this->comment('Caches busted for '.count($affectedStatusIds).' affected status(es).');
|
||||
$this->comment('Tip: run `php artisan cache:clear` if any stale URLs remain cached elsewhere.');
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rebuild any stale URL field on a single media row from its storage path.
|
||||
* Only writes when a field's host differs from the target host.
|
||||
*
|
||||
* @return bool whether the row was (or would be) changed
|
||||
*/
|
||||
protected function migrateOne(Media $media): bool
|
||||
{
|
||||
// Never touch remote media or rows whose media_path is an absolute URL.
|
||||
if ($media->remote_media || Str::startsWith((string) $media->media_path, 'http')) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$changes = [];
|
||||
|
||||
// cdn_url and optimized_url are both derived from media_path;
|
||||
// thumbnail_url is derived from thumbnail_path.
|
||||
$map = [
|
||||
'cdn_url' => $media->media_path,
|
||||
'optimized_url' => $media->media_path,
|
||||
'thumbnail_url' => $media->thumbnail_path,
|
||||
];
|
||||
|
||||
foreach ($map as $field => $path) {
|
||||
$current = $media->{$field};
|
||||
if (! $current) {
|
||||
// Field not set; leave it as-is (nothing to migrate).
|
||||
continue;
|
||||
}
|
||||
if (! $path) {
|
||||
// No source path to rebuild from; skip.
|
||||
continue;
|
||||
}
|
||||
$host = parse_url($current, PHP_URL_HOST);
|
||||
if (! $this->shouldRewrite($host)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$rebuilt = $this->targetUrl($path);
|
||||
if (! $rebuilt) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$changes[$field] = ['from' => $current, 'to' => $rebuilt];
|
||||
}
|
||||
|
||||
if (empty($changes)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$this->warn('media '.$media->id.(($media->status_id) ? ' (status '.$media->status_id.')' : '').':');
|
||||
foreach ($changes as $field => $c) {
|
||||
$fromHost = parse_url($c['from'], PHP_URL_HOST);
|
||||
$this->line(' '.$field.': '.$fromHost.' -> '.$this->newHost);
|
||||
}
|
||||
|
||||
if ($this->option('dry-run')) {
|
||||
return true;
|
||||
}
|
||||
|
||||
foreach ($changes as $field => $c) {
|
||||
$media->{$field} = $c['to'];
|
||||
}
|
||||
$media->save();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
protected function bustCaches($statusId): void
|
||||
{
|
||||
MediaService::del($statusId);
|
||||
StatusService::del($statusId, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Decide whether a URL on $currentHost should be rewritten.
|
||||
* Skips when already on the target host, and honours the optional
|
||||
* --oldDomain filter.
|
||||
*/
|
||||
protected function shouldRewrite(?string $currentHost): bool
|
||||
{
|
||||
if (! $currentHost) {
|
||||
return false;
|
||||
}
|
||||
// Already on the target host.
|
||||
if (strcasecmp($currentHost, $this->newHost) === 0) {
|
||||
return false;
|
||||
}
|
||||
// With --oldDomain, only rewrite that specific host.
|
||||
if ($this->oldHost !== null && strcasecmp($currentHost, $this->oldHost) !== 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the target URL for a storage path against the target host.
|
||||
* Uses the configured cloud disk to produce the correct path, then
|
||||
* swaps in --newDomain when it overrides the configured host.
|
||||
*/
|
||||
protected function targetUrl(string $path): ?string
|
||||
{
|
||||
$url = $this->diskUrl($path);
|
||||
if (! $url) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$diskHost = parse_url($url, PHP_URL_HOST);
|
||||
if ($diskHost && strcasecmp($diskHost, $this->newHost) !== 0) {
|
||||
// Override host was requested; swap it into the disk-built URL.
|
||||
$url = preg_replace('#^(https?://)'.preg_quote($diskHost, '#').'#i', '$1'.$this->newHost, $url);
|
||||
}
|
||||
|
||||
return $url;
|
||||
}
|
||||
|
||||
protected function diskUrl(string $path): ?string
|
||||
{
|
||||
try {
|
||||
return (string) Storage::disk(config('filesystems.cloud'))->url($path);
|
||||
} catch (\Throwable $e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
protected function cloudHost(): ?string
|
||||
{
|
||||
try {
|
||||
$url = Storage::disk(config('filesystems.cloud'))->url('probe');
|
||||
$host = parse_url($url, PHP_URL_HOST);
|
||||
|
||||
return $host ?: null;
|
||||
} catch (\Throwable $e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
protected function resolveStatusId(string $input): ?string
|
||||
{
|
||||
$input = trim($input);
|
||||
if (ctype_digit($input)) {
|
||||
return $input;
|
||||
}
|
||||
if (preg_match('#/p/[^/]+/(\d+)#', $input, $m)) {
|
||||
return $m[1];
|
||||
}
|
||||
if (preg_match('#(\d{6,})#', $input, $m)) {
|
||||
return $m[1];
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Admin;
|
||||
|
||||
use App\Jobs\ImageOptimizePipeline\ImageThumbnail;
|
||||
use App\Models\Media;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Admin;
|
||||
|
||||
use App\Models\Instance;
|
||||
use App\Models\Profile;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Admin;
|
||||
|
||||
use App\Jobs\VideoPipeline\VideoThumbnail as Pipeline;
|
||||
use App\Models\Media;
|
||||
@ -1,102 +0,0 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Models\Media;
|
||||
use App\Services\MediaStorageService;
|
||||
use App\Util\Lexer\PrettyNumber;
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Symfony\Component\HttpFoundation\File\Exception\FileNotFoundException;
|
||||
use Symfony\Component\HttpKernel\Exception\NotFoundHttpException;
|
||||
|
||||
class CloudMediaMigrate extends Command
|
||||
{
|
||||
public $totalSize = 0;
|
||||
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'media:migrate2cloud {--limit=200} {--huge}';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Move older media to cloud storage';
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
*
|
||||
* @return int
|
||||
*/
|
||||
public function handle()
|
||||
{
|
||||
$enabled = (bool) config_cache('pixelfed.cloud_storage');
|
||||
if (! $enabled) {
|
||||
$this->error('Cloud storage not enabled. Exiting...');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (! $this->confirm('Are you sure you want to proceed?')) {
|
||||
return;
|
||||
}
|
||||
|
||||
$limit = $this->option('limit');
|
||||
$hugeMode = $this->option('huge');
|
||||
|
||||
if ($limit > 500 && ! $hugeMode) {
|
||||
$this->error('Max limit exceeded, use a limit lower than 500 or run again with the --huge flag');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$bar = $this->output->createProgressBar($limit);
|
||||
$bar->start();
|
||||
|
||||
Media::whereNot('version', '4')
|
||||
->where('created_at', '<', now()->subDays(2))
|
||||
->whereRemoteMedia(false)
|
||||
->whereNotNull(['status_id', 'profile_id'])
|
||||
->whereNull(['cdn_url', 'replicated_at'])
|
||||
->orderByDesc('size')
|
||||
->take($limit)
|
||||
->get()
|
||||
->each(function ($media) use ($bar) {
|
||||
if (Storage::disk('local')->exists($media->media_path)) {
|
||||
$this->totalSize = $this->totalSize + $media->size;
|
||||
try {
|
||||
MediaStorageService::store($media);
|
||||
} catch (FileNotFoundException $e) {
|
||||
$this->error('Error migrating media '.$media->id.' to cloud storage: '.$e->getMessage());
|
||||
|
||||
return;
|
||||
} catch (NotFoundHttpException $e) {
|
||||
$this->error('Error migrating media '.$media->id.' to cloud storage: '.$e->getMessage());
|
||||
|
||||
return;
|
||||
} catch (\Exception $e) {
|
||||
$this->error('Error migrating media '.$media->id.' to cloud storage: '.$e->getMessage());
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
$bar->advance();
|
||||
});
|
||||
|
||||
$bar->finish();
|
||||
$this->line(' ');
|
||||
$this->info('Finished!');
|
||||
if ($this->totalSize) {
|
||||
$this->info('Uploaded '.PrettyNumber::size($this->totalSize).' of media to cloud storage!');
|
||||
$this->line(' ');
|
||||
$this->info('These files are still stored locally, and will be automatically removed.');
|
||||
}
|
||||
|
||||
return Command::SUCCESS;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,122 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands\Concerns;
|
||||
|
||||
use App\Services\ConfigCacheService;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
|
||||
/**
|
||||
* Shared helpers for the media storage migration commands.
|
||||
*
|
||||
* Handles reading/writing the .env storage flags atomically (reusing the
|
||||
* installer's vetted approach) and applying the change to the live runtime
|
||||
* so that new uploads route to the correct backend during a migration on a
|
||||
* hot (running) server, without requiring a restart.
|
||||
*/
|
||||
trait ManagesMediaStorageEnv
|
||||
{
|
||||
/**
|
||||
* Read the raw current value of an .env key (unquoted), or null.
|
||||
*/
|
||||
protected function readEnvValue(string $key): ?string
|
||||
{
|
||||
$envPath = app()->environmentFilePath();
|
||||
if (! is_file($envPath)) {
|
||||
return null;
|
||||
}
|
||||
$payload = file_get_contents($envPath);
|
||||
if ($payload === false) {
|
||||
return null;
|
||||
}
|
||||
if (! preg_match("/^{$key}=([^\r\n]*)/m", $payload, $m)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return trim($m[1], " \t\"'");
|
||||
}
|
||||
|
||||
/**
|
||||
* Set an .env key + the live runtime config + config-cache entry so the
|
||||
* change takes effect immediately on a running server.
|
||||
*
|
||||
* @param string $configKey dotted config key kept in sync (e.g. 'pixelfed.cloud_storage')
|
||||
* @param mixed $configValue the typed runtime value (e.g. true/false)
|
||||
*/
|
||||
protected function setStorageEnv(string $envKey, string $envValue, string $configKey, $configValue): void
|
||||
{
|
||||
// 1. Persist to .env atomically (survives restarts).
|
||||
$this->updateEnvFile($envKey, $envValue);
|
||||
|
||||
// 2. Update the live runtime config for the current process.
|
||||
config([$configKey => $configValue]);
|
||||
|
||||
// 3. Update the DB-backed config cache so other workers/requests
|
||||
// reading via config_cache() see the new value (hot server).
|
||||
try {
|
||||
ConfigCacheService::put($configKey, $configValue);
|
||||
} catch (\Throwable $e) {
|
||||
$this->warn('Could not update config cache for '.$configKey.': '.$e->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The configured cloud disk host (used to sanity check cloud config).
|
||||
*/
|
||||
protected function cloudHost(): ?string
|
||||
{
|
||||
try {
|
||||
$url = Storage::disk(config('filesystems.cloud'))->url('probe');
|
||||
$host = parse_url($url, PHP_URL_HOST);
|
||||
|
||||
return $host ?: null;
|
||||
} catch (\Throwable $e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Atomic .env writer (adapted from Installer) ---------------------
|
||||
|
||||
protected function updateEnvFile($key, $value): void
|
||||
{
|
||||
$envPath = app()->environmentFilePath();
|
||||
$payload = file_get_contents($envPath);
|
||||
|
||||
$value = str_replace(['\\', '"', "\n", "\r"], ['\\\\', '\\"', '\\n', '\\r'], $value);
|
||||
|
||||
if (($existing = $this->existingEnv($key, $payload)) !== false) {
|
||||
$payload = str_replace("{$key}={$existing}", "{$key}=\"{$value}\"", $payload);
|
||||
} else {
|
||||
$payload = $payload."\n{$key}=\"{$value}\"\n";
|
||||
}
|
||||
|
||||
$this->storeEnv($payload);
|
||||
}
|
||||
|
||||
protected function existingEnv($needle, $haystack)
|
||||
{
|
||||
preg_match("/^{$needle}=[^\r\n]*/m", $haystack, $matches);
|
||||
if ($matches && count($matches)) {
|
||||
return substr($matches[0], strlen($needle) + 1);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
protected function storeEnv($payload): void
|
||||
{
|
||||
$envPath = app()->environmentFilePath();
|
||||
$tempPath = $envPath.'.tmp';
|
||||
|
||||
$file = fopen($tempPath, 'w');
|
||||
if ($file === false) {
|
||||
throw new \RuntimeException("Cannot write to {$tempPath}");
|
||||
}
|
||||
fwrite($file, $payload);
|
||||
fclose($file);
|
||||
|
||||
if (! rename($tempPath, $envPath)) {
|
||||
@unlink($tempPath);
|
||||
throw new \RuntimeException('Cannot update .env file');
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Deprecated;
|
||||
|
||||
use App\Models\Avatar;
|
||||
use Illuminate\Console\Command;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Deprecated;
|
||||
|
||||
use App\Jobs\StatusPipeline\StatusDelete;
|
||||
use App\Models\Status;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Dev;
|
||||
|
||||
use Illuminate\Console\Command;
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Dev;
|
||||
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Support\Facades\File;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Dev;
|
||||
|
||||
use App\Models\User;
|
||||
use Illuminate\Console\Command;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Dev;
|
||||
|
||||
use App\Jobs\FollowPipeline\FollowPipeline;
|
||||
use App\Models\Follower;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\FixBugs;
|
||||
|
||||
use App\Jobs\AvatarPipeline\RemoteAvatarFetch;
|
||||
use App\Models\Avatar;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\FixBugs;
|
||||
|
||||
use App\Jobs\AvatarPipeline\AvatarStorageCleanup;
|
||||
use App\Models\Avatar;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\FixBugs;
|
||||
|
||||
use App\Jobs\ImageOptimizePipeline\ImageOptimize;
|
||||
use App\Models\Media;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\FixBugs;
|
||||
|
||||
use App\Models\Media;
|
||||
use App\Services\MediaService;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\FixBugs;
|
||||
|
||||
use App\Models\Avatar;
|
||||
use App\Models\Bookmark;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\FixBugs;
|
||||
|
||||
use App\Models\Hashtag;
|
||||
use App\Models\StatusHashtag;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\FixBugs;
|
||||
|
||||
use App\Models\Like;
|
||||
use App\Models\Status;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\FixBugs;
|
||||
|
||||
use App\Jobs\ImageOptimizePipeline\ImageOptimize;
|
||||
use App\Jobs\MediaPipeline\MediaFixLocalFilesystemCleanupPipeline;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\FixBugs;
|
||||
|
||||
use App\Jobs\AvatarPipeline\CreateAvatar;
|
||||
use App\Jobs\FollowPipeline\FollowPipeline;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\FixBugs;
|
||||
|
||||
use App\Jobs\FollowPipeline\FollowServiceWarmCache;
|
||||
use App\Models\Profile;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\FixBugs;
|
||||
|
||||
use App\Models\Profile;
|
||||
use App\Models\User;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\FixBugs;
|
||||
|
||||
use App\Models\Hashtag;
|
||||
use App\Models\HashtagRelated;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\FixBugs;
|
||||
|
||||
use App\Models\Media;
|
||||
use App\Util\Media\Filter;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Install;
|
||||
|
||||
use App\Models\InstanceActor;
|
||||
use Illuminate\Console\Command;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Install;
|
||||
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Foundation\Bootstrap\LoadEnvironmentVariables;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Install;
|
||||
|
||||
use Illuminate\Console\Command;
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Internal;
|
||||
|
||||
use App\Models\Profile;
|
||||
use App\Services\Account\AccountStatService;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Internal;
|
||||
|
||||
use App\Models\AppRegister;
|
||||
use Illuminate\Console\Command;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Internal;
|
||||
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Internal;
|
||||
|
||||
use App\Models\FailedJob;
|
||||
use Illuminate\Console\Command;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Internal;
|
||||
|
||||
use App\Models\Hashtag;
|
||||
use Illuminate\Console\Command;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Internal;
|
||||
|
||||
use App\Models\ImportPost;
|
||||
use App\Models\User;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Internal;
|
||||
|
||||
use App\Models\User;
|
||||
use Illuminate\Console\Command;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Internal;
|
||||
|
||||
use App\Models\ImportPost;
|
||||
use App\Services\ImportService;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Internal;
|
||||
|
||||
use App\Jobs\ImportPipeline\ImportMediaToCloudPipeline;
|
||||
use App\Models\ImportPost;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Internal;
|
||||
|
||||
use App\Services\ConfigCacheService;
|
||||
use Illuminate\Console\Command;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Internal;
|
||||
|
||||
use App\Models\Media;
|
||||
use App\Services\MediaStorageService;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Internal;
|
||||
|
||||
use App\Jobs\InternalPipeline\NotificationEpochUpdatePipeline;
|
||||
use Illuminate\Console\Command;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Internal;
|
||||
|
||||
use App\Models\EmailVerification;
|
||||
use Illuminate\Console\Command;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Internal;
|
||||
|
||||
use App\Services\NotificationAppGatewayService;
|
||||
use Illuminate\Console\Command;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Internal;
|
||||
|
||||
use App\Services\Internal\SoftwareUpdateService;
|
||||
use Illuminate\Console\Command;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Internal;
|
||||
|
||||
use App\Jobs\StoryPipeline\StoryExpire;
|
||||
use App\Jobs\StoryPipeline\StoryRotateMedia;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Internal;
|
||||
|
||||
use App\Models\ImportPost;
|
||||
use App\Models\Media;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\Internal;
|
||||
|
||||
use App\Jobs\InstancePipeline\FetchNodeinfoPipeline;
|
||||
use App\Models\Instance;
|
||||
@ -1,209 +0,0 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Models\Avatar;
|
||||
use App\Models\Media;
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Contracts\Console\PromptsForMissingInput;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
|
||||
use function Laravel\Prompts\select;
|
||||
|
||||
class MediaCloudUrlRewrite extends Command implements PromptsForMissingInput
|
||||
{
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'media:cloud-url-rewrite {oldDomain} {newDomain}';
|
||||
|
||||
/**
|
||||
* Prompt for missing input arguments using the returned questions.
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
protected function promptForMissingArgumentsUsing()
|
||||
{
|
||||
return [
|
||||
'oldDomain' => 'The old S3 domain',
|
||||
'newDomain' => 'The new S3 domain',
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Rewrite S3 media urls from local users';
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
*/
|
||||
public function handle()
|
||||
{
|
||||
$this->preflightCheck();
|
||||
$this->bootMessage();
|
||||
$this->confirmCloudUrl();
|
||||
$this->handleTask();
|
||||
}
|
||||
|
||||
protected function preflightCheck()
|
||||
{
|
||||
if (! (bool) config_cache('pixelfed.cloud_storage')) {
|
||||
$this->info('Error: Cloud storage is not enabled! Please enable before proceeding.');
|
||||
$this->error('Aborting...');
|
||||
exit;
|
||||
}
|
||||
}
|
||||
|
||||
protected function bootMessage()
|
||||
{
|
||||
$this->info(' ____ _ ______ __ ');
|
||||
$this->info(' / __ \(_) _____ / / __/__ ____/ / ');
|
||||
$this->info(' / /_/ / / |/_/ _ \/ / /_/ _ \/ __ / ');
|
||||
$this->info(' / ____/ /> </ __/ / __/ __/ /_/ / ');
|
||||
$this->info(' /_/ /_/_/|_|\___/_/_/ \___/\__,_/ ');
|
||||
$this->info(' ');
|
||||
$this->info(' Media Cloud Url Rewrite Tool');
|
||||
$this->info(' ===');
|
||||
$this->info(' Old S3: '.trim($this->argument('oldDomain')));
|
||||
$this->info(' New S3: '.trim($this->argument('newDomain')));
|
||||
$this->info(' ');
|
||||
}
|
||||
|
||||
protected function confirmCloudUrl()
|
||||
{
|
||||
$disk = Storage::disk(config('filesystems.cloud'))->url('test');
|
||||
$domain = parse_url($disk, PHP_URL_HOST);
|
||||
if (trim($this->argument('newDomain')) !== $domain) {
|
||||
$this->error('Error: The new S3 domain you entered is not currently configured');
|
||||
exit;
|
||||
}
|
||||
|
||||
if (! $this->confirm('Confirm this is correct')) {
|
||||
$this->error('Aborting...');
|
||||
exit;
|
||||
}
|
||||
}
|
||||
|
||||
protected function handleTask()
|
||||
{
|
||||
$task = select(
|
||||
label: 'What action would you like to perform?',
|
||||
options: ['Migrate All', 'Migrate Media', 'Migrate Avatars']
|
||||
);
|
||||
|
||||
switch ($task) {
|
||||
case 'Migrate All':
|
||||
$this->updateMediaUrls();
|
||||
$this->updateAvatarUrls();
|
||||
break;
|
||||
case 'Migrate Media':
|
||||
$this->updateMediaUrls();
|
||||
break;
|
||||
case 'Migrate Avatars':
|
||||
$this->updateAvatarUrls();
|
||||
break;
|
||||
default:
|
||||
$this->error('Invalid selection');
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
protected function updateMediaUrls()
|
||||
{
|
||||
$this->info('Updating media urls...');
|
||||
$oldDomain = trim($this->argument('oldDomain'));
|
||||
$newDomain = trim($this->argument('newDomain'));
|
||||
$disk = Storage::disk(config('filesystems.cloud'));
|
||||
$count = Media::whereNotNull('cdn_url')->count();
|
||||
$bar = $this->output->createProgressBar($count);
|
||||
$counter = 0;
|
||||
$bar->start();
|
||||
foreach (Media::whereNotNull('cdn_url')->lazyById(1000, 'id') as $media) {
|
||||
if (strncmp($media->media_path, 'http', 4) === 0) {
|
||||
$bar->advance();
|
||||
|
||||
continue;
|
||||
}
|
||||
$cdnHost = parse_url($media->cdn_url, PHP_URL_HOST);
|
||||
if ($oldDomain != $cdnHost || $newDomain == $cdnHost) {
|
||||
$bar->advance();
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
$media->cdn_url = str_replace($oldDomain, $newDomain, $media->cdn_url);
|
||||
|
||||
if ($media->thumbnail_url != null) {
|
||||
$thumbHost = parse_url($media->thumbnail_url, PHP_URL_HOST);
|
||||
if ($thumbHost == $oldDomain) {
|
||||
$thumbUrl = $disk->url($media->thumbnail_path);
|
||||
$media->thumbnail_url = $thumbUrl;
|
||||
}
|
||||
}
|
||||
|
||||
if ($media->optimized_url != null) {
|
||||
$optiHost = parse_url($media->optimized_url, PHP_URL_HOST);
|
||||
if ($optiHost == $oldDomain) {
|
||||
$optiUrl = str_replace($oldDomain, $newDomain, $media->optimized_url);
|
||||
$media->optimized_url = $optiUrl;
|
||||
}
|
||||
}
|
||||
|
||||
$media->save();
|
||||
$counter++;
|
||||
$bar->advance();
|
||||
}
|
||||
|
||||
$bar->finish();
|
||||
|
||||
$this->line(' ');
|
||||
$this->info('Finished! Updated '.$counter.' total records!');
|
||||
$this->line(' ');
|
||||
$this->info('Tip: Run `php artisan cache:clear` to purge cached urls');
|
||||
}
|
||||
|
||||
protected function updateAvatarUrls()
|
||||
{
|
||||
$this->info('Updating avatar urls...');
|
||||
$oldDomain = trim($this->argument('oldDomain'));
|
||||
$newDomain = trim($this->argument('newDomain'));
|
||||
$disk = Storage::disk(config('filesystems.cloud'));
|
||||
$count = Avatar::count();
|
||||
$bar = $this->output->createProgressBar($count);
|
||||
$counter = 0;
|
||||
$bar->start();
|
||||
foreach (Avatar::lazyById(1000, 'id') as $avatar) {
|
||||
if (! $avatar->cdn_url) {
|
||||
$bar->advance();
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
$cdnHost = parse_url($avatar->cdn_url, PHP_URL_HOST);
|
||||
if (strcasecmp($oldDomain, $cdnHost) !== 0 || strcasecmp($newDomain, $cdnHost) === 0) {
|
||||
$bar->advance();
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
$avatar->cdn_url = str_replace($oldDomain, $newDomain, $avatar->cdn_url);
|
||||
|
||||
$avatar->save();
|
||||
$counter++;
|
||||
$bar->advance();
|
||||
}
|
||||
|
||||
$bar->finish();
|
||||
|
||||
$this->line(' ');
|
||||
$this->info('Finished! Updated '.$counter.' total records!');
|
||||
$this->line(' ');
|
||||
$this->info('Tip: Run `php artisan cache:clear` to purge cached urls');
|
||||
}
|
||||
}
|
||||
@ -1,204 +0,0 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Models\Media;
|
||||
use App\Services\MediaService;
|
||||
use App\Services\StatusService;
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Symfony\Component\HttpFoundation\File\Exception\FileNotFoundException;
|
||||
use Symfony\Component\HttpKernel\Exception\NotFoundHttpException;
|
||||
|
||||
class MediaS3GarbageCollector extends Command
|
||||
{
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'media:s3gc {--limit=200} {--huge} {--log-errors}';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Delete (local) media uploads that exist on S3';
|
||||
|
||||
/**
|
||||
* Create a new command instance.
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function __construct()
|
||||
{
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
*
|
||||
* @return int
|
||||
*/
|
||||
public function handle()
|
||||
{
|
||||
$enabled = (bool) config_cache('pixelfed.cloud_storage');
|
||||
if (! $enabled) {
|
||||
$this->error('Cloud storage not enabled. Exiting...');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$deleteEnabled = config('media.delete_local_after_cloud');
|
||||
if (! $deleteEnabled) {
|
||||
$this->error('Delete local storage after cloud upload is not enabled');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$limit = $this->option('limit');
|
||||
$hugeMode = $this->option('huge');
|
||||
$log = $this->option('log-errors');
|
||||
|
||||
if ($limit > 2000 && ! $hugeMode) {
|
||||
$this->error('Limit exceeded, please use a limit under 2000 or run again with the --huge flag');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$minId = Media::orderByDesc('id')->where('created_at', '<', now()->subHours(12))->first();
|
||||
|
||||
if (! $minId) {
|
||||
return;
|
||||
} else {
|
||||
$minId = $minId->id;
|
||||
}
|
||||
|
||||
return $hugeMode ?
|
||||
$this->hugeMode($minId, $limit, $log) :
|
||||
$this->regularMode($minId, $limit, $log);
|
||||
}
|
||||
|
||||
protected function regularMode($minId, $limit, $log)
|
||||
{
|
||||
$gc = Media::whereRemoteMedia(false)
|
||||
->whereNotNull(['status_id', 'cdn_url', 'replicated_at'])
|
||||
->whereNot('version', '4')
|
||||
->where('id', '<', $minId)
|
||||
->inRandomOrder()
|
||||
->take($limit)
|
||||
->get();
|
||||
|
||||
$totalSize = 0;
|
||||
$bar = $this->output->createProgressBar($gc->count());
|
||||
$bar->start();
|
||||
$cloudDisk = Storage::disk(config('filesystems.cloud'));
|
||||
$localDisk = Storage::disk('local');
|
||||
|
||||
foreach ($gc as $media) {
|
||||
try {
|
||||
if (
|
||||
$cloudDisk->exists($media->media_path)
|
||||
) {
|
||||
if ($localDisk->exists($media->media_path)) {
|
||||
$localDisk->delete($media->media_path);
|
||||
$media->version = 4;
|
||||
$media->save();
|
||||
$totalSize = $totalSize + $media->size;
|
||||
MediaService::del($media->status_id);
|
||||
StatusService::del($media->status_id, false);
|
||||
if ($localDisk->exists($media->thumbnail_path)) {
|
||||
$localDisk->delete($media->thumbnail_path);
|
||||
}
|
||||
} else {
|
||||
$media->version = 4;
|
||||
$media->save();
|
||||
}
|
||||
} else {
|
||||
if ($log) {
|
||||
Log::channel('media')->info('[GC] Local media not properly persisted to cloud storage', ['media_id' => $media->id]);
|
||||
}
|
||||
}
|
||||
$bar->advance();
|
||||
} catch (FileNotFoundException $e) {
|
||||
$bar->advance();
|
||||
|
||||
continue;
|
||||
} catch (NotFoundHttpException $e) {
|
||||
$bar->advance();
|
||||
|
||||
continue;
|
||||
} catch (\Exception $e) {
|
||||
$bar->advance();
|
||||
|
||||
continue;
|
||||
}
|
||||
}
|
||||
$bar->finish();
|
||||
$this->line(' ');
|
||||
$this->info('Finished!');
|
||||
if ($totalSize) {
|
||||
$this->info('Cleared '.$totalSize.' bytes of media from local disk!');
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
protected function hugeMode($minId, $limit, $log)
|
||||
{
|
||||
$cloudDisk = Storage::disk(config('filesystems.cloud'));
|
||||
$localDisk = Storage::disk('local');
|
||||
|
||||
$bar = $this->output->createProgressBar($limit);
|
||||
$bar->start();
|
||||
|
||||
Media::whereRemoteMedia(false)
|
||||
->whereNotNull(['status_id', 'cdn_url', 'replicated_at'])
|
||||
->whereNot('version', '4')
|
||||
->where('id', '<', $minId)
|
||||
->chunk(50, function ($medias) use ($cloudDisk, $localDisk, $bar, $log) {
|
||||
foreach ($medias as $media) {
|
||||
try {
|
||||
if ($cloudDisk->exists($media->media_path)) {
|
||||
if ($localDisk->exists($media->media_path)) {
|
||||
$localDisk->delete($media->media_path);
|
||||
$media->version = 4;
|
||||
$media->save();
|
||||
MediaService::del($media->status_id);
|
||||
StatusService::del($media->status_id, false);
|
||||
if ($localDisk->exists($media->thumbnail_path)) {
|
||||
$localDisk->delete($media->thumbnail_path);
|
||||
}
|
||||
} else {
|
||||
$media->version = 4;
|
||||
$media->save();
|
||||
}
|
||||
} else {
|
||||
if ($log) {
|
||||
Log::channel('media')->info('[GC] Local media not properly persisted to cloud storage', ['media_id' => $media->id]);
|
||||
}
|
||||
}
|
||||
$bar->advance();
|
||||
} catch (FileNotFoundException $e) {
|
||||
$bar->advance();
|
||||
|
||||
continue;
|
||||
} catch (NotFoundHttpException $e) {
|
||||
$bar->advance();
|
||||
|
||||
continue;
|
||||
} catch (\Exception $e) {
|
||||
$bar->advance();
|
||||
|
||||
continue;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
$bar->finish();
|
||||
$this->line(' ');
|
||||
$this->info('Finished!');
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,150 @@
|
||||
# Artisan Commands
|
||||
|
||||
This directory contains Pixelfed's custom Artisan console commands, grouped into
|
||||
subfolders by purpose. Laravel auto-discovers every command in this tree, so the
|
||||
subfolder is purely organizational — the command name is defined by each class's
|
||||
`$signature`.
|
||||
|
||||
Run any command with `php artisan <command>`, and append `--help` to see its
|
||||
full argument and option list.
|
||||
|
||||
## Folder layout
|
||||
|
||||
| Folder | Namespace | Purpose |
|
||||
| --- | --- | --- |
|
||||
| `Admin/` | `App\Console\Commands\Admin` | Instance administration and operator tooling |
|
||||
| `Deprecated/` | `App\Console\Commands\Deprecated` | Historical one-off migrations whose root issue is already resolved; kept for reference only |
|
||||
| `Dev/` | `App\Console\Commands\Dev` | Local development and localization build helpers |
|
||||
| `FixBugs/` | `App\Console\Commands\FixBugs` | One-off repair and data-fix utilities |
|
||||
| `Install/` | `App\Console\Commands\Install` | Installation and upgrade |
|
||||
| `Internal/` | `App\Console\Commands\Internal` | Scheduled/background maintenance (mostly run by the scheduler) |
|
||||
| `Status/` | `App\Console\Commands\Status` | Read-only debug/diagnostic inspectors |
|
||||
| `User/` | `App\Console\Commands\User` | User account management |
|
||||
| `Concerns/` | `App\Console\Commands\Concerns` | Shared traits used by commands (not commands themselves) |
|
||||
|
||||
---
|
||||
|
||||
## Admin
|
||||
|
||||
| Command | Description |
|
||||
| --- | --- |
|
||||
| `admin:invite` | Create an invite link. |
|
||||
| `backup:cloud` | Send backups to cloud storage. |
|
||||
| `email:bancheck` | Check user emails against banned domains. |
|
||||
| `app:captcha-toggle-command` | Show captcha status and optionally disable it. |
|
||||
| `app:curated-onboarding` | Manage curated onboarding applications. |
|
||||
| `app:delete-remote-profile` | Delete a remote profile. |
|
||||
| `import:cities` | Import the cities dataset into the database. |
|
||||
| `import:emojis` | Import custom emojis from a `tar.gz` archive (supports `--prefix`/`--suffix`). |
|
||||
| `app:instance-manager` | Manage federated instances. |
|
||||
| `admin:MediaMoveStorageCloudToCloud` | Cold-migrate media from an old S3 bucket to the current cloud bucket, verifying each copy. |
|
||||
| `admin:MediaMoveStorageCloudToLocal` | Migrate cloud media back to local storage (download, verify, rewrite URLs, optionally delete cloud copy). |
|
||||
| `admin:MediaMoveStorageLocalToCloud` | Migrate local media to cloud storage (upload, verify, rewrite URLs, delete local copy). |
|
||||
| `admin:MigrateLocalS3MediaURL` | Rewrite stale local media cloud URLs from storage paths to the configured S3 host. Replaces the old `media:cloud-url-rewrite`. |
|
||||
| `regenerate:thumbnails` | Regenerate image thumbnails for all image media. |
|
||||
| `ap:update-actors` | Send Update Actor activities to known remote servers (`--force`). |
|
||||
| `video:thumbnail` | Generate missing video thumbnails. |
|
||||
|
||||
## Dev
|
||||
|
||||
| Command | Description |
|
||||
| --- | --- |
|
||||
| `i18n:export` | Build and export the JS localization files. |
|
||||
| `localization:generate` | Generate JSON files for all available localizations. |
|
||||
| `seed:devusers` | Seed dev users (admin + regular) with random passwords. |
|
||||
| `seed:follows` | Seed follow relationships for testing. |
|
||||
|
||||
## FixBugs
|
||||
|
||||
| Command | Description |
|
||||
| --- | --- |
|
||||
| `fix:avatars` | Replace old SVG identicon avatars with the default PNG avatar. |
|
||||
| `avatar:storage` | Manage avatar storage. |
|
||||
| `avatar:storage-deep-clean` | Clean up orphaned avatar storage. |
|
||||
| `media:optimize` | Find and optimize media that has not yet been optimized. |
|
||||
| `app:fetch-missing-media-mime-type` | Backfill missing MIME types on remote media by issuing HEAD requests. |
|
||||
| `fix:profile:duplicates` | Fix duplicate profiles. |
|
||||
| `fix:hashtags` | Fix hashtag records. |
|
||||
| `fix:likes` | Recalculate like counts. |
|
||||
| `media:fix-nonlocal-driver` | Repair filesystem records when `FILESYSTEM_DRIVER` is not set to local. |
|
||||
| `app:fix-missing-user-profile` | Interactively create a missing profile for an affected user. |
|
||||
| `admin:fixProfileCounts` | Resync a profile's cached counts (followers, following, statuses) from source tables; supports bulk `--all`/`--active`. |
|
||||
| `fix:usernames` | Fix invalid usernames. |
|
||||
| `app:hashtag-related-generate` | Generate related-hashtag data for a given tag. |
|
||||
| `media:fix` | Null out media `filter_class` values no longer present in `Filter::classes()`. Still relevant: image filters remain an active feature. |
|
||||
|
||||
## Install
|
||||
|
||||
| Command | Description |
|
||||
| --- | --- |
|
||||
| `instance:actor` | Generate the instance actor. |
|
||||
| `install` | CLI installer (`--dangerously-overwrite-env`, `--domain`, `--name`). |
|
||||
| `update` | Run Pixelfed schema updates between versions. |
|
||||
|
||||
## Internal
|
||||
|
||||
These are primarily invoked by the scheduler (see `bootstrap/app.php`) rather than run by hand.
|
||||
|
||||
| Command | Description |
|
||||
| --- | --- |
|
||||
| `app:account-post-count-stat-update` | Update post counts from recent activity. |
|
||||
| `app:cleanup-expired-app-registrations` | Delete app registrations older than 90 days. |
|
||||
| `gc:sessions` | Garbage-collect database sessions. |
|
||||
| `gc:failedjobs` | Delete failed jobs older than one month. |
|
||||
| `app:hashtag-cached-count-update` | Update cached hashtag counters (`--limit`). |
|
||||
| `app:import-remove-deleted-accounts` | Remove import data belonging to deleted accounts. |
|
||||
| `app:import-upload-clean-storage` | Delete import storage directories for non-active users. |
|
||||
| `app:import-upload-garbage-collection` | Garbage-collect skipped Instagram import posts. |
|
||||
| `app:import-upload-media-to-cloud-storage` | Migrate imported Instagram media to S3 (`--limit`). |
|
||||
| `app:instance-update-total-local-posts` | Update the total local post count. |
|
||||
| `media:gc` | Delete media uploads not attached to any active status. |
|
||||
| `app:notification-epoch-update` | Update the notification epoch. |
|
||||
| `gc:passwordreset` | Delete password reset tokens older than 24 hours. |
|
||||
| `app:push-gateway-refresh` | Refresh push-notification gateway support. |
|
||||
| `app:software-update-refresh` | Refresh latest software version data. |
|
||||
| `story:gc` | Clear expired stories. |
|
||||
| `app:transform-imports` | Transform completed imports into statuses. |
|
||||
| `app:weekly-instance-scan` | Scan instance nodeinfo weekly. |
|
||||
|
||||
## Deprecated
|
||||
|
||||
Historical one-off migrations whose underlying issue is already resolved in the
|
||||
current codebase. Retained for reference and rare legacy-data recovery; not
|
||||
expected to be needed on a current install.
|
||||
|
||||
| Command | Description | Why deprecated |
|
||||
| --- | --- | --- |
|
||||
| `status:dedup` | Remove duplicate statuses from before the unique-URI migration. | `statuses.uri` has had a unique index since the 2019 `add_unique_to_statuses_table` migration, so duplicate URIs can no longer be inserted. |
|
||||
| `fix:avatars` | Replace old SVG identicon avatars with the default PNG/JPG avatar. | SVG identicon avatars are no longer generated anywhere; new avatars default to `public/avatars/default.jpg`. |
|
||||
|
||||
## Status (debug/diagnostics)
|
||||
|
||||
Read-only inspectors for troubleshooting. They do not modify data.
|
||||
|
||||
| Command | Description |
|
||||
| --- | --- |
|
||||
| `status:post` | Show detailed metadata for a post and its media, including stored vs expected media URLs. |
|
||||
| `status:profile` | Show detailed metadata for a local or remote profile (federation-aware). |
|
||||
| `status:user` | Show detailed diagnostics for a user account (login & password reset), with recent account logs (`--logs`). |
|
||||
|
||||
## User
|
||||
|
||||
| Command | Description |
|
||||
| --- | --- |
|
||||
| `app:add-user-domain-block` | Apply a domain block for all users. |
|
||||
| `app:delete-user-domain-block` | Remove a domain block for all users. |
|
||||
| `app:reclaim-username` | Force-delete a user and profile to reclaim a username. |
|
||||
| `app:user-account-delete` | Federate an account deletion (`--concurrency`, `--chunk`, `--attempts`, `--target`, `--dry-run`). |
|
||||
| `user:admin` | Grant or remove admin privileges for a user. |
|
||||
| `user:avatar-delete` | Delete a user avatar and reset to default (`--force`). |
|
||||
| `user:checkpassword` | Read-only: verify a candidate password against the stored hash and diagnose login rejection. |
|
||||
| `user:create` | Create a new user. |
|
||||
| `user:delete` | Delete an account (`--force`). |
|
||||
| `user:app-magic-link` | Get the app magic link for in-app registrations missing a confirmation email. |
|
||||
| `user:setpassword` | Set/reset a user password (prompts securely, bcrypt). |
|
||||
| `user:show` | Show user info. |
|
||||
| `user:suspend` | Suspend a local user. |
|
||||
| `user:table` | Display the latest users. |
|
||||
| `user:2fa` | Disable two-factor authentication for a username. |
|
||||
| `user:unsuspend` | Unsuspend a local user. |
|
||||
| `user:verifyemail` | Verify a user's email address. |
|
||||
@ -0,0 +1,319 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands\Status;
|
||||
|
||||
use App\Models\Media;
|
||||
use App\Models\Status;
|
||||
use App\Services\AccountService;
|
||||
use App\Services\MediaService;
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Illuminate\Support\Str;
|
||||
|
||||
class StatusPost extends Command
|
||||
{
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'status:post {id : Status id, or a post URL like https://host/p/username/ID}';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Show detailed debug/metadata for a post (Status) and its media, including stored vs expected media URLs';
|
||||
|
||||
/**
|
||||
* Sensitive/long status columns to redact or trim.
|
||||
*
|
||||
* @var array<int, string>
|
||||
*/
|
||||
protected $longStatusCols = ['caption', 'cw_summary'];
|
||||
|
||||
public function handle()
|
||||
{
|
||||
$id = $this->resolveId($this->argument('id'));
|
||||
|
||||
if (! $id) {
|
||||
$this->error('Could not extract a status id from "'.$this->argument('id').'".');
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
$status = Status::withTrashed()->find($id);
|
||||
|
||||
if (! $status) {
|
||||
$this->error('No status found with id '.$id.'.');
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
$this->line(str_repeat('=', 64));
|
||||
$this->info('STATUS ROW (table: statuses)');
|
||||
$this->line(str_repeat('=', 64));
|
||||
$this->dumpStatus($status);
|
||||
|
||||
$this->newLine();
|
||||
$this->line(str_repeat('=', 64));
|
||||
$this->info('AUTHOR');
|
||||
$this->line(str_repeat('=', 64));
|
||||
$this->dumpAuthor($status);
|
||||
|
||||
$this->newLine();
|
||||
$this->line(str_repeat('=', 64));
|
||||
$this->info('MEDIA');
|
||||
$this->line(str_repeat('=', 64));
|
||||
$this->dumpMedia($status);
|
||||
|
||||
$this->newLine();
|
||||
$this->line(str_repeat('=', 64));
|
||||
$this->info('URL HEALTH CHECK');
|
||||
$this->line(str_repeat('=', 64));
|
||||
$this->urlHealth($status);
|
||||
|
||||
$this->newLine();
|
||||
$this->line(str_repeat('=', 64));
|
||||
$this->info('CACHE');
|
||||
$this->line(str_repeat('=', 64));
|
||||
$this->dumpCache($status);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Accept a numeric id or a post URL (…/p/username/ID) and return the id.
|
||||
*/
|
||||
protected function resolveId(string $input): ?string
|
||||
{
|
||||
$input = trim($input);
|
||||
|
||||
if (ctype_digit($input)) {
|
||||
return $input;
|
||||
}
|
||||
|
||||
// Extract the last numeric path segment from a URL.
|
||||
if (preg_match('#/p/[^/]+/(\d+)#', $input, $m)) {
|
||||
return $m[1];
|
||||
}
|
||||
|
||||
if (preg_match('#(\d{6,})#', $input, $m)) {
|
||||
return $m[1];
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
protected function dumpStatus(Status $status): void
|
||||
{
|
||||
$rows = [];
|
||||
foreach ($status->getAttributes() as $key => $value) {
|
||||
$display = $this->format($value);
|
||||
if (in_array($key, $this->longStatusCols, true) && is_string($value) && strlen($value) > 60) {
|
||||
$display = mb_strimwidth($value, 0, 60, '…');
|
||||
}
|
||||
$rows[] = [$key, $display];
|
||||
}
|
||||
$this->table(['Column', 'Value'], $rows);
|
||||
|
||||
$this->comment('Computed:');
|
||||
$this->line(' url(): '.$this->safe(fn () => $status->url()));
|
||||
$this->line(' permalink(): '.$this->safe(fn () => $status->permalink()));
|
||||
$this->line(' is remote: '.($status->uri ? 'yes ('.$status->uri.')' : 'no (local)'));
|
||||
if ($status->deleted_at) {
|
||||
$this->error(' ✗ Status is soft-deleted ('.$status->deleted_at.').');
|
||||
}
|
||||
}
|
||||
|
||||
protected function dumpAuthor(Status $status): void
|
||||
{
|
||||
$acct = AccountService::get($status->profile_id, true);
|
||||
if (! $acct) {
|
||||
$this->error('No account found for profile_id '.$status->profile_id.'.');
|
||||
|
||||
return;
|
||||
}
|
||||
$this->table(['Field', 'Value'], [
|
||||
['profile_id', $status->profile_id],
|
||||
['username', $acct['username'] ?? 'null'],
|
||||
['acct', $acct['acct'] ?? 'null'],
|
||||
['url', $acct['url'] ?? 'null'],
|
||||
['local', ($acct['local'] ?? null) ? 'true' : 'false'],
|
||||
]);
|
||||
}
|
||||
|
||||
protected function dumpMedia(Status $status): void
|
||||
{
|
||||
$media = Media::withTrashed()->whereStatusId($status->id)->orderBy('order')->get();
|
||||
|
||||
if ($media->isEmpty()) {
|
||||
$this->comment('No media attached to this status.');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$cloudHost = $this->cloudHost();
|
||||
|
||||
foreach ($media as $i => $m) {
|
||||
$this->newLine();
|
||||
$this->comment('Media #'.($i + 1).' (id '.$m->id.', order '.$m->order.')');
|
||||
$rows = [];
|
||||
$cols = [
|
||||
'media_path', 'thumbnail_path', 'cdn_url', 'thumbnail_url',
|
||||
'optimized_url', 'remote_url', 'remote_media', 'mime', 'size',
|
||||
'version', 'replicated_at', 'original_sha256', 'processed_at',
|
||||
'deleted_at',
|
||||
];
|
||||
$attrs = $m->getAttributes();
|
||||
foreach ($cols as $c) {
|
||||
if (array_key_exists($c, $attrs)) {
|
||||
$rows[] = [$c, $this->format($attrs[$c])];
|
||||
}
|
||||
}
|
||||
$this->table(['Media Column', 'Value'], $rows);
|
||||
|
||||
$this->line(' computed url(): '.$this->safe(fn () => $m->url()));
|
||||
$this->line(' computed thumbnailUrl(): '.$this->safe(fn () => $m->thumbnailUrl()));
|
||||
$this->line(' expected (from path): '.$this->expectedUrl($m->media_path));
|
||||
|
||||
// Per-field host comparison.
|
||||
$this->compareHost(' cdn_url', $m->cdn_url, $cloudHost);
|
||||
$this->compareHost(' thumbnail_url', $m->thumbnail_url, $cloudHost);
|
||||
$this->compareHost(' optimized_url', $m->optimized_url, $cloudHost);
|
||||
}
|
||||
}
|
||||
|
||||
protected function urlHealth(Status $status): void
|
||||
{
|
||||
$cloudHost = $this->cloudHost();
|
||||
if (! $cloudHost) {
|
||||
$this->comment('Cloud storage not configured (or no cloud disk url); skipping host comparison.');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$this->line('Configured cloud host (correct base): '.$cloudHost);
|
||||
$this->newLine();
|
||||
|
||||
$media = Media::withTrashed()->whereStatusId($status->id)->get();
|
||||
$stale = [];
|
||||
|
||||
foreach ($media as $m) {
|
||||
if ($m->remote_media || Str::startsWith((string) $m->media_path, 'http')) {
|
||||
continue;
|
||||
}
|
||||
foreach (['cdn_url', 'thumbnail_url', 'optimized_url'] as $field) {
|
||||
$val = $m->{$field};
|
||||
if (! $val) {
|
||||
continue;
|
||||
}
|
||||
$host = parse_url($val, PHP_URL_HOST);
|
||||
if ($host && $cloudHost && strcasecmp($host, $cloudHost) !== 0) {
|
||||
$stale[] = 'media '.$m->id.' '.$field.' points at '.$host.' (expected '.$cloudHost.')';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($stale) {
|
||||
$this->error('STALE MEDIA URLS DETECTED:');
|
||||
foreach ($stale as $s) {
|
||||
$this->line(' ✗ '.$s);
|
||||
}
|
||||
$this->newLine();
|
||||
$this->comment('Fix with: php artisan admin:MigrateLocalMediaURL '.$status->id);
|
||||
$this->comment('(or --all to scan every local media row)');
|
||||
} else {
|
||||
$this->info('All local media URLs point at the configured cloud host. ✓');
|
||||
}
|
||||
}
|
||||
|
||||
protected function dumpCache(Status $status): void
|
||||
{
|
||||
$cached = MediaService::get($status->id);
|
||||
if (empty($cached)) {
|
||||
$this->comment('No cached media_attachments entry (MediaService).');
|
||||
|
||||
return;
|
||||
}
|
||||
$this->comment('Cached media_attachments (MediaService, 6h TTL) — served to clients:');
|
||||
foreach ($cached as $i => $item) {
|
||||
$this->line(' ['.$i.'] url: '.($item['url'] ?? 'null'));
|
||||
$this->line(' ['.$i.'] preview_url: '.($item['preview_url'] ?? 'null'));
|
||||
}
|
||||
$this->newLine();
|
||||
$this->comment('If these still show a stale host after a DB fix, run: php artisan cache:clear');
|
||||
}
|
||||
|
||||
protected function compareHost(string $label, ?string $url, ?string $cloudHost): void
|
||||
{
|
||||
if (! $url) {
|
||||
return;
|
||||
}
|
||||
$host = parse_url($url, PHP_URL_HOST);
|
||||
if (! $host || ! $cloudHost) {
|
||||
return;
|
||||
}
|
||||
if (strcasecmp($host, $cloudHost) !== 0) {
|
||||
$this->line('<fg=red>'.$label.' host: '.$host.' ✗ (expected '.$cloudHost.')</>');
|
||||
} else {
|
||||
$this->line('<fg=green>'.$label.' host: '.$host.' ✓</>');
|
||||
}
|
||||
}
|
||||
|
||||
protected function expectedUrl(?string $mediaPath): string
|
||||
{
|
||||
if (! $mediaPath || Str::startsWith($mediaPath, 'http')) {
|
||||
return $mediaPath ?? 'null';
|
||||
}
|
||||
|
||||
try {
|
||||
return (string) Storage::disk(config('filesystems.cloud'))->url($mediaPath);
|
||||
} catch (\Throwable $e) {
|
||||
return '(cloud disk not resolvable in this environment)';
|
||||
}
|
||||
}
|
||||
|
||||
protected function cloudHost(): ?string
|
||||
{
|
||||
try {
|
||||
if (! (bool) config_cache('pixelfed.cloud_storage')) {
|
||||
// Still try to read the configured cloud disk host for reference.
|
||||
}
|
||||
$url = Storage::disk(config('filesystems.cloud'))->url('probe');
|
||||
$host = parse_url($url, PHP_URL_HOST);
|
||||
|
||||
return $host ?: null;
|
||||
} catch (\Throwable $e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
protected function safe(callable $fn): string
|
||||
{
|
||||
try {
|
||||
return (string) ($fn() ?? 'null');
|
||||
} catch (\Throwable $e) {
|
||||
return 'error: '.$e->getMessage();
|
||||
}
|
||||
}
|
||||
|
||||
protected function format($value): string
|
||||
{
|
||||
if ($value === null) {
|
||||
return 'null';
|
||||
}
|
||||
if (is_bool($value)) {
|
||||
return $value ? 'true' : 'false';
|
||||
}
|
||||
if ($value === '') {
|
||||
return '(empty string)';
|
||||
}
|
||||
if ($value instanceof \DateTimeInterface) {
|
||||
return $value->format('Y-m-d H:i:s');
|
||||
}
|
||||
|
||||
return (string) $value;
|
||||
}
|
||||
}
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\User;
|
||||
|
||||
use App\Models\DefaultDomainBlock;
|
||||
use App\Models\User;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\User;
|
||||
|
||||
use App\Models\DefaultDomainBlock;
|
||||
use App\Models\UserDomainBlock;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\User;
|
||||
|
||||
use App\Models\Profile;
|
||||
use App\Models\User;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\User;
|
||||
|
||||
use App\Models\Instance;
|
||||
use App\Models\Profile;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\User;
|
||||
|
||||
use App\Models\User;
|
||||
use Illuminate\Console\Command;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\User;
|
||||
|
||||
use App\Models\Avatar;
|
||||
use App\Models\User;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\User;
|
||||
|
||||
use App\Models\User;
|
||||
use Illuminate\Console\Command;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\User;
|
||||
|
||||
use App\Models\User;
|
||||
use Illuminate\Console\Command;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\User;
|
||||
|
||||
use App\Jobs\DeletePipeline\DeleteAccountPipeline;
|
||||
use App\Models\Profile;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\User;
|
||||
|
||||
use App\Models\EmailVerification;
|
||||
use App\Models\User;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\User;
|
||||
|
||||
use App\Models\User;
|
||||
use Illuminate\Console\Command;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\User;
|
||||
|
||||
use App\Models\User;
|
||||
use Illuminate\Console\Command;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\User;
|
||||
|
||||
use App\Models\User;
|
||||
use Illuminate\Console\Command;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\User;
|
||||
|
||||
use App\Models\User;
|
||||
use Illuminate\Console\Command;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\User;
|
||||
|
||||
use App\Models\User;
|
||||
use Illuminate\Console\Command;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\User;
|
||||
|
||||
use App\Models\User;
|
||||
use Illuminate\Console\Command;
|
||||
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
namespace App\Console\Commands\User;
|
||||
|
||||
use App\Models\User;
|
||||
use Illuminate\Console\Command;
|
||||
@ -0,0 +1,247 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services;
|
||||
|
||||
use App\Util\ActivityPub\Helpers;
|
||||
use Illuminate\Http\Client\ConnectionException;
|
||||
use Illuminate\Http\Client\RequestException;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
use League\Uri\BaseUri;
|
||||
use Psr\Http\Message\ResponseInterface;
|
||||
|
||||
/**
|
||||
* SSRF-hardened fetcher for remote media (avatars, attachments).
|
||||
*
|
||||
* Unlike a bare Guzzle client or file_get_contents(), this service:
|
||||
* - validates the URL (https-only, no userinfo, normalized host);
|
||||
* - resolves the host and refuses any non-global (private/reserved/
|
||||
* link-local) IP, closing the metadata.google.internal style bypass
|
||||
* where a public-looking hostname resolves into a reserved range;
|
||||
* - pins the connection to the already-validated IPs via CURLOPT_RESOLVE
|
||||
* so the address we validated is the address we connect to (no
|
||||
* TOCTOU / DNS-rebinding window);
|
||||
* - never lets the HTTP client follow redirects automatically, instead
|
||||
* re-validating and re-resolving every hop;
|
||||
* - enforces a hard byte cap.
|
||||
*/
|
||||
class SecureMediaFetchService
|
||||
{
|
||||
private const MAX_REDIRECTS = 2;
|
||||
|
||||
private const CONNECT_TIMEOUT = 5;
|
||||
|
||||
private const TIMEOUT = 15;
|
||||
|
||||
/**
|
||||
* Perform a HEAD request through the pinned, validated path.
|
||||
*
|
||||
* @return array{length:int,mime:string}|false
|
||||
*/
|
||||
public static function head(string $url, ?int $maxBytes = null)
|
||||
{
|
||||
$maxBytes = $maxBytes ?? self::defaultMaxBytes();
|
||||
|
||||
return (new self)->request($url, 'head', $maxBytes);
|
||||
}
|
||||
|
||||
/**
|
||||
* Download up to $maxBytes of the resource through the pinned,
|
||||
* validated path. Returns the raw body string, or false.
|
||||
*
|
||||
* @return string|false
|
||||
*/
|
||||
public static function get(string $url, ?int $maxBytes = null, ?int $expectedLength = null)
|
||||
{
|
||||
$maxBytes = $maxBytes ?? self::defaultMaxBytes();
|
||||
$result = (new self)->request($url, 'get', $maxBytes, $expectedLength);
|
||||
|
||||
if (! is_array($result)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return $result['body'] ?? false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shared request loop: validate -> resolve public IPs -> pin -> issue
|
||||
* request with redirects disabled -> re-validate each hop.
|
||||
*
|
||||
* @return array|false For 'head': ['length'=>int,'mime'=>string].
|
||||
* For 'get': ['body'=>string,'length'=>int,'mime'=>string].
|
||||
*/
|
||||
protected function request(string $url, string $method, int $maxBytes, ?int $expectedLength = null)
|
||||
{
|
||||
$currentUrl = $url;
|
||||
|
||||
for ($redirects = 0; $redirects <= self::MAX_REDIRECTS; $redirects++) {
|
||||
$currentUrl = Helpers::validateUrl($currentUrl);
|
||||
|
||||
if (! $currentUrl) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$host = parse_url($currentUrl, PHP_URL_HOST);
|
||||
$scheme = parse_url($currentUrl, PHP_URL_SCHEME);
|
||||
$port = parse_url($currentUrl, PHP_URL_PORT) ?: 443;
|
||||
|
||||
if (! $host || strtolower((string) $scheme) !== 'https') {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Resolve the host and reject if ANY resolved address is
|
||||
// non-global. Fail-closed: empty means unresolved or private.
|
||||
$ips = Helpers::resolvePublicIps($host);
|
||||
|
||||
if (empty($ips)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
$res = Http::withOptions([
|
||||
'allow_redirects' => false,
|
||||
'sink' => null,
|
||||
'curl' => [
|
||||
CURLOPT_RESOLVE => [
|
||||
$this->buildResolveEntry($host, (int) $port, $ips),
|
||||
],
|
||||
CURLOPT_FRESH_CONNECT => true,
|
||||
CURLOPT_FORBID_REUSE => true,
|
||||
CURLOPT_PROTOCOLS => CURLPROTO_HTTPS,
|
||||
CURLOPT_REDIR_PROTOCOLS => CURLPROTO_HTTPS,
|
||||
],
|
||||
'on_headers' => function (ResponseInterface $response) use ($maxBytes) {
|
||||
$length = $response->getHeaderLine('Content-Length');
|
||||
if ($length !== '' && ctype_digit($length) && (int) $length > $maxBytes) {
|
||||
throw new \RuntimeException('Remote media exceeds maximum size');
|
||||
}
|
||||
},
|
||||
])
|
||||
->withHeaders(['User-Agent' => self::userAgent()])
|
||||
->timeout(self::TIMEOUT)
|
||||
->connectTimeout(self::CONNECT_TIMEOUT)
|
||||
->{$method}($currentUrl);
|
||||
} catch (RequestException $e) {
|
||||
return false;
|
||||
} catch (ConnectionException $e) {
|
||||
return false;
|
||||
} catch (\Throwable $e) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Manual redirect handling: re-validate + re-resolve the next hop.
|
||||
if (in_array($res->status(), [301, 302, 303, 307, 308], true)) {
|
||||
if ($redirects >= self::MAX_REDIRECTS) {
|
||||
return false;
|
||||
}
|
||||
$location = $res->header('Location');
|
||||
if (! $location) {
|
||||
return false;
|
||||
}
|
||||
$nextUrl = $this->resolveRedirect($currentUrl, $location);
|
||||
if (! $nextUrl) {
|
||||
return false;
|
||||
}
|
||||
$currentUrl = $nextUrl;
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if (! $res->successful()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$mime = $this->normalizeMime($res->header('Content-Type'));
|
||||
$declaredLength = $res->header('Content-Length');
|
||||
$declaredLength = ($declaredLength !== null && ctype_digit((string) $declaredLength))
|
||||
? (int) $declaredLength
|
||||
: null;
|
||||
|
||||
if ($method === 'head') {
|
||||
if ($declaredLength === null || $mime === null) {
|
||||
return false;
|
||||
}
|
||||
if ($declaredLength < 10 || $declaredLength > $maxBytes) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return ['length' => $declaredLength, 'mime' => $mime];
|
||||
}
|
||||
|
||||
// GET: enforce the cap against the actual body we received.
|
||||
$body = $res->body();
|
||||
$len = strlen($body);
|
||||
|
||||
if ($len === 0 || $len > $maxBytes) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if ($expectedLength !== null && $len < $expectedLength) {
|
||||
// Received less than the HEAD promised; treat as truncated.
|
||||
// Still return what we have, capped, but never more than asked.
|
||||
$body = substr($body, 0, $expectedLength);
|
||||
$len = strlen($body);
|
||||
}
|
||||
|
||||
return [
|
||||
'body' => $body,
|
||||
'length' => $len,
|
||||
'mime' => $mime ?? ($declaredLength !== null ? $mime : null),
|
||||
];
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
protected function buildResolveEntry(string $host, int $port, array $ips): string
|
||||
{
|
||||
$addresses = array_map(function ($ip) {
|
||||
return str_contains($ip, ':') ? '['.$ip.']' : $ip;
|
||||
}, $ips);
|
||||
|
||||
return $host.':'.$port.':'.implode(',', $addresses);
|
||||
}
|
||||
|
||||
protected function resolveRedirect(string $baseUrl, string $location): ?string
|
||||
{
|
||||
$location = trim($location);
|
||||
|
||||
if ($location === '' || preg_match('/[\x00-\x20\x7f]/', $location)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
$resolved = (string) BaseUri::from($baseUrl)->resolve($location);
|
||||
|
||||
return Helpers::validateUrl($resolved) ? $resolved : null;
|
||||
} catch (\Throwable $e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
protected function normalizeMime(?string $contentType): ?string
|
||||
{
|
||||
if (! $contentType) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$mime = strtolower(trim(explode(';', $contentType)[0]));
|
||||
|
||||
return $mime === '' ? null : $mime;
|
||||
}
|
||||
|
||||
protected static function defaultMaxBytes(): int
|
||||
{
|
||||
// Cap on the larger of avatar/photo config limits (kB -> bytes),
|
||||
// with a sane fallback.
|
||||
$photo = (int) config_cache('pixelfed.max_photo_size');
|
||||
$avatar = (int) config('pixelfed.max_avatar_size');
|
||||
$maxKb = max($photo, $avatar, 1000);
|
||||
|
||||
return $maxKb * 1000;
|
||||
}
|
||||
|
||||
protected static function userAgent(): string
|
||||
{
|
||||
return 'PixelFedBot/1.0.0 (Pixelfed/'.config('pixelfed.version').'; +'.config('app.url').')';
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,115 @@
|
||||
<?php
|
||||
|
||||
use App\Models\Media;
|
||||
use App\Models\Status;
|
||||
use App\Models\User;
|
||||
use Illuminate\Foundation\Testing\LazilyRefreshDatabase;
|
||||
use Illuminate\Support\Facades\Config;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
|
||||
uses(LazilyRefreshDatabase::class);
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| admin:MediaMoveStorageCloudToCloud
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Cold S3->S3 migration: copy existing objects from the old bucket (s3-old)
|
||||
| to the current cloud bucket (s3), verify, rewrite media URLs, GC the source.
|
||||
|
|
||||
*/
|
||||
|
||||
beforeEach(function () {
|
||||
Config::set('filesystems.cloud', 's3');
|
||||
// Destination (new) bucket.
|
||||
Storage::fake('s3', ['url' => 'https://cdneast.pixelfed.au']);
|
||||
// Source (old) bucket.
|
||||
Storage::fake('s3-old', ['url' => 'https://cdn.pixelfed.au']);
|
||||
});
|
||||
|
||||
function makeOldBucketMedia(string $oldHost = 'https://cdn.pixelfed.au'): Media
|
||||
{
|
||||
$user = User::factory()->create();
|
||||
$user->refresh();
|
||||
$pid = $user->profile->id;
|
||||
$status = Status::factory()->create(['profile_id' => $pid, 'type' => 'photo']);
|
||||
|
||||
$path = 'public/m/_v2/'.$pid.'/aa/bb/file.jpg';
|
||||
$thumb = 'public/m/_v2/'.$pid.'/aa/bb/file_thumb.jpeg';
|
||||
|
||||
// Files exist only on the OLD bucket.
|
||||
Storage::disk('s3-old')->put($path, 'PRIMARY-BYTES-1234567890');
|
||||
Storage::disk('s3-old')->put($thumb, 'THUMB-BYTES');
|
||||
|
||||
return Media::create([
|
||||
'status_id' => $status->id,
|
||||
'profile_id' => $pid,
|
||||
'user_id' => $user->id,
|
||||
'media_path' => $path,
|
||||
'thumbnail_path' => $thumb,
|
||||
'cdn_url' => $oldHost.'/'.$path,
|
||||
'thumbnail_url' => $oldHost.'/'.$thumb,
|
||||
'optimized_url' => $oldHost.'/'.$path,
|
||||
'mime' => 'image/jpeg',
|
||||
'size' => strlen('PRIMARY-BYTES-1234567890'),
|
||||
'remote_media' => false,
|
||||
'version' => 4,
|
||||
'replicated_at' => now(),
|
||||
'order' => 0,
|
||||
]);
|
||||
}
|
||||
|
||||
it('copies old-bucket media to the new bucket, rewrites urls and GCs the source', function () {
|
||||
$media = makeOldBucketMedia();
|
||||
|
||||
$this->artisan('admin:MediaMoveStorageCloudToCloud', ['--force' => true])
|
||||
->assertExitCode(0);
|
||||
|
||||
// Copied to destination.
|
||||
expect(Storage::disk('s3')->exists($media->media_path))->toBeTrue();
|
||||
// Removed from source (GC).
|
||||
expect(Storage::disk('s3-old')->exists($media->media_path))->toBeFalse();
|
||||
|
||||
$media->refresh();
|
||||
expect(parse_url($media->cdn_url, PHP_URL_HOST))->toBe('cdneast.pixelfed.au');
|
||||
expect(parse_url($media->thumbnail_url, PHP_URL_HOST))->toBe('cdneast.pixelfed.au');
|
||||
expect(parse_url($media->optimized_url, PHP_URL_HOST))->toBe('cdneast.pixelfed.au');
|
||||
});
|
||||
|
||||
it('keeps the source objects with --keep-source', function () {
|
||||
$media = makeOldBucketMedia();
|
||||
|
||||
$this->artisan('admin:MediaMoveStorageCloudToCloud', ['--force' => true, '--keep-source' => true])
|
||||
->assertExitCode(0);
|
||||
|
||||
expect(Storage::disk('s3')->exists($media->media_path))->toBeTrue();
|
||||
expect(Storage::disk('s3-old')->exists($media->media_path))->toBeTrue();
|
||||
});
|
||||
|
||||
it('makes no changes in dry-run', function () {
|
||||
$media = makeOldBucketMedia();
|
||||
|
||||
$this->artisan('admin:MediaMoveStorageCloudToCloud', ['--force' => true, '--dry-run' => true])
|
||||
->assertExitCode(0);
|
||||
|
||||
expect(Storage::disk('s3')->exists($media->media_path))->toBeFalse();
|
||||
expect(parse_url($media->fresh()->cdn_url, PHP_URL_HOST))->toBe('cdn.pixelfed.au');
|
||||
});
|
||||
|
||||
it('skips media already pointing at the destination host', function () {
|
||||
// cdn_url already on the destination host -> nothing to do.
|
||||
$media = makeOldBucketMedia('https://cdneast.pixelfed.au');
|
||||
$before = $media->cdn_url;
|
||||
|
||||
$this->artisan('admin:MediaMoveStorageCloudToCloud', ['--force' => true])
|
||||
->assertExitCode(0);
|
||||
|
||||
expect($media->fresh()->cdn_url)->toBe($before);
|
||||
// Not copied (was skipped).
|
||||
expect(Storage::disk('s3')->exists($media->media_path))->toBeFalse();
|
||||
});
|
||||
|
||||
it('errors when source and destination are the same disk', function () {
|
||||
$this->artisan('admin:MediaMoveStorageCloudToCloud', ['--sourceDisk' => 's3', '--force' => true])
|
||||
->assertExitCode(1);
|
||||
});
|
||||
@ -0,0 +1,150 @@
|
||||
<?php
|
||||
|
||||
use App\Models\Media;
|
||||
use App\Models\Status;
|
||||
use App\Models\User;
|
||||
use Illuminate\Foundation\Testing\LazilyRefreshDatabase;
|
||||
use Illuminate\Support\Facades\Config;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
|
||||
uses(LazilyRefreshDatabase::class);
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Media storage migration commands
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| admin:MediaMoveStorageLocalToCloud / admin:MediaMoveStorageCloudToLocal
|
||||
| Move media between local and cloud disks, verify by size/sha256, GC the
|
||||
| source, and manage the PF_ENABLE_CLOUD .env flag for hot migrations.
|
||||
|
|
||||
*/
|
||||
|
||||
beforeEach(function () {
|
||||
Config::set('filesystems.cloud', 's3');
|
||||
Storage::fake('local');
|
||||
Storage::fake('s3', ['url' => 'https://cdn.test']);
|
||||
|
||||
// Use a throwaway env file so the command's env edits don't touch the
|
||||
// real one. App::useEnvironmentPath expects a directory; the app resolves
|
||||
// the environment-specific filename (e.g. .env.testing) itself.
|
||||
$this->originalEnvPath = app()->environmentPath();
|
||||
$dir = sys_get_temp_dir().'/pf-env-test-'.uniqid();
|
||||
mkdir($dir);
|
||||
app()->useEnvironmentPath($dir);
|
||||
file_put_contents(app()->environmentFilePath(), "APP_KEY=base64:test\nPF_ENABLE_CLOUD=false\n");
|
||||
});
|
||||
|
||||
afterEach(function () {
|
||||
// Restore the real environment path so we don't leak into other test files.
|
||||
if (isset($this->originalEnvPath)) {
|
||||
app()->useEnvironmentPath($this->originalEnvPath);
|
||||
}
|
||||
});
|
||||
|
||||
function makeCloudMedia(): Media
|
||||
{
|
||||
$user = User::factory()->create();
|
||||
$user->refresh();
|
||||
$pid = $user->profile->id;
|
||||
$status = Status::factory()->create(['profile_id' => $pid, 'type' => 'photo']);
|
||||
|
||||
$path = 'public/m/_v2/'.$pid.'/aa/bb/file.jpg';
|
||||
$thumb = 'public/m/_v2/'.$pid.'/aa/bb/file_thumb.jpeg';
|
||||
|
||||
// Put the files on the cloud disk only.
|
||||
Storage::disk('s3')->put($path, 'PRIMARY-BYTES-1234567890');
|
||||
Storage::disk('s3')->put($thumb, 'THUMB-BYTES');
|
||||
|
||||
return Media::create([
|
||||
'status_id' => $status->id,
|
||||
'profile_id' => $pid,
|
||||
'user_id' => $user->id,
|
||||
'media_path' => $path,
|
||||
'thumbnail_path' => $thumb,
|
||||
'cdn_url' => Storage::disk('s3')->url($path),
|
||||
'thumbnail_url' => Storage::disk('s3')->url($thumb),
|
||||
'optimized_url' => Storage::disk('s3')->url($path),
|
||||
'mime' => 'image/jpeg',
|
||||
'size' => strlen('PRIMARY-BYTES-1234567890'),
|
||||
'remote_media' => false,
|
||||
'version' => 4,
|
||||
'replicated_at' => now(),
|
||||
'order' => 0,
|
||||
]);
|
||||
}
|
||||
|
||||
describe('admin:MediaMoveStorageCloudToLocal', function () {
|
||||
it('downloads cloud media to local, clears cloud urls and deletes the cloud copy', function () {
|
||||
$media = makeCloudMedia();
|
||||
|
||||
$this->artisan('admin:MediaMoveStorageCloudToLocal', ['--force' => true])
|
||||
->assertExitCode(0);
|
||||
|
||||
// File is now on local disk.
|
||||
expect(Storage::disk('local')->exists($media->media_path))->toBeTrue();
|
||||
// Cloud copy removed (GC), thumbnail too.
|
||||
expect(Storage::disk('s3')->exists($media->media_path))->toBeFalse();
|
||||
|
||||
$media->refresh();
|
||||
expect($media->cdn_url)->toBeNull();
|
||||
expect($media->optimized_url)->toBeNull();
|
||||
expect($media->thumbnail_url)->toBeNull();
|
||||
expect($media->replicated_at)->toBeNull();
|
||||
expect((string) $media->version)->toBe('3');
|
||||
});
|
||||
|
||||
it('keeps the cloud copy with --keep-cloud', function () {
|
||||
$media = makeCloudMedia();
|
||||
|
||||
$this->artisan('admin:MediaMoveStorageCloudToLocal', ['--force' => true, '--keep-cloud' => true])
|
||||
->assertExitCode(0);
|
||||
|
||||
expect(Storage::disk('local')->exists($media->media_path))->toBeTrue();
|
||||
expect(Storage::disk('s3')->exists($media->media_path))->toBeTrue();
|
||||
});
|
||||
|
||||
it('does not modify anything in dry-run', function () {
|
||||
$media = makeCloudMedia();
|
||||
|
||||
$this->artisan('admin:MediaMoveStorageCloudToLocal', ['--force' => true, '--dry-run' => true])
|
||||
->assertExitCode(0);
|
||||
|
||||
expect(Storage::disk('local')->exists($media->media_path))->toBeFalse();
|
||||
expect($media->fresh()->cdn_url)->not->toBeNull();
|
||||
});
|
||||
|
||||
it('sets PF_ENABLE_CLOUD=false in .env and runtime when cloud is enabled', function () {
|
||||
// Start with cloud enabled.
|
||||
file_put_contents(app()->environmentFilePath(), "APP_KEY=base64:test\nPF_ENABLE_CLOUD=true\n");
|
||||
Config::set('pixelfed.cloud_storage', true);
|
||||
makeCloudMedia();
|
||||
|
||||
$this->artisan('admin:MediaMoveStorageCloudToLocal', ['--force' => true])
|
||||
->assertExitCode(0);
|
||||
|
||||
expect(file_get_contents(app()->environmentFilePath()))->toContain('PF_ENABLE_CLOUD="false"');
|
||||
expect(config('pixelfed.cloud_storage'))->toBeFalse();
|
||||
});
|
||||
});
|
||||
|
||||
describe('admin:MediaMoveStorageLocalToCloud', function () {
|
||||
it('requires a configured cloud disk', function () {
|
||||
// Fake s3 disk has no url() host resolvable? Storage::fake provides a
|
||||
// url, so instead point cloud at a disk that throws.
|
||||
Config::set('filesystems.cloud', 'does-not-exist');
|
||||
|
||||
$this->artisan('admin:MediaMoveStorageLocalToCloud', ['--force' => true])
|
||||
->assertExitCode(1);
|
||||
});
|
||||
|
||||
it('flips PF_ENABLE_CLOUD to true before migrating (dry-run reports it)', function () {
|
||||
// cloud currently false in the temp .env from beforeEach.
|
||||
$this->artisan('admin:MediaMoveStorageLocalToCloud', ['--dry-run' => true])
|
||||
->expectsOutputToContain('PF_ENABLE_CLOUD')
|
||||
->assertExitCode(0);
|
||||
|
||||
// dry-run must not write the .env.
|
||||
expect(file_get_contents(app()->environmentFilePath()))->toContain('PF_ENABLE_CLOUD=false');
|
||||
});
|
||||
});
|
||||
@ -0,0 +1,189 @@
|
||||
<?php
|
||||
|
||||
use App\Models\Media;
|
||||
use App\Models\Status;
|
||||
use App\Models\User;
|
||||
use App\Services\ConfigCacheService;
|
||||
use Illuminate\Foundation\Testing\LazilyRefreshDatabase;
|
||||
use Illuminate\Support\Facades\Config;
|
||||
|
||||
uses(LazilyRefreshDatabase::class);
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| admin:MigrateLocalS3MediaURL
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Rebuilds stale local media URLs (cdn_url, thumbnail_url, optimized_url)
|
||||
| from their storage paths using the configured cloud disk host.
|
||||
|
|
||||
*/
|
||||
|
||||
beforeEach(function () {
|
||||
// Point the cloud disk at a deterministic host with path-style urls so
|
||||
// Storage::disk('s3')->url($path) == https://cdn.test/<path>.
|
||||
Config::set('filesystems.cloud', 's3');
|
||||
Config::set('filesystems.disks.s3', [
|
||||
'driver' => 's3',
|
||||
'key' => 'test',
|
||||
'secret' => 'test',
|
||||
'region' => 'us-east-1',
|
||||
'bucket' => 'bucket',
|
||||
'url' => 'https://cdn.test',
|
||||
'endpoint' => 'https://cdn.test',
|
||||
'use_path_style_endpoint' => true,
|
||||
'visibility' => 'public',
|
||||
]);
|
||||
// Enable cloud storage. config_cache() falls through to config() when
|
||||
// instance.enable_cc is off (as in CI), so set both to be safe.
|
||||
Config::set('pixelfed.cloud_storage', true);
|
||||
ConfigCacheService::put('pixelfed.cloud_storage', true);
|
||||
});
|
||||
|
||||
function makeStatusWithStaleMedia(string $staleHost = 'https://s3.old.example'): Media
|
||||
{
|
||||
$user = User::factory()->create();
|
||||
$user->refresh();
|
||||
$pid = $user->profile->id;
|
||||
|
||||
$status = Status::factory()->create(['profile_id' => $pid, 'type' => 'video']);
|
||||
|
||||
$path = 'public/m/_v2/'.$pid.'/aa/bb/file.mp4';
|
||||
$thumbPath = 'public/m/_v2/'.$pid.'/aa/bb/file_thumb.jpeg';
|
||||
|
||||
return Media::create([
|
||||
'status_id' => $status->id,
|
||||
'profile_id' => $pid,
|
||||
'user_id' => $user->id,
|
||||
'media_path' => $path,
|
||||
'thumbnail_path' => $thumbPath,
|
||||
'cdn_url' => 'https://cdn.test/'.$path, // already correct
|
||||
'thumbnail_url' => $staleHost.'/'.$thumbPath, // stale
|
||||
'optimized_url' => $staleHost.'/'.$path, // stale
|
||||
'mime' => 'video/mp4',
|
||||
'remote_media' => false,
|
||||
'order' => 0,
|
||||
]);
|
||||
}
|
||||
|
||||
it('rebuilds stale thumbnail_url and optimized_url but leaves correct cdn_url', function () {
|
||||
$media = makeStatusWithStaleMedia();
|
||||
|
||||
$this->artisan('admin:MigrateLocalS3MediaURL', ['id' => (string) $media->status_id, '--force' => true])
|
||||
->assertExitCode(0);
|
||||
|
||||
$media->refresh();
|
||||
expect($media->cdn_url)->toBe('https://cdn.test/'.$media->media_path);
|
||||
expect($media->thumbnail_url)->toBe('https://cdn.test/'.$media->thumbnail_path);
|
||||
expect($media->optimized_url)->toBe('https://cdn.test/'.$media->media_path);
|
||||
expect(parse_url($media->thumbnail_url, PHP_URL_HOST))->toBe('cdn.test');
|
||||
expect(parse_url($media->optimized_url, PHP_URL_HOST))->toBe('cdn.test');
|
||||
});
|
||||
|
||||
it('does not change anything in dry-run mode', function () {
|
||||
$media = makeStatusWithStaleMedia();
|
||||
$originalThumb = $media->thumbnail_url;
|
||||
|
||||
$this->artisan('admin:MigrateLocalS3MediaURL', ['id' => (string) $media->status_id, '--dry-run' => true])
|
||||
->assertExitCode(0);
|
||||
|
||||
expect($media->fresh()->thumbnail_url)->toBe($originalThumb);
|
||||
});
|
||||
|
||||
it('leaves already-correct media untouched', function () {
|
||||
$user = User::factory()->create();
|
||||
$user->refresh();
|
||||
$pid = $user->profile->id;
|
||||
$status = Status::factory()->create(['profile_id' => $pid, 'type' => 'photo']);
|
||||
$path = 'public/m/_v2/'.$pid.'/aa/bb/ok.jpg';
|
||||
|
||||
$media = Media::create([
|
||||
'status_id' => $status->id,
|
||||
'profile_id' => $pid,
|
||||
'media_path' => $path,
|
||||
'cdn_url' => 'https://cdn.test/'.$path,
|
||||
'thumbnail_url' => 'https://cdn.test/'.$path,
|
||||
'mime' => 'image/jpeg',
|
||||
'remote_media' => false,
|
||||
'order' => 0,
|
||||
]);
|
||||
|
||||
$updatedAt = $media->fresh()->updated_at;
|
||||
|
||||
$this->artisan('admin:MigrateLocalS3MediaURL', ['id' => (string) $status->id, '--force' => true])
|
||||
->assertExitCode(0);
|
||||
|
||||
expect($media->fresh()->updated_at->eq($updatedAt))->toBeTrue();
|
||||
});
|
||||
|
||||
it('never rewrites remote media', function () {
|
||||
$user = User::factory()->create();
|
||||
$user->refresh();
|
||||
$pid = $user->profile->id;
|
||||
$status = Status::factory()->create(['profile_id' => $pid, 'type' => 'photo']);
|
||||
|
||||
$media = Media::create([
|
||||
'status_id' => $status->id,
|
||||
'profile_id' => $pid,
|
||||
'media_path' => 'https://remote.example/image.jpg',
|
||||
'cdn_url' => 'https://s3.old.example/image.jpg',
|
||||
'remote_media' => true,
|
||||
'remote_url' => 'https://remote.example/image.jpg',
|
||||
'mime' => 'image/jpeg',
|
||||
'order' => 0,
|
||||
]);
|
||||
|
||||
$this->artisan('admin:MigrateLocalS3MediaURL', ['id' => (string) $status->id, '--force' => true])
|
||||
->assertExitCode(0);
|
||||
|
||||
// Unchanged: remote media is skipped.
|
||||
expect($media->fresh()->cdn_url)->toBe('https://s3.old.example/image.jpg');
|
||||
});
|
||||
|
||||
it('requires an id or --all', function () {
|
||||
$this->artisan('admin:MigrateLocalS3MediaURL')
|
||||
->assertExitCode(1);
|
||||
});
|
||||
|
||||
it('refuses to run on a local-storage instance', function () {
|
||||
// Simulate local storage: cloud disabled (set both, see beforeEach).
|
||||
Config::set('pixelfed.cloud_storage', false);
|
||||
ConfigCacheService::put('pixelfed.cloud_storage', false);
|
||||
|
||||
$this->artisan('admin:MigrateLocalS3MediaURL', ['--all' => true, '--force' => true])
|
||||
->expectsOutputToContain('Cloud storage is not enabled')
|
||||
->assertExitCode(1);
|
||||
});
|
||||
|
||||
it('with --oldDomain only rewrites URLs on that host', function () {
|
||||
// thumbnail_url on s3.old.example, optimized_url on other.example.
|
||||
$media = makeStatusWithStaleMedia('https://s3.old.example');
|
||||
$media->optimized_url = 'https://other.example/'.$media->media_path;
|
||||
$media->save();
|
||||
|
||||
$this->artisan('admin:MigrateLocalS3MediaURL', [
|
||||
'id' => (string) $media->status_id,
|
||||
'--oldDomain' => 's3.old.example',
|
||||
'--force' => true,
|
||||
])->assertExitCode(0);
|
||||
|
||||
$media->refresh();
|
||||
// Matched the filter -> rewritten.
|
||||
expect(parse_url($media->thumbnail_url, PHP_URL_HOST))->toBe('cdn.test');
|
||||
// Did NOT match the filter -> left as-is.
|
||||
expect(parse_url($media->optimized_url, PHP_URL_HOST))->toBe('other.example');
|
||||
});
|
||||
|
||||
it('with --newDomain override rewrites to the given host', function () {
|
||||
$media = makeStatusWithStaleMedia('https://s3.old.example');
|
||||
|
||||
$this->artisan('admin:MigrateLocalS3MediaURL', [
|
||||
'id' => (string) $media->status_id,
|
||||
'--newDomain' => 'media.example',
|
||||
'--force' => true,
|
||||
])->assertExitCode(0);
|
||||
|
||||
$media->refresh();
|
||||
expect(parse_url($media->thumbnail_url, PHP_URL_HOST))->toBe('media.example');
|
||||
expect(parse_url($media->optimized_url, PHP_URL_HOST))->toBe('media.example');
|
||||
});
|
||||
@ -0,0 +1,90 @@
|
||||
<?php
|
||||
|
||||
use App\Models\Media;
|
||||
use App\Models\Status;
|
||||
use App\Models\User;
|
||||
use App\Util\ActivityPub\Helpers;
|
||||
use Illuminate\Foundation\Testing\LazilyRefreshDatabase;
|
||||
|
||||
uses(LazilyRefreshDatabase::class);
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Duplicate media attachment on remote import
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Regression test for the 1062 duplicate-key violation on
|
||||
| media_status_id_media_path_unique when importing remote status attachments
|
||||
| (e.g. an Announce racing another inbox job, or a re-fetch). Media import
|
||||
| must be idempotent on (status_id, media_path).
|
||||
|
|
||||
*/
|
||||
|
||||
function attachmentPayload(string $url): array
|
||||
{
|
||||
return [
|
||||
'type' => 'Document',
|
||||
'mediaType' => 'image/jpeg',
|
||||
'url' => $url,
|
||||
'name' => 'alt text',
|
||||
'blurhash' => 'UREVf}R:E2WB~qNKWBs.XURkxZofD+n~oJR-',
|
||||
'width' => 768,
|
||||
'height' => 1024,
|
||||
];
|
||||
}
|
||||
|
||||
it('does not create a duplicate media row for the same status and url', function () {
|
||||
$user = User::factory()->create();
|
||||
$user->refresh();
|
||||
$status = Status::factory()->create(['profile_id' => $user->profile->id, 'type' => 'photo']);
|
||||
|
||||
$url = 'https://files.mastodon.social/media_attachments/files/117/original/e9ab6f0314043b12.jpeg';
|
||||
$payload = attachmentPayload($url);
|
||||
|
||||
$first = Helpers::createMediaAttachment($payload, $status, 0);
|
||||
// Second call (simulating re-import / race) must not throw and must be a no-op.
|
||||
$second = Helpers::createMediaAttachment($payload, $status, 0);
|
||||
|
||||
expect($first)->not->toBeNull();
|
||||
expect($second)->toBeNull();
|
||||
expect(Media::whereStatusId($status->id)->whereMediaPath($url)->count())->toBe(1);
|
||||
});
|
||||
|
||||
it('creates distinct rows for different urls on the same status', function () {
|
||||
$user = User::factory()->create();
|
||||
$user->refresh();
|
||||
$status = Status::factory()->create(['profile_id' => $user->profile->id, 'type' => 'photo:album']);
|
||||
|
||||
$a = Helpers::createMediaAttachment(attachmentPayload('https://files.mastodon.social/a.jpeg'), $status, 0);
|
||||
$b = Helpers::createMediaAttachment(attachmentPayload('https://files.mastodon.social/b.jpeg'), $status, 1);
|
||||
|
||||
expect($a)->not->toBeNull();
|
||||
expect($b)->not->toBeNull();
|
||||
expect(Media::whereStatusId($status->id)->count())->toBe(2);
|
||||
});
|
||||
|
||||
it('returns null when the row was inserted concurrently after the existence check', function () {
|
||||
$user = User::factory()->create();
|
||||
$user->refresh();
|
||||
$status = Status::factory()->create(['profile_id' => $user->profile->id, 'type' => 'photo']);
|
||||
|
||||
$url = 'https://files.mastodon.social/race.jpeg';
|
||||
|
||||
// Pre-insert the row to simulate the concurrent winner.
|
||||
Media::create([
|
||||
'remote_media' => true,
|
||||
'status_id' => $status->id,
|
||||
'profile_id' => $status->profile_id,
|
||||
'media_path' => $url,
|
||||
'remote_url' => $url,
|
||||
'mime' => 'image/jpeg',
|
||||
'version' => 3,
|
||||
'order' => 1,
|
||||
]);
|
||||
|
||||
// Should detect the existing row and return null without throwing.
|
||||
$result = Helpers::createMediaAttachment(attachmentPayload($url), $status, 0);
|
||||
|
||||
expect($result)->toBeNull();
|
||||
expect(Media::whereStatusId($status->id)->whereMediaPath($url)->count())->toBe(1);
|
||||
});
|
||||
@ -0,0 +1,139 @@
|
||||
<?php
|
||||
|
||||
namespace Tests\Unit\ActivityPub;
|
||||
|
||||
use App\Services\SecureMediaFetchService;
|
||||
use App\Util\ActivityPub\Helpers;
|
||||
use PHPUnit\Framework\Attributes\DataProvider;
|
||||
use PHPUnit\Framework\Attributes\Test;
|
||||
use Tests\TestCase;
|
||||
|
||||
/**
|
||||
* Regression tests for the SSRF hardening in the remote media/avatar fetch
|
||||
* path (variant of CVE-2026-71246).
|
||||
*
|
||||
* These cover the deterministic, network-free guards:
|
||||
* - isPublicIp() rejects private/reserved/link-local ranges (incl. the
|
||||
* 169.254.169.254 cloud-metadata address).
|
||||
* - normalizeHost() rejects IP literals and localhost domains.
|
||||
* - validateUrl() rejects non-https, userinfo, control chars, backslashes
|
||||
* and IP-literal URLs.
|
||||
* - SecureMediaFetchService fails closed (no network call) for URLs that
|
||||
* cannot pass validation.
|
||||
*/
|
||||
class SsrfUrlValidationTest extends TestCase
|
||||
{
|
||||
// ---- isPublicIp -------------------------------------------------------
|
||||
|
||||
public static function privateAndReservedIps(): array
|
||||
{
|
||||
return [
|
||||
'aws/gcp metadata' => ['169.254.169.254'],
|
||||
'link-local' => ['169.254.0.1'],
|
||||
'loopback v4' => ['127.0.0.1'],
|
||||
'rfc1918 10/8' => ['10.0.0.1'],
|
||||
'rfc1918 172.16/12' => ['172.18.0.1'],
|
||||
'rfc1918 192.168/16' => ['192.168.1.1'],
|
||||
'loopback v6' => ['::1'],
|
||||
'unique local v6' => ['fd00::1'],
|
||||
'unspecified' => ['0.0.0.0'],
|
||||
];
|
||||
}
|
||||
|
||||
#[Test]
|
||||
#[DataProvider('privateAndReservedIps')]
|
||||
public function it_rejects_private_and_reserved_ips(string $ip): void
|
||||
{
|
||||
$this->assertFalse(Helpers::isPublicIp($ip), $ip.' should be treated as non-public');
|
||||
}
|
||||
|
||||
public static function publicIps(): array
|
||||
{
|
||||
return [
|
||||
'cloudflare dns' => ['1.1.1.1'],
|
||||
'google dns' => ['8.8.8.8'],
|
||||
'public v6' => ['2606:4700:4700::1111'],
|
||||
];
|
||||
}
|
||||
|
||||
#[Test]
|
||||
#[DataProvider('publicIps')]
|
||||
public function it_accepts_public_ips(string $ip): void
|
||||
{
|
||||
$this->assertTrue(Helpers::isPublicIp($ip), $ip.' should be public');
|
||||
}
|
||||
|
||||
// ---- normalizeHost ----------------------------------------------------
|
||||
|
||||
#[Test]
|
||||
public function it_rejects_ip_literal_hosts(): void
|
||||
{
|
||||
$this->assertNull(Helpers::normalizeHost('169.254.169.254'));
|
||||
$this->assertNull(Helpers::normalizeHost('127.0.0.1'));
|
||||
$this->assertNull(Helpers::normalizeHost('::1'));
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function it_rejects_localhost_domains(): void
|
||||
{
|
||||
$this->assertNull(Helpers::normalizeHost('localhost'));
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function it_normalizes_regular_hosts(): void
|
||||
{
|
||||
$this->assertSame('example.com', Helpers::normalizeHost('Example.com.'));
|
||||
}
|
||||
|
||||
// ---- validateUrl ------------------------------------------------------
|
||||
|
||||
public static function invalidUrls(): array
|
||||
{
|
||||
return [
|
||||
'http scheme' => ['http://example.com/avatar.jpg'],
|
||||
'ftp scheme' => ['ftp://example.com/avatar.jpg'],
|
||||
'ip literal https' => ['https://169.254.169.254/latest/meta-data/'],
|
||||
'private ip literal' => ['https://172.18.0.1:9000/internal.jpg'],
|
||||
'userinfo smuggling' => ['https://user:pass@example.com/a.jpg'],
|
||||
'userinfo at-trick' => ['https://example.com@169.254.169.254/a.jpg'],
|
||||
'control char' => ["https://example.com/\r\n/a.jpg"],
|
||||
'backslash' => ['https://example.com\\@evil.com/a.jpg'],
|
||||
'no dot host' => ['https://localhost/a.jpg'],
|
||||
'empty' => [''],
|
||||
];
|
||||
}
|
||||
|
||||
#[Test]
|
||||
#[DataProvider('invalidUrls')]
|
||||
public function it_rejects_unsafe_urls(string $url): void
|
||||
{
|
||||
$this->assertFalse(Helpers::validateUrl($url), $url.' should be rejected');
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function it_accepts_a_normal_https_url_in_non_prod(): void
|
||||
{
|
||||
// In the local/testing environment shouldCheckDNS()/shouldCheckBans()
|
||||
// are off, so a well-formed public https URL normalizes successfully.
|
||||
$url = 'https://example.com/avatar.jpg';
|
||||
$this->assertSame($url, Helpers::validateUrl($url));
|
||||
}
|
||||
|
||||
// ---- SecureMediaFetchService fails closed -----------------------------
|
||||
|
||||
#[Test]
|
||||
public function secure_fetch_head_fails_closed_on_invalid_url(): void
|
||||
{
|
||||
// These never resolve/connect because validateUrl rejects them first.
|
||||
$this->assertFalse(SecureMediaFetchService::head('http://169.254.169.254/'));
|
||||
$this->assertFalse(SecureMediaFetchService::head('https://172.18.0.1:9000/internal.jpg'));
|
||||
$this->assertFalse(SecureMediaFetchService::head('https://example.com@169.254.169.254/a.jpg'));
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function secure_fetch_get_fails_closed_on_invalid_url(): void
|
||||
{
|
||||
$this->assertFalse(SecureMediaFetchService::get('http://169.254.169.254/'));
|
||||
$this->assertFalse(SecureMediaFetchService::get('https://172.18.0.1:9000/internal.jpg'));
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue