diff --git a/CHANGELOG.md b/CHANGELOG.md index b04e6fcf3..53ad5cb9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,11 @@ - ([#6856](https://github.com/pixelfed/pixelfed/pull/6856)) - Testing: Refactored the testing environment - Testing: Added 250+ tests +- ([#6930](https://github.com/pixelfed/pixelfed/pull/6930)) + - added admin:fixProfileCounts command to fix the followers/following/statuses cache count locally and remotely + - added admin:MigrateLocalS3MediaURL command for fixing dead CDN storage paths. + - added status:user status:profile status:post command for diagnostic purposes. + - added SecureMediaFetchService to enhance/harden media downloading from remote servers ## [v0.12.9 (2026-08-25)](https://github.com/pixelfed/pixelfed/compare/v0.12.9...dev) diff --git a/app/Console/Commands/AdminInviteCommand.php b/app/Console/Commands/Admin/AdminInviteCommand.php similarity index 99% rename from app/Console/Commands/AdminInviteCommand.php rename to app/Console/Commands/Admin/AdminInviteCommand.php index e2a6a1d47..6f094032a 100644 --- a/app/Console/Commands/AdminInviteCommand.php +++ b/app/Console/Commands/Admin/AdminInviteCommand.php @@ -1,6 +1,6 @@ option('sourceDisk'); + $destName = config('filesystems.cloud'); + + if ($sourceName === $destName) { + $this->error('Source disk and destination (cloud) disk are the same ('.$sourceName.').'); + $this->line('Point AWS_* at the NEW bucket and keep the OLD bucket creds in the source disk.'); + + return 1; + } + + try { + $sourceDisk = Storage::disk($sourceName); + } catch (\Throwable $e) { + $this->error('Source disk "'.$sourceName.'" could not be resolved: '.$e->getMessage()); + + return 1; + } + + try { + $destDisk = Storage::disk($destName); + } catch (\Throwable $e) { + $this->error('Destination cloud disk "'.$destName.'" could not be resolved: '.$e->getMessage()); + + return 1; + } + + $sourceHost = $this->diskHost($sourceDisk); + $destHost = $this->diskHost($destDisk); + + if (! $sourceHost) { + $this->error('Source disk "'.$sourceName.'" is not configured (no resolvable URL). Set AWS_OLD_* in your .env.'); + + return 1; + } + if (! $destHost) { + $this->error('Destination cloud disk "'.$destName.'" is not configured (no resolvable URL). Set AWS_* in your .env.'); + + return 1; + } + if (strcasecmp($sourceHost, $destHost) === 0) { + $this->error('Source and destination resolve to the same host ('.$sourceHost.'); nothing to migrate.'); + + return 1; + } + + $this->info('Source (old): '.$sourceName.' -> '.$sourceHost); + $this->info('Destination (new): '.$destName.' -> '.$destHost); + $this->newLine(); + + if (! $this->option('dry-run') && ! $this->option('force')) { + if (! $this->confirm('Copy existing media from "'.$sourceHost.'" to "'.$destHost.'" and rewrite URLs?', true)) { + $this->comment('Aborted.'); + + return 0; + } + } + + $limit = (int) $this->option('limit'); + $moved = 0; + $skipped = 0; + $failed = 0; + + // Candidates: non-remote media whose stored URL still points at the + // source host (i.e. not yet migrated to the destination). + $query = Media::whereRemoteMedia(false) + ->whereNotNull(['media_path', 'cdn_url']) + ->orderByDesc('id') + ->limit($limit); + + $bar = $this->output->createProgressBar($query->count()); + $bar->start(); + + foreach ($query->get() as $media) { + $result = $this->migrateOne($media, $sourceDisk, $destDisk, $sourceHost, $destHost); + match ($result) { + 'moved' => $moved++, + 'skipped' => $skipped++, + default => $failed++, + }; + $bar->advance(); + } + + $bar->finish(); + $this->newLine(2); + $this->info(($this->option('dry-run') ? '[dry-run] ' : '').'Done. moved='.$moved.' skipped='.$skipped.' failed='.$failed.'.'); + if ($this->movedBytes) { + $this->info('Transferred '.PrettyNumber::size($this->movedBytes).' to the new bucket.'); + } + if ($moved > 0 && ! $this->option('dry-run')) { + $this->comment('Tip: run `php artisan cache:clear` if any stale URLs remain cached elsewhere.'); + } + + return 0; + } + + /** + * @return string one of moved|skipped|failed + */ + protected function migrateOne(Media $media, $sourceDisk, $destDisk, string $sourceHost, string $destHost): string + { + if (Str::startsWith((string) $media->media_path, 'http')) { + return 'skipped'; + } + + // Only act on rows whose URL still references the source host. + $currentHost = parse_url((string) $media->cdn_url, PHP_URL_HOST); + if (! $currentHost || strcasecmp($currentHost, $sourceHost) !== 0) { + return 'skipped'; + } + + // If already present on the destination, just rewrite the URLs. + $onDest = $destDisk->exists($media->media_path); + $onSource = $sourceDisk->exists($media->media_path); + + if (! $onDest && ! $onSource) { + // File missing from both buckets; leave URLs untouched. + return 'skipped'; + } + + if ($this->option('dry-run')) { + return 'moved'; + } + + try { + if (! $onDest) { + // Copy primary + thumbnail source -> destination. + $this->copy($media->media_path, $sourceDisk, $destDisk); + if ($media->thumbnail_path && $sourceDisk->exists($media->thumbnail_path)) { + $this->copy($media->thumbnail_path, $sourceDisk, $destDisk); + } + + if (! $this->verify($media->media_path, $sourceDisk, $destDisk, $media->original_sha256)) { + $this->warn(PHP_EOL.'Verify failed for media '.$media->id.' ('.$media->media_path.'); left source intact, URLs unchanged.'); + + return 'failed'; + } + } + + // Rewrite URLs to the destination bucket. + $media->cdn_url = $destDisk->url($media->media_path); + $media->optimized_url = $media->cdn_url; + if ($media->thumbnail_path && $destDisk->exists($media->thumbnail_path)) { + $media->thumbnail_url = $destDisk->url($media->thumbnail_path); + } + $media->replicated_at = now(); + $media->save(); + + $this->movedBytes += (int) $media->size; + + // Integrated GC on the OLD bucket, unless kept. + if (! $this->option('keep-source') && $onSource) { + $sourceDisk->delete($media->media_path); + if ($media->thumbnail_path && $sourceDisk->exists($media->thumbnail_path)) { + $sourceDisk->delete($media->thumbnail_path); + } + } + + if ($media->status_id) { + MediaService::del($media->status_id); + StatusService::del($media->status_id, false); + } + + return 'moved'; + } catch (\Throwable $e) { + $this->warn(PHP_EOL.'Error migrating media '.$media->id.': '.$e->getMessage()); + + return 'failed'; + } + } + + protected function copy(string $path, $sourceDisk, $destDisk): void + { + $stream = $sourceDisk->readStream($path); + if ($stream === false || $stream === null) { + throw new \RuntimeException('Could not open source stream for '.$path); + } + $destDisk->writeStream($path, $stream); + if (is_resource($stream)) { + fclose($stream); + } + } + + /** + * Verify the destination copy matches the source by size, and by sha256 + * against the stored original checksum when available. Fails closed. + */ + protected function verify(string $path, $sourceDisk, $destDisk, ?string $expectedSha = null): bool + { + if (! $destDisk->exists($path)) { + return false; + } + + $sourceSize = $sourceDisk->size($path); + $destSize = $destDisk->size($path); + if ($sourceSize === false || $destSize === false || $sourceSize !== $destSize) { + return false; + } + + if ($expectedSha) { + // Hash the freshly written destination object to confirm integrity. + $stream = $destDisk->readStream($path); + if ($stream === false || $stream === null) { + return false; + } + $ctx = hash_init('sha256'); + hash_update_stream($ctx, $stream); + if (is_resource($stream)) { + fclose($stream); + } + $destSha = hash_final($ctx); + if (! hash_equals($expectedSha, $destSha)) { + return false; + } + } + + return true; + } + + protected function diskHost($disk): ?string + { + try { + $url = $disk->url('probe'); + $host = parse_url($url, PHP_URL_HOST); + + return $host ?: null; + } catch (\Throwable $e) { + return null; + } + } +} diff --git a/app/Console/Commands/Admin/MediaMoveStorageCloudToLocal.php b/app/Console/Commands/Admin/MediaMoveStorageCloudToLocal.php new file mode 100644 index 000000000..4177e5247 --- /dev/null +++ b/app/Console/Commands/Admin/MediaMoveStorageCloudToLocal.php @@ -0,0 +1,227 @@ +readEnvValue('PF_ENABLE_CLOUD'); + $cloudEnabled = filter_var($envCloud, FILTER_VALIDATE_BOOLEAN); + + if ($cloudEnabled) { + $this->warn('PF_ENABLE_CLOUD is currently true.'); + $this->line('New uploads would keep landing on CLOUD storage during this migration.'); + if ($this->option('dry-run')) { + $this->line('[dry-run] Would set PF_ENABLE_CLOUD=false (.env + runtime + config cache).'); + } elseif ($this->option('force') || $this->confirm('Set PF_ENABLE_CLOUD=false now so new uploads stay local?', true)) { + $this->setStorageEnv('PF_ENABLE_CLOUD', 'false', 'pixelfed.cloud_storage', false); + $this->info('PF_ENABLE_CLOUD set to false (.env + live runtime + config cache).'); + } else { + $this->error('Aborting: refusing to migrate to local while new uploads go to cloud.'); + + return 1; + } + } else { + $this->info('PF_ENABLE_CLOUD is already false; new uploads stay local. ✓'); + } + + $this->newLine(); + if (! $this->option('dry-run') && ! $this->option('force')) { + if (! $this->confirm('Begin migrating cloud media to local?', true)) { + $this->comment('Aborted.'); + + return 0; + } + } + + $limit = (int) $this->option('limit'); + $moved = 0; + $skipped = 0; + $failed = 0; + + // Candidates: non-remote media that has a cloud copy (cdn_url set). + $query = Media::whereRemoteMedia(false) + ->whereNotNull(['media_path', 'cdn_url']) + ->orderByDesc('id') + ->limit($limit); + + $bar = $this->output->createProgressBar($query->count()); + $bar->start(); + + foreach ($query->get() as $media) { + $result = $this->migrateOne($media, $localDisk, $cloudDisk); + match ($result) { + 'moved' => $moved++, + 'skipped' => $skipped++, + default => $failed++, + }; + $bar->advance(); + } + + $bar->finish(); + $this->newLine(2); + $this->info(($this->option('dry-run') ? '[dry-run] ' : '').'Done. moved='.$moved.' skipped='.$skipped.' failed='.$failed.'.'); + if ($this->movedBytes) { + $this->info('Transferred '.PrettyNumber::size($this->movedBytes).' back to local storage.'); + } + + return 0; + } + + /** + * @return string one of moved|skipped|failed + */ + protected function migrateOne(Media $media, $localDisk, $cloudDisk): string + { + if (Str::startsWith((string) $media->media_path, 'http')) { + return 'skipped'; + } + + // Must exist on cloud to pull down. + if (! $cloudDisk->exists($media->media_path)) { + // Already local-only? just clear the cloud url fields. + if ($localDisk->exists($media->media_path)) { + if (! $this->option('dry-run')) { + $this->clearCloudFields($media); + $media->save(); + } + + return 'skipped'; + } + + return 'failed'; + } + + if ($this->option('dry-run')) { + return 'moved'; + } + + try { + $this->copyToLocal($media->media_path, $localDisk, $cloudDisk); + if ($media->thumbnail_path && $cloudDisk->exists($media->thumbnail_path)) { + $this->copyToLocal($media->thumbnail_path, $localDisk, $cloudDisk); + } + + if (! $this->verify($media->media_path, $localDisk, $cloudDisk, $media->original_sha256)) { + $this->warn(PHP_EOL.'Verify failed for media '.$media->id.' ('.$media->media_path.'); left cloud copy intact.'); + + return 'failed'; + } + + // Point URLs back at local storage. + $this->clearCloudFields($media); + + // Integrated GC: delete the verified cloud copy unless --keep-cloud. + if (! $this->option('keep-cloud')) { + $cloudDisk->delete($media->media_path); + if ($media->thumbnail_path && $cloudDisk->exists($media->thumbnail_path)) { + $cloudDisk->delete($media->thumbnail_path); + } + } + + $media->save(); + $this->movedBytes += (int) $media->size; + + if ($media->status_id) { + MediaService::del($media->status_id); + StatusService::del($media->status_id, false); + } + + return 'moved'; + } catch (\Throwable $e) { + $this->warn(PHP_EOL.'Error migrating media '.$media->id.': '.$e->getMessage()); + + return 'failed'; + } + } + + /** + * Reset a media row to local-served state. + */ + protected function clearCloudFields(Media $media): void + { + $media->cdn_url = null; + $media->optimized_url = null; + $media->thumbnail_url = null; + $media->replicated_at = null; + // version 4 meant "local deleted, cloud only"; reset so the file is + // treated as locally present again. + if ($media->version === '4' || $media->version === 4) { + $media->version = 3; + } + } + + protected function copyToLocal(string $path, $localDisk, $cloudDisk): void + { + $stream = $cloudDisk->readStream($path); + if ($stream === false || $stream === null) { + throw new \RuntimeException('Could not open cloud stream for '.$path); + } + $localDisk->writeStream($path, $stream); + if (is_resource($stream)) { + fclose($stream); + } + } + + /** + * Verify the local copy matches the cloud source by size, and by sha256 + * against the stored original checksum when available. Fails closed. + */ + protected function verify(string $path, $localDisk, $cloudDisk, ?string $expectedSha = null): bool + { + if (! $localDisk->exists($path)) { + return false; + } + + $localSize = $localDisk->size($path); + $cloudSize = $cloudDisk->size($path); + if ($localSize === false || $cloudSize === false || $localSize !== $cloudSize) { + return false; + } + + if ($expectedSha) { + $localSha = @hash_file('sha256', $localDisk->path($path)); + if ($localSha && ! hash_equals($expectedSha, $localSha)) { + return false; + } + } + + return true; + } +} diff --git a/app/Console/Commands/Admin/MediaMoveStorageLocalToCloud.php b/app/Console/Commands/Admin/MediaMoveStorageLocalToCloud.php new file mode 100644 index 000000000..ef686d297 --- /dev/null +++ b/app/Console/Commands/Admin/MediaMoveStorageLocalToCloud.php @@ -0,0 +1,239 @@ +error('Cloud disk ('.config('filesystems.cloud').') could not be resolved: '.$e->getMessage()); + + return 1; + } + + if (! $this->cloudHost()) { + $this->error('Cloud disk ('.config('filesystems.cloud').') is not configured (no resolvable URL).'); + $this->line('Set AWS_URL / AWS_* in your .env before migrating to cloud.'); + + return 1; + } + + // --- Ensure new uploads route to cloud during the migration -------- + $envCloud = $this->readEnvValue('PF_ENABLE_CLOUD'); + $cloudEnabled = filter_var($envCloud, FILTER_VALIDATE_BOOLEAN); + + if (! $cloudEnabled) { + $this->warn('PF_ENABLE_CLOUD is currently "'.($envCloud ?? 'unset').'".'); + $this->line('New uploads would keep landing on LOCAL storage during this migration.'); + if ($this->option('dry-run')) { + $this->line('[dry-run] Would set PF_ENABLE_CLOUD=true (.env + runtime + config cache).'); + } elseif ($this->option('force') || $this->confirm('Set PF_ENABLE_CLOUD=true now so new uploads go to cloud?', true)) { + $this->setStorageEnv('PF_ENABLE_CLOUD', 'true', 'pixelfed.cloud_storage', true); + $this->info('PF_ENABLE_CLOUD set to true (.env + live runtime + config cache).'); + } else { + $this->error('Aborting: refusing to migrate to cloud while new uploads stay local.'); + + return 1; + } + } else { + $this->info('PF_ENABLE_CLOUD is already true; new uploads route to cloud. ✓'); + } + + $this->newLine(); + if (! $this->option('dry-run') && ! $this->option('force')) { + if (! $this->confirm('Begin migrating local media to cloud?', true)) { + $this->comment('Aborted.'); + + return 0; + } + } + + $limit = (int) $this->option('limit'); + $moved = 0; + $skipped = 0; + $failed = 0; + + // Candidates: local, non-remote media not yet replicated to cloud. + $query = Media::whereRemoteMedia(false) + ->whereNotNull('media_path') + ->where(function ($q) { + $q->whereNull('cdn_url')->orWhereNull('replicated_at')->orWhereNot('version', '4'); + }) + ->orderByDesc('id') + ->limit($limit); + + $bar = $this->output->createProgressBar($query->count()); + $bar->start(); + + foreach ($query->get() as $media) { + $result = $this->migrateOne($media, $localDisk, $cloudDisk); + match ($result) { + 'moved' => $moved++, + 'skipped' => $skipped++, + default => $failed++, + }; + $bar->advance(); + } + + $bar->finish(); + $this->newLine(2); + $this->info(($this->option('dry-run') ? '[dry-run] ' : '').'Done. moved='.$moved.' skipped='.$skipped.' failed='.$failed.'.'); + if ($this->movedBytes) { + $this->info('Transferred '.PrettyNumber::size($this->movedBytes).' to cloud storage.'); + } + + return 0; + } + + /** + * @return string one of moved|skipped|failed + */ + protected function migrateOne(Media $media, $localDisk, $cloudDisk): string + { + if (Str::startsWith((string) $media->media_path, 'http')) { + return 'skipped'; + } + + // Nothing to do if the local file is gone. + if (! $localDisk->exists($media->media_path)) { + // Already on cloud only? mark version and move on. + if ($cloudDisk->exists($media->media_path)) { + if (! $this->option('dry-run') && $media->version !== '4') { + $media->version = 4; + $media->save(); + } + + return 'skipped'; + } + + return 'skipped'; + } + + if ($this->option('dry-run')) { + return 'moved'; + } + + try { + // Copy the primary file (and thumbnail) to cloud. + $this->copyToCloud($media->media_path, $localDisk, $cloudDisk); + if ($media->thumbnail_path && $localDisk->exists($media->thumbnail_path)) { + $this->copyToCloud($media->thumbnail_path, $localDisk, $cloudDisk); + } + + // Verify the primary file before touching anything else. + if (! $this->verify($media->media_path, $localDisk, $cloudDisk, $media->original_sha256)) { + $this->warn(PHP_EOL.'Verify failed for media '.$media->id.' ('.$media->media_path.'); left local copy intact.'); + + return 'failed'; + } + + // Update URL fields to the cloud disk. + $media->cdn_url = $cloudDisk->url($media->media_path); + $media->optimized_url = $media->cdn_url; + if ($media->thumbnail_path && $cloudDisk->exists($media->thumbnail_path)) { + $media->thumbnail_url = $cloudDisk->url($media->thumbnail_path); + } + $media->replicated_at = now(); + + // Integrated GC: delete the verified local copy unless --keep-local. + if (! $this->option('keep-local')) { + $localDisk->delete($media->media_path); + if ($media->thumbnail_path && $localDisk->exists($media->thumbnail_path)) { + $localDisk->delete($media->thumbnail_path); + } + $media->version = 4; + } + + $media->save(); + $this->movedBytes += (int) $media->size; + + if ($media->status_id) { + MediaService::del($media->status_id); + StatusService::del($media->status_id, false); + } + + return 'moved'; + } catch (\Throwable $e) { + $this->warn(PHP_EOL.'Error migrating media '.$media->id.': '.$e->getMessage()); + + return 'failed'; + } + } + + protected function copyToCloud(string $path, $localDisk, $cloudDisk): void + { + $p = explode('/', $path); + $name = array_pop($p); + $storagePath = implode('/', $p); + + // Reuse the resilient uploader (handles alt disks + retries). + ResilientMediaStorageService::store($storagePath, $localDisk->path($path), $name); + } + + /** + * Verify the cloud copy matches the local source by size, and by sha256 + * when a checksum is available/cheap. Fails closed. + */ + protected function verify(string $path, $localDisk, $cloudDisk, ?string $expectedSha = null): bool + { + if (! $cloudDisk->exists($path)) { + return false; + } + + $localSize = $localDisk->size($path); + $cloudSize = $cloudDisk->size($path); + if ($localSize === false || $cloudSize === false || $localSize !== $cloudSize) { + return false; + } + + // If we already have the original checksum, verify the local file still + // matches it (so we never delete a locally-corrupted-but-uploaded file + // without noticing). Cloud content hashing would require a full + // download, which we avoid for large media; size parity + known sha + // is a strong signal. + if ($expectedSha) { + $localSha = @hash_file('sha256', $localDisk->path($path)); + if ($localSha && ! hash_equals($expectedSha, $localSha)) { + return false; + } + } + + return true; + } +} diff --git a/app/Console/Commands/Admin/MigrateLocalS3MediaURL.php b/app/Console/Commands/Admin/MigrateLocalS3MediaURL.php new file mode 100644 index 000000000..3a2f4e6bb --- /dev/null +++ b/app/Console/Commands/Admin/MigrateLocalS3MediaURL.php @@ -0,0 +1,382 @@ +error('Cloud storage is not enabled (PF_ENABLE_CLOUD is false).'); + $this->line('This instance serves media from local storage; there are no cloud media URLs to migrate.'); + + return 1; + } + + // Safe default target = the currently configured cloud disk host, + // driven by AWS_URL in .env. Allow explicit override via --newDomain. + $configuredHost = $this->cloudHost(); + $override = $this->normalizeHost($this->option('newDomain')); + $this->newHost = $override ?: $configuredHost; + + if (! $this->newHost) { + $this->error('Could not resolve a target host.'); + $this->line('The cloud disk ('.config('filesystems.cloud').') did not return a usable URL.'); + $this->line('Set AWS_URL in your .env, or pass --newDomain explicitly.'); + + return 1; + } + + // Defensive guard: when auto-detecting the target (no --newDomain + // override), never rewrite media URLs to the app's own domain. That + // would indicate local storage or a misconfigured cloud disk URL + // (AWS_URL). An explicit --newDomain is treated as a deliberate choice. + if (! $override) { + $appHost = parse_url(config('app.url'), PHP_URL_HOST); + if ($appHost && strcasecmp($this->newHost, $appHost) === 0) { + $this->error('Refusing to run: auto-detected target host ('.$this->newHost.') is the app domain.'); + $this->line('That indicates local storage or a misconfigured cloud disk URL (AWS_URL).'); + $this->line('If you really intend this, pass an explicit --newDomain.'); + + return 1; + } + } + + $this->oldHost = $this->normalizeHost($this->option('oldDomain')); + + $id = $this->argument('id'); + $all = $this->option('all'); + + if (! $id && ! $all) { + $this->error('Provide a status id/URL, or pass --all.'); + + return 1; + } + + if ($id && $all) { + $this->error('Pass either a status id or --all, not both.'); + + return 1; + } + + // Show the plan and require explicit approval of the target host. + $this->info('Target host (newDomain): '.$this->newHost.($override ? ' (override)' : ' (from configured cloud disk)')); + $this->info('Filter (oldDomain): '.($this->oldHost ?: 'none — rewriting all stale hosts')); + if ($this->newHost !== $configuredHost) { + $this->warn('Note: target host differs from the configured cloud disk host ('.($configuredHost ?: 'unresolved').').'); + } + + if (! $this->option('dry-run') && ! $this->option('force')) { + if (! $this->confirm('Rewrite media URLs to "'.$this->newHost.'"?', false)) { + $this->comment('Aborted.'); + + return 0; + } + } + $this->newLine(); + + if ($id) { + return $this->handleSingle($id); + } + + return $this->handleAll(); + } + + /** + * Extract a bare host from a domain/URL option value. + */ + protected function normalizeHost(?string $value): ?string + { + $value = trim((string) $value); + if ($value === '') { + return null; + } + // Accept full URLs or bare hosts. + if (str_contains($value, '://')) { + $host = parse_url($value, PHP_URL_HOST); + + return $host ?: null; + } + + // Strip any accidental path/scheme fragments. + $host = parse_url('https://'.$value, PHP_URL_HOST); + + return $host ?: null; + } + + protected function handleSingle(string $id): int + { + $statusId = $this->resolveStatusId($id); + if (! $statusId) { + $this->error('Could not extract a status id from "'.$id.'".'); + + return 1; + } + + $status = Status::withTrashed()->find($statusId); + if (! $status) { + $this->error('No status found with id '.$statusId.'.'); + + return 1; + } + + $media = Media::whereStatusId($status->id)->get(); + if ($media->isEmpty()) { + $this->comment('Status '.$status->id.' has no media.'); + + return 0; + } + + $fixed = 0; + foreach ($media as $m) { + if ($this->migrateOne($m)) { + $fixed++; + } + } + + if ($fixed > 0 && ! $this->option('dry-run')) { + $this->bustCaches($status->id); + } + + $this->newLine(); + $this->info(($this->option('dry-run') ? 'Would fix ' : 'Fixed ').$fixed.' media row(s) for status '.$status->id.'.'); + if ($fixed > 0 && ! $this->option('dry-run')) { + $this->comment('Caches busted for this status.'); + } + + return 0; + } + + protected function handleAll(): int + { + $fixed = 0; + $scanned = 0; + $affectedStatusIds = []; + + Media::whereNull('remote_url') + ->where(function ($q) { + $q->whereNull('remote_media')->orWhere('remote_media', false); + }) + ->lazyById(1000, 'id') + ->each(function ($m) use (&$fixed, &$scanned, &$affectedStatusIds) { + $scanned++; + if ($this->migrateOne($m)) { + $fixed++; + if ($m->status_id) { + $affectedStatusIds[$m->status_id] = true; + } + } + }); + + if (! $this->option('dry-run')) { + foreach (array_keys($affectedStatusIds) as $sid) { + $this->bustCaches($sid); + } + } + + $this->newLine(); + $this->info('Scanned '.$scanned.' local media rows; '.($this->option('dry-run') ? 'would fix ' : 'fixed ').$fixed.'.'); + if ($fixed > 0 && ! $this->option('dry-run')) { + $this->comment('Caches busted for '.count($affectedStatusIds).' affected status(es).'); + $this->comment('Tip: run `php artisan cache:clear` if any stale URLs remain cached elsewhere.'); + } + + return 0; + } + + /** + * Rebuild any stale URL field on a single media row from its storage path. + * Only writes when a field's host differs from the target host. + * + * @return bool whether the row was (or would be) changed + */ + protected function migrateOne(Media $media): bool + { + // Never touch remote media or rows whose media_path is an absolute URL. + if ($media->remote_media || Str::startsWith((string) $media->media_path, 'http')) { + return false; + } + + $changes = []; + + // cdn_url and optimized_url are both derived from media_path; + // thumbnail_url is derived from thumbnail_path. + $map = [ + 'cdn_url' => $media->media_path, + 'optimized_url' => $media->media_path, + 'thumbnail_url' => $media->thumbnail_path, + ]; + + foreach ($map as $field => $path) { + $current = $media->{$field}; + if (! $current) { + // Field not set; leave it as-is (nothing to migrate). + continue; + } + if (! $path) { + // No source path to rebuild from; skip. + continue; + } + $host = parse_url($current, PHP_URL_HOST); + if (! $this->shouldRewrite($host)) { + continue; + } + + $rebuilt = $this->targetUrl($path); + if (! $rebuilt) { + continue; + } + + $changes[$field] = ['from' => $current, 'to' => $rebuilt]; + } + + if (empty($changes)) { + return false; + } + + $this->warn('media '.$media->id.(($media->status_id) ? ' (status '.$media->status_id.')' : '').':'); + foreach ($changes as $field => $c) { + $fromHost = parse_url($c['from'], PHP_URL_HOST); + $this->line(' '.$field.': '.$fromHost.' -> '.$this->newHost); + } + + if ($this->option('dry-run')) { + return true; + } + + foreach ($changes as $field => $c) { + $media->{$field} = $c['to']; + } + $media->save(); + + return true; + } + + protected function bustCaches($statusId): void + { + MediaService::del($statusId); + StatusService::del($statusId, true); + } + + /** + * Decide whether a URL on $currentHost should be rewritten. + * Skips when already on the target host, and honours the optional + * --oldDomain filter. + */ + protected function shouldRewrite(?string $currentHost): bool + { + if (! $currentHost) { + return false; + } + // Already on the target host. + if (strcasecmp($currentHost, $this->newHost) === 0) { + return false; + } + // With --oldDomain, only rewrite that specific host. + if ($this->oldHost !== null && strcasecmp($currentHost, $this->oldHost) !== 0) { + return false; + } + + return true; + } + + /** + * Build the target URL for a storage path against the target host. + * Uses the configured cloud disk to produce the correct path, then + * swaps in --newDomain when it overrides the configured host. + */ + protected function targetUrl(string $path): ?string + { + $url = $this->diskUrl($path); + if (! $url) { + return null; + } + + $diskHost = parse_url($url, PHP_URL_HOST); + if ($diskHost && strcasecmp($diskHost, $this->newHost) !== 0) { + // Override host was requested; swap it into the disk-built URL. + $url = preg_replace('#^(https?://)'.preg_quote($diskHost, '#').'#i', '$1'.$this->newHost, $url); + } + + return $url; + } + + protected function diskUrl(string $path): ?string + { + try { + return (string) Storage::disk(config('filesystems.cloud'))->url($path); + } catch (\Throwable $e) { + return null; + } + } + + protected function cloudHost(): ?string + { + try { + $url = Storage::disk(config('filesystems.cloud'))->url('probe'); + $host = parse_url($url, PHP_URL_HOST); + + return $host ?: null; + } catch (\Throwable $e) { + return null; + } + } + + protected function resolveStatusId(string $input): ?string + { + $input = trim($input); + if (ctype_digit($input)) { + return $input; + } + if (preg_match('#/p/[^/]+/(\d+)#', $input, $m)) { + return $m[1]; + } + if (preg_match('#(\d{6,})#', $input, $m)) { + return $m[1]; + } + + return null; + } +} diff --git a/app/Console/Commands/RegenerateThumbnails.php b/app/Console/Commands/Admin/RegenerateThumbnails.php similarity index 96% rename from app/Console/Commands/RegenerateThumbnails.php rename to app/Console/Commands/Admin/RegenerateThumbnails.php index cb8327baf..384338c56 100644 --- a/app/Console/Commands/RegenerateThumbnails.php +++ b/app/Console/Commands/Admin/RegenerateThumbnails.php @@ -1,6 +1,6 @@ error('Cloud storage not enabled. Exiting...'); - - return; - } - - if (! $this->confirm('Are you sure you want to proceed?')) { - return; - } - - $limit = $this->option('limit'); - $hugeMode = $this->option('huge'); - - if ($limit > 500 && ! $hugeMode) { - $this->error('Max limit exceeded, use a limit lower than 500 or run again with the --huge flag'); - - return; - } - - $bar = $this->output->createProgressBar($limit); - $bar->start(); - - Media::whereNot('version', '4') - ->where('created_at', '<', now()->subDays(2)) - ->whereRemoteMedia(false) - ->whereNotNull(['status_id', 'profile_id']) - ->whereNull(['cdn_url', 'replicated_at']) - ->orderByDesc('size') - ->take($limit) - ->get() - ->each(function ($media) use ($bar) { - if (Storage::disk('local')->exists($media->media_path)) { - $this->totalSize = $this->totalSize + $media->size; - try { - MediaStorageService::store($media); - } catch (FileNotFoundException $e) { - $this->error('Error migrating media '.$media->id.' to cloud storage: '.$e->getMessage()); - - return; - } catch (NotFoundHttpException $e) { - $this->error('Error migrating media '.$media->id.' to cloud storage: '.$e->getMessage()); - - return; - } catch (\Exception $e) { - $this->error('Error migrating media '.$media->id.' to cloud storage: '.$e->getMessage()); - - return; - } - } - $bar->advance(); - }); - - $bar->finish(); - $this->line(' '); - $this->info('Finished!'); - if ($this->totalSize) { - $this->info('Uploaded '.PrettyNumber::size($this->totalSize).' of media to cloud storage!'); - $this->line(' '); - $this->info('These files are still stored locally, and will be automatically removed.'); - } - - return Command::SUCCESS; - } -} diff --git a/app/Console/Commands/Concerns/ManagesMediaStorageEnv.php b/app/Console/Commands/Concerns/ManagesMediaStorageEnv.php new file mode 100644 index 000000000..716a2a02d --- /dev/null +++ b/app/Console/Commands/Concerns/ManagesMediaStorageEnv.php @@ -0,0 +1,122 @@ +environmentFilePath(); + if (! is_file($envPath)) { + return null; + } + $payload = file_get_contents($envPath); + if ($payload === false) { + return null; + } + if (! preg_match("/^{$key}=([^\r\n]*)/m", $payload, $m)) { + return null; + } + + return trim($m[1], " \t\"'"); + } + + /** + * Set an .env key + the live runtime config + config-cache entry so the + * change takes effect immediately on a running server. + * + * @param string $configKey dotted config key kept in sync (e.g. 'pixelfed.cloud_storage') + * @param mixed $configValue the typed runtime value (e.g. true/false) + */ + protected function setStorageEnv(string $envKey, string $envValue, string $configKey, $configValue): void + { + // 1. Persist to .env atomically (survives restarts). + $this->updateEnvFile($envKey, $envValue); + + // 2. Update the live runtime config for the current process. + config([$configKey => $configValue]); + + // 3. Update the DB-backed config cache so other workers/requests + // reading via config_cache() see the new value (hot server). + try { + ConfigCacheService::put($configKey, $configValue); + } catch (\Throwable $e) { + $this->warn('Could not update config cache for '.$configKey.': '.$e->getMessage()); + } + } + + /** + * The configured cloud disk host (used to sanity check cloud config). + */ + protected function cloudHost(): ?string + { + try { + $url = Storage::disk(config('filesystems.cloud'))->url('probe'); + $host = parse_url($url, PHP_URL_HOST); + + return $host ?: null; + } catch (\Throwable $e) { + return null; + } + } + + // ---- Atomic .env writer (adapted from Installer) --------------------- + + protected function updateEnvFile($key, $value): void + { + $envPath = app()->environmentFilePath(); + $payload = file_get_contents($envPath); + + $value = str_replace(['\\', '"', "\n", "\r"], ['\\\\', '\\"', '\\n', '\\r'], $value); + + if (($existing = $this->existingEnv($key, $payload)) !== false) { + $payload = str_replace("{$key}={$existing}", "{$key}=\"{$value}\"", $payload); + } else { + $payload = $payload."\n{$key}=\"{$value}\"\n"; + } + + $this->storeEnv($payload); + } + + protected function existingEnv($needle, $haystack) + { + preg_match("/^{$needle}=[^\r\n]*/m", $haystack, $matches); + if ($matches && count($matches)) { + return substr($matches[0], strlen($needle) + 1); + } + + return false; + } + + protected function storeEnv($payload): void + { + $envPath = app()->environmentFilePath(); + $tempPath = $envPath.'.tmp'; + + $file = fopen($tempPath, 'w'); + if ($file === false) { + throw new \RuntimeException("Cannot write to {$tempPath}"); + } + fwrite($file, $payload); + fclose($file); + + if (! rename($tempPath, $envPath)) { + @unlink($tempPath); + throw new \RuntimeException('Cannot update .env file'); + } + } +} diff --git a/app/Console/Commands/AvatarDefaultMigration.php b/app/Console/Commands/Deprecated/AvatarDefaultMigration.php similarity index 98% rename from app/Console/Commands/AvatarDefaultMigration.php rename to app/Console/Commands/Deprecated/AvatarDefaultMigration.php index dc981e077..772555af2 100644 --- a/app/Console/Commands/AvatarDefaultMigration.php +++ b/app/Console/Commands/Deprecated/AvatarDefaultMigration.php @@ -1,6 +1,6 @@ 'The old S3 domain', - 'newDomain' => 'The new S3 domain', - ]; - } - - /** - * The console command description. - * - * @var string - */ - protected $description = 'Rewrite S3 media urls from local users'; - - /** - * Execute the console command. - */ - public function handle() - { - $this->preflightCheck(); - $this->bootMessage(); - $this->confirmCloudUrl(); - $this->handleTask(); - } - - protected function preflightCheck() - { - if (! (bool) config_cache('pixelfed.cloud_storage')) { - $this->info('Error: Cloud storage is not enabled! Please enable before proceeding.'); - $this->error('Aborting...'); - exit; - } - } - - protected function bootMessage() - { - $this->info(' ____ _ ______ __ '); - $this->info(' / __ \(_) _____ / / __/__ ____/ / '); - $this->info(' / /_/ / / |/_/ _ \/ / /_/ _ \/ __ / '); - $this->info(' / ____/ /> info(' /_/ /_/_/|_|\___/_/_/ \___/\__,_/ '); - $this->info(' '); - $this->info(' Media Cloud Url Rewrite Tool'); - $this->info(' ==='); - $this->info(' Old S3: '.trim($this->argument('oldDomain'))); - $this->info(' New S3: '.trim($this->argument('newDomain'))); - $this->info(' '); - } - - protected function confirmCloudUrl() - { - $disk = Storage::disk(config('filesystems.cloud'))->url('test'); - $domain = parse_url($disk, PHP_URL_HOST); - if (trim($this->argument('newDomain')) !== $domain) { - $this->error('Error: The new S3 domain you entered is not currently configured'); - exit; - } - - if (! $this->confirm('Confirm this is correct')) { - $this->error('Aborting...'); - exit; - } - } - - protected function handleTask() - { - $task = select( - label: 'What action would you like to perform?', - options: ['Migrate All', 'Migrate Media', 'Migrate Avatars'] - ); - - switch ($task) { - case 'Migrate All': - $this->updateMediaUrls(); - $this->updateAvatarUrls(); - break; - case 'Migrate Media': - $this->updateMediaUrls(); - break; - case 'Migrate Avatars': - $this->updateAvatarUrls(); - break; - default: - $this->error('Invalid selection'); - - return; - } - } - - protected function updateMediaUrls() - { - $this->info('Updating media urls...'); - $oldDomain = trim($this->argument('oldDomain')); - $newDomain = trim($this->argument('newDomain')); - $disk = Storage::disk(config('filesystems.cloud')); - $count = Media::whereNotNull('cdn_url')->count(); - $bar = $this->output->createProgressBar($count); - $counter = 0; - $bar->start(); - foreach (Media::whereNotNull('cdn_url')->lazyById(1000, 'id') as $media) { - if (strncmp($media->media_path, 'http', 4) === 0) { - $bar->advance(); - - continue; - } - $cdnHost = parse_url($media->cdn_url, PHP_URL_HOST); - if ($oldDomain != $cdnHost || $newDomain == $cdnHost) { - $bar->advance(); - - continue; - } - - $media->cdn_url = str_replace($oldDomain, $newDomain, $media->cdn_url); - - if ($media->thumbnail_url != null) { - $thumbHost = parse_url($media->thumbnail_url, PHP_URL_HOST); - if ($thumbHost == $oldDomain) { - $thumbUrl = $disk->url($media->thumbnail_path); - $media->thumbnail_url = $thumbUrl; - } - } - - if ($media->optimized_url != null) { - $optiHost = parse_url($media->optimized_url, PHP_URL_HOST); - if ($optiHost == $oldDomain) { - $optiUrl = str_replace($oldDomain, $newDomain, $media->optimized_url); - $media->optimized_url = $optiUrl; - } - } - - $media->save(); - $counter++; - $bar->advance(); - } - - $bar->finish(); - - $this->line(' '); - $this->info('Finished! Updated '.$counter.' total records!'); - $this->line(' '); - $this->info('Tip: Run `php artisan cache:clear` to purge cached urls'); - } - - protected function updateAvatarUrls() - { - $this->info('Updating avatar urls...'); - $oldDomain = trim($this->argument('oldDomain')); - $newDomain = trim($this->argument('newDomain')); - $disk = Storage::disk(config('filesystems.cloud')); - $count = Avatar::count(); - $bar = $this->output->createProgressBar($count); - $counter = 0; - $bar->start(); - foreach (Avatar::lazyById(1000, 'id') as $avatar) { - if (! $avatar->cdn_url) { - $bar->advance(); - - continue; - } - - $cdnHost = parse_url($avatar->cdn_url, PHP_URL_HOST); - if (strcasecmp($oldDomain, $cdnHost) !== 0 || strcasecmp($newDomain, $cdnHost) === 0) { - $bar->advance(); - - continue; - } - - $avatar->cdn_url = str_replace($oldDomain, $newDomain, $avatar->cdn_url); - - $avatar->save(); - $counter++; - $bar->advance(); - } - - $bar->finish(); - - $this->line(' '); - $this->info('Finished! Updated '.$counter.' total records!'); - $this->line(' '); - $this->info('Tip: Run `php artisan cache:clear` to purge cached urls'); - } -} diff --git a/app/Console/Commands/MediaS3GarbageCollector.php b/app/Console/Commands/MediaS3GarbageCollector.php deleted file mode 100644 index d569659fa..000000000 --- a/app/Console/Commands/MediaS3GarbageCollector.php +++ /dev/null @@ -1,204 +0,0 @@ -error('Cloud storage not enabled. Exiting...'); - - return; - } - - $deleteEnabled = config('media.delete_local_after_cloud'); - if (! $deleteEnabled) { - $this->error('Delete local storage after cloud upload is not enabled'); - - return; - } - - $limit = $this->option('limit'); - $hugeMode = $this->option('huge'); - $log = $this->option('log-errors'); - - if ($limit > 2000 && ! $hugeMode) { - $this->error('Limit exceeded, please use a limit under 2000 or run again with the --huge flag'); - - return; - } - - $minId = Media::orderByDesc('id')->where('created_at', '<', now()->subHours(12))->first(); - - if (! $minId) { - return; - } else { - $minId = $minId->id; - } - - return $hugeMode ? - $this->hugeMode($minId, $limit, $log) : - $this->regularMode($minId, $limit, $log); - } - - protected function regularMode($minId, $limit, $log) - { - $gc = Media::whereRemoteMedia(false) - ->whereNotNull(['status_id', 'cdn_url', 'replicated_at']) - ->whereNot('version', '4') - ->where('id', '<', $minId) - ->inRandomOrder() - ->take($limit) - ->get(); - - $totalSize = 0; - $bar = $this->output->createProgressBar($gc->count()); - $bar->start(); - $cloudDisk = Storage::disk(config('filesystems.cloud')); - $localDisk = Storage::disk('local'); - - foreach ($gc as $media) { - try { - if ( - $cloudDisk->exists($media->media_path) - ) { - if ($localDisk->exists($media->media_path)) { - $localDisk->delete($media->media_path); - $media->version = 4; - $media->save(); - $totalSize = $totalSize + $media->size; - MediaService::del($media->status_id); - StatusService::del($media->status_id, false); - if ($localDisk->exists($media->thumbnail_path)) { - $localDisk->delete($media->thumbnail_path); - } - } else { - $media->version = 4; - $media->save(); - } - } else { - if ($log) { - Log::channel('media')->info('[GC] Local media not properly persisted to cloud storage', ['media_id' => $media->id]); - } - } - $bar->advance(); - } catch (FileNotFoundException $e) { - $bar->advance(); - - continue; - } catch (NotFoundHttpException $e) { - $bar->advance(); - - continue; - } catch (\Exception $e) { - $bar->advance(); - - continue; - } - } - $bar->finish(); - $this->line(' '); - $this->info('Finished!'); - if ($totalSize) { - $this->info('Cleared '.$totalSize.' bytes of media from local disk!'); - } - - return 0; - } - - protected function hugeMode($minId, $limit, $log) - { - $cloudDisk = Storage::disk(config('filesystems.cloud')); - $localDisk = Storage::disk('local'); - - $bar = $this->output->createProgressBar($limit); - $bar->start(); - - Media::whereRemoteMedia(false) - ->whereNotNull(['status_id', 'cdn_url', 'replicated_at']) - ->whereNot('version', '4') - ->where('id', '<', $minId) - ->chunk(50, function ($medias) use ($cloudDisk, $localDisk, $bar, $log) { - foreach ($medias as $media) { - try { - if ($cloudDisk->exists($media->media_path)) { - if ($localDisk->exists($media->media_path)) { - $localDisk->delete($media->media_path); - $media->version = 4; - $media->save(); - MediaService::del($media->status_id); - StatusService::del($media->status_id, false); - if ($localDisk->exists($media->thumbnail_path)) { - $localDisk->delete($media->thumbnail_path); - } - } else { - $media->version = 4; - $media->save(); - } - } else { - if ($log) { - Log::channel('media')->info('[GC] Local media not properly persisted to cloud storage', ['media_id' => $media->id]); - } - } - $bar->advance(); - } catch (FileNotFoundException $e) { - $bar->advance(); - - continue; - } catch (NotFoundHttpException $e) { - $bar->advance(); - - continue; - } catch (\Exception $e) { - $bar->advance(); - - continue; - } - } - }); - - $bar->finish(); - $this->line(' '); - $this->info('Finished!'); - } -} diff --git a/app/Console/Commands/README.md b/app/Console/Commands/README.md new file mode 100644 index 000000000..4ce9dec65 --- /dev/null +++ b/app/Console/Commands/README.md @@ -0,0 +1,150 @@ +# Artisan Commands + +This directory contains Pixelfed's custom Artisan console commands, grouped into +subfolders by purpose. Laravel auto-discovers every command in this tree, so the +subfolder is purely organizational — the command name is defined by each class's +`$signature`. + +Run any command with `php artisan `, and append `--help` to see its +full argument and option list. + +## Folder layout + +| Folder | Namespace | Purpose | +| --- | --- | --- | +| `Admin/` | `App\Console\Commands\Admin` | Instance administration and operator tooling | +| `Deprecated/` | `App\Console\Commands\Deprecated` | Historical one-off migrations whose root issue is already resolved; kept for reference only | +| `Dev/` | `App\Console\Commands\Dev` | Local development and localization build helpers | +| `FixBugs/` | `App\Console\Commands\FixBugs` | One-off repair and data-fix utilities | +| `Install/` | `App\Console\Commands\Install` | Installation and upgrade | +| `Internal/` | `App\Console\Commands\Internal` | Scheduled/background maintenance (mostly run by the scheduler) | +| `Status/` | `App\Console\Commands\Status` | Read-only debug/diagnostic inspectors | +| `User/` | `App\Console\Commands\User` | User account management | +| `Concerns/` | `App\Console\Commands\Concerns` | Shared traits used by commands (not commands themselves) | + +--- + +## Admin + +| Command | Description | +| --- | --- | +| `admin:invite` | Create an invite link. | +| `backup:cloud` | Send backups to cloud storage. | +| `email:bancheck` | Check user emails against banned domains. | +| `app:captcha-toggle-command` | Show captcha status and optionally disable it. | +| `app:curated-onboarding` | Manage curated onboarding applications. | +| `app:delete-remote-profile` | Delete a remote profile. | +| `import:cities` | Import the cities dataset into the database. | +| `import:emojis` | Import custom emojis from a `tar.gz` archive (supports `--prefix`/`--suffix`). | +| `app:instance-manager` | Manage federated instances. | +| `admin:MediaMoveStorageCloudToCloud` | Cold-migrate media from an old S3 bucket to the current cloud bucket, verifying each copy. | +| `admin:MediaMoveStorageCloudToLocal` | Migrate cloud media back to local storage (download, verify, rewrite URLs, optionally delete cloud copy). | +| `admin:MediaMoveStorageLocalToCloud` | Migrate local media to cloud storage (upload, verify, rewrite URLs, delete local copy). | +| `admin:MigrateLocalS3MediaURL` | Rewrite stale local media cloud URLs from storage paths to the configured S3 host. Replaces the old `media:cloud-url-rewrite`. | +| `regenerate:thumbnails` | Regenerate image thumbnails for all image media. | +| `ap:update-actors` | Send Update Actor activities to known remote servers (`--force`). | +| `video:thumbnail` | Generate missing video thumbnails. | + +## Dev + +| Command | Description | +| --- | --- | +| `i18n:export` | Build and export the JS localization files. | +| `localization:generate` | Generate JSON files for all available localizations. | +| `seed:devusers` | Seed dev users (admin + regular) with random passwords. | +| `seed:follows` | Seed follow relationships for testing. | + +## FixBugs + +| Command | Description | +| --- | --- | +| `fix:avatars` | Replace old SVG identicon avatars with the default PNG avatar. | +| `avatar:storage` | Manage avatar storage. | +| `avatar:storage-deep-clean` | Clean up orphaned avatar storage. | +| `media:optimize` | Find and optimize media that has not yet been optimized. | +| `app:fetch-missing-media-mime-type` | Backfill missing MIME types on remote media by issuing HEAD requests. | +| `fix:profile:duplicates` | Fix duplicate profiles. | +| `fix:hashtags` | Fix hashtag records. | +| `fix:likes` | Recalculate like counts. | +| `media:fix-nonlocal-driver` | Repair filesystem records when `FILESYSTEM_DRIVER` is not set to local. | +| `app:fix-missing-user-profile` | Interactively create a missing profile for an affected user. | +| `admin:fixProfileCounts` | Resync a profile's cached counts (followers, following, statuses) from source tables; supports bulk `--all`/`--active`. | +| `fix:usernames` | Fix invalid usernames. | +| `app:hashtag-related-generate` | Generate related-hashtag data for a given tag. | +| `media:fix` | Null out media `filter_class` values no longer present in `Filter::classes()`. Still relevant: image filters remain an active feature. | + +## Install + +| Command | Description | +| --- | --- | +| `instance:actor` | Generate the instance actor. | +| `install` | CLI installer (`--dangerously-overwrite-env`, `--domain`, `--name`). | +| `update` | Run Pixelfed schema updates between versions. | + +## Internal + +These are primarily invoked by the scheduler (see `bootstrap/app.php`) rather than run by hand. + +| Command | Description | +| --- | --- | +| `app:account-post-count-stat-update` | Update post counts from recent activity. | +| `app:cleanup-expired-app-registrations` | Delete app registrations older than 90 days. | +| `gc:sessions` | Garbage-collect database sessions. | +| `gc:failedjobs` | Delete failed jobs older than one month. | +| `app:hashtag-cached-count-update` | Update cached hashtag counters (`--limit`). | +| `app:import-remove-deleted-accounts` | Remove import data belonging to deleted accounts. | +| `app:import-upload-clean-storage` | Delete import storage directories for non-active users. | +| `app:import-upload-garbage-collection` | Garbage-collect skipped Instagram import posts. | +| `app:import-upload-media-to-cloud-storage` | Migrate imported Instagram media to S3 (`--limit`). | +| `app:instance-update-total-local-posts` | Update the total local post count. | +| `media:gc` | Delete media uploads not attached to any active status. | +| `app:notification-epoch-update` | Update the notification epoch. | +| `gc:passwordreset` | Delete password reset tokens older than 24 hours. | +| `app:push-gateway-refresh` | Refresh push-notification gateway support. | +| `app:software-update-refresh` | Refresh latest software version data. | +| `story:gc` | Clear expired stories. | +| `app:transform-imports` | Transform completed imports into statuses. | +| `app:weekly-instance-scan` | Scan instance nodeinfo weekly. | + +## Deprecated + +Historical one-off migrations whose underlying issue is already resolved in the +current codebase. Retained for reference and rare legacy-data recovery; not +expected to be needed on a current install. + +| Command | Description | Why deprecated | +| --- | --- | --- | +| `status:dedup` | Remove duplicate statuses from before the unique-URI migration. | `statuses.uri` has had a unique index since the 2019 `add_unique_to_statuses_table` migration, so duplicate URIs can no longer be inserted. | +| `fix:avatars` | Replace old SVG identicon avatars with the default PNG/JPG avatar. | SVG identicon avatars are no longer generated anywhere; new avatars default to `public/avatars/default.jpg`. | + +## Status (debug/diagnostics) + +Read-only inspectors for troubleshooting. They do not modify data. + +| Command | Description | +| --- | --- | +| `status:post` | Show detailed metadata for a post and its media, including stored vs expected media URLs. | +| `status:profile` | Show detailed metadata for a local or remote profile (federation-aware). | +| `status:user` | Show detailed diagnostics for a user account (login & password reset), with recent account logs (`--logs`). | + +## User + +| Command | Description | +| --- | --- | +| `app:add-user-domain-block` | Apply a domain block for all users. | +| `app:delete-user-domain-block` | Remove a domain block for all users. | +| `app:reclaim-username` | Force-delete a user and profile to reclaim a username. | +| `app:user-account-delete` | Federate an account deletion (`--concurrency`, `--chunk`, `--attempts`, `--target`, `--dry-run`). | +| `user:admin` | Grant or remove admin privileges for a user. | +| `user:avatar-delete` | Delete a user avatar and reset to default (`--force`). | +| `user:checkpassword` | Read-only: verify a candidate password against the stored hash and diagnose login rejection. | +| `user:create` | Create a new user. | +| `user:delete` | Delete an account (`--force`). | +| `user:app-magic-link` | Get the app magic link for in-app registrations missing a confirmation email. | +| `user:setpassword` | Set/reset a user password (prompts securely, bcrypt). | +| `user:show` | Show user info. | +| `user:suspend` | Suspend a local user. | +| `user:table` | Display the latest users. | +| `user:2fa` | Disable two-factor authentication for a username. | +| `user:unsuspend` | Unsuspend a local user. | +| `user:verifyemail` | Verify a user's email address. | diff --git a/app/Console/Commands/Status/StatusPost.php b/app/Console/Commands/Status/StatusPost.php new file mode 100644 index 000000000..fe75585e7 --- /dev/null +++ b/app/Console/Commands/Status/StatusPost.php @@ -0,0 +1,319 @@ + + */ + protected $longStatusCols = ['caption', 'cw_summary']; + + public function handle() + { + $id = $this->resolveId($this->argument('id')); + + if (! $id) { + $this->error('Could not extract a status id from "'.$this->argument('id').'".'); + + return 1; + } + + $status = Status::withTrashed()->find($id); + + if (! $status) { + $this->error('No status found with id '.$id.'.'); + + return 1; + } + + $this->line(str_repeat('=', 64)); + $this->info('STATUS ROW (table: statuses)'); + $this->line(str_repeat('=', 64)); + $this->dumpStatus($status); + + $this->newLine(); + $this->line(str_repeat('=', 64)); + $this->info('AUTHOR'); + $this->line(str_repeat('=', 64)); + $this->dumpAuthor($status); + + $this->newLine(); + $this->line(str_repeat('=', 64)); + $this->info('MEDIA'); + $this->line(str_repeat('=', 64)); + $this->dumpMedia($status); + + $this->newLine(); + $this->line(str_repeat('=', 64)); + $this->info('URL HEALTH CHECK'); + $this->line(str_repeat('=', 64)); + $this->urlHealth($status); + + $this->newLine(); + $this->line(str_repeat('=', 64)); + $this->info('CACHE'); + $this->line(str_repeat('=', 64)); + $this->dumpCache($status); + + return 0; + } + + /** + * Accept a numeric id or a post URL (…/p/username/ID) and return the id. + */ + protected function resolveId(string $input): ?string + { + $input = trim($input); + + if (ctype_digit($input)) { + return $input; + } + + // Extract the last numeric path segment from a URL. + if (preg_match('#/p/[^/]+/(\d+)#', $input, $m)) { + return $m[1]; + } + + if (preg_match('#(\d{6,})#', $input, $m)) { + return $m[1]; + } + + return null; + } + + protected function dumpStatus(Status $status): void + { + $rows = []; + foreach ($status->getAttributes() as $key => $value) { + $display = $this->format($value); + if (in_array($key, $this->longStatusCols, true) && is_string($value) && strlen($value) > 60) { + $display = mb_strimwidth($value, 0, 60, '…'); + } + $rows[] = [$key, $display]; + } + $this->table(['Column', 'Value'], $rows); + + $this->comment('Computed:'); + $this->line(' url(): '.$this->safe(fn () => $status->url())); + $this->line(' permalink(): '.$this->safe(fn () => $status->permalink())); + $this->line(' is remote: '.($status->uri ? 'yes ('.$status->uri.')' : 'no (local)')); + if ($status->deleted_at) { + $this->error(' ✗ Status is soft-deleted ('.$status->deleted_at.').'); + } + } + + protected function dumpAuthor(Status $status): void + { + $acct = AccountService::get($status->profile_id, true); + if (! $acct) { + $this->error('No account found for profile_id '.$status->profile_id.'.'); + + return; + } + $this->table(['Field', 'Value'], [ + ['profile_id', $status->profile_id], + ['username', $acct['username'] ?? 'null'], + ['acct', $acct['acct'] ?? 'null'], + ['url', $acct['url'] ?? 'null'], + ['local', ($acct['local'] ?? null) ? 'true' : 'false'], + ]); + } + + protected function dumpMedia(Status $status): void + { + $media = Media::withTrashed()->whereStatusId($status->id)->orderBy('order')->get(); + + if ($media->isEmpty()) { + $this->comment('No media attached to this status.'); + + return; + } + + $cloudHost = $this->cloudHost(); + + foreach ($media as $i => $m) { + $this->newLine(); + $this->comment('Media #'.($i + 1).' (id '.$m->id.', order '.$m->order.')'); + $rows = []; + $cols = [ + 'media_path', 'thumbnail_path', 'cdn_url', 'thumbnail_url', + 'optimized_url', 'remote_url', 'remote_media', 'mime', 'size', + 'version', 'replicated_at', 'original_sha256', 'processed_at', + 'deleted_at', + ]; + $attrs = $m->getAttributes(); + foreach ($cols as $c) { + if (array_key_exists($c, $attrs)) { + $rows[] = [$c, $this->format($attrs[$c])]; + } + } + $this->table(['Media Column', 'Value'], $rows); + + $this->line(' computed url(): '.$this->safe(fn () => $m->url())); + $this->line(' computed thumbnailUrl(): '.$this->safe(fn () => $m->thumbnailUrl())); + $this->line(' expected (from path): '.$this->expectedUrl($m->media_path)); + + // Per-field host comparison. + $this->compareHost(' cdn_url', $m->cdn_url, $cloudHost); + $this->compareHost(' thumbnail_url', $m->thumbnail_url, $cloudHost); + $this->compareHost(' optimized_url', $m->optimized_url, $cloudHost); + } + } + + protected function urlHealth(Status $status): void + { + $cloudHost = $this->cloudHost(); + if (! $cloudHost) { + $this->comment('Cloud storage not configured (or no cloud disk url); skipping host comparison.'); + + return; + } + + $this->line('Configured cloud host (correct base): '.$cloudHost); + $this->newLine(); + + $media = Media::withTrashed()->whereStatusId($status->id)->get(); + $stale = []; + + foreach ($media as $m) { + if ($m->remote_media || Str::startsWith((string) $m->media_path, 'http')) { + continue; + } + foreach (['cdn_url', 'thumbnail_url', 'optimized_url'] as $field) { + $val = $m->{$field}; + if (! $val) { + continue; + } + $host = parse_url($val, PHP_URL_HOST); + if ($host && $cloudHost && strcasecmp($host, $cloudHost) !== 0) { + $stale[] = 'media '.$m->id.' '.$field.' points at '.$host.' (expected '.$cloudHost.')'; + } + } + } + + if ($stale) { + $this->error('STALE MEDIA URLS DETECTED:'); + foreach ($stale as $s) { + $this->line(' ✗ '.$s); + } + $this->newLine(); + $this->comment('Fix with: php artisan admin:MigrateLocalMediaURL '.$status->id); + $this->comment('(or --all to scan every local media row)'); + } else { + $this->info('All local media URLs point at the configured cloud host. ✓'); + } + } + + protected function dumpCache(Status $status): void + { + $cached = MediaService::get($status->id); + if (empty($cached)) { + $this->comment('No cached media_attachments entry (MediaService).'); + + return; + } + $this->comment('Cached media_attachments (MediaService, 6h TTL) — served to clients:'); + foreach ($cached as $i => $item) { + $this->line(' ['.$i.'] url: '.($item['url'] ?? 'null')); + $this->line(' ['.$i.'] preview_url: '.($item['preview_url'] ?? 'null')); + } + $this->newLine(); + $this->comment('If these still show a stale host after a DB fix, run: php artisan cache:clear'); + } + + protected function compareHost(string $label, ?string $url, ?string $cloudHost): void + { + if (! $url) { + return; + } + $host = parse_url($url, PHP_URL_HOST); + if (! $host || ! $cloudHost) { + return; + } + if (strcasecmp($host, $cloudHost) !== 0) { + $this->line(''.$label.' host: '.$host.' ✗ (expected '.$cloudHost.')'); + } else { + $this->line(''.$label.' host: '.$host.' ✓'); + } + } + + protected function expectedUrl(?string $mediaPath): string + { + if (! $mediaPath || Str::startsWith($mediaPath, 'http')) { + return $mediaPath ?? 'null'; + } + + try { + return (string) Storage::disk(config('filesystems.cloud'))->url($mediaPath); + } catch (\Throwable $e) { + return '(cloud disk not resolvable in this environment)'; + } + } + + protected function cloudHost(): ?string + { + try { + if (! (bool) config_cache('pixelfed.cloud_storage')) { + // Still try to read the configured cloud disk host for reference. + } + $url = Storage::disk(config('filesystems.cloud'))->url('probe'); + $host = parse_url($url, PHP_URL_HOST); + + return $host ?: null; + } catch (\Throwable $e) { + return null; + } + } + + protected function safe(callable $fn): string + { + try { + return (string) ($fn() ?? 'null'); + } catch (\Throwable $e) { + return 'error: '.$e->getMessage(); + } + } + + protected function format($value): string + { + if ($value === null) { + return 'null'; + } + if (is_bool($value)) { + return $value ? 'true' : 'false'; + } + if ($value === '') { + return '(empty string)'; + } + if ($value instanceof \DateTimeInterface) { + return $value->format('Y-m-d H:i:s'); + } + + return (string) $value; + } +} diff --git a/app/Console/Commands/ProfileStatus.php b/app/Console/Commands/Status/StatusProfile.php similarity index 98% rename from app/Console/Commands/ProfileStatus.php rename to app/Console/Commands/Status/StatusProfile.php index 9322678c3..9049996cb 100644 --- a/app/Console/Commands/ProfileStatus.php +++ b/app/Console/Commands/Status/StatusProfile.php @@ -1,6 +1,6 @@ format($user->last_active_at)], ]; $this->table(['User Field', 'Value'], $rows); - $this->comment('Tip: run `user:status '.$user->username.'` for full auth diagnostics.'); + $this->comment('Tip: run `status:user '.$user->username.'` for full auth diagnostics.'); } protected function dumpInstance(Profile $profile): void diff --git a/app/Console/Commands/UserStatus.php b/app/Console/Commands/Status/StatusUser.php similarity index 99% rename from app/Console/Commands/UserStatus.php rename to app/Console/Commands/Status/StatusUser.php index 96702463a..7236a863d 100644 --- a/app/Console/Commands/UserStatus.php +++ b/app/Console/Commands/Status/StatusUser.php @@ -1,6 +1,6 @@ get($url); + $res = Http::acceptJson() + ->withOptions([ + 'allow_redirects' => false, + 'curl' => [ + CURLOPT_RESOLVE => [ + $host.':'.((int) $port).':'.implode(',', array_map( + fn ($ip) => str_contains($ip, ':') ? '['.$ip.']' : $ip, + $ips + )), + ], + CURLOPT_FRESH_CONNECT => true, + CURLOPT_FORBID_REUSE => true, + CURLOPT_PROTOCOLS => CURLPROTO_HTTPS, + CURLOPT_REDIR_PROTOCOLS => CURLPROTO_HTTPS, + ], + ]) + ->timeout(15) + ->connectTimeout(5) + ->get($url); } catch (RequestException $e) { return; } catch (\Exception $e) { @@ -56,11 +96,15 @@ class CustomEmojiService ! isset($json['icon']['url']) || ! isset($json['icon']['type']) || $json['icon']['type'] !== 'Image' || - ! in_array($json['icon']['mediaType'], ['image/jpeg', 'image/png', 'image/jpg']) + ! in_array($json['icon']['mediaType'], self::ALLOWED_MIME_TYPES, true) ) { return; } + if (Helpers::validateUrl($json['icon']['url']) == false) { + return; + } + if (! self::headCheck($json['icon']['url'])) { return; } @@ -83,21 +127,16 @@ class CustomEmojiService $mediaPath = 'emoji/'.$emoji->id.$ext; try { - $response = Http::timeout(30) - ->withOptions(['max_redirects' => 0]) - ->get($json['icon']['url']); - - if (! $response->successful()) { - return; - } + // SSRF-hardened: validated URL, resolved+pinned public IP, + // no internal redirects, size-capped. + $maxSize = (int) config('federation.custom_emoji.max_size'); + $body = SecureMediaFetchService::get($json['icon']['url'], $maxSize > 0 ? $maxSize : null); - // Validate actual content type from response - $contentType = $response->header('Content-Type'); - if (! in_array($contentType, ['image/jpeg', 'image/png', 'image/jpg'])) { + if ($body === false) { return; } - Storage::put('public/'.$mediaPath, $response->body()); + Storage::put('public/'.$mediaPath, $body); $emoji->media_path = $mediaPath; $emoji->save(); @@ -121,27 +160,20 @@ class CustomEmojiService public static function headCheck($url) { - try { - $res = Http::head($url); - } catch (RequestException $e) { - return false; - } catch (\Exception $e) { + $maxSize = (int) config('federation.custom_emoji.max_size'); + // SSRF-hardened HEAD: validated URL, resolved+pinned public IP, no + // internal redirects. + $head = SecureMediaFetchService::head($url, $maxSize > 0 ? $maxSize : null); + + if (! $head) { return false; } - if (! $res->successful()) { + if (! in_array($head['mime'], self::ALLOWED_MIME_TYPES, true)) { return false; } - $type = $res->header('content-type'); - $length = $res->header('content-length'); - - if ( - ! $type || - ! $length || - ! in_array($type, ['image/jpeg', 'image/png', 'image/jpg']) || - $length > config('federation.custom_emoji.max_size') - ) { + if ($maxSize > 0 && $head['length'] > $maxSize) { return false; } diff --git a/app/Services/FetchCacheService.php b/app/Services/FetchCacheService.php index c28ace3bb..ded9fb988 100644 --- a/app/Services/FetchCacheService.php +++ b/app/Services/FetchCacheService.php @@ -22,29 +22,47 @@ class FetchCacheService } if ($verifyCheck) { - if (! Helpers::validateUrl($url)) { + $validated = Helpers::validateUrl($url); + if (! $validated) { Cache::put($key, 1, $ttl); return false; } + $url = $validated; } $headers = [ 'User-Agent' => '(Pixelfed/'.config('pixelfed.version').'; +'.config('app.url').')', ]; - if ($allowRedirects) { - $options = [ - 'allow_redirects' => [ - 'max' => 2, - 'strict' => true, - ], - ]; - } else { - $options = [ - 'allow_redirects' => false, - ]; + // SSRF-hardening: resolve the host and pin the connection to a + // validated public IP. Auto-redirects are disabled so a remote host + // cannot steer the request into an internal address on a later hop. + $host = parse_url($url, PHP_URL_HOST); + $port = parse_url($url, PHP_URL_PORT) ?: 443; + $ips = $host ? Helpers::resolvePublicIps($host) : []; + if (empty($ips)) { + Cache::put($key, 1, $ttl); + + return false; } + + $options = [ + 'allow_redirects' => false, + 'curl' => [ + CURLOPT_RESOLVE => [ + $host.':'.((int) $port).':'.implode(',', array_map( + fn ($ip) => str_contains($ip, ':') ? '['.$ip.']' : $ip, + $ips + )), + ], + CURLOPT_FRESH_CONNECT => true, + CURLOPT_FORBID_REUSE => true, + CURLOPT_PROTOCOLS => CURLPROTO_HTTPS, + CURLOPT_REDIR_PROTOCOLS => CURLPROTO_HTTPS, + ], + ]; + try { $res = Http::withOptions($options) ->retry(3, function (int $attempt, $exception) { diff --git a/app/Services/MediaStorageService.php b/app/Services/MediaStorageService.php index 322cc29f1..d52063e3e 100644 --- a/app/Services/MediaStorageService.php +++ b/app/Services/MediaStorageService.php @@ -8,11 +8,8 @@ use App\Jobs\StatusPipeline\NewStatusPipeline; use App\Models\Media; use App\Models\Status; use App\Util\ActivityPub\Helpers; -use Illuminate\Http\Client\ConnectionException; use Illuminate\Http\File; -use Illuminate\Support\Arr; use Illuminate\Support\Facades\Cache; -use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Storage; use Illuminate\Support\Str; @@ -44,35 +41,10 @@ class MediaStorageService public static function head($url) { - try { - $r = Http::head($url); - } catch (ConnectionException $e) { - return false; - } - - if (! $r->successful()) { - return false; - } - - $h = Arr::mapWithKeys($r->headers(), function ($item, $key) { - return [strtolower($key) => last($item)]; - }); - - if (! isset($h['content-length'], $h['content-type'])) { - return false; - } - - $len = (int) $h['content-length']; - $mime = $h['content-type']; - - if ($len < 10 || $len > ((config_cache('pixelfed.max_photo_size') * 1000))) { - return false; - } - - return [ - 'length' => $len, - 'mime' => $mime, - ]; + // SSRF-hardened: validates URL, resolves + rejects private/reserved + // IPs, pins the connection to the validated address, and refuses to + // follow redirects into internal networks. See SecureMediaFetchService. + return SecureMediaFetchService::head($url, (int) config_cache('pixelfed.max_photo_size') * 1000); } protected function cloudStore($media) @@ -155,7 +127,8 @@ class MediaStorageService return; } - $head = $this->head($media->remote_url); + // Hardened HEAD (IP-validated, pinned, no internal redirects). + $head = $this->head($url); if (! $head) { return; @@ -206,7 +179,11 @@ class MediaStorageService $tmpBase = storage_path('app/remcache/'); $tmpPath = $media->profile_id.'-'.$path; $tmpName = $tmpBase.$tmpPath; - $data = file_get_contents($url, false, null, 0, $head['length']); + // Hardened byte fetch through the same validated, pinned, redirect-safe path. + $data = SecureMediaFetchService::get($url, $max_size, $head['length']); + if ($data === false) { + return; + } file_put_contents($tmpName, $data); $hash = hash_file('sha256', $tmpName); @@ -281,7 +258,8 @@ class MediaStorageService $tmpBase = storage_path('app/remcache/'); $tmpPath = 'avatar_'.$avatar->profile_id.'-'.$path; $tmpName = $tmpBase.$tmpPath; - $data = @file_get_contents($url, false, null, 0, $head['length']); + // Hardened byte fetch: validated URL, pinned IP, no internal redirects, size-capped. + $data = SecureMediaFetchService::get($url, $max_size, $head['length']); if (! $data) { return; } diff --git a/app/Services/SecureMediaFetchService.php b/app/Services/SecureMediaFetchService.php new file mode 100644 index 000000000..88d560c45 --- /dev/null +++ b/app/Services/SecureMediaFetchService.php @@ -0,0 +1,247 @@ +request($url, 'head', $maxBytes); + } + + /** + * Download up to $maxBytes of the resource through the pinned, + * validated path. Returns the raw body string, or false. + * + * @return string|false + */ + public static function get(string $url, ?int $maxBytes = null, ?int $expectedLength = null) + { + $maxBytes = $maxBytes ?? self::defaultMaxBytes(); + $result = (new self)->request($url, 'get', $maxBytes, $expectedLength); + + if (! is_array($result)) { + return false; + } + + return $result['body'] ?? false; + } + + /** + * Shared request loop: validate -> resolve public IPs -> pin -> issue + * request with redirects disabled -> re-validate each hop. + * + * @return array|false For 'head': ['length'=>int,'mime'=>string]. + * For 'get': ['body'=>string,'length'=>int,'mime'=>string]. + */ + protected function request(string $url, string $method, int $maxBytes, ?int $expectedLength = null) + { + $currentUrl = $url; + + for ($redirects = 0; $redirects <= self::MAX_REDIRECTS; $redirects++) { + $currentUrl = Helpers::validateUrl($currentUrl); + + if (! $currentUrl) { + return false; + } + + $host = parse_url($currentUrl, PHP_URL_HOST); + $scheme = parse_url($currentUrl, PHP_URL_SCHEME); + $port = parse_url($currentUrl, PHP_URL_PORT) ?: 443; + + if (! $host || strtolower((string) $scheme) !== 'https') { + return false; + } + + // Resolve the host and reject if ANY resolved address is + // non-global. Fail-closed: empty means unresolved or private. + $ips = Helpers::resolvePublicIps($host); + + if (empty($ips)) { + return false; + } + + try { + $res = Http::withOptions([ + 'allow_redirects' => false, + 'sink' => null, + 'curl' => [ + CURLOPT_RESOLVE => [ + $this->buildResolveEntry($host, (int) $port, $ips), + ], + CURLOPT_FRESH_CONNECT => true, + CURLOPT_FORBID_REUSE => true, + CURLOPT_PROTOCOLS => CURLPROTO_HTTPS, + CURLOPT_REDIR_PROTOCOLS => CURLPROTO_HTTPS, + ], + 'on_headers' => function (ResponseInterface $response) use ($maxBytes) { + $length = $response->getHeaderLine('Content-Length'); + if ($length !== '' && ctype_digit($length) && (int) $length > $maxBytes) { + throw new \RuntimeException('Remote media exceeds maximum size'); + } + }, + ]) + ->withHeaders(['User-Agent' => self::userAgent()]) + ->timeout(self::TIMEOUT) + ->connectTimeout(self::CONNECT_TIMEOUT) + ->{$method}($currentUrl); + } catch (RequestException $e) { + return false; + } catch (ConnectionException $e) { + return false; + } catch (\Throwable $e) { + return false; + } + + // Manual redirect handling: re-validate + re-resolve the next hop. + if (in_array($res->status(), [301, 302, 303, 307, 308], true)) { + if ($redirects >= self::MAX_REDIRECTS) { + return false; + } + $location = $res->header('Location'); + if (! $location) { + return false; + } + $nextUrl = $this->resolveRedirect($currentUrl, $location); + if (! $nextUrl) { + return false; + } + $currentUrl = $nextUrl; + + continue; + } + + if (! $res->successful()) { + return false; + } + + $mime = $this->normalizeMime($res->header('Content-Type')); + $declaredLength = $res->header('Content-Length'); + $declaredLength = ($declaredLength !== null && ctype_digit((string) $declaredLength)) + ? (int) $declaredLength + : null; + + if ($method === 'head') { + if ($declaredLength === null || $mime === null) { + return false; + } + if ($declaredLength < 10 || $declaredLength > $maxBytes) { + return false; + } + + return ['length' => $declaredLength, 'mime' => $mime]; + } + + // GET: enforce the cap against the actual body we received. + $body = $res->body(); + $len = strlen($body); + + if ($len === 0 || $len > $maxBytes) { + return false; + } + + if ($expectedLength !== null && $len < $expectedLength) { + // Received less than the HEAD promised; treat as truncated. + // Still return what we have, capped, but never more than asked. + $body = substr($body, 0, $expectedLength); + $len = strlen($body); + } + + return [ + 'body' => $body, + 'length' => $len, + 'mime' => $mime ?? ($declaredLength !== null ? $mime : null), + ]; + } + + return false; + } + + protected function buildResolveEntry(string $host, int $port, array $ips): string + { + $addresses = array_map(function ($ip) { + return str_contains($ip, ':') ? '['.$ip.']' : $ip; + }, $ips); + + return $host.':'.$port.':'.implode(',', $addresses); + } + + protected function resolveRedirect(string $baseUrl, string $location): ?string + { + $location = trim($location); + + if ($location === '' || preg_match('/[\x00-\x20\x7f]/', $location)) { + return null; + } + + try { + $resolved = (string) BaseUri::from($baseUrl)->resolve($location); + + return Helpers::validateUrl($resolved) ? $resolved : null; + } catch (\Throwable $e) { + return null; + } + } + + protected function normalizeMime(?string $contentType): ?string + { + if (! $contentType) { + return null; + } + + $mime = strtolower(trim(explode(';', $contentType)[0])); + + return $mime === '' ? null : $mime; + } + + protected static function defaultMaxBytes(): int + { + // Cap on the larger of avatar/photo config limits (kB -> bytes), + // with a sane fallback. + $photo = (int) config_cache('pixelfed.max_photo_size'); + $avatar = (int) config('pixelfed.max_avatar_size'); + $maxKb = max($photo, $avatar, 1000); + + return $maxKb * 1000; + } + + protected static function userAgent(): string + { + return 'PixelFedBot/1.0.0 (Pixelfed/'.config('pixelfed.version').'; +'.config('app.url').')'; + } +} diff --git a/app/Util/ActivityPub/DiscoverActor.php b/app/Util/ActivityPub/DiscoverActor.php index 4e41cf059..734e053e1 100644 --- a/app/Util/ActivityPub/DiscoverActor.php +++ b/app/Util/ActivityPub/DiscoverActor.php @@ -2,7 +2,7 @@ namespace App\Util\ActivityPub; -use Illuminate\Support\Facades\Http; +use App\Services\ActivityPubFetchService; class DiscoverActor { @@ -17,17 +17,20 @@ class DiscoverActor public function fetch() { - $res = Http::withHeaders([ - 'Accept' => 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"', - 'User-Agent' => 'PixelfedBot - https://pixelfed.org', - ])->get($this->url); - $this->response = $res->body(); + // SSRF-hardened: route through the validated, IP-pinned, + // redirect-revalidating ActivityPub fetch path instead of a raw + // Http::get on an unvalidated URL. + $this->response = ActivityPubFetchService::get($this->url) ?: null; return $this; } public function getResponse() { + if (! $this->response) { + return null; + } + return json_decode($this->response, true); } diff --git a/app/Util/ActivityPub/Helpers.php b/app/Util/ActivityPub/Helpers.php index f96d0e037..c10452c17 100644 --- a/app/Util/ActivityPub/Helpers.php +++ b/app/Util/ActivityPub/Helpers.php @@ -25,6 +25,7 @@ use App\Services\SanitizeService; use App\Services\UserFilterService; use App\Util\Media\License; use Carbon\Carbon; +use Illuminate\Database\UniqueConstraintViolationException; use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Validator; use Illuminate\Validation\Rule; @@ -199,6 +200,18 @@ class Helpers } } + // SSRF guard: when DNS verification is enabled, reject any host that + // resolves into a non-global (private/reserved/link-local) range. This + // closes the bypass where a public-looking hostname (e.g. + // metadata.google.internal) resolves to a reserved address such as + // 169.254.169.254. resolvePublicIps() fails closed: it returns an empty + // array if the host does not resolve or any resolved IP is non-global. + if ($disableDNSCheck !== true && self::shouldCheckDNS()) { + if (empty(self::resolvePublicIps($host))) { + return false; + } + } + return $uri->toString(); } @@ -1129,7 +1142,9 @@ class Helpers } $mediaModel = self::createMediaAttachment($media, $status, $key); - self::handleMediaStorage($mediaModel); + if ($mediaModel) { + self::handleMediaStorage($mediaModel); + } } $status->viewType(); @@ -1158,16 +1173,35 @@ class Helpers } /** - * Create media attachment record + * Create media attachment record. + * + * Idempotent on the (status_id, media_path) unique key: if a row already + * exists (e.g. a re-fetch, an Announce racing another inbox job, or a + * duplicate url within one activity's attachments) the existing row is + * returned instead of triggering a duplicate-key violation. + * + * @return Media|null the newly created model, or null when the attachment + * already existed (so the caller can skip re-storage) */ - public static function createMediaAttachment(array $media, Status $status, int $key): Media + public static function createMediaAttachment(array $media, Status $status, int $key): ?Media { + // Fast path: already imported for this status. + if (Media::whereStatusId($status->id)->whereMediaPath($media['url'])->exists()) { + return null; + } + $mediaModel = new Media; self::setBasicMediaAttributes($mediaModel, $media, $status, $key); self::setOptionalMediaAttributes($mediaModel, $media); - $mediaModel->save(); + try { + $mediaModel->save(); + } catch (UniqueConstraintViolationException $e) { + // Lost a race with a concurrent inbox job that inserted the same + // (status_id, media_path). Treat as already-imported. + return null; + } return $mediaModel; } diff --git a/bootstrap/app.php b/bootstrap/app.php index b9dd57a76..9d4f06c0b 100644 --- a/bootstrap/app.php +++ b/bootstrap/app.php @@ -143,7 +143,8 @@ return Application::configure(basePath: dirname(__DIR__)) $schedule->command('passport:purge')->everyFourHours(20)->onOneServer(); if ((bool) config_cache('pixelfed.cloud_storage') && (bool) config_cache('media.delete_local_after_cloud')) { - $schedule->command('media:s3gc')->hourlyAt(15); + // Upload any local stragglers to cloud and GC verified local copies. + $schedule->command('admin:MediaMoveStorageLocalToCloud --force --limit=500')->hourlyAt(15); } if (config('import.instagram.enabled')) { diff --git a/config/auth.php b/config/auth.php index 8eaff6a80..5ee6fc3cd 100644 --- a/config/auth.php +++ b/config/auth.php @@ -82,6 +82,7 @@ return [ // 'database' => [ // 'model' => App\User::class, // 'sync_passwords' => false, + // 'locate_users_by' => 'mail', // 'sync_attributes' => [ // 'name' => 'cn', // 'email' => 'mail', diff --git a/config/filesystems.php b/config/filesystems.php index 81ee0547b..b4cea06a8 100644 --- a/config/filesystems.php +++ b/config/filesystems.php @@ -83,6 +83,27 @@ return [ ], ], + // Source disk for admin:MediaMoveStorageCloudToCloud (cold migration). + // After pointing AWS_* at the NEW bucket, keep the OLD bucket's + // credentials here as AWS_OLD_* so existing data can be copied across + // to the new bucket and media URLs rewritten. + 's3-old' => [ + 'driver' => 's3', + 'key' => env('AWS_OLD_ACCESS_KEY_ID'), + 'secret' => env('AWS_OLD_SECRET_ACCESS_KEY'), + 'region' => env('AWS_OLD_DEFAULT_REGION'), + 'bucket' => env('AWS_OLD_BUCKET'), + 'visibility' => env('AWS_OLD_VISIBILITY', 'public'), + 'url' => env('AWS_OLD_URL'), + 'endpoint' => env('AWS_OLD_ENDPOINT'), + 'use_path_style_endpoint' => env('AWS_OLD_USE_PATH_STYLE_ENDPOINT', false), + 'throw' => true, + 'options' => [ + 'request_checksum_calculation' => env('AWS_OLD_REQUEST_CHECKSUM_CALCULATION', 'WHEN_SUPPORTED'), + 'response_checksum_validation' => env('AWS_OLD_RESPONSE_CHECKSUM_VALIDATION', 'WHEN_SUPPORTED'), + ], + ], + 'alt-primary' => [ 'enabled' => env('ALT_PRI_ENABLED', false), 'driver' => 's3', diff --git a/tests/Feature/Account/MediaMoveStorageCloudToCloudTest.php b/tests/Feature/Account/MediaMoveStorageCloudToCloudTest.php new file mode 100644 index 000000000..be9c34599 --- /dev/null +++ b/tests/Feature/Account/MediaMoveStorageCloudToCloudTest.php @@ -0,0 +1,115 @@ +S3 migration: copy existing objects from the old bucket (s3-old) +| to the current cloud bucket (s3), verify, rewrite media URLs, GC the source. +| +*/ + +beforeEach(function () { + Config::set('filesystems.cloud', 's3'); + // Destination (new) bucket. + Storage::fake('s3', ['url' => 'https://cdneast.pixelfed.au']); + // Source (old) bucket. + Storage::fake('s3-old', ['url' => 'https://cdn.pixelfed.au']); +}); + +function makeOldBucketMedia(string $oldHost = 'https://cdn.pixelfed.au'): Media +{ + $user = User::factory()->create(); + $user->refresh(); + $pid = $user->profile->id; + $status = Status::factory()->create(['profile_id' => $pid, 'type' => 'photo']); + + $path = 'public/m/_v2/'.$pid.'/aa/bb/file.jpg'; + $thumb = 'public/m/_v2/'.$pid.'/aa/bb/file_thumb.jpeg'; + + // Files exist only on the OLD bucket. + Storage::disk('s3-old')->put($path, 'PRIMARY-BYTES-1234567890'); + Storage::disk('s3-old')->put($thumb, 'THUMB-BYTES'); + + return Media::create([ + 'status_id' => $status->id, + 'profile_id' => $pid, + 'user_id' => $user->id, + 'media_path' => $path, + 'thumbnail_path' => $thumb, + 'cdn_url' => $oldHost.'/'.$path, + 'thumbnail_url' => $oldHost.'/'.$thumb, + 'optimized_url' => $oldHost.'/'.$path, + 'mime' => 'image/jpeg', + 'size' => strlen('PRIMARY-BYTES-1234567890'), + 'remote_media' => false, + 'version' => 4, + 'replicated_at' => now(), + 'order' => 0, + ]); +} + +it('copies old-bucket media to the new bucket, rewrites urls and GCs the source', function () { + $media = makeOldBucketMedia(); + + $this->artisan('admin:MediaMoveStorageCloudToCloud', ['--force' => true]) + ->assertExitCode(0); + + // Copied to destination. + expect(Storage::disk('s3')->exists($media->media_path))->toBeTrue(); + // Removed from source (GC). + expect(Storage::disk('s3-old')->exists($media->media_path))->toBeFalse(); + + $media->refresh(); + expect(parse_url($media->cdn_url, PHP_URL_HOST))->toBe('cdneast.pixelfed.au'); + expect(parse_url($media->thumbnail_url, PHP_URL_HOST))->toBe('cdneast.pixelfed.au'); + expect(parse_url($media->optimized_url, PHP_URL_HOST))->toBe('cdneast.pixelfed.au'); +}); + +it('keeps the source objects with --keep-source', function () { + $media = makeOldBucketMedia(); + + $this->artisan('admin:MediaMoveStorageCloudToCloud', ['--force' => true, '--keep-source' => true]) + ->assertExitCode(0); + + expect(Storage::disk('s3')->exists($media->media_path))->toBeTrue(); + expect(Storage::disk('s3-old')->exists($media->media_path))->toBeTrue(); +}); + +it('makes no changes in dry-run', function () { + $media = makeOldBucketMedia(); + + $this->artisan('admin:MediaMoveStorageCloudToCloud', ['--force' => true, '--dry-run' => true]) + ->assertExitCode(0); + + expect(Storage::disk('s3')->exists($media->media_path))->toBeFalse(); + expect(parse_url($media->fresh()->cdn_url, PHP_URL_HOST))->toBe('cdn.pixelfed.au'); +}); + +it('skips media already pointing at the destination host', function () { + // cdn_url already on the destination host -> nothing to do. + $media = makeOldBucketMedia('https://cdneast.pixelfed.au'); + $before = $media->cdn_url; + + $this->artisan('admin:MediaMoveStorageCloudToCloud', ['--force' => true]) + ->assertExitCode(0); + + expect($media->fresh()->cdn_url)->toBe($before); + // Not copied (was skipped). + expect(Storage::disk('s3')->exists($media->media_path))->toBeFalse(); +}); + +it('errors when source and destination are the same disk', function () { + $this->artisan('admin:MediaMoveStorageCloudToCloud', ['--sourceDisk' => 's3', '--force' => true]) + ->assertExitCode(1); +}); diff --git a/tests/Feature/Account/MediaMoveStorageTest.php b/tests/Feature/Account/MediaMoveStorageTest.php new file mode 100644 index 000000000..5017292ec --- /dev/null +++ b/tests/Feature/Account/MediaMoveStorageTest.php @@ -0,0 +1,150 @@ + 'https://cdn.test']); + + // Use a throwaway env file so the command's env edits don't touch the + // real one. App::useEnvironmentPath expects a directory; the app resolves + // the environment-specific filename (e.g. .env.testing) itself. + $this->originalEnvPath = app()->environmentPath(); + $dir = sys_get_temp_dir().'/pf-env-test-'.uniqid(); + mkdir($dir); + app()->useEnvironmentPath($dir); + file_put_contents(app()->environmentFilePath(), "APP_KEY=base64:test\nPF_ENABLE_CLOUD=false\n"); +}); + +afterEach(function () { + // Restore the real environment path so we don't leak into other test files. + if (isset($this->originalEnvPath)) { + app()->useEnvironmentPath($this->originalEnvPath); + } +}); + +function makeCloudMedia(): Media +{ + $user = User::factory()->create(); + $user->refresh(); + $pid = $user->profile->id; + $status = Status::factory()->create(['profile_id' => $pid, 'type' => 'photo']); + + $path = 'public/m/_v2/'.$pid.'/aa/bb/file.jpg'; + $thumb = 'public/m/_v2/'.$pid.'/aa/bb/file_thumb.jpeg'; + + // Put the files on the cloud disk only. + Storage::disk('s3')->put($path, 'PRIMARY-BYTES-1234567890'); + Storage::disk('s3')->put($thumb, 'THUMB-BYTES'); + + return Media::create([ + 'status_id' => $status->id, + 'profile_id' => $pid, + 'user_id' => $user->id, + 'media_path' => $path, + 'thumbnail_path' => $thumb, + 'cdn_url' => Storage::disk('s3')->url($path), + 'thumbnail_url' => Storage::disk('s3')->url($thumb), + 'optimized_url' => Storage::disk('s3')->url($path), + 'mime' => 'image/jpeg', + 'size' => strlen('PRIMARY-BYTES-1234567890'), + 'remote_media' => false, + 'version' => 4, + 'replicated_at' => now(), + 'order' => 0, + ]); +} + +describe('admin:MediaMoveStorageCloudToLocal', function () { + it('downloads cloud media to local, clears cloud urls and deletes the cloud copy', function () { + $media = makeCloudMedia(); + + $this->artisan('admin:MediaMoveStorageCloudToLocal', ['--force' => true]) + ->assertExitCode(0); + + // File is now on local disk. + expect(Storage::disk('local')->exists($media->media_path))->toBeTrue(); + // Cloud copy removed (GC), thumbnail too. + expect(Storage::disk('s3')->exists($media->media_path))->toBeFalse(); + + $media->refresh(); + expect($media->cdn_url)->toBeNull(); + expect($media->optimized_url)->toBeNull(); + expect($media->thumbnail_url)->toBeNull(); + expect($media->replicated_at)->toBeNull(); + expect((string) $media->version)->toBe('3'); + }); + + it('keeps the cloud copy with --keep-cloud', function () { + $media = makeCloudMedia(); + + $this->artisan('admin:MediaMoveStorageCloudToLocal', ['--force' => true, '--keep-cloud' => true]) + ->assertExitCode(0); + + expect(Storage::disk('local')->exists($media->media_path))->toBeTrue(); + expect(Storage::disk('s3')->exists($media->media_path))->toBeTrue(); + }); + + it('does not modify anything in dry-run', function () { + $media = makeCloudMedia(); + + $this->artisan('admin:MediaMoveStorageCloudToLocal', ['--force' => true, '--dry-run' => true]) + ->assertExitCode(0); + + expect(Storage::disk('local')->exists($media->media_path))->toBeFalse(); + expect($media->fresh()->cdn_url)->not->toBeNull(); + }); + + it('sets PF_ENABLE_CLOUD=false in .env and runtime when cloud is enabled', function () { + // Start with cloud enabled. + file_put_contents(app()->environmentFilePath(), "APP_KEY=base64:test\nPF_ENABLE_CLOUD=true\n"); + Config::set('pixelfed.cloud_storage', true); + makeCloudMedia(); + + $this->artisan('admin:MediaMoveStorageCloudToLocal', ['--force' => true]) + ->assertExitCode(0); + + expect(file_get_contents(app()->environmentFilePath()))->toContain('PF_ENABLE_CLOUD="false"'); + expect(config('pixelfed.cloud_storage'))->toBeFalse(); + }); +}); + +describe('admin:MediaMoveStorageLocalToCloud', function () { + it('requires a configured cloud disk', function () { + // Fake s3 disk has no url() host resolvable? Storage::fake provides a + // url, so instead point cloud at a disk that throws. + Config::set('filesystems.cloud', 'does-not-exist'); + + $this->artisan('admin:MediaMoveStorageLocalToCloud', ['--force' => true]) + ->assertExitCode(1); + }); + + it('flips PF_ENABLE_CLOUD to true before migrating (dry-run reports it)', function () { + // cloud currently false in the temp .env from beforeEach. + $this->artisan('admin:MediaMoveStorageLocalToCloud', ['--dry-run' => true]) + ->expectsOutputToContain('PF_ENABLE_CLOUD') + ->assertExitCode(0); + + // dry-run must not write the .env. + expect(file_get_contents(app()->environmentFilePath()))->toContain('PF_ENABLE_CLOUD=false'); + }); +}); diff --git a/tests/Feature/Account/MigrateLocalS3MediaUrlTest.php b/tests/Feature/Account/MigrateLocalS3MediaUrlTest.php new file mode 100644 index 000000000..6479f1661 --- /dev/null +++ b/tests/Feature/Account/MigrateLocalS3MediaUrlTest.php @@ -0,0 +1,189 @@ +url($path) == https://cdn.test/. + Config::set('filesystems.cloud', 's3'); + Config::set('filesystems.disks.s3', [ + 'driver' => 's3', + 'key' => 'test', + 'secret' => 'test', + 'region' => 'us-east-1', + 'bucket' => 'bucket', + 'url' => 'https://cdn.test', + 'endpoint' => 'https://cdn.test', + 'use_path_style_endpoint' => true, + 'visibility' => 'public', + ]); + // Enable cloud storage. config_cache() falls through to config() when + // instance.enable_cc is off (as in CI), so set both to be safe. + Config::set('pixelfed.cloud_storage', true); + ConfigCacheService::put('pixelfed.cloud_storage', true); +}); + +function makeStatusWithStaleMedia(string $staleHost = 'https://s3.old.example'): Media +{ + $user = User::factory()->create(); + $user->refresh(); + $pid = $user->profile->id; + + $status = Status::factory()->create(['profile_id' => $pid, 'type' => 'video']); + + $path = 'public/m/_v2/'.$pid.'/aa/bb/file.mp4'; + $thumbPath = 'public/m/_v2/'.$pid.'/aa/bb/file_thumb.jpeg'; + + return Media::create([ + 'status_id' => $status->id, + 'profile_id' => $pid, + 'user_id' => $user->id, + 'media_path' => $path, + 'thumbnail_path' => $thumbPath, + 'cdn_url' => 'https://cdn.test/'.$path, // already correct + 'thumbnail_url' => $staleHost.'/'.$thumbPath, // stale + 'optimized_url' => $staleHost.'/'.$path, // stale + 'mime' => 'video/mp4', + 'remote_media' => false, + 'order' => 0, + ]); +} + +it('rebuilds stale thumbnail_url and optimized_url but leaves correct cdn_url', function () { + $media = makeStatusWithStaleMedia(); + + $this->artisan('admin:MigrateLocalS3MediaURL', ['id' => (string) $media->status_id, '--force' => true]) + ->assertExitCode(0); + + $media->refresh(); + expect($media->cdn_url)->toBe('https://cdn.test/'.$media->media_path); + expect($media->thumbnail_url)->toBe('https://cdn.test/'.$media->thumbnail_path); + expect($media->optimized_url)->toBe('https://cdn.test/'.$media->media_path); + expect(parse_url($media->thumbnail_url, PHP_URL_HOST))->toBe('cdn.test'); + expect(parse_url($media->optimized_url, PHP_URL_HOST))->toBe('cdn.test'); +}); + +it('does not change anything in dry-run mode', function () { + $media = makeStatusWithStaleMedia(); + $originalThumb = $media->thumbnail_url; + + $this->artisan('admin:MigrateLocalS3MediaURL', ['id' => (string) $media->status_id, '--dry-run' => true]) + ->assertExitCode(0); + + expect($media->fresh()->thumbnail_url)->toBe($originalThumb); +}); + +it('leaves already-correct media untouched', function () { + $user = User::factory()->create(); + $user->refresh(); + $pid = $user->profile->id; + $status = Status::factory()->create(['profile_id' => $pid, 'type' => 'photo']); + $path = 'public/m/_v2/'.$pid.'/aa/bb/ok.jpg'; + + $media = Media::create([ + 'status_id' => $status->id, + 'profile_id' => $pid, + 'media_path' => $path, + 'cdn_url' => 'https://cdn.test/'.$path, + 'thumbnail_url' => 'https://cdn.test/'.$path, + 'mime' => 'image/jpeg', + 'remote_media' => false, + 'order' => 0, + ]); + + $updatedAt = $media->fresh()->updated_at; + + $this->artisan('admin:MigrateLocalS3MediaURL', ['id' => (string) $status->id, '--force' => true]) + ->assertExitCode(0); + + expect($media->fresh()->updated_at->eq($updatedAt))->toBeTrue(); +}); + +it('never rewrites remote media', function () { + $user = User::factory()->create(); + $user->refresh(); + $pid = $user->profile->id; + $status = Status::factory()->create(['profile_id' => $pid, 'type' => 'photo']); + + $media = Media::create([ + 'status_id' => $status->id, + 'profile_id' => $pid, + 'media_path' => 'https://remote.example/image.jpg', + 'cdn_url' => 'https://s3.old.example/image.jpg', + 'remote_media' => true, + 'remote_url' => 'https://remote.example/image.jpg', + 'mime' => 'image/jpeg', + 'order' => 0, + ]); + + $this->artisan('admin:MigrateLocalS3MediaURL', ['id' => (string) $status->id, '--force' => true]) + ->assertExitCode(0); + + // Unchanged: remote media is skipped. + expect($media->fresh()->cdn_url)->toBe('https://s3.old.example/image.jpg'); +}); + +it('requires an id or --all', function () { + $this->artisan('admin:MigrateLocalS3MediaURL') + ->assertExitCode(1); +}); + +it('refuses to run on a local-storage instance', function () { + // Simulate local storage: cloud disabled (set both, see beforeEach). + Config::set('pixelfed.cloud_storage', false); + ConfigCacheService::put('pixelfed.cloud_storage', false); + + $this->artisan('admin:MigrateLocalS3MediaURL', ['--all' => true, '--force' => true]) + ->expectsOutputToContain('Cloud storage is not enabled') + ->assertExitCode(1); +}); + +it('with --oldDomain only rewrites URLs on that host', function () { + // thumbnail_url on s3.old.example, optimized_url on other.example. + $media = makeStatusWithStaleMedia('https://s3.old.example'); + $media->optimized_url = 'https://other.example/'.$media->media_path; + $media->save(); + + $this->artisan('admin:MigrateLocalS3MediaURL', [ + 'id' => (string) $media->status_id, + '--oldDomain' => 's3.old.example', + '--force' => true, + ])->assertExitCode(0); + + $media->refresh(); + // Matched the filter -> rewritten. + expect(parse_url($media->thumbnail_url, PHP_URL_HOST))->toBe('cdn.test'); + // Did NOT match the filter -> left as-is. + expect(parse_url($media->optimized_url, PHP_URL_HOST))->toBe('other.example'); +}); + +it('with --newDomain override rewrites to the given host', function () { + $media = makeStatusWithStaleMedia('https://s3.old.example'); + + $this->artisan('admin:MigrateLocalS3MediaURL', [ + 'id' => (string) $media->status_id, + '--newDomain' => 'media.example', + '--force' => true, + ])->assertExitCode(0); + + $media->refresh(); + expect(parse_url($media->thumbnail_url, PHP_URL_HOST))->toBe('media.example'); + expect(parse_url($media->optimized_url, PHP_URL_HOST))->toBe('media.example'); +}); diff --git a/tests/Feature/Federation/DuplicateMediaAttachmentTest.php b/tests/Feature/Federation/DuplicateMediaAttachmentTest.php new file mode 100644 index 000000000..a4d76f872 --- /dev/null +++ b/tests/Feature/Federation/DuplicateMediaAttachmentTest.php @@ -0,0 +1,90 @@ + 'Document', + 'mediaType' => 'image/jpeg', + 'url' => $url, + 'name' => 'alt text', + 'blurhash' => 'UREVf}R:E2WB~qNKWBs.XURkxZofD+n~oJR-', + 'width' => 768, + 'height' => 1024, + ]; +} + +it('does not create a duplicate media row for the same status and url', function () { + $user = User::factory()->create(); + $user->refresh(); + $status = Status::factory()->create(['profile_id' => $user->profile->id, 'type' => 'photo']); + + $url = 'https://files.mastodon.social/media_attachments/files/117/original/e9ab6f0314043b12.jpeg'; + $payload = attachmentPayload($url); + + $first = Helpers::createMediaAttachment($payload, $status, 0); + // Second call (simulating re-import / race) must not throw and must be a no-op. + $second = Helpers::createMediaAttachment($payload, $status, 0); + + expect($first)->not->toBeNull(); + expect($second)->toBeNull(); + expect(Media::whereStatusId($status->id)->whereMediaPath($url)->count())->toBe(1); +}); + +it('creates distinct rows for different urls on the same status', function () { + $user = User::factory()->create(); + $user->refresh(); + $status = Status::factory()->create(['profile_id' => $user->profile->id, 'type' => 'photo:album']); + + $a = Helpers::createMediaAttachment(attachmentPayload('https://files.mastodon.social/a.jpeg'), $status, 0); + $b = Helpers::createMediaAttachment(attachmentPayload('https://files.mastodon.social/b.jpeg'), $status, 1); + + expect($a)->not->toBeNull(); + expect($b)->not->toBeNull(); + expect(Media::whereStatusId($status->id)->count())->toBe(2); +}); + +it('returns null when the row was inserted concurrently after the existence check', function () { + $user = User::factory()->create(); + $user->refresh(); + $status = Status::factory()->create(['profile_id' => $user->profile->id, 'type' => 'photo']); + + $url = 'https://files.mastodon.social/race.jpeg'; + + // Pre-insert the row to simulate the concurrent winner. + Media::create([ + 'remote_media' => true, + 'status_id' => $status->id, + 'profile_id' => $status->profile_id, + 'media_path' => $url, + 'remote_url' => $url, + 'mime' => 'image/jpeg', + 'version' => 3, + 'order' => 1, + ]); + + // Should detect the existing row and return null without throwing. + $result = Helpers::createMediaAttachment(attachmentPayload($url), $status, 0); + + expect($result)->toBeNull(); + expect(Media::whereStatusId($status->id)->whereMediaPath($url)->count())->toBe(1); +}); diff --git a/tests/Unit/ActivityPub/SsrfUrlValidationTest.php b/tests/Unit/ActivityPub/SsrfUrlValidationTest.php new file mode 100644 index 000000000..a946b71e4 --- /dev/null +++ b/tests/Unit/ActivityPub/SsrfUrlValidationTest.php @@ -0,0 +1,139 @@ + ['169.254.169.254'], + 'link-local' => ['169.254.0.1'], + 'loopback v4' => ['127.0.0.1'], + 'rfc1918 10/8' => ['10.0.0.1'], + 'rfc1918 172.16/12' => ['172.18.0.1'], + 'rfc1918 192.168/16' => ['192.168.1.1'], + 'loopback v6' => ['::1'], + 'unique local v6' => ['fd00::1'], + 'unspecified' => ['0.0.0.0'], + ]; + } + + #[Test] + #[DataProvider('privateAndReservedIps')] + public function it_rejects_private_and_reserved_ips(string $ip): void + { + $this->assertFalse(Helpers::isPublicIp($ip), $ip.' should be treated as non-public'); + } + + public static function publicIps(): array + { + return [ + 'cloudflare dns' => ['1.1.1.1'], + 'google dns' => ['8.8.8.8'], + 'public v6' => ['2606:4700:4700::1111'], + ]; + } + + #[Test] + #[DataProvider('publicIps')] + public function it_accepts_public_ips(string $ip): void + { + $this->assertTrue(Helpers::isPublicIp($ip), $ip.' should be public'); + } + + // ---- normalizeHost ---------------------------------------------------- + + #[Test] + public function it_rejects_ip_literal_hosts(): void + { + $this->assertNull(Helpers::normalizeHost('169.254.169.254')); + $this->assertNull(Helpers::normalizeHost('127.0.0.1')); + $this->assertNull(Helpers::normalizeHost('::1')); + } + + #[Test] + public function it_rejects_localhost_domains(): void + { + $this->assertNull(Helpers::normalizeHost('localhost')); + } + + #[Test] + public function it_normalizes_regular_hosts(): void + { + $this->assertSame('example.com', Helpers::normalizeHost('Example.com.')); + } + + // ---- validateUrl ------------------------------------------------------ + + public static function invalidUrls(): array + { + return [ + 'http scheme' => ['http://example.com/avatar.jpg'], + 'ftp scheme' => ['ftp://example.com/avatar.jpg'], + 'ip literal https' => ['https://169.254.169.254/latest/meta-data/'], + 'private ip literal' => ['https://172.18.0.1:9000/internal.jpg'], + 'userinfo smuggling' => ['https://user:pass@example.com/a.jpg'], + 'userinfo at-trick' => ['https://example.com@169.254.169.254/a.jpg'], + 'control char' => ["https://example.com/\r\n/a.jpg"], + 'backslash' => ['https://example.com\\@evil.com/a.jpg'], + 'no dot host' => ['https://localhost/a.jpg'], + 'empty' => [''], + ]; + } + + #[Test] + #[DataProvider('invalidUrls')] + public function it_rejects_unsafe_urls(string $url): void + { + $this->assertFalse(Helpers::validateUrl($url), $url.' should be rejected'); + } + + #[Test] + public function it_accepts_a_normal_https_url_in_non_prod(): void + { + // In the local/testing environment shouldCheckDNS()/shouldCheckBans() + // are off, so a well-formed public https URL normalizes successfully. + $url = 'https://example.com/avatar.jpg'; + $this->assertSame($url, Helpers::validateUrl($url)); + } + + // ---- SecureMediaFetchService fails closed ----------------------------- + + #[Test] + public function secure_fetch_head_fails_closed_on_invalid_url(): void + { + // These never resolve/connect because validateUrl rejects them first. + $this->assertFalse(SecureMediaFetchService::head('http://169.254.169.254/')); + $this->assertFalse(SecureMediaFetchService::head('https://172.18.0.1:9000/internal.jpg')); + $this->assertFalse(SecureMediaFetchService::head('https://example.com@169.254.169.254/a.jpg')); + } + + #[Test] + public function secure_fetch_get_fails_closed_on_invalid_url(): void + { + $this->assertFalse(SecureMediaFetchService::get('http://169.254.169.254/')); + $this->assertFalse(SecureMediaFetchService::get('https://172.18.0.1:9000/internal.jpg')); + } +}