From d9e55199f51d4af5c8aeee1e668afbf58f3b6cff Mon Sep 17 00:00:00 2001 From: Shlee Date: Tue, 25 Aug 2026 13:01:11 +0930 Subject: [PATCH 01/19] Update auth.php --- config/auth.php | 1 + 1 file changed, 1 insertion(+) diff --git a/config/auth.php b/config/auth.php index 6fa4beba0..de2b6259c 100644 --- a/config/auth.php +++ b/config/auth.php @@ -80,6 +80,7 @@ return [ // 'database' => [ // 'model' => App\User::class, // 'sync_passwords' => false, + // 'locate_users_by' => 'mail', // 'sync_attributes' => [ // 'name' => 'cn', // 'email' => 'mail', From 3d82a8e8b206e5e7a0ca69626827b1263e45c9a5 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 29 Aug 2026 16:06:57 +0930 Subject: [PATCH 02/19] Fix unauthenticated SSRF in remote media/avatar fetch (variant of CVE-2026-71246) The remote media path validated URLs only as strings (Helpers::validateUrl normalizes the host + checks a ban list) and then downloaded them with Http::head + file_get_contents($url), which resolve DNS themselves and follow redirects with no private-IP checks and no address pinning. A remote actor whose icon.url redirected to an internal address (e.g. 172.18.0.1 or 169.254.169.254) made the queue worker fetch internal content and, for image responses, republish it at a public avatar URL. No account required. Fixes: - Add SecureMediaFetchService: validates URL, resolves + rejects non-global IPs (fail-closed), pins the connection to the validated IP via CURLOPT_RESOLVE, disables auto-redirects with per-hop re-validation, and enforces https-only + a byte cap. Mirrors the ActivityPubFetchService hardening from CVE-2026-71246. - Route MediaStorageService head()/fetchAvatar()/remoteToCloud() through it, removing the bare Http::head and file_get_contents($url) sinks. - validateUrl(): when DNS verification is enabled, reject hosts that resolve into reserved ranges, closing the metadata.google.internal bypass. - Harden adjacent same-class sinks: CustomEmojiService (emoji doc + image + head), FetchCacheService/webfinger, and DiscoverActor. - Add regression tests (tests/Unit/ActivityPub/SsrfUrlValidationTest.php). --- app/Services/CustomEmojiService.php | 77 ++++-- app/Services/FetchCacheService.php | 42 ++- app/Services/MediaStorageService.php | 48 +--- app/Services/SecureMediaFetchService.php | 250 ++++++++++++++++++ app/Util/ActivityPub/DiscoverActor.php | 15 +- app/Util/ActivityPub/Helpers.php | 12 + .../ActivityPub/SsrfUrlValidationTest.php | 139 ++++++++++ 7 files changed, 502 insertions(+), 81 deletions(-) create mode 100644 app/Services/SecureMediaFetchService.php create mode 100644 tests/Unit/ActivityPub/SsrfUrlValidationTest.php diff --git a/app/Services/CustomEmojiService.php b/app/Services/CustomEmojiService.php index 482c58ce6..8e480bef3 100644 --- a/app/Services/CustomEmojiService.php +++ b/app/Services/CustomEmojiService.php @@ -26,7 +26,8 @@ class CustomEmojiService return; } - if (Helpers::validateUrl($url) == false) { + $url = Helpers::validateUrl($url); + if ($url == false) { return; } @@ -35,8 +36,36 @@ class CustomEmojiService return; } + // SSRF-hardened JSON fetch: resolve + pin the host to a validated + // public IP and refuse redirects so the emoji-document request cannot + // be steered into internal addresses. + $host = parse_url($url, PHP_URL_HOST); + $port = parse_url($url, PHP_URL_PORT) ?: 443; + $ips = $host ? Helpers::resolvePublicIps($host) : []; + if (empty($ips)) { + return; + } + try { - $res = Http::acceptJson()->get($url); + $res = Http::acceptJson() + ->withOptions([ + 'allow_redirects' => false, + 'curl' => [ + CURLOPT_RESOLVE => [ + $host.':'.((int) $port).':'.implode(',', array_map( + fn ($ip) => str_contains($ip, ':') ? '['.$ip.']' : $ip, + $ips + )), + ], + CURLOPT_FRESH_CONNECT => true, + CURLOPT_FORBID_REUSE => true, + CURLOPT_PROTOCOLS => CURLPROTO_HTTPS, + CURLOPT_REDIR_PROTOCOLS => CURLPROTO_HTTPS, + ], + ]) + ->timeout(15) + ->connectTimeout(5) + ->get($url); } catch (RequestException $e) { return; } catch (\Exception $e) { @@ -61,6 +90,10 @@ class CustomEmojiService return; } + if (Helpers::validateUrl($json['icon']['url']) == false) { + return; + } + if (! self::headCheck($json['icon']['url'])) { return; } @@ -83,21 +116,16 @@ class CustomEmojiService $mediaPath = 'emoji/'.$emoji->id.$ext; try { - $response = Http::timeout(30) - ->withOptions(['max_redirects' => 0]) - ->get($json['icon']['url']); + // SSRF-hardened: validated URL, resolved+pinned public IP, + // no internal redirects, size-capped. + $maxSize = (int) config('federation.custom_emoji.max_size'); + $body = SecureMediaFetchService::get($json['icon']['url'], $maxSize > 0 ? $maxSize : null); - if (! $response->successful()) { + if ($body === false) { return; } - // Validate actual content type from response - $contentType = $response->header('Content-Type'); - if (! in_array($contentType, ['image/jpeg', 'image/png', 'image/jpg'])) { - return; - } - - Storage::put('public/'.$mediaPath, $response->body()); + Storage::put('public/'.$mediaPath, $body); $emoji->media_path = $mediaPath; $emoji->save(); @@ -121,27 +149,20 @@ class CustomEmojiService public static function headCheck($url) { - try { - $res = Http::head($url); - } catch (RequestException $e) { - return false; - } catch (\Exception $e) { + $maxSize = (int) config('federation.custom_emoji.max_size'); + // SSRF-hardened HEAD: validated URL, resolved+pinned public IP, no + // internal redirects. + $head = SecureMediaFetchService::head($url, $maxSize > 0 ? $maxSize : null); + + if (! $head) { return false; } - if (! $res->successful()) { + if (! in_array($head['mime'], ['image/jpeg', 'image/png', 'image/jpg'], true)) { return false; } - $type = $res->header('content-type'); - $length = $res->header('content-length'); - - if ( - ! $type || - ! $length || - ! in_array($type, ['image/jpeg', 'image/png', 'image/jpg']) || - $length > config('federation.custom_emoji.max_size') - ) { + if ($maxSize > 0 && $head['length'] > $maxSize) { return false; } diff --git a/app/Services/FetchCacheService.php b/app/Services/FetchCacheService.php index c28ace3bb..ded9fb988 100644 --- a/app/Services/FetchCacheService.php +++ b/app/Services/FetchCacheService.php @@ -22,29 +22,47 @@ class FetchCacheService } if ($verifyCheck) { - if (! Helpers::validateUrl($url)) { + $validated = Helpers::validateUrl($url); + if (! $validated) { Cache::put($key, 1, $ttl); return false; } + $url = $validated; } $headers = [ 'User-Agent' => '(Pixelfed/'.config('pixelfed.version').'; +'.config('app.url').')', ]; - if ($allowRedirects) { - $options = [ - 'allow_redirects' => [ - 'max' => 2, - 'strict' => true, - ], - ]; - } else { - $options = [ - 'allow_redirects' => false, - ]; + // SSRF-hardening: resolve the host and pin the connection to a + // validated public IP. Auto-redirects are disabled so a remote host + // cannot steer the request into an internal address on a later hop. + $host = parse_url($url, PHP_URL_HOST); + $port = parse_url($url, PHP_URL_PORT) ?: 443; + $ips = $host ? Helpers::resolvePublicIps($host) : []; + if (empty($ips)) { + Cache::put($key, 1, $ttl); + + return false; } + + $options = [ + 'allow_redirects' => false, + 'curl' => [ + CURLOPT_RESOLVE => [ + $host.':'.((int) $port).':'.implode(',', array_map( + fn ($ip) => str_contains($ip, ':') ? '['.$ip.']' : $ip, + $ips + )), + ], + CURLOPT_FRESH_CONNECT => true, + CURLOPT_FORBID_REUSE => true, + CURLOPT_PROTOCOLS => CURLPROTO_HTTPS, + CURLOPT_REDIR_PROTOCOLS => CURLPROTO_HTTPS, + ], + ]; + try { $res = Http::withOptions($options) ->retry(3, function (int $attempt, $exception) { diff --git a/app/Services/MediaStorageService.php b/app/Services/MediaStorageService.php index 322cc29f1..d52063e3e 100644 --- a/app/Services/MediaStorageService.php +++ b/app/Services/MediaStorageService.php @@ -8,11 +8,8 @@ use App\Jobs\StatusPipeline\NewStatusPipeline; use App\Models\Media; use App\Models\Status; use App\Util\ActivityPub\Helpers; -use Illuminate\Http\Client\ConnectionException; use Illuminate\Http\File; -use Illuminate\Support\Arr; use Illuminate\Support\Facades\Cache; -use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Storage; use Illuminate\Support\Str; @@ -44,35 +41,10 @@ class MediaStorageService public static function head($url) { - try { - $r = Http::head($url); - } catch (ConnectionException $e) { - return false; - } - - if (! $r->successful()) { - return false; - } - - $h = Arr::mapWithKeys($r->headers(), function ($item, $key) { - return [strtolower($key) => last($item)]; - }); - - if (! isset($h['content-length'], $h['content-type'])) { - return false; - } - - $len = (int) $h['content-length']; - $mime = $h['content-type']; - - if ($len < 10 || $len > ((config_cache('pixelfed.max_photo_size') * 1000))) { - return false; - } - - return [ - 'length' => $len, - 'mime' => $mime, - ]; + // SSRF-hardened: validates URL, resolves + rejects private/reserved + // IPs, pins the connection to the validated address, and refuses to + // follow redirects into internal networks. See SecureMediaFetchService. + return SecureMediaFetchService::head($url, (int) config_cache('pixelfed.max_photo_size') * 1000); } protected function cloudStore($media) @@ -155,7 +127,8 @@ class MediaStorageService return; } - $head = $this->head($media->remote_url); + // Hardened HEAD (IP-validated, pinned, no internal redirects). + $head = $this->head($url); if (! $head) { return; @@ -206,7 +179,11 @@ class MediaStorageService $tmpBase = storage_path('app/remcache/'); $tmpPath = $media->profile_id.'-'.$path; $tmpName = $tmpBase.$tmpPath; - $data = file_get_contents($url, false, null, 0, $head['length']); + // Hardened byte fetch through the same validated, pinned, redirect-safe path. + $data = SecureMediaFetchService::get($url, $max_size, $head['length']); + if ($data === false) { + return; + } file_put_contents($tmpName, $data); $hash = hash_file('sha256', $tmpName); @@ -281,7 +258,8 @@ class MediaStorageService $tmpBase = storage_path('app/remcache/'); $tmpPath = 'avatar_'.$avatar->profile_id.'-'.$path; $tmpName = $tmpBase.$tmpPath; - $data = @file_get_contents($url, false, null, 0, $head['length']); + // Hardened byte fetch: validated URL, pinned IP, no internal redirects, size-capped. + $data = SecureMediaFetchService::get($url, $max_size, $head['length']); if (! $data) { return; } diff --git a/app/Services/SecureMediaFetchService.php b/app/Services/SecureMediaFetchService.php new file mode 100644 index 000000000..78214824d --- /dev/null +++ b/app/Services/SecureMediaFetchService.php @@ -0,0 +1,250 @@ +request($url, 'head', $maxBytes); + } + + /** + * Download up to $maxBytes of the resource through the pinned, + * validated path. Returns the raw body string, or false. + * + * @return string|false + */ + public static function get(string $url, ?int $maxBytes = null, ?int $expectedLength = null) + { + $maxBytes = $maxBytes ?? self::defaultMaxBytes(); + $result = (new self)->request($url, 'get', $maxBytes, $expectedLength); + + if (! is_array($result)) { + return false; + } + + return $result['body'] ?? false; + } + + /** + * Shared request loop: validate -> resolve public IPs -> pin -> issue + * request with redirects disabled -> re-validate each hop. + * + * @return array|false For 'head': ['length'=>int,'mime'=>string]. + * For 'get': ['body'=>string,'length'=>int,'mime'=>string]. + */ + protected function request(string $url, string $method, int $maxBytes, ?int $expectedLength = null) + { + $currentUrl = $url; + + for ($redirects = 0; $redirects <= self::MAX_REDIRECTS; $redirects++) { + $currentUrl = Helpers::validateUrl($currentUrl); + + if (! $currentUrl) { + return false; + } + + $host = parse_url($currentUrl, PHP_URL_HOST); + $scheme = parse_url($currentUrl, PHP_URL_SCHEME); + $port = parse_url($currentUrl, PHP_URL_PORT) ?: 443; + + if (! $host || strtolower((string) $scheme) !== 'https') { + return false; + } + + // Resolve the host and reject if ANY resolved address is + // non-global. Fail-closed: empty means unresolved or private. + $ips = Helpers::resolvePublicIps($host); + + if (empty($ips)) { + return false; + } + + try { + $res = Http::withOptions([ + 'allow_redirects' => false, + 'sink' => null, + 'curl' => [ + CURLOPT_RESOLVE => [ + $this->buildResolveEntry($host, (int) $port, $ips), + ], + CURLOPT_FRESH_CONNECT => true, + CURLOPT_FORBID_REUSE => true, + CURLOPT_PROTOCOLS => CURLPROTO_HTTPS, + CURLOPT_REDIR_PROTOCOLS => CURLPROTO_HTTPS, + ], + 'on_headers' => function (ResponseInterface $response) use ($maxBytes) { + $length = $response->getHeaderLine('Content-Length'); + if ($length !== '' && ctype_digit($length) && (int) $length > $maxBytes) { + throw new \RuntimeException('Remote media exceeds maximum size'); + } + }, + ]) + ->withHeaders(['User-Agent' => self::userAgent()]) + ->timeout(self::TIMEOUT) + ->connectTimeout(self::CONNECT_TIMEOUT) + ->{$method}($currentUrl); + } catch (RequestException $e) { + return false; + } catch (ConnectionException $e) { + return false; + } catch (\Throwable $e) { + return false; + } + + // Manual redirect handling: re-validate + re-resolve the next hop. + if (in_array($res->status(), [301, 302, 303, 307, 308], true)) { + if ($redirects >= self::MAX_REDIRECTS) { + return false; + } + $location = $res->header('Location'); + if (! $location) { + return false; + } + $nextUrl = $this->resolveRedirect($currentUrl, $location); + if (! $nextUrl) { + return false; + } + $currentUrl = $nextUrl; + + continue; + } + + if (! $res->successful()) { + return false; + } + + $mime = $this->normalizeMime($res->header('Content-Type')); + $declaredLength = $res->header('Content-Length'); + $declaredLength = ($declaredLength !== null && ctype_digit((string) $declaredLength)) + ? (int) $declaredLength + : null; + + if ($method === 'head') { + if ($declaredLength === null || $mime === null) { + return false; + } + if ($declaredLength < 10 || $declaredLength > $maxBytes) { + return false; + } + + return ['length' => $declaredLength, 'mime' => $mime]; + } + + // GET: enforce the cap against the actual body we received. + $body = $res->body(); + $len = strlen($body); + + if ($len === 0 || $len > $maxBytes) { + return false; + } + + if ($expectedLength !== null && $len < $expectedLength) { + // Received less than the HEAD promised; treat as truncated. + // Still return what we have, capped, but never more than asked. + $body = substr($body, 0, $expectedLength); + $len = strlen($body); + } + + return [ + 'body' => $body, + 'length' => $len, + 'mime' => $mime ?? ($declaredLength !== null ? $mime : null), + ]; + } + + return false; + } + + protected function buildResolveEntry(string $host, int $port, array $ips): string + { + $addresses = array_map(function ($ip) { + return str_contains($ip, ':') ? '['.$ip.']' : $ip; + }, $ips); + + return $host.':'.$port.':'.implode(',', $addresses); + } + + protected function resolveRedirect(string $baseUrl, string $location): ?string + { + $location = trim($location); + + if ($location === '' || preg_match('/[\x00-\x20\x7f]/', $location)) { + return null; + } + + try { + $resolved = (string) BaseUri::from($baseUrl)->resolve($location); + + return Helpers::validateUrl($resolved) ? $resolved : null; + } catch (\Throwable $e) { + return null; + } + } + + protected function normalizeMime(?string $contentType): ?string + { + if (! $contentType) { + return null; + } + + $mime = strtolower(trim(explode(';', $contentType)[0])); + + return $mime === '' ? null : $mime; + } + + protected static function defaultMaxBytes(): int + { + // Cap on the larger of avatar/photo config limits (kB -> bytes), + // with a sane fallback. + $photo = (int) config_cache('pixelfed.max_photo_size'); + $avatar = (int) config('pixelfed.max_avatar_size'); + $maxKb = max($photo, $avatar, 1000); + + return $maxKb * 1000; + } + + protected static function userAgent(): string + { + return 'PixelFedBot/1.0.0 (Pixelfed/'.config('pixelfed.version').'; +'.config('app.url').')'; + } +} diff --git a/app/Util/ActivityPub/DiscoverActor.php b/app/Util/ActivityPub/DiscoverActor.php index 4e41cf059..734e053e1 100644 --- a/app/Util/ActivityPub/DiscoverActor.php +++ b/app/Util/ActivityPub/DiscoverActor.php @@ -2,7 +2,7 @@ namespace App\Util\ActivityPub; -use Illuminate\Support\Facades\Http; +use App\Services\ActivityPubFetchService; class DiscoverActor { @@ -17,17 +17,20 @@ class DiscoverActor public function fetch() { - $res = Http::withHeaders([ - 'Accept' => 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"', - 'User-Agent' => 'PixelfedBot - https://pixelfed.org', - ])->get($this->url); - $this->response = $res->body(); + // SSRF-hardened: route through the validated, IP-pinned, + // redirect-revalidating ActivityPub fetch path instead of a raw + // Http::get on an unvalidated URL. + $this->response = ActivityPubFetchService::get($this->url) ?: null; return $this; } public function getResponse() { + if (! $this->response) { + return null; + } + return json_decode($this->response, true); } diff --git a/app/Util/ActivityPub/Helpers.php b/app/Util/ActivityPub/Helpers.php index f96d0e037..7d9926744 100644 --- a/app/Util/ActivityPub/Helpers.php +++ b/app/Util/ActivityPub/Helpers.php @@ -199,6 +199,18 @@ class Helpers } } + // SSRF guard: when DNS verification is enabled, reject any host that + // resolves into a non-global (private/reserved/link-local) range. This + // closes the bypass where a public-looking hostname (e.g. + // metadata.google.internal) resolves to a reserved address such as + // 169.254.169.254. resolvePublicIps() fails closed: it returns an empty + // array if the host does not resolve or any resolved IP is non-global. + if ($disableDNSCheck !== true && self::shouldCheckDNS()) { + if (empty(self::resolvePublicIps($host))) { + return false; + } + } + return $uri->toString(); } diff --git a/tests/Unit/ActivityPub/SsrfUrlValidationTest.php b/tests/Unit/ActivityPub/SsrfUrlValidationTest.php new file mode 100644 index 000000000..a946b71e4 --- /dev/null +++ b/tests/Unit/ActivityPub/SsrfUrlValidationTest.php @@ -0,0 +1,139 @@ + ['169.254.169.254'], + 'link-local' => ['169.254.0.1'], + 'loopback v4' => ['127.0.0.1'], + 'rfc1918 10/8' => ['10.0.0.1'], + 'rfc1918 172.16/12' => ['172.18.0.1'], + 'rfc1918 192.168/16' => ['192.168.1.1'], + 'loopback v6' => ['::1'], + 'unique local v6' => ['fd00::1'], + 'unspecified' => ['0.0.0.0'], + ]; + } + + #[Test] + #[DataProvider('privateAndReservedIps')] + public function it_rejects_private_and_reserved_ips(string $ip): void + { + $this->assertFalse(Helpers::isPublicIp($ip), $ip.' should be treated as non-public'); + } + + public static function publicIps(): array + { + return [ + 'cloudflare dns' => ['1.1.1.1'], + 'google dns' => ['8.8.8.8'], + 'public v6' => ['2606:4700:4700::1111'], + ]; + } + + #[Test] + #[DataProvider('publicIps')] + public function it_accepts_public_ips(string $ip): void + { + $this->assertTrue(Helpers::isPublicIp($ip), $ip.' should be public'); + } + + // ---- normalizeHost ---------------------------------------------------- + + #[Test] + public function it_rejects_ip_literal_hosts(): void + { + $this->assertNull(Helpers::normalizeHost('169.254.169.254')); + $this->assertNull(Helpers::normalizeHost('127.0.0.1')); + $this->assertNull(Helpers::normalizeHost('::1')); + } + + #[Test] + public function it_rejects_localhost_domains(): void + { + $this->assertNull(Helpers::normalizeHost('localhost')); + } + + #[Test] + public function it_normalizes_regular_hosts(): void + { + $this->assertSame('example.com', Helpers::normalizeHost('Example.com.')); + } + + // ---- validateUrl ------------------------------------------------------ + + public static function invalidUrls(): array + { + return [ + 'http scheme' => ['http://example.com/avatar.jpg'], + 'ftp scheme' => ['ftp://example.com/avatar.jpg'], + 'ip literal https' => ['https://169.254.169.254/latest/meta-data/'], + 'private ip literal' => ['https://172.18.0.1:9000/internal.jpg'], + 'userinfo smuggling' => ['https://user:pass@example.com/a.jpg'], + 'userinfo at-trick' => ['https://example.com@169.254.169.254/a.jpg'], + 'control char' => ["https://example.com/\r\n/a.jpg"], + 'backslash' => ['https://example.com\\@evil.com/a.jpg'], + 'no dot host' => ['https://localhost/a.jpg'], + 'empty' => [''], + ]; + } + + #[Test] + #[DataProvider('invalidUrls')] + public function it_rejects_unsafe_urls(string $url): void + { + $this->assertFalse(Helpers::validateUrl($url), $url.' should be rejected'); + } + + #[Test] + public function it_accepts_a_normal_https_url_in_non_prod(): void + { + // In the local/testing environment shouldCheckDNS()/shouldCheckBans() + // are off, so a well-formed public https URL normalizes successfully. + $url = 'https://example.com/avatar.jpg'; + $this->assertSame($url, Helpers::validateUrl($url)); + } + + // ---- SecureMediaFetchService fails closed ----------------------------- + + #[Test] + public function secure_fetch_head_fails_closed_on_invalid_url(): void + { + // These never resolve/connect because validateUrl rejects them first. + $this->assertFalse(SecureMediaFetchService::head('http://169.254.169.254/')); + $this->assertFalse(SecureMediaFetchService::head('https://172.18.0.1:9000/internal.jpg')); + $this->assertFalse(SecureMediaFetchService::head('https://example.com@169.254.169.254/a.jpg')); + } + + #[Test] + public function secure_fetch_get_fails_closed_on_invalid_url(): void + { + $this->assertFalse(SecureMediaFetchService::get('http://169.254.169.254/')); + $this->assertFalse(SecureMediaFetchService::get('https://172.18.0.1:9000/internal.jpg')); + } +} From 7482befd8f08845c7b79631e39cd50a8c491a980 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 29 Aug 2026 16:09:37 +0930 Subject: [PATCH 03/19] Allow gif and webp mime types for custom emoji import Add image/gif and image/webp to the accepted custom emoji image types via a shared CustomEmojiService::ALLOWED_MIME_TYPES constant used by both the ActivityPub mediaType check and the response-content headCheck, so the allowlist stays in sync. File extension derives from the mime type. --- app/Services/CustomEmojiService.php | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/app/Services/CustomEmojiService.php b/app/Services/CustomEmojiService.php index 8e480bef3..438965def 100644 --- a/app/Services/CustomEmojiService.php +++ b/app/Services/CustomEmojiService.php @@ -11,6 +11,17 @@ use Illuminate\Support\Facades\Storage; class CustomEmojiService { + /** + * Allowed image mime types for imported custom emoji. + */ + public const ALLOWED_MIME_TYPES = [ + 'image/jpeg', + 'image/jpg', + 'image/png', + 'image/gif', + 'image/webp', + ]; + public static function get($shortcode) { if ((bool) config_cache('federation.custom_emoji.enabled') == false) { @@ -85,7 +96,7 @@ class CustomEmojiService ! isset($json['icon']['url']) || ! isset($json['icon']['type']) || $json['icon']['type'] !== 'Image' || - ! in_array($json['icon']['mediaType'], ['image/jpeg', 'image/png', 'image/jpg']) + ! in_array($json['icon']['mediaType'], self::ALLOWED_MIME_TYPES, true) ) { return; } @@ -158,7 +169,7 @@ class CustomEmojiService return false; } - if (! in_array($head['mime'], ['image/jpeg', 'image/png', 'image/jpg'], true)) { + if (! in_array($head['mime'], self::ALLOWED_MIME_TYPES, true)) { return false; } From 8123dcf9342c1c296bbeea5ca8066dca89b72db1 Mon Sep 17 00:00:00 2001 From: Shlee Date: Sat, 29 Aug 2026 16:16:37 +0930 Subject: [PATCH 04/19] Update SecureMediaFetchService.php --- app/Services/SecureMediaFetchService.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/Services/SecureMediaFetchService.php b/app/Services/SecureMediaFetchService.php index 78214824d..2987ee083 100644 --- a/app/Services/SecureMediaFetchService.php +++ b/app/Services/SecureMediaFetchService.php @@ -24,8 +24,6 @@ use Psr\Http\Message\ResponseInterface; * re-validating and re-resolving every hop; * - enforces a hard byte cap. * - * This mirrors the hardening in ActivityPubFetchService (the CVE-2026-71246 - * fix) but returns raw media bytes / headers instead of JSON. */ class SecureMediaFetchService { From aadde946d267594c8468f74c2eb176f310718a0e Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 29 Aug 2026 16:25:56 +0930 Subject: [PATCH 05/19] Apply Pint formatting to SecureMediaFetchService --- app/Services/SecureMediaFetchService.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/Services/SecureMediaFetchService.php b/app/Services/SecureMediaFetchService.php index 2987ee083..88d560c45 100644 --- a/app/Services/SecureMediaFetchService.php +++ b/app/Services/SecureMediaFetchService.php @@ -23,7 +23,6 @@ use Psr\Http\Message\ResponseInterface; * - never lets the HTTP client follow redirects automatically, instead * re-validating and re-resolving every hop; * - enforces a hard byte cap. - * */ class SecureMediaFetchService { From 4aa7b572807cc00c41252f97223d1e59671fa1cd Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 29 Aug 2026 17:21:39 +0930 Subject: [PATCH 06/19] Add post:status command for post/media diagnostics Dumps a Status and its media for debugging. Accepts a post id or URL (/p/username/ID). Shows status columns, author, every media row's storage fields (media_path, thumbnail_path, cdn_url, thumbnail_url, optimized_url, remote_url, etc.), computed url()/thumbnailUrl()/expected-from-path, a URL health check comparing stored URL hosts against the configured cloud disk host (flags stale hosts), and the cached MediaService media_attachments actually served to clients. --- app/Console/Commands/PostStatus.php | 319 ++++++++++++++++++++++++++++ 1 file changed, 319 insertions(+) create mode 100644 app/Console/Commands/PostStatus.php diff --git a/app/Console/Commands/PostStatus.php b/app/Console/Commands/PostStatus.php new file mode 100644 index 000000000..ddfce8164 --- /dev/null +++ b/app/Console/Commands/PostStatus.php @@ -0,0 +1,319 @@ + + */ + protected $longStatusCols = ['caption', 'cw_summary']; + + public function handle() + { + $id = $this->resolveId($this->argument('id')); + + if (! $id) { + $this->error('Could not extract a status id from "'.$this->argument('id').'".'); + + return 1; + } + + $status = Status::withTrashed()->find($id); + + if (! $status) { + $this->error('No status found with id '.$id.'.'); + + return 1; + } + + $this->line(str_repeat('=', 64)); + $this->info('STATUS ROW (table: statuses)'); + $this->line(str_repeat('=', 64)); + $this->dumpStatus($status); + + $this->newLine(); + $this->line(str_repeat('=', 64)); + $this->info('AUTHOR'); + $this->line(str_repeat('=', 64)); + $this->dumpAuthor($status); + + $this->newLine(); + $this->line(str_repeat('=', 64)); + $this->info('MEDIA'); + $this->line(str_repeat('=', 64)); + $this->dumpMedia($status); + + $this->newLine(); + $this->line(str_repeat('=', 64)); + $this->info('URL HEALTH CHECK'); + $this->line(str_repeat('=', 64)); + $this->urlHealth($status); + + $this->newLine(); + $this->line(str_repeat('=', 64)); + $this->info('CACHE'); + $this->line(str_repeat('=', 64)); + $this->dumpCache($status); + + return 0; + } + + /** + * Accept a numeric id or a post URL (…/p/username/ID) and return the id. + */ + protected function resolveId(string $input): ?string + { + $input = trim($input); + + if (ctype_digit($input)) { + return $input; + } + + // Extract the last numeric path segment from a URL. + if (preg_match('#/p/[^/]+/(\d+)#', $input, $m)) { + return $m[1]; + } + + if (preg_match('#(\d{6,})#', $input, $m)) { + return $m[1]; + } + + return null; + } + + protected function dumpStatus(Status $status): void + { + $rows = []; + foreach ($status->getAttributes() as $key => $value) { + $display = $this->format($value); + if (in_array($key, $this->longStatusCols, true) && is_string($value) && strlen($value) > 60) { + $display = mb_strimwidth($value, 0, 60, '…'); + } + $rows[] = [$key, $display]; + } + $this->table(['Column', 'Value'], $rows); + + $this->comment('Computed:'); + $this->line(' url(): '.$this->safe(fn () => $status->url())); + $this->line(' permalink(): '.$this->safe(fn () => $status->permalink())); + $this->line(' is remote: '.($status->uri ? 'yes ('.$status->uri.')' : 'no (local)')); + if ($status->deleted_at) { + $this->error(' ✗ Status is soft-deleted ('.$status->deleted_at.').'); + } + } + + protected function dumpAuthor(Status $status): void + { + $acct = AccountService::get($status->profile_id, true); + if (! $acct) { + $this->error('No account found for profile_id '.$status->profile_id.'.'); + + return; + } + $this->table(['Field', 'Value'], [ + ['profile_id', $status->profile_id], + ['username', $acct['username'] ?? 'null'], + ['acct', $acct['acct'] ?? 'null'], + ['url', $acct['url'] ?? 'null'], + ['local', ($acct['local'] ?? null) ? 'true' : 'false'], + ]); + } + + protected function dumpMedia(Status $status): void + { + $media = Media::withTrashed()->whereStatusId($status->id)->orderBy('order')->get(); + + if ($media->isEmpty()) { + $this->comment('No media attached to this status.'); + + return; + } + + $cloudHost = $this->cloudHost(); + + foreach ($media as $i => $m) { + $this->newLine(); + $this->comment('Media #'.($i + 1).' (id '.$m->id.', order '.$m->order.')'); + $rows = []; + $cols = [ + 'media_path', 'thumbnail_path', 'cdn_url', 'thumbnail_url', + 'optimized_url', 'remote_url', 'remote_media', 'mime', 'size', + 'version', 'replicated_at', 'original_sha256', 'processed_at', + 'deleted_at', + ]; + $attrs = $m->getAttributes(); + foreach ($cols as $c) { + if (array_key_exists($c, $attrs)) { + $rows[] = [$c, $this->format($attrs[$c])]; + } + } + $this->table(['Media Column', 'Value'], $rows); + + $this->line(' computed url(): '.$this->safe(fn () => $m->url())); + $this->line(' computed thumbnailUrl(): '.$this->safe(fn () => $m->thumbnailUrl())); + $this->line(' expected (from path): '.$this->expectedUrl($m->media_path)); + + // Per-field host comparison. + $this->compareHost(' cdn_url', $m->cdn_url, $cloudHost); + $this->compareHost(' thumbnail_url', $m->thumbnail_url, $cloudHost); + $this->compareHost(' optimized_url', $m->optimized_url, $cloudHost); + } + } + + protected function urlHealth(Status $status): void + { + $cloudHost = $this->cloudHost(); + if (! $cloudHost) { + $this->comment('Cloud storage not configured (or no cloud disk url); skipping host comparison.'); + + return; + } + + $this->line('Configured cloud host (correct base): '.$cloudHost); + $this->newLine(); + + $media = Media::withTrashed()->whereStatusId($status->id)->get(); + $stale = []; + + foreach ($media as $m) { + if ($m->remote_media || Str::startsWith((string) $m->media_path, 'http')) { + continue; + } + foreach (['cdn_url', 'thumbnail_url', 'optimized_url'] as $field) { + $val = $m->{$field}; + if (! $val) { + continue; + } + $host = parse_url($val, PHP_URL_HOST); + if ($host && $cloudHost && strcasecmp($host, $cloudHost) !== 0) { + $stale[] = 'media '.$m->id.' '.$field.' points at '.$host.' (expected '.$cloudHost.')'; + } + } + } + + if ($stale) { + $this->error('STALE MEDIA URLS DETECTED:'); + foreach ($stale as $s) { + $this->line(' ✗ '.$s); + } + $this->newLine(); + $this->comment('Fix with: php artisan admin:MigrateLocalMediaURL '.$status->id); + $this->comment('(or --all to scan every local media row)'); + } else { + $this->info('All local media URLs point at the configured cloud host. ✓'); + } + } + + protected function dumpCache(Status $status): void + { + $cached = MediaService::get($status->id); + if (empty($cached)) { + $this->comment('No cached media_attachments entry (MediaService).'); + + return; + } + $this->comment('Cached media_attachments (MediaService, 6h TTL) — served to clients:'); + foreach ($cached as $i => $item) { + $this->line(' ['.$i.'] url: '.($item['url'] ?? 'null')); + $this->line(' ['.$i.'] preview_url: '.($item['preview_url'] ?? 'null')); + } + $this->newLine(); + $this->comment('If these still show a stale host after a DB fix, run: php artisan cache:clear'); + } + + protected function compareHost(string $label, ?string $url, ?string $cloudHost): void + { + if (! $url) { + return; + } + $host = parse_url($url, PHP_URL_HOST); + if (! $host || ! $cloudHost) { + return; + } + if (strcasecmp($host, $cloudHost) !== 0) { + $this->line(''.$label.' host: '.$host.' ✗ (expected '.$cloudHost.')'); + } else { + $this->line(''.$label.' host: '.$host.' ✓'); + } + } + + protected function expectedUrl(?string $mediaPath): string + { + if (! $mediaPath || Str::startsWith($mediaPath, 'http')) { + return $mediaPath ?? 'null'; + } + + try { + return (string) Storage::disk(config('filesystems.cloud'))->url($mediaPath); + } catch (\Throwable $e) { + return '(cloud disk not resolvable in this environment)'; + } + } + + protected function cloudHost(): ?string + { + try { + if (! (bool) config_cache('pixelfed.cloud_storage')) { + // Still try to read the configured cloud disk host for reference. + } + $url = Storage::disk(config('filesystems.cloud'))->url('probe'); + $host = parse_url($url, PHP_URL_HOST); + + return $host ?: null; + } catch (\Throwable $e) { + return null; + } + } + + protected function safe(callable $fn): string + { + try { + return (string) ($fn() ?? 'null'); + } catch (\Throwable $e) { + return 'error: '.$e->getMessage(); + } + } + + protected function format($value): string + { + if ($value === null) { + return 'null'; + } + if (is_bool($value)) { + return $value ? 'true' : 'false'; + } + if ($value === '') { + return '(empty string)'; + } + if ($value instanceof \DateTimeInterface) { + return $value->format('Y-m-d H:i:s'); + } + + return (string) $value; + } +} From 04536a6e322a2c007d2f37d08b937364c76b83fe Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 29 Aug 2026 17:38:56 +0930 Subject: [PATCH 07/19] Add admin:MigrateLocalMediaURL; replace media:cloud-url-rewrite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebuilds stale local media URLs (cdn_url, thumbnail_url, optimized_url) and avatar cdn_urls from their storage paths via the configured cloud disk. - Default target host comes from the configured cloud disk (AWS_URL); requires confirmation (or --force) and can be overridden with --newDomain. - Optional --oldDomain filters to a single old backend host; by default all stale hosts are rewritten. - Refuses to run when PF_ENABLE_CLOUD is false (local storage) and, when auto-detecting, refuses a target equal to the app domain — so local-storage instances are never rewritten. - Single status id / post URL, --all, --avatars; --dry-run; busts MediaService/StatusService caches for affected statuses. - Removes the superseded media:cloud-url-rewrite command. - Adds feature tests covering rewrite/skip/dry-run/oldDomain/newDomain/ remote-skip/local-storage-refusal. --- app/Console/Commands/MediaCloudUrlRewrite.php | 209 -------- app/Console/Commands/MigrateLocalMediaURL.php | 450 ++++++++++++++++++ .../Account/MigrateLocalMediaUrlTest.php | 186 ++++++++ 3 files changed, 636 insertions(+), 209 deletions(-) delete mode 100644 app/Console/Commands/MediaCloudUrlRewrite.php create mode 100644 app/Console/Commands/MigrateLocalMediaURL.php create mode 100644 tests/Feature/Account/MigrateLocalMediaUrlTest.php diff --git a/app/Console/Commands/MediaCloudUrlRewrite.php b/app/Console/Commands/MediaCloudUrlRewrite.php deleted file mode 100644 index 337fee178..000000000 --- a/app/Console/Commands/MediaCloudUrlRewrite.php +++ /dev/null @@ -1,209 +0,0 @@ - 'The old S3 domain', - 'newDomain' => 'The new S3 domain', - ]; - } - - /** - * The console command description. - * - * @var string - */ - protected $description = 'Rewrite S3 media urls from local users'; - - /** - * Execute the console command. - */ - public function handle() - { - $this->preflightCheck(); - $this->bootMessage(); - $this->confirmCloudUrl(); - $this->handleTask(); - } - - protected function preflightCheck() - { - if (! (bool) config_cache('pixelfed.cloud_storage')) { - $this->info('Error: Cloud storage is not enabled! Please enable before proceeding.'); - $this->error('Aborting...'); - exit; - } - } - - protected function bootMessage() - { - $this->info(' ____ _ ______ __ '); - $this->info(' / __ \(_) _____ / / __/__ ____/ / '); - $this->info(' / /_/ / / |/_/ _ \/ / /_/ _ \/ __ / '); - $this->info(' / ____/ /> info(' /_/ /_/_/|_|\___/_/_/ \___/\__,_/ '); - $this->info(' '); - $this->info(' Media Cloud Url Rewrite Tool'); - $this->info(' ==='); - $this->info(' Old S3: '.trim($this->argument('oldDomain'))); - $this->info(' New S3: '.trim($this->argument('newDomain'))); - $this->info(' '); - } - - protected function confirmCloudUrl() - { - $disk = Storage::disk(config('filesystems.cloud'))->url('test'); - $domain = parse_url($disk, PHP_URL_HOST); - if (trim($this->argument('newDomain')) !== $domain) { - $this->error('Error: The new S3 domain you entered is not currently configured'); - exit; - } - - if (! $this->confirm('Confirm this is correct')) { - $this->error('Aborting...'); - exit; - } - } - - protected function handleTask() - { - $task = select( - label: 'What action would you like to perform?', - options: ['Migrate All', 'Migrate Media', 'Migrate Avatars'] - ); - - switch ($task) { - case 'Migrate All': - $this->updateMediaUrls(); - $this->updateAvatarUrls(); - break; - case 'Migrate Media': - $this->updateMediaUrls(); - break; - case 'Migrate Avatars': - $this->updateAvatarUrls(); - break; - default: - $this->error('Invalid selection'); - - return; - } - } - - protected function updateMediaUrls() - { - $this->info('Updating media urls...'); - $oldDomain = trim($this->argument('oldDomain')); - $newDomain = trim($this->argument('newDomain')); - $disk = Storage::disk(config('filesystems.cloud')); - $count = Media::whereNotNull('cdn_url')->count(); - $bar = $this->output->createProgressBar($count); - $counter = 0; - $bar->start(); - foreach (Media::whereNotNull('cdn_url')->lazyById(1000, 'id') as $media) { - if (strncmp($media->media_path, 'http', 4) === 0) { - $bar->advance(); - - continue; - } - $cdnHost = parse_url($media->cdn_url, PHP_URL_HOST); - if ($oldDomain != $cdnHost || $newDomain == $cdnHost) { - $bar->advance(); - - continue; - } - - $media->cdn_url = str_replace($oldDomain, $newDomain, $media->cdn_url); - - if ($media->thumbnail_url != null) { - $thumbHost = parse_url($media->thumbnail_url, PHP_URL_HOST); - if ($thumbHost == $oldDomain) { - $thumbUrl = $disk->url($media->thumbnail_path); - $media->thumbnail_url = $thumbUrl; - } - } - - if ($media->optimized_url != null) { - $optiHost = parse_url($media->optimized_url, PHP_URL_HOST); - if ($optiHost == $oldDomain) { - $optiUrl = str_replace($oldDomain, $newDomain, $media->optimized_url); - $media->optimized_url = $optiUrl; - } - } - - $media->save(); - $counter++; - $bar->advance(); - } - - $bar->finish(); - - $this->line(' '); - $this->info('Finished! Updated '.$counter.' total records!'); - $this->line(' '); - $this->info('Tip: Run `php artisan cache:clear` to purge cached urls'); - } - - protected function updateAvatarUrls() - { - $this->info('Updating avatar urls...'); - $oldDomain = trim($this->argument('oldDomain')); - $newDomain = trim($this->argument('newDomain')); - $disk = Storage::disk(config('filesystems.cloud')); - $count = Avatar::count(); - $bar = $this->output->createProgressBar($count); - $counter = 0; - $bar->start(); - foreach (Avatar::lazyById(1000, 'id') as $avatar) { - if (! $avatar->cdn_url) { - $bar->advance(); - - continue; - } - - $cdnHost = parse_url($avatar->cdn_url, PHP_URL_HOST); - if (strcasecmp($oldDomain, $cdnHost) !== 0 || strcasecmp($newDomain, $cdnHost) === 0) { - $bar->advance(); - - continue; - } - - $avatar->cdn_url = str_replace($oldDomain, $newDomain, $avatar->cdn_url); - - $avatar->save(); - $counter++; - $bar->advance(); - } - - $bar->finish(); - - $this->line(' '); - $this->info('Finished! Updated '.$counter.' total records!'); - $this->line(' '); - $this->info('Tip: Run `php artisan cache:clear` to purge cached urls'); - } -} diff --git a/app/Console/Commands/MigrateLocalMediaURL.php b/app/Console/Commands/MigrateLocalMediaURL.php new file mode 100644 index 000000000..ae675f5f6 --- /dev/null +++ b/app/Console/Commands/MigrateLocalMediaURL.php @@ -0,0 +1,450 @@ +error('Cloud storage is not enabled (PF_ENABLE_CLOUD is false).'); + $this->line('This instance serves media from local storage; there are no cloud media URLs to migrate.'); + + return 1; + } + + // Safe default target = the currently configured cloud disk host, + // driven by AWS_URL in .env. Allow explicit override via --newDomain. + $configuredHost = $this->cloudHost(); + $override = $this->normalizeHost($this->option('newDomain')); + $this->newHost = $override ?: $configuredHost; + + if (! $this->newHost) { + $this->error('Could not resolve a target host.'); + $this->line('The cloud disk ('.config('filesystems.cloud').') did not return a usable URL.'); + $this->line('Set AWS_URL in your .env, or pass --newDomain explicitly.'); + + return 1; + } + + // Defensive guard: when auto-detecting the target (no --newDomain + // override), never rewrite media URLs to the app's own domain. That + // would indicate local storage or a misconfigured cloud disk URL + // (AWS_URL). An explicit --newDomain is treated as a deliberate choice. + if (! $override) { + $appHost = parse_url(config('app.url'), PHP_URL_HOST); + if ($appHost && strcasecmp($this->newHost, $appHost) === 0) { + $this->error('Refusing to run: auto-detected target host ('.$this->newHost.') is the app domain.'); + $this->line('That indicates local storage or a misconfigured cloud disk URL (AWS_URL).'); + $this->line('If you really intend this, pass an explicit --newDomain.'); + + return 1; + } + } + + $this->oldHost = $this->normalizeHost($this->option('oldDomain')); + + $id = $this->argument('id'); + $all = $this->option('all'); + $avatarsOnly = $this->option('avatars') && ! $all && ! $id; + + if (! $id && ! $all && ! $avatarsOnly) { + $this->error('Provide a status id/URL, or pass --all (optionally --avatars).'); + + return 1; + } + + if ($id && $all) { + $this->error('Pass either a status id or --all, not both.'); + + return 1; + } + + // Show the plan and require explicit approval of the target host. + $this->info('Target host (newDomain): '.$this->newHost.($override ? ' (override)' : ' (from configured cloud disk)')); + $this->info('Filter (oldDomain): '.($this->oldHost ?: 'none — rewriting all stale hosts')); + if ($this->newHost !== $configuredHost) { + $this->warn('Note: target host differs from the configured cloud disk host ('.($configuredHost ?: 'unresolved').').'); + } + + if (! $this->option('dry-run') && ! $this->option('force')) { + if (! $this->confirm('Rewrite media URLs to "'.$this->newHost.'"?', false)) { + $this->comment('Aborted.'); + + return 0; + } + } + $this->newLine(); + + if ($id) { + return $this->handleSingle($id); + } + + if ($avatarsOnly) { + $this->migrateAvatars(); + + return 0; + } + + return $this->handleAll(); + } + + /** + * Extract a bare host from a domain/URL option value. + */ + protected function normalizeHost(?string $value): ?string + { + $value = trim((string) $value); + if ($value === '') { + return null; + } + // Accept full URLs or bare hosts. + if (str_contains($value, '://')) { + $host = parse_url($value, PHP_URL_HOST); + + return $host ?: null; + } + + // Strip any accidental path/scheme fragments. + $host = parse_url('https://'.$value, PHP_URL_HOST); + + return $host ?: null; + } + + protected function handleSingle(string $id): int + { + $statusId = $this->resolveStatusId($id); + if (! $statusId) { + $this->error('Could not extract a status id from "'.$id.'".'); + + return 1; + } + + $status = Status::withTrashed()->find($statusId); + if (! $status) { + $this->error('No status found with id '.$statusId.'.'); + + return 1; + } + + $media = Media::whereStatusId($status->id)->get(); + if ($media->isEmpty()) { + $this->comment('Status '.$status->id.' has no media.'); + + return 0; + } + + $fixed = 0; + foreach ($media as $m) { + if ($this->migrateOne($m)) { + $fixed++; + } + } + + if ($fixed > 0 && ! $this->option('dry-run')) { + $this->bustCaches($status->id); + } + + $this->newLine(); + $this->info(($this->option('dry-run') ? 'Would fix ' : 'Fixed ').$fixed.' media row(s) for status '.$status->id.'.'); + if ($fixed > 0 && ! $this->option('dry-run')) { + $this->comment('Caches busted for this status.'); + } + + return 0; + } + + protected function handleAll(): int + { + if (! $this->option('dry-run') && ! $this->option('force')) { + if (! $this->confirm('Rebuild stale URLs for all local media rows?', true)) { + $this->comment('Aborted.'); + + return 0; + } + } + + $fixed = 0; + $scanned = 0; + $affectedStatusIds = []; + + Media::whereNull('remote_url') + ->where(function ($q) { + $q->whereNull('remote_media')->orWhere('remote_media', false); + }) + ->lazyById(1000, 'id') + ->each(function ($m) use (&$fixed, &$scanned, &$affectedStatusIds) { + $scanned++; + if ($this->migrateOne($m)) { + $fixed++; + if ($m->status_id) { + $affectedStatusIds[$m->status_id] = true; + } + } + }); + + if (! $this->option('dry-run')) { + foreach (array_keys($affectedStatusIds) as $sid) { + $this->bustCaches($sid); + } + } + + $this->newLine(); + $this->info('Scanned '.$scanned.' local media rows; '.($this->option('dry-run') ? 'would fix ' : 'fixed ').$fixed.'.'); + if ($fixed > 0 && ! $this->option('dry-run')) { + $this->comment('Caches busted for '.count($affectedStatusIds).' affected status(es).'); + } + + // --all always includes avatars (parity with the old command's + // "Migrate All"); --avatars can also be passed explicitly. + $this->migrateAvatars(); + + if (! $this->option('dry-run')) { + $this->comment('Tip: run `php artisan cache:clear` if any stale URLs remain cached elsewhere.'); + } + + return 0; + } + + /** + * Rebuild stale avatar cdn_urls from their media_path. + */ + protected function migrateAvatars(): void + { + $this->newLine(); + $this->info('Checking avatars...'); + + $fixed = 0; + $scanned = 0; + + Avatar::whereNotNull('cdn_url')->lazyById(1000, 'id')->each(function ($avatar) use (&$fixed, &$scanned) { + $scanned++; + + if (! $avatar->cdn_url || ! $avatar->media_path) { + return; + } + if (Str::startsWith((string) $avatar->media_path, 'http')) { + return; + } + $host = parse_url($avatar->cdn_url, PHP_URL_HOST); + if (! $this->shouldRewrite($host)) { + return; + } + + $rebuilt = $this->targetUrl($avatar->media_path); + if (! $rebuilt) { + return; + } + + $this->line(' avatar '.$avatar->id.' (profile '.$avatar->profile_id.'): '.$host.' -> '.$this->newHost); + + if (! $this->option('dry-run')) { + $avatar->cdn_url = $rebuilt; + $avatar->save(); + AccountService::del($avatar->profile_id); + } + $fixed++; + }); + + $this->info('Scanned '.$scanned.' avatars; '.($this->option('dry-run') ? 'would fix ' : 'fixed ').$fixed.'.'); + } + + /** + * Rebuild any stale URL field on a single media row from its storage path. + * Only writes when a field's host differs from the cloud host. + * + * @return bool whether the row was (or would be) changed + */ + protected function migrateOne(Media $media): bool + { + // Never touch remote media or rows whose media_path is an absolute URL. + if ($media->remote_media || Str::startsWith((string) $media->media_path, 'http')) { + return false; + } + + $changes = []; + + // cdn_url and optimized_url are both derived from media_path; + // thumbnail_url is derived from thumbnail_path. + $map = [ + 'cdn_url' => $media->media_path, + 'optimized_url' => $media->media_path, + 'thumbnail_url' => $media->thumbnail_path, + ]; + + foreach ($map as $field => $path) { + $current = $media->{$field}; + if (! $current) { + // Field not set; leave it as-is (nothing to migrate). + continue; + } + if (! $path) { + // No source path to rebuild from; skip. + continue; + } + $host = parse_url($current, PHP_URL_HOST); + if (! $this->shouldRewrite($host)) { + continue; + } + + $rebuilt = $this->targetUrl($path); + if (! $rebuilt) { + continue; + } + + $changes[$field] = ['from' => $current, 'to' => $rebuilt]; + } + + if (empty($changes)) { + return false; + } + + $this->warn('media '.$media->id.(($media->status_id) ? ' (status '.$media->status_id.')' : '').':'); + foreach ($changes as $field => $c) { + $fromHost = parse_url($c['from'], PHP_URL_HOST); + $this->line(' '.$field.': '.$fromHost.' -> '.$this->newHost); + } + + if ($this->option('dry-run')) { + return true; + } + + foreach ($changes as $field => $c) { + $media->{$field} = $c['to']; + } + $media->save(); + + return true; + } + + protected function bustCaches($statusId): void + { + MediaService::del($statusId); + StatusService::del($statusId, true); + } + + /** + * Decide whether a URL on $currentHost should be rewritten. + * Skips when already on the target host, and honours the optional + * --oldDomain filter. + */ + protected function shouldRewrite(?string $currentHost): bool + { + if (! $currentHost) { + return false; + } + // Already on the target host. + if (strcasecmp($currentHost, $this->newHost) === 0) { + return false; + } + // With --oldDomain, only rewrite that specific host. + if ($this->oldHost !== null && strcasecmp($currentHost, $this->oldHost) !== 0) { + return false; + } + + return true; + } + + /** + * Build the target URL for a storage path against the target host. + * Uses the configured cloud disk to produce the correct path, then + * swaps in --newDomain when it overrides the configured host. + */ + protected function targetUrl(string $path): ?string + { + $url = $this->diskUrl($path); + if (! $url) { + return null; + } + + $diskHost = parse_url($url, PHP_URL_HOST); + if ($diskHost && strcasecmp($diskHost, $this->newHost) !== 0) { + // Override host was requested; swap it into the disk-built URL. + $url = preg_replace('#^(https?://)'.preg_quote($diskHost, '#').'#i', '$1'.$this->newHost, $url); + } + + return $url; + } + + protected function diskUrl(string $path): ?string + { + try { + return (string) Storage::disk(config('filesystems.cloud'))->url($path); + } catch (\Throwable $e) { + return null; + } + } + + protected function cloudHost(): ?string + { + try { + $url = Storage::disk(config('filesystems.cloud'))->url('probe'); + $host = parse_url($url, PHP_URL_HOST); + + return $host ?: null; + } catch (\Throwable $e) { + return null; + } + } + + protected function resolveStatusId(string $input): ?string + { + $input = trim($input); + if (ctype_digit($input)) { + return $input; + } + if (preg_match('#/p/[^/]+/(\d+)#', $input, $m)) { + return $m[1]; + } + if (preg_match('#(\d{6,})#', $input, $m)) { + return $m[1]; + } + + return null; + } +} diff --git a/tests/Feature/Account/MigrateLocalMediaUrlTest.php b/tests/Feature/Account/MigrateLocalMediaUrlTest.php new file mode 100644 index 000000000..27027b085 --- /dev/null +++ b/tests/Feature/Account/MigrateLocalMediaUrlTest.php @@ -0,0 +1,186 @@ +url($path) == https://cdn.test/. + Config::set('filesystems.cloud', 's3'); + Config::set('filesystems.disks.s3', [ + 'driver' => 's3', + 'key' => 'test', + 'secret' => 'test', + 'region' => 'us-east-1', + 'bucket' => 'bucket', + 'url' => 'https://cdn.test', + 'endpoint' => 'https://cdn.test', + 'use_path_style_endpoint' => true, + 'visibility' => 'public', + ]); + // Enable cloud storage (config_cache reads from ConfigCacheService). + ConfigCacheService::put('pixelfed.cloud_storage', true); +}); + +function makeStatusWithStaleMedia(string $staleHost = 'https://s3.old.example'): Media +{ + $user = User::factory()->create(); + $user->refresh(); + $pid = $user->profile->id; + + $status = Status::factory()->create(['profile_id' => $pid, 'type' => 'video']); + + $path = 'public/m/_v2/'.$pid.'/aa/bb/file.mp4'; + $thumbPath = 'public/m/_v2/'.$pid.'/aa/bb/file_thumb.jpeg'; + + return Media::create([ + 'status_id' => $status->id, + 'profile_id' => $pid, + 'user_id' => $user->id, + 'media_path' => $path, + 'thumbnail_path' => $thumbPath, + 'cdn_url' => 'https://cdn.test/'.$path, // already correct + 'thumbnail_url' => $staleHost.'/'.$thumbPath, // stale + 'optimized_url' => $staleHost.'/'.$path, // stale + 'mime' => 'video/mp4', + 'remote_media' => false, + 'order' => 0, + ]); +} + +it('rebuilds stale thumbnail_url and optimized_url but leaves correct cdn_url', function () { + $media = makeStatusWithStaleMedia(); + + $this->artisan('admin:MigrateLocalMediaURL', ['id' => (string) $media->status_id, '--force' => true]) + ->assertExitCode(0); + + $media->refresh(); + expect($media->cdn_url)->toBe('https://cdn.test/'.$media->media_path); + expect($media->thumbnail_url)->toBe('https://cdn.test/'.$media->thumbnail_path); + expect($media->optimized_url)->toBe('https://cdn.test/'.$media->media_path); + expect(parse_url($media->thumbnail_url, PHP_URL_HOST))->toBe('cdn.test'); + expect(parse_url($media->optimized_url, PHP_URL_HOST))->toBe('cdn.test'); +}); + +it('does not change anything in dry-run mode', function () { + $media = makeStatusWithStaleMedia(); + $originalThumb = $media->thumbnail_url; + + $this->artisan('admin:MigrateLocalMediaURL', ['id' => (string) $media->status_id, '--dry-run' => true]) + ->assertExitCode(0); + + expect($media->fresh()->thumbnail_url)->toBe($originalThumb); +}); + +it('leaves already-correct media untouched', function () { + $user = User::factory()->create(); + $user->refresh(); + $pid = $user->profile->id; + $status = Status::factory()->create(['profile_id' => $pid, 'type' => 'photo']); + $path = 'public/m/_v2/'.$pid.'/aa/bb/ok.jpg'; + + $media = Media::create([ + 'status_id' => $status->id, + 'profile_id' => $pid, + 'media_path' => $path, + 'cdn_url' => 'https://cdn.test/'.$path, + 'thumbnail_url' => 'https://cdn.test/'.$path, + 'mime' => 'image/jpeg', + 'remote_media' => false, + 'order' => 0, + ]); + + $updatedAt = $media->fresh()->updated_at; + + $this->artisan('admin:MigrateLocalMediaURL', ['id' => (string) $status->id, '--force' => true]) + ->assertExitCode(0); + + expect($media->fresh()->updated_at->eq($updatedAt))->toBeTrue(); +}); + +it('never rewrites remote media', function () { + $user = User::factory()->create(); + $user->refresh(); + $pid = $user->profile->id; + $status = Status::factory()->create(['profile_id' => $pid, 'type' => 'photo']); + + $media = Media::create([ + 'status_id' => $status->id, + 'profile_id' => $pid, + 'media_path' => 'https://remote.example/image.jpg', + 'cdn_url' => 'https://s3.old.example/image.jpg', + 'remote_media' => true, + 'remote_url' => 'https://remote.example/image.jpg', + 'mime' => 'image/jpeg', + 'order' => 0, + ]); + + $this->artisan('admin:MigrateLocalMediaURL', ['id' => (string) $status->id, '--force' => true]) + ->assertExitCode(0); + + // Unchanged: remote media is skipped. + expect($media->fresh()->cdn_url)->toBe('https://s3.old.example/image.jpg'); +}); + +it('requires an id or --all', function () { + $this->artisan('admin:MigrateLocalMediaURL') + ->assertExitCode(1); +}); + +it('refuses to run on a local-storage instance', function () { + // Simulate local storage: cloud disabled. + ConfigCacheService::put('pixelfed.cloud_storage', false); + + $this->artisan('admin:MigrateLocalMediaURL', ['--all' => true, '--force' => true]) + ->expectsOutputToContain('Cloud storage is not enabled') + ->assertExitCode(1); +}); + +it('with --oldDomain only rewrites URLs on that host', function () { + // thumbnail_url on s3.old.example, optimized_url on other.example. + $media = makeStatusWithStaleMedia('https://s3.old.example'); + $media->optimized_url = 'https://other.example/'.$media->media_path; + $media->save(); + + $this->artisan('admin:MigrateLocalMediaURL', [ + 'id' => (string) $media->status_id, + '--oldDomain' => 's3.old.example', + '--force' => true, + ])->assertExitCode(0); + + $media->refresh(); + // Matched the filter -> rewritten. + expect(parse_url($media->thumbnail_url, PHP_URL_HOST))->toBe('cdn.test'); + // Did NOT match the filter -> left as-is. + expect(parse_url($media->optimized_url, PHP_URL_HOST))->toBe('other.example'); +}); + +it('with --newDomain override rewrites to the given host', function () { + $media = makeStatusWithStaleMedia('https://s3.old.example'); + + $this->artisan('admin:MigrateLocalMediaURL', [ + 'id' => (string) $media->status_id, + '--newDomain' => 'media.example', + '--force' => true, + ])->assertExitCode(0); + + $media->refresh(); + expect(parse_url($media->thumbnail_url, PHP_URL_HOST))->toBe('media.example'); + expect(parse_url($media->optimized_url, PHP_URL_HOST))->toBe('media.example'); +}); From da9e73dd2207b7eea7ee1e0b0a78db91415f6f50 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 29 Aug 2026 17:43:21 +0930 Subject: [PATCH 08/19] Rename to admin:MigrateLocalS3MediaURL and drop --avatars Rename the command (and test) to admin:MigrateLocalS3MediaURL to reflect its scope: rewriting stale S3/cloud media URLs only. Remove avatar handling and the --avatars option; the command now focuses solely on status media (cdn_url, thumbnail_url, optimized_url). --- ...ediaURL.php => MigrateLocalS3MediaURL.php} | 80 ++----------------- ...est.php => MigrateLocalS3MediaUrlTest.php} | 18 ++--- 2 files changed, 15 insertions(+), 83 deletions(-) rename app/Console/Commands/{MigrateLocalMediaURL.php => MigrateLocalS3MediaURL.php} (82%) rename tests/Feature/Account/{MigrateLocalMediaUrlTest.php => MigrateLocalS3MediaUrlTest.php} (89%) diff --git a/app/Console/Commands/MigrateLocalMediaURL.php b/app/Console/Commands/MigrateLocalS3MediaURL.php similarity index 82% rename from app/Console/Commands/MigrateLocalMediaURL.php rename to app/Console/Commands/MigrateLocalS3MediaURL.php index ae675f5f6..dc5195f88 100644 --- a/app/Console/Commands/MigrateLocalMediaURL.php +++ b/app/Console/Commands/MigrateLocalS3MediaURL.php @@ -2,27 +2,24 @@ namespace App\Console\Commands; -use App\Models\Avatar; use App\Models\Media; use App\Models\Status; -use App\Services\AccountService; use App\Services\MediaService; use App\Services\StatusService; use Illuminate\Console\Command; use Illuminate\Support\Facades\Storage; use Illuminate\Support\Str; -class MigrateLocalMediaURL extends Command +class MigrateLocalS3MediaURL extends Command { /** * The name and signature of the console command. * * @var string */ - protected $signature = 'admin:MigrateLocalMediaURL + protected $signature = 'admin:MigrateLocalS3MediaURL {id? : A status id (or post URL) to fix; omit with --all} {--all : Scan every local media row and fix any with a stale host} - {--avatars : Also rebuild stale avatar cdn_urls (implied by --all)} {--oldDomain= : Only rewrite URLs whose host matches this old backend (default: rewrite all stale hosts)} {--newDomain= : Target host to rewrite to (default: the configured cloud disk host from .env)} {--dry-run : Report what would change without writing} @@ -33,7 +30,7 @@ class MigrateLocalMediaURL extends Command * * @var string */ - protected $description = 'Rebuild stale local media URLs (cdn_url, thumbnail_url, optimized_url, avatars) from their storage paths using the configured cloud disk. Replaces media:cloud-url-rewrite.'; + protected $description = 'Rewrite stale local media cloud URLs (cdn_url, thumbnail_url, optimized_url) from their storage paths to the configured S3/cloud host. Replaces media:cloud-url-rewrite.'; /** * The target host to rewrite URLs to. @@ -91,10 +88,9 @@ class MigrateLocalMediaURL extends Command $id = $this->argument('id'); $all = $this->option('all'); - $avatarsOnly = $this->option('avatars') && ! $all && ! $id; - if (! $id && ! $all && ! $avatarsOnly) { - $this->error('Provide a status id/URL, or pass --all (optionally --avatars).'); + if (! $id && ! $all) { + $this->error('Provide a status id/URL, or pass --all.'); return 1; } @@ -125,12 +121,6 @@ class MigrateLocalMediaURL extends Command return $this->handleSingle($id); } - if ($avatarsOnly) { - $this->migrateAvatars(); - - return 0; - } - return $this->handleAll(); } @@ -201,14 +191,6 @@ class MigrateLocalMediaURL extends Command protected function handleAll(): int { - if (! $this->option('dry-run') && ! $this->option('force')) { - if (! $this->confirm('Rebuild stale URLs for all local media rows?', true)) { - $this->comment('Aborted.'); - - return 0; - } - } - $fixed = 0; $scanned = 0; $affectedStatusIds = []; @@ -238,65 +220,15 @@ class MigrateLocalMediaURL extends Command $this->info('Scanned '.$scanned.' local media rows; '.($this->option('dry-run') ? 'would fix ' : 'fixed ').$fixed.'.'); if ($fixed > 0 && ! $this->option('dry-run')) { $this->comment('Caches busted for '.count($affectedStatusIds).' affected status(es).'); - } - - // --all always includes avatars (parity with the old command's - // "Migrate All"); --avatars can also be passed explicitly. - $this->migrateAvatars(); - - if (! $this->option('dry-run')) { $this->comment('Tip: run `php artisan cache:clear` if any stale URLs remain cached elsewhere.'); } return 0; } - /** - * Rebuild stale avatar cdn_urls from their media_path. - */ - protected function migrateAvatars(): void - { - $this->newLine(); - $this->info('Checking avatars...'); - - $fixed = 0; - $scanned = 0; - - Avatar::whereNotNull('cdn_url')->lazyById(1000, 'id')->each(function ($avatar) use (&$fixed, &$scanned) { - $scanned++; - - if (! $avatar->cdn_url || ! $avatar->media_path) { - return; - } - if (Str::startsWith((string) $avatar->media_path, 'http')) { - return; - } - $host = parse_url($avatar->cdn_url, PHP_URL_HOST); - if (! $this->shouldRewrite($host)) { - return; - } - - $rebuilt = $this->targetUrl($avatar->media_path); - if (! $rebuilt) { - return; - } - - $this->line(' avatar '.$avatar->id.' (profile '.$avatar->profile_id.'): '.$host.' -> '.$this->newHost); - - if (! $this->option('dry-run')) { - $avatar->cdn_url = $rebuilt; - $avatar->save(); - AccountService::del($avatar->profile_id); - } - $fixed++; - }); - - $this->info('Scanned '.$scanned.' avatars; '.($this->option('dry-run') ? 'would fix ' : 'fixed ').$fixed.'.'); - } - /** * Rebuild any stale URL field on a single media row from its storage path. - * Only writes when a field's host differs from the cloud host. + * Only writes when a field's host differs from the target host. * * @return bool whether the row was (or would be) changed */ diff --git a/tests/Feature/Account/MigrateLocalMediaUrlTest.php b/tests/Feature/Account/MigrateLocalS3MediaUrlTest.php similarity index 89% rename from tests/Feature/Account/MigrateLocalMediaUrlTest.php rename to tests/Feature/Account/MigrateLocalS3MediaUrlTest.php index 27027b085..f453941bb 100644 --- a/tests/Feature/Account/MigrateLocalMediaUrlTest.php +++ b/tests/Feature/Account/MigrateLocalS3MediaUrlTest.php @@ -11,7 +11,7 @@ uses(LazilyRefreshDatabase::class); /* |-------------------------------------------------------------------------- -| admin:MigrateLocalMediaURL +| admin:MigrateLocalS3MediaURL |-------------------------------------------------------------------------- | | Rebuilds stale local media URLs (cdn_url, thumbnail_url, optimized_url) @@ -67,7 +67,7 @@ function makeStatusWithStaleMedia(string $staleHost = 'https://s3.old.example'): it('rebuilds stale thumbnail_url and optimized_url but leaves correct cdn_url', function () { $media = makeStatusWithStaleMedia(); - $this->artisan('admin:MigrateLocalMediaURL', ['id' => (string) $media->status_id, '--force' => true]) + $this->artisan('admin:MigrateLocalS3MediaURL', ['id' => (string) $media->status_id, '--force' => true]) ->assertExitCode(0); $media->refresh(); @@ -82,7 +82,7 @@ it('does not change anything in dry-run mode', function () { $media = makeStatusWithStaleMedia(); $originalThumb = $media->thumbnail_url; - $this->artisan('admin:MigrateLocalMediaURL', ['id' => (string) $media->status_id, '--dry-run' => true]) + $this->artisan('admin:MigrateLocalS3MediaURL', ['id' => (string) $media->status_id, '--dry-run' => true]) ->assertExitCode(0); expect($media->fresh()->thumbnail_url)->toBe($originalThumb); @@ -108,7 +108,7 @@ it('leaves already-correct media untouched', function () { $updatedAt = $media->fresh()->updated_at; - $this->artisan('admin:MigrateLocalMediaURL', ['id' => (string) $status->id, '--force' => true]) + $this->artisan('admin:MigrateLocalS3MediaURL', ['id' => (string) $status->id, '--force' => true]) ->assertExitCode(0); expect($media->fresh()->updated_at->eq($updatedAt))->toBeTrue(); @@ -131,7 +131,7 @@ it('never rewrites remote media', function () { 'order' => 0, ]); - $this->artisan('admin:MigrateLocalMediaURL', ['id' => (string) $status->id, '--force' => true]) + $this->artisan('admin:MigrateLocalS3MediaURL', ['id' => (string) $status->id, '--force' => true]) ->assertExitCode(0); // Unchanged: remote media is skipped. @@ -139,7 +139,7 @@ it('never rewrites remote media', function () { }); it('requires an id or --all', function () { - $this->artisan('admin:MigrateLocalMediaURL') + $this->artisan('admin:MigrateLocalS3MediaURL') ->assertExitCode(1); }); @@ -147,7 +147,7 @@ it('refuses to run on a local-storage instance', function () { // Simulate local storage: cloud disabled. ConfigCacheService::put('pixelfed.cloud_storage', false); - $this->artisan('admin:MigrateLocalMediaURL', ['--all' => true, '--force' => true]) + $this->artisan('admin:MigrateLocalS3MediaURL', ['--all' => true, '--force' => true]) ->expectsOutputToContain('Cloud storage is not enabled') ->assertExitCode(1); }); @@ -158,7 +158,7 @@ it('with --oldDomain only rewrites URLs on that host', function () { $media->optimized_url = 'https://other.example/'.$media->media_path; $media->save(); - $this->artisan('admin:MigrateLocalMediaURL', [ + $this->artisan('admin:MigrateLocalS3MediaURL', [ 'id' => (string) $media->status_id, '--oldDomain' => 's3.old.example', '--force' => true, @@ -174,7 +174,7 @@ it('with --oldDomain only rewrites URLs on that host', function () { it('with --newDomain override rewrites to the given host', function () { $media = makeStatusWithStaleMedia('https://s3.old.example'); - $this->artisan('admin:MigrateLocalMediaURL', [ + $this->artisan('admin:MigrateLocalS3MediaURL', [ 'id' => (string) $media->status_id, '--newDomain' => 'media.example', '--force' => true, From 34d6fb31f945eb733455110ab1c4397988d5065c Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 29 Aug 2026 17:50:28 +0930 Subject: [PATCH 09/19] Fix MigrateLocalS3MediaUrl tests failing in CI config_cache() falls through to config() when instance.enable_cc is off (ENABLE_CONFIG_CACHE=false, as in CI/.env.testing), so ConfigCacheService::put() alone did not toggle pixelfed.cloud_storage and the command's cloud-enabled guard aborted with exit 1. Set the underlying config value too (both in beforeEach and the local-storage refusal test). --- tests/Feature/Account/MigrateLocalS3MediaUrlTest.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/Feature/Account/MigrateLocalS3MediaUrlTest.php b/tests/Feature/Account/MigrateLocalS3MediaUrlTest.php index f453941bb..6479f1661 100644 --- a/tests/Feature/Account/MigrateLocalS3MediaUrlTest.php +++ b/tests/Feature/Account/MigrateLocalS3MediaUrlTest.php @@ -34,7 +34,9 @@ beforeEach(function () { 'use_path_style_endpoint' => true, 'visibility' => 'public', ]); - // Enable cloud storage (config_cache reads from ConfigCacheService). + // Enable cloud storage. config_cache() falls through to config() when + // instance.enable_cc is off (as in CI), so set both to be safe. + Config::set('pixelfed.cloud_storage', true); ConfigCacheService::put('pixelfed.cloud_storage', true); }); @@ -144,7 +146,8 @@ it('requires an id or --all', function () { }); it('refuses to run on a local-storage instance', function () { - // Simulate local storage: cloud disabled. + // Simulate local storage: cloud disabled (set both, see beforeEach). + Config::set('pixelfed.cloud_storage', false); ConfigCacheService::put('pixelfed.cloud_storage', false); $this->artisan('admin:MigrateLocalS3MediaURL', ['--all' => true, '--force' => true]) From 45918bbb1a23b6982f873dceb923c4edf75e825d Mon Sep 17 00:00:00 2001 From: Shlee Date: Sat, 29 Aug 2026 18:01:12 +0930 Subject: [PATCH 10/19] Update CHANGELOG.md --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b04e6fcf3..53ad5cb9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,11 @@ - ([#6856](https://github.com/pixelfed/pixelfed/pull/6856)) - Testing: Refactored the testing environment - Testing: Added 250+ tests +- ([#6930](https://github.com/pixelfed/pixelfed/pull/6930)) + - added admin:fixProfileCounts command to fix the followers/following/statuses cache count locally and remotely + - added admin:MigrateLocalS3MediaURL command for fixing dead CDN storage paths. + - added status:user status:profile status:post command for diagnostic purposes. + - added SecureMediaFetchService to enhance/harden media downloading from remote servers ## [v0.12.9 (2026-08-25)](https://github.com/pixelfed/pixelfed/compare/v0.12.9...dev) From 6ff9ffbbb8f406fae09b06669da9acf36b136703 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 29 Aug 2026 18:05:40 +0930 Subject: [PATCH 11/19] Add media storage migration commands (local<->cloud) with integrated GC Add admin:MediaMoveStorageLocalToCloud and admin:MediaMoveStorageCloudToLocal: - Copy media (+thumbnail) between local and cloud disks, verify by size (and sha256 against original_sha256 when present) before deleting the source. - Integrated GC: delete the verified source copy (local on upload, cloud on download), set version=4 / reset to 3, and bust MediaService/StatusService caches. --keep-local / --keep-cloud opt out. - Manage PF_ENABLE_CLOUD in .env AND the live runtime + config cache so new uploads route to the correct backend mid-migration on a hot server. Uses the installer's atomic .env writer (shared ManagesMediaStorageEnv trait). - --limit / --dry-run / --force. Replaces media:migrate2cloud (CloudMediaMigrate) and media:s3gc (MediaS3GarbageCollector); scheduler now runs MediaMoveStorageLocalToCloud hourly for straggler upload + GC. Keeps media:fix-nonlocal-driver. Adds feature tests (download+GC, --keep-cloud, dry-run, env-flag flip both directions, unknown-disk guard). --- app/Console/Commands/CloudMediaMigrate.php | 102 -------- .../Concerns/ManagesMediaStorageEnv.php | 122 +++++++++ .../Commands/MediaMoveStorageCloudToLocal.php | 227 +++++++++++++++++ .../Commands/MediaMoveStorageLocalToCloud.php | 239 ++++++++++++++++++ .../Commands/MediaS3GarbageCollector.php | 204 --------------- bootstrap/app.php | 3 +- .../Feature/Account/MediaMoveStorageTest.php | 150 +++++++++++ 7 files changed, 740 insertions(+), 307 deletions(-) delete mode 100644 app/Console/Commands/CloudMediaMigrate.php create mode 100644 app/Console/Commands/Concerns/ManagesMediaStorageEnv.php create mode 100644 app/Console/Commands/MediaMoveStorageCloudToLocal.php create mode 100644 app/Console/Commands/MediaMoveStorageLocalToCloud.php delete mode 100644 app/Console/Commands/MediaS3GarbageCollector.php create mode 100644 tests/Feature/Account/MediaMoveStorageTest.php diff --git a/app/Console/Commands/CloudMediaMigrate.php b/app/Console/Commands/CloudMediaMigrate.php deleted file mode 100644 index fbb068731..000000000 --- a/app/Console/Commands/CloudMediaMigrate.php +++ /dev/null @@ -1,102 +0,0 @@ -error('Cloud storage not enabled. Exiting...'); - - return; - } - - if (! $this->confirm('Are you sure you want to proceed?')) { - return; - } - - $limit = $this->option('limit'); - $hugeMode = $this->option('huge'); - - if ($limit > 500 && ! $hugeMode) { - $this->error('Max limit exceeded, use a limit lower than 500 or run again with the --huge flag'); - - return; - } - - $bar = $this->output->createProgressBar($limit); - $bar->start(); - - Media::whereNot('version', '4') - ->where('created_at', '<', now()->subDays(2)) - ->whereRemoteMedia(false) - ->whereNotNull(['status_id', 'profile_id']) - ->whereNull(['cdn_url', 'replicated_at']) - ->orderByDesc('size') - ->take($limit) - ->get() - ->each(function ($media) use ($bar) { - if (Storage::disk('local')->exists($media->media_path)) { - $this->totalSize = $this->totalSize + $media->size; - try { - MediaStorageService::store($media); - } catch (FileNotFoundException $e) { - $this->error('Error migrating media '.$media->id.' to cloud storage: '.$e->getMessage()); - - return; - } catch (NotFoundHttpException $e) { - $this->error('Error migrating media '.$media->id.' to cloud storage: '.$e->getMessage()); - - return; - } catch (\Exception $e) { - $this->error('Error migrating media '.$media->id.' to cloud storage: '.$e->getMessage()); - - return; - } - } - $bar->advance(); - }); - - $bar->finish(); - $this->line(' '); - $this->info('Finished!'); - if ($this->totalSize) { - $this->info('Uploaded '.PrettyNumber::size($this->totalSize).' of media to cloud storage!'); - $this->line(' '); - $this->info('These files are still stored locally, and will be automatically removed.'); - } - - return Command::SUCCESS; - } -} diff --git a/app/Console/Commands/Concerns/ManagesMediaStorageEnv.php b/app/Console/Commands/Concerns/ManagesMediaStorageEnv.php new file mode 100644 index 000000000..716a2a02d --- /dev/null +++ b/app/Console/Commands/Concerns/ManagesMediaStorageEnv.php @@ -0,0 +1,122 @@ +environmentFilePath(); + if (! is_file($envPath)) { + return null; + } + $payload = file_get_contents($envPath); + if ($payload === false) { + return null; + } + if (! preg_match("/^{$key}=([^\r\n]*)/m", $payload, $m)) { + return null; + } + + return trim($m[1], " \t\"'"); + } + + /** + * Set an .env key + the live runtime config + config-cache entry so the + * change takes effect immediately on a running server. + * + * @param string $configKey dotted config key kept in sync (e.g. 'pixelfed.cloud_storage') + * @param mixed $configValue the typed runtime value (e.g. true/false) + */ + protected function setStorageEnv(string $envKey, string $envValue, string $configKey, $configValue): void + { + // 1. Persist to .env atomically (survives restarts). + $this->updateEnvFile($envKey, $envValue); + + // 2. Update the live runtime config for the current process. + config([$configKey => $configValue]); + + // 3. Update the DB-backed config cache so other workers/requests + // reading via config_cache() see the new value (hot server). + try { + ConfigCacheService::put($configKey, $configValue); + } catch (\Throwable $e) { + $this->warn('Could not update config cache for '.$configKey.': '.$e->getMessage()); + } + } + + /** + * The configured cloud disk host (used to sanity check cloud config). + */ + protected function cloudHost(): ?string + { + try { + $url = Storage::disk(config('filesystems.cloud'))->url('probe'); + $host = parse_url($url, PHP_URL_HOST); + + return $host ?: null; + } catch (\Throwable $e) { + return null; + } + } + + // ---- Atomic .env writer (adapted from Installer) --------------------- + + protected function updateEnvFile($key, $value): void + { + $envPath = app()->environmentFilePath(); + $payload = file_get_contents($envPath); + + $value = str_replace(['\\', '"', "\n", "\r"], ['\\\\', '\\"', '\\n', '\\r'], $value); + + if (($existing = $this->existingEnv($key, $payload)) !== false) { + $payload = str_replace("{$key}={$existing}", "{$key}=\"{$value}\"", $payload); + } else { + $payload = $payload."\n{$key}=\"{$value}\"\n"; + } + + $this->storeEnv($payload); + } + + protected function existingEnv($needle, $haystack) + { + preg_match("/^{$needle}=[^\r\n]*/m", $haystack, $matches); + if ($matches && count($matches)) { + return substr($matches[0], strlen($needle) + 1); + } + + return false; + } + + protected function storeEnv($payload): void + { + $envPath = app()->environmentFilePath(); + $tempPath = $envPath.'.tmp'; + + $file = fopen($tempPath, 'w'); + if ($file === false) { + throw new \RuntimeException("Cannot write to {$tempPath}"); + } + fwrite($file, $payload); + fclose($file); + + if (! rename($tempPath, $envPath)) { + @unlink($tempPath); + throw new \RuntimeException('Cannot update .env file'); + } + } +} diff --git a/app/Console/Commands/MediaMoveStorageCloudToLocal.php b/app/Console/Commands/MediaMoveStorageCloudToLocal.php new file mode 100644 index 000000000..bfbdfc2bb --- /dev/null +++ b/app/Console/Commands/MediaMoveStorageCloudToLocal.php @@ -0,0 +1,227 @@ +readEnvValue('PF_ENABLE_CLOUD'); + $cloudEnabled = filter_var($envCloud, FILTER_VALIDATE_BOOLEAN); + + if ($cloudEnabled) { + $this->warn('PF_ENABLE_CLOUD is currently true.'); + $this->line('New uploads would keep landing on CLOUD storage during this migration.'); + if ($this->option('dry-run')) { + $this->line('[dry-run] Would set PF_ENABLE_CLOUD=false (.env + runtime + config cache).'); + } elseif ($this->option('force') || $this->confirm('Set PF_ENABLE_CLOUD=false now so new uploads stay local?', true)) { + $this->setStorageEnv('PF_ENABLE_CLOUD', 'false', 'pixelfed.cloud_storage', false); + $this->info('PF_ENABLE_CLOUD set to false (.env + live runtime + config cache).'); + } else { + $this->error('Aborting: refusing to migrate to local while new uploads go to cloud.'); + + return 1; + } + } else { + $this->info('PF_ENABLE_CLOUD is already false; new uploads stay local. ✓'); + } + + $this->newLine(); + if (! $this->option('dry-run') && ! $this->option('force')) { + if (! $this->confirm('Begin migrating cloud media to local?', true)) { + $this->comment('Aborted.'); + + return 0; + } + } + + $limit = (int) $this->option('limit'); + $moved = 0; + $skipped = 0; + $failed = 0; + + // Candidates: non-remote media that has a cloud copy (cdn_url set). + $query = Media::whereRemoteMedia(false) + ->whereNotNull(['media_path', 'cdn_url']) + ->orderByDesc('id') + ->limit($limit); + + $bar = $this->output->createProgressBar($query->count()); + $bar->start(); + + foreach ($query->get() as $media) { + $result = $this->migrateOne($media, $localDisk, $cloudDisk); + match ($result) { + 'moved' => $moved++, + 'skipped' => $skipped++, + default => $failed++, + }; + $bar->advance(); + } + + $bar->finish(); + $this->newLine(2); + $this->info(($this->option('dry-run') ? '[dry-run] ' : '').'Done. moved='.$moved.' skipped='.$skipped.' failed='.$failed.'.'); + if ($this->movedBytes) { + $this->info('Transferred '.PrettyNumber::size($this->movedBytes).' back to local storage.'); + } + + return 0; + } + + /** + * @return string one of moved|skipped|failed + */ + protected function migrateOne(Media $media, $localDisk, $cloudDisk): string + { + if (Str::startsWith((string) $media->media_path, 'http')) { + return 'skipped'; + } + + // Must exist on cloud to pull down. + if (! $cloudDisk->exists($media->media_path)) { + // Already local-only? just clear the cloud url fields. + if ($localDisk->exists($media->media_path)) { + if (! $this->option('dry-run')) { + $this->clearCloudFields($media); + $media->save(); + } + + return 'skipped'; + } + + return 'failed'; + } + + if ($this->option('dry-run')) { + return 'moved'; + } + + try { + $this->copyToLocal($media->media_path, $localDisk, $cloudDisk); + if ($media->thumbnail_path && $cloudDisk->exists($media->thumbnail_path)) { + $this->copyToLocal($media->thumbnail_path, $localDisk, $cloudDisk); + } + + if (! $this->verify($media->media_path, $localDisk, $cloudDisk, $media->original_sha256)) { + $this->warn(PHP_EOL.'Verify failed for media '.$media->id.' ('.$media->media_path.'); left cloud copy intact.'); + + return 'failed'; + } + + // Point URLs back at local storage. + $this->clearCloudFields($media); + + // Integrated GC: delete the verified cloud copy unless --keep-cloud. + if (! $this->option('keep-cloud')) { + $cloudDisk->delete($media->media_path); + if ($media->thumbnail_path && $cloudDisk->exists($media->thumbnail_path)) { + $cloudDisk->delete($media->thumbnail_path); + } + } + + $media->save(); + $this->movedBytes += (int) $media->size; + + if ($media->status_id) { + MediaService::del($media->status_id); + StatusService::del($media->status_id, false); + } + + return 'moved'; + } catch (\Throwable $e) { + $this->warn(PHP_EOL.'Error migrating media '.$media->id.': '.$e->getMessage()); + + return 'failed'; + } + } + + /** + * Reset a media row to local-served state. + */ + protected function clearCloudFields(Media $media): void + { + $media->cdn_url = null; + $media->optimized_url = null; + $media->thumbnail_url = null; + $media->replicated_at = null; + // version 4 meant "local deleted, cloud only"; reset so the file is + // treated as locally present again. + if ($media->version === '4' || $media->version === 4) { + $media->version = 3; + } + } + + protected function copyToLocal(string $path, $localDisk, $cloudDisk): void + { + $stream = $cloudDisk->readStream($path); + if ($stream === false || $stream === null) { + throw new \RuntimeException('Could not open cloud stream for '.$path); + } + $localDisk->writeStream($path, $stream); + if (is_resource($stream)) { + fclose($stream); + } + } + + /** + * Verify the local copy matches the cloud source by size, and by sha256 + * against the stored original checksum when available. Fails closed. + */ + protected function verify(string $path, $localDisk, $cloudDisk, ?string $expectedSha = null): bool + { + if (! $localDisk->exists($path)) { + return false; + } + + $localSize = $localDisk->size($path); + $cloudSize = $cloudDisk->size($path); + if ($localSize === false || $cloudSize === false || $localSize !== $cloudSize) { + return false; + } + + if ($expectedSha) { + $localSha = @hash_file('sha256', $localDisk->path($path)); + if ($localSha && ! hash_equals($expectedSha, $localSha)) { + return false; + } + } + + return true; + } +} diff --git a/app/Console/Commands/MediaMoveStorageLocalToCloud.php b/app/Console/Commands/MediaMoveStorageLocalToCloud.php new file mode 100644 index 000000000..185f73739 --- /dev/null +++ b/app/Console/Commands/MediaMoveStorageLocalToCloud.php @@ -0,0 +1,239 @@ +error('Cloud disk ('.config('filesystems.cloud').') could not be resolved: '.$e->getMessage()); + + return 1; + } + + if (! $this->cloudHost()) { + $this->error('Cloud disk ('.config('filesystems.cloud').') is not configured (no resolvable URL).'); + $this->line('Set AWS_URL / AWS_* in your .env before migrating to cloud.'); + + return 1; + } + + // --- Ensure new uploads route to cloud during the migration -------- + $envCloud = $this->readEnvValue('PF_ENABLE_CLOUD'); + $cloudEnabled = filter_var($envCloud, FILTER_VALIDATE_BOOLEAN); + + if (! $cloudEnabled) { + $this->warn('PF_ENABLE_CLOUD is currently "'.($envCloud ?? 'unset').'".'); + $this->line('New uploads would keep landing on LOCAL storage during this migration.'); + if ($this->option('dry-run')) { + $this->line('[dry-run] Would set PF_ENABLE_CLOUD=true (.env + runtime + config cache).'); + } elseif ($this->option('force') || $this->confirm('Set PF_ENABLE_CLOUD=true now so new uploads go to cloud?', true)) { + $this->setStorageEnv('PF_ENABLE_CLOUD', 'true', 'pixelfed.cloud_storage', true); + $this->info('PF_ENABLE_CLOUD set to true (.env + live runtime + config cache).'); + } else { + $this->error('Aborting: refusing to migrate to cloud while new uploads stay local.'); + + return 1; + } + } else { + $this->info('PF_ENABLE_CLOUD is already true; new uploads route to cloud. ✓'); + } + + $this->newLine(); + if (! $this->option('dry-run') && ! $this->option('force')) { + if (! $this->confirm('Begin migrating local media to cloud?', true)) { + $this->comment('Aborted.'); + + return 0; + } + } + + $limit = (int) $this->option('limit'); + $moved = 0; + $skipped = 0; + $failed = 0; + + // Candidates: local, non-remote media not yet replicated to cloud. + $query = Media::whereRemoteMedia(false) + ->whereNotNull('media_path') + ->where(function ($q) { + $q->whereNull('cdn_url')->orWhereNull('replicated_at')->orWhereNot('version', '4'); + }) + ->orderByDesc('id') + ->limit($limit); + + $bar = $this->output->createProgressBar($query->count()); + $bar->start(); + + foreach ($query->get() as $media) { + $result = $this->migrateOne($media, $localDisk, $cloudDisk); + match ($result) { + 'moved' => $moved++, + 'skipped' => $skipped++, + default => $failed++, + }; + $bar->advance(); + } + + $bar->finish(); + $this->newLine(2); + $this->info(($this->option('dry-run') ? '[dry-run] ' : '').'Done. moved='.$moved.' skipped='.$skipped.' failed='.$failed.'.'); + if ($this->movedBytes) { + $this->info('Transferred '.PrettyNumber::size($this->movedBytes).' to cloud storage.'); + } + + return 0; + } + + /** + * @return string one of moved|skipped|failed + */ + protected function migrateOne(Media $media, $localDisk, $cloudDisk): string + { + if (Str::startsWith((string) $media->media_path, 'http')) { + return 'skipped'; + } + + // Nothing to do if the local file is gone. + if (! $localDisk->exists($media->media_path)) { + // Already on cloud only? mark version and move on. + if ($cloudDisk->exists($media->media_path)) { + if (! $this->option('dry-run') && $media->version !== '4') { + $media->version = 4; + $media->save(); + } + + return 'skipped'; + } + + return 'skipped'; + } + + if ($this->option('dry-run')) { + return 'moved'; + } + + try { + // Copy the primary file (and thumbnail) to cloud. + $this->copyToCloud($media->media_path, $localDisk, $cloudDisk); + if ($media->thumbnail_path && $localDisk->exists($media->thumbnail_path)) { + $this->copyToCloud($media->thumbnail_path, $localDisk, $cloudDisk); + } + + // Verify the primary file before touching anything else. + if (! $this->verify($media->media_path, $localDisk, $cloudDisk, $media->original_sha256)) { + $this->warn(PHP_EOL.'Verify failed for media '.$media->id.' ('.$media->media_path.'); left local copy intact.'); + + return 'failed'; + } + + // Update URL fields to the cloud disk. + $media->cdn_url = $cloudDisk->url($media->media_path); + $media->optimized_url = $media->cdn_url; + if ($media->thumbnail_path && $cloudDisk->exists($media->thumbnail_path)) { + $media->thumbnail_url = $cloudDisk->url($media->thumbnail_path); + } + $media->replicated_at = now(); + + // Integrated GC: delete the verified local copy unless --keep-local. + if (! $this->option('keep-local')) { + $localDisk->delete($media->media_path); + if ($media->thumbnail_path && $localDisk->exists($media->thumbnail_path)) { + $localDisk->delete($media->thumbnail_path); + } + $media->version = 4; + } + + $media->save(); + $this->movedBytes += (int) $media->size; + + if ($media->status_id) { + MediaService::del($media->status_id); + StatusService::del($media->status_id, false); + } + + return 'moved'; + } catch (\Throwable $e) { + $this->warn(PHP_EOL.'Error migrating media '.$media->id.': '.$e->getMessage()); + + return 'failed'; + } + } + + protected function copyToCloud(string $path, $localDisk, $cloudDisk): void + { + $p = explode('/', $path); + $name = array_pop($p); + $storagePath = implode('/', $p); + + // Reuse the resilient uploader (handles alt disks + retries). + ResilientMediaStorageService::store($storagePath, $localDisk->path($path), $name); + } + + /** + * Verify the cloud copy matches the local source by size, and by sha256 + * when a checksum is available/cheap. Fails closed. + */ + protected function verify(string $path, $localDisk, $cloudDisk, ?string $expectedSha = null): bool + { + if (! $cloudDisk->exists($path)) { + return false; + } + + $localSize = $localDisk->size($path); + $cloudSize = $cloudDisk->size($path); + if ($localSize === false || $cloudSize === false || $localSize !== $cloudSize) { + return false; + } + + // If we already have the original checksum, verify the local file still + // matches it (so we never delete a locally-corrupted-but-uploaded file + // without noticing). Cloud content hashing would require a full + // download, which we avoid for large media; size parity + known sha + // is a strong signal. + if ($expectedSha) { + $localSha = @hash_file('sha256', $localDisk->path($path)); + if ($localSha && ! hash_equals($expectedSha, $localSha)) { + return false; + } + } + + return true; + } +} diff --git a/app/Console/Commands/MediaS3GarbageCollector.php b/app/Console/Commands/MediaS3GarbageCollector.php deleted file mode 100644 index d569659fa..000000000 --- a/app/Console/Commands/MediaS3GarbageCollector.php +++ /dev/null @@ -1,204 +0,0 @@ -error('Cloud storage not enabled. Exiting...'); - - return; - } - - $deleteEnabled = config('media.delete_local_after_cloud'); - if (! $deleteEnabled) { - $this->error('Delete local storage after cloud upload is not enabled'); - - return; - } - - $limit = $this->option('limit'); - $hugeMode = $this->option('huge'); - $log = $this->option('log-errors'); - - if ($limit > 2000 && ! $hugeMode) { - $this->error('Limit exceeded, please use a limit under 2000 or run again with the --huge flag'); - - return; - } - - $minId = Media::orderByDesc('id')->where('created_at', '<', now()->subHours(12))->first(); - - if (! $minId) { - return; - } else { - $minId = $minId->id; - } - - return $hugeMode ? - $this->hugeMode($minId, $limit, $log) : - $this->regularMode($minId, $limit, $log); - } - - protected function regularMode($minId, $limit, $log) - { - $gc = Media::whereRemoteMedia(false) - ->whereNotNull(['status_id', 'cdn_url', 'replicated_at']) - ->whereNot('version', '4') - ->where('id', '<', $minId) - ->inRandomOrder() - ->take($limit) - ->get(); - - $totalSize = 0; - $bar = $this->output->createProgressBar($gc->count()); - $bar->start(); - $cloudDisk = Storage::disk(config('filesystems.cloud')); - $localDisk = Storage::disk('local'); - - foreach ($gc as $media) { - try { - if ( - $cloudDisk->exists($media->media_path) - ) { - if ($localDisk->exists($media->media_path)) { - $localDisk->delete($media->media_path); - $media->version = 4; - $media->save(); - $totalSize = $totalSize + $media->size; - MediaService::del($media->status_id); - StatusService::del($media->status_id, false); - if ($localDisk->exists($media->thumbnail_path)) { - $localDisk->delete($media->thumbnail_path); - } - } else { - $media->version = 4; - $media->save(); - } - } else { - if ($log) { - Log::channel('media')->info('[GC] Local media not properly persisted to cloud storage', ['media_id' => $media->id]); - } - } - $bar->advance(); - } catch (FileNotFoundException $e) { - $bar->advance(); - - continue; - } catch (NotFoundHttpException $e) { - $bar->advance(); - - continue; - } catch (\Exception $e) { - $bar->advance(); - - continue; - } - } - $bar->finish(); - $this->line(' '); - $this->info('Finished!'); - if ($totalSize) { - $this->info('Cleared '.$totalSize.' bytes of media from local disk!'); - } - - return 0; - } - - protected function hugeMode($minId, $limit, $log) - { - $cloudDisk = Storage::disk(config('filesystems.cloud')); - $localDisk = Storage::disk('local'); - - $bar = $this->output->createProgressBar($limit); - $bar->start(); - - Media::whereRemoteMedia(false) - ->whereNotNull(['status_id', 'cdn_url', 'replicated_at']) - ->whereNot('version', '4') - ->where('id', '<', $minId) - ->chunk(50, function ($medias) use ($cloudDisk, $localDisk, $bar, $log) { - foreach ($medias as $media) { - try { - if ($cloudDisk->exists($media->media_path)) { - if ($localDisk->exists($media->media_path)) { - $localDisk->delete($media->media_path); - $media->version = 4; - $media->save(); - MediaService::del($media->status_id); - StatusService::del($media->status_id, false); - if ($localDisk->exists($media->thumbnail_path)) { - $localDisk->delete($media->thumbnail_path); - } - } else { - $media->version = 4; - $media->save(); - } - } else { - if ($log) { - Log::channel('media')->info('[GC] Local media not properly persisted to cloud storage', ['media_id' => $media->id]); - } - } - $bar->advance(); - } catch (FileNotFoundException $e) { - $bar->advance(); - - continue; - } catch (NotFoundHttpException $e) { - $bar->advance(); - - continue; - } catch (\Exception $e) { - $bar->advance(); - - continue; - } - } - }); - - $bar->finish(); - $this->line(' '); - $this->info('Finished!'); - } -} diff --git a/bootstrap/app.php b/bootstrap/app.php index b9dd57a76..9d4f06c0b 100644 --- a/bootstrap/app.php +++ b/bootstrap/app.php @@ -143,7 +143,8 @@ return Application::configure(basePath: dirname(__DIR__)) $schedule->command('passport:purge')->everyFourHours(20)->onOneServer(); if ((bool) config_cache('pixelfed.cloud_storage') && (bool) config_cache('media.delete_local_after_cloud')) { - $schedule->command('media:s3gc')->hourlyAt(15); + // Upload any local stragglers to cloud and GC verified local copies. + $schedule->command('admin:MediaMoveStorageLocalToCloud --force --limit=500')->hourlyAt(15); } if (config('import.instagram.enabled')) { diff --git a/tests/Feature/Account/MediaMoveStorageTest.php b/tests/Feature/Account/MediaMoveStorageTest.php new file mode 100644 index 000000000..5017292ec --- /dev/null +++ b/tests/Feature/Account/MediaMoveStorageTest.php @@ -0,0 +1,150 @@ + 'https://cdn.test']); + + // Use a throwaway env file so the command's env edits don't touch the + // real one. App::useEnvironmentPath expects a directory; the app resolves + // the environment-specific filename (e.g. .env.testing) itself. + $this->originalEnvPath = app()->environmentPath(); + $dir = sys_get_temp_dir().'/pf-env-test-'.uniqid(); + mkdir($dir); + app()->useEnvironmentPath($dir); + file_put_contents(app()->environmentFilePath(), "APP_KEY=base64:test\nPF_ENABLE_CLOUD=false\n"); +}); + +afterEach(function () { + // Restore the real environment path so we don't leak into other test files. + if (isset($this->originalEnvPath)) { + app()->useEnvironmentPath($this->originalEnvPath); + } +}); + +function makeCloudMedia(): Media +{ + $user = User::factory()->create(); + $user->refresh(); + $pid = $user->profile->id; + $status = Status::factory()->create(['profile_id' => $pid, 'type' => 'photo']); + + $path = 'public/m/_v2/'.$pid.'/aa/bb/file.jpg'; + $thumb = 'public/m/_v2/'.$pid.'/aa/bb/file_thumb.jpeg'; + + // Put the files on the cloud disk only. + Storage::disk('s3')->put($path, 'PRIMARY-BYTES-1234567890'); + Storage::disk('s3')->put($thumb, 'THUMB-BYTES'); + + return Media::create([ + 'status_id' => $status->id, + 'profile_id' => $pid, + 'user_id' => $user->id, + 'media_path' => $path, + 'thumbnail_path' => $thumb, + 'cdn_url' => Storage::disk('s3')->url($path), + 'thumbnail_url' => Storage::disk('s3')->url($thumb), + 'optimized_url' => Storage::disk('s3')->url($path), + 'mime' => 'image/jpeg', + 'size' => strlen('PRIMARY-BYTES-1234567890'), + 'remote_media' => false, + 'version' => 4, + 'replicated_at' => now(), + 'order' => 0, + ]); +} + +describe('admin:MediaMoveStorageCloudToLocal', function () { + it('downloads cloud media to local, clears cloud urls and deletes the cloud copy', function () { + $media = makeCloudMedia(); + + $this->artisan('admin:MediaMoveStorageCloudToLocal', ['--force' => true]) + ->assertExitCode(0); + + // File is now on local disk. + expect(Storage::disk('local')->exists($media->media_path))->toBeTrue(); + // Cloud copy removed (GC), thumbnail too. + expect(Storage::disk('s3')->exists($media->media_path))->toBeFalse(); + + $media->refresh(); + expect($media->cdn_url)->toBeNull(); + expect($media->optimized_url)->toBeNull(); + expect($media->thumbnail_url)->toBeNull(); + expect($media->replicated_at)->toBeNull(); + expect((string) $media->version)->toBe('3'); + }); + + it('keeps the cloud copy with --keep-cloud', function () { + $media = makeCloudMedia(); + + $this->artisan('admin:MediaMoveStorageCloudToLocal', ['--force' => true, '--keep-cloud' => true]) + ->assertExitCode(0); + + expect(Storage::disk('local')->exists($media->media_path))->toBeTrue(); + expect(Storage::disk('s3')->exists($media->media_path))->toBeTrue(); + }); + + it('does not modify anything in dry-run', function () { + $media = makeCloudMedia(); + + $this->artisan('admin:MediaMoveStorageCloudToLocal', ['--force' => true, '--dry-run' => true]) + ->assertExitCode(0); + + expect(Storage::disk('local')->exists($media->media_path))->toBeFalse(); + expect($media->fresh()->cdn_url)->not->toBeNull(); + }); + + it('sets PF_ENABLE_CLOUD=false in .env and runtime when cloud is enabled', function () { + // Start with cloud enabled. + file_put_contents(app()->environmentFilePath(), "APP_KEY=base64:test\nPF_ENABLE_CLOUD=true\n"); + Config::set('pixelfed.cloud_storage', true); + makeCloudMedia(); + + $this->artisan('admin:MediaMoveStorageCloudToLocal', ['--force' => true]) + ->assertExitCode(0); + + expect(file_get_contents(app()->environmentFilePath()))->toContain('PF_ENABLE_CLOUD="false"'); + expect(config('pixelfed.cloud_storage'))->toBeFalse(); + }); +}); + +describe('admin:MediaMoveStorageLocalToCloud', function () { + it('requires a configured cloud disk', function () { + // Fake s3 disk has no url() host resolvable? Storage::fake provides a + // url, so instead point cloud at a disk that throws. + Config::set('filesystems.cloud', 'does-not-exist'); + + $this->artisan('admin:MediaMoveStorageLocalToCloud', ['--force' => true]) + ->assertExitCode(1); + }); + + it('flips PF_ENABLE_CLOUD to true before migrating (dry-run reports it)', function () { + // cloud currently false in the temp .env from beforeEach. + $this->artisan('admin:MediaMoveStorageLocalToCloud', ['--dry-run' => true]) + ->expectsOutputToContain('PF_ENABLE_CLOUD') + ->assertExitCode(0); + + // dry-run must not write the .env. + expect(file_get_contents(app()->environmentFilePath()))->toContain('PF_ENABLE_CLOUD=false'); + }); +}); From 70b4a05b5c56525d88e894db63cef7480c72db2f Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 29 Aug 2026 18:22:36 +0930 Subject: [PATCH 12/19] Add admin:MediaMoveStorageCloudToCloud for cold S3->S3 migration Cold-migrate existing media from an old S3 bucket to the current cloud bucket, one media row at a time (like MigrateLocalS3MediaURL): - Source = --sourceDisk (default s3-old, reads AWS_OLD_*); destination = the current cloud disk (config filesystems.cloud). No .env editing: operators point AWS_* at the new bucket first (restarting workers as usual) so new uploads/downloads land on the new bucket, then run this to backfill old data. - Copies media (+thumbnail) source->destination, verifies by size and by sha256 of the freshly-written destination object (against original_sha256), rewrites cdn_url/optimized_url/thumbnail_url to the destination host, and GCs the source objects (unless --keep-source). Busts caches. - Only touches rows whose cdn_url still points at the source host; idempotent. - --sourceDisk / --limit / --dry-run / --force. - Adds the s3-old disk (AWS_OLD_*) to config/filesystems.php and feature tests. --- .../Commands/MediaMoveStorageCloudToCloud.php | 267 ++++++++++++++++++ config/filesystems.php | 21 ++ .../MediaMoveStorageCloudToCloudTest.php | 115 ++++++++ 3 files changed, 403 insertions(+) create mode 100644 app/Console/Commands/MediaMoveStorageCloudToCloud.php create mode 100644 tests/Feature/Account/MediaMoveStorageCloudToCloudTest.php diff --git a/app/Console/Commands/MediaMoveStorageCloudToCloud.php b/app/Console/Commands/MediaMoveStorageCloudToCloud.php new file mode 100644 index 000000000..90311d10d --- /dev/null +++ b/app/Console/Commands/MediaMoveStorageCloudToCloud.php @@ -0,0 +1,267 @@ +option('sourceDisk'); + $destName = config('filesystems.cloud'); + + if ($sourceName === $destName) { + $this->error('Source disk and destination (cloud) disk are the same ('.$sourceName.').'); + $this->line('Point AWS_* at the NEW bucket and keep the OLD bucket creds in the source disk.'); + + return 1; + } + + try { + $sourceDisk = Storage::disk($sourceName); + } catch (\Throwable $e) { + $this->error('Source disk "'.$sourceName.'" could not be resolved: '.$e->getMessage()); + + return 1; + } + + try { + $destDisk = Storage::disk($destName); + } catch (\Throwable $e) { + $this->error('Destination cloud disk "'.$destName.'" could not be resolved: '.$e->getMessage()); + + return 1; + } + + $sourceHost = $this->diskHost($sourceDisk); + $destHost = $this->diskHost($destDisk); + + if (! $sourceHost) { + $this->error('Source disk "'.$sourceName.'" is not configured (no resolvable URL). Set AWS_OLD_* in your .env.'); + + return 1; + } + if (! $destHost) { + $this->error('Destination cloud disk "'.$destName.'" is not configured (no resolvable URL). Set AWS_* in your .env.'); + + return 1; + } + if (strcasecmp($sourceHost, $destHost) === 0) { + $this->error('Source and destination resolve to the same host ('.$sourceHost.'); nothing to migrate.'); + + return 1; + } + + $this->info('Source (old): '.$sourceName.' -> '.$sourceHost); + $this->info('Destination (new): '.$destName.' -> '.$destHost); + $this->newLine(); + + if (! $this->option('dry-run') && ! $this->option('force')) { + if (! $this->confirm('Copy existing media from "'.$sourceHost.'" to "'.$destHost.'" and rewrite URLs?', true)) { + $this->comment('Aborted.'); + + return 0; + } + } + + $limit = (int) $this->option('limit'); + $moved = 0; + $skipped = 0; + $failed = 0; + + // Candidates: non-remote media whose stored URL still points at the + // source host (i.e. not yet migrated to the destination). + $query = Media::whereRemoteMedia(false) + ->whereNotNull(['media_path', 'cdn_url']) + ->orderByDesc('id') + ->limit($limit); + + $bar = $this->output->createProgressBar($query->count()); + $bar->start(); + + foreach ($query->get() as $media) { + $result = $this->migrateOne($media, $sourceDisk, $destDisk, $sourceHost, $destHost); + match ($result) { + 'moved' => $moved++, + 'skipped' => $skipped++, + default => $failed++, + }; + $bar->advance(); + } + + $bar->finish(); + $this->newLine(2); + $this->info(($this->option('dry-run') ? '[dry-run] ' : '').'Done. moved='.$moved.' skipped='.$skipped.' failed='.$failed.'.'); + if ($this->movedBytes) { + $this->info('Transferred '.PrettyNumber::size($this->movedBytes).' to the new bucket.'); + } + if ($moved > 0 && ! $this->option('dry-run')) { + $this->comment('Tip: run `php artisan cache:clear` if any stale URLs remain cached elsewhere.'); + } + + return 0; + } + + /** + * @return string one of moved|skipped|failed + */ + protected function migrateOne(Media $media, $sourceDisk, $destDisk, string $sourceHost, string $destHost): string + { + if (Str::startsWith((string) $media->media_path, 'http')) { + return 'skipped'; + } + + // Only act on rows whose URL still references the source host. + $currentHost = parse_url((string) $media->cdn_url, PHP_URL_HOST); + if (! $currentHost || strcasecmp($currentHost, $sourceHost) !== 0) { + return 'skipped'; + } + + // If already present on the destination, just rewrite the URLs. + $onDest = $destDisk->exists($media->media_path); + $onSource = $sourceDisk->exists($media->media_path); + + if (! $onDest && ! $onSource) { + // File missing from both buckets; leave URLs untouched. + return 'skipped'; + } + + if ($this->option('dry-run')) { + return 'moved'; + } + + try { + if (! $onDest) { + // Copy primary + thumbnail source -> destination. + $this->copy($media->media_path, $sourceDisk, $destDisk); + if ($media->thumbnail_path && $sourceDisk->exists($media->thumbnail_path)) { + $this->copy($media->thumbnail_path, $sourceDisk, $destDisk); + } + + if (! $this->verify($media->media_path, $sourceDisk, $destDisk, $media->original_sha256)) { + $this->warn(PHP_EOL.'Verify failed for media '.$media->id.' ('.$media->media_path.'); left source intact, URLs unchanged.'); + + return 'failed'; + } + } + + // Rewrite URLs to the destination bucket. + $media->cdn_url = $destDisk->url($media->media_path); + $media->optimized_url = $media->cdn_url; + if ($media->thumbnail_path && $destDisk->exists($media->thumbnail_path)) { + $media->thumbnail_url = $destDisk->url($media->thumbnail_path); + } + $media->replicated_at = now(); + $media->save(); + + $this->movedBytes += (int) $media->size; + + // Integrated GC on the OLD bucket, unless kept. + if (! $this->option('keep-source') && $onSource) { + $sourceDisk->delete($media->media_path); + if ($media->thumbnail_path && $sourceDisk->exists($media->thumbnail_path)) { + $sourceDisk->delete($media->thumbnail_path); + } + } + + if ($media->status_id) { + MediaService::del($media->status_id); + StatusService::del($media->status_id, false); + } + + return 'moved'; + } catch (\Throwable $e) { + $this->warn(PHP_EOL.'Error migrating media '.$media->id.': '.$e->getMessage()); + + return 'failed'; + } + } + + protected function copy(string $path, $sourceDisk, $destDisk): void + { + $stream = $sourceDisk->readStream($path); + if ($stream === false || $stream === null) { + throw new \RuntimeException('Could not open source stream for '.$path); + } + $destDisk->writeStream($path, $stream); + if (is_resource($stream)) { + fclose($stream); + } + } + + /** + * Verify the destination copy matches the source by size, and by sha256 + * against the stored original checksum when available. Fails closed. + */ + protected function verify(string $path, $sourceDisk, $destDisk, ?string $expectedSha = null): bool + { + if (! $destDisk->exists($path)) { + return false; + } + + $sourceSize = $sourceDisk->size($path); + $destSize = $destDisk->size($path); + if ($sourceSize === false || $destSize === false || $sourceSize !== $destSize) { + return false; + } + + if ($expectedSha) { + // Hash the freshly written destination object to confirm integrity. + $stream = $destDisk->readStream($path); + if ($stream === false || $stream === null) { + return false; + } + $ctx = hash_init('sha256'); + hash_update_stream($ctx, $stream); + if (is_resource($stream)) { + fclose($stream); + } + $destSha = hash_final($ctx); + if (! hash_equals($expectedSha, $destSha)) { + return false; + } + } + + return true; + } + + protected function diskHost($disk): ?string + { + try { + $url = $disk->url('probe'); + $host = parse_url($url, PHP_URL_HOST); + + return $host ?: null; + } catch (\Throwable $e) { + return null; + } + } +} diff --git a/config/filesystems.php b/config/filesystems.php index 81ee0547b..7fbb0be58 100644 --- a/config/filesystems.php +++ b/config/filesystems.php @@ -83,6 +83,27 @@ return [ ], ], + // Source disk for admin:MediaMoveStorageCloudToCloud (cold migration). + // After pointing AWS_* at the NEW bucket, keep the OLD bucket's + // credentials here as AWS_OLD_* so existing data can be copied across + // to the new bucket and media URLs rewritten. + 's3-old' => [ + 'driver' => 's3', + 'key' => env('AWS_OLD_ACCESS_KEY_ID'), + 'secret' => env('AWS_OLD_SECRET_ACCESS_KEY'), + 'region' => env('AWS_OLD_DEFAULT_REGION'), + 'bucket' => env('AWS_OLD_BUCKET'), + 'visibility' => env('AWS_OLD_VISIBILITY', 'public'), + 'url' => env('AWS_OLD_URL'), + 'endpoint' => env('AWS_OLD_ENDPOINT'), + 'use_path_style_endpoint' => env('AWS_OLD_USE_PATH_STYLE_ENDPOINT', false), + 'throw' => true, + 'options' => [ + 'request_checksum_calculation' => env('AWS_REQUEST_CHECKSUM_CALCULATION', 'WHEN_SUPPORTED'), + 'response_checksum_validation' => env('AWS_RESPONSE_CHECKSUM_VALIDATION', 'WHEN_SUPPORTED'), + ], + ], + 'alt-primary' => [ 'enabled' => env('ALT_PRI_ENABLED', false), 'driver' => 's3', diff --git a/tests/Feature/Account/MediaMoveStorageCloudToCloudTest.php b/tests/Feature/Account/MediaMoveStorageCloudToCloudTest.php new file mode 100644 index 000000000..be9c34599 --- /dev/null +++ b/tests/Feature/Account/MediaMoveStorageCloudToCloudTest.php @@ -0,0 +1,115 @@ +S3 migration: copy existing objects from the old bucket (s3-old) +| to the current cloud bucket (s3), verify, rewrite media URLs, GC the source. +| +*/ + +beforeEach(function () { + Config::set('filesystems.cloud', 's3'); + // Destination (new) bucket. + Storage::fake('s3', ['url' => 'https://cdneast.pixelfed.au']); + // Source (old) bucket. + Storage::fake('s3-old', ['url' => 'https://cdn.pixelfed.au']); +}); + +function makeOldBucketMedia(string $oldHost = 'https://cdn.pixelfed.au'): Media +{ + $user = User::factory()->create(); + $user->refresh(); + $pid = $user->profile->id; + $status = Status::factory()->create(['profile_id' => $pid, 'type' => 'photo']); + + $path = 'public/m/_v2/'.$pid.'/aa/bb/file.jpg'; + $thumb = 'public/m/_v2/'.$pid.'/aa/bb/file_thumb.jpeg'; + + // Files exist only on the OLD bucket. + Storage::disk('s3-old')->put($path, 'PRIMARY-BYTES-1234567890'); + Storage::disk('s3-old')->put($thumb, 'THUMB-BYTES'); + + return Media::create([ + 'status_id' => $status->id, + 'profile_id' => $pid, + 'user_id' => $user->id, + 'media_path' => $path, + 'thumbnail_path' => $thumb, + 'cdn_url' => $oldHost.'/'.$path, + 'thumbnail_url' => $oldHost.'/'.$thumb, + 'optimized_url' => $oldHost.'/'.$path, + 'mime' => 'image/jpeg', + 'size' => strlen('PRIMARY-BYTES-1234567890'), + 'remote_media' => false, + 'version' => 4, + 'replicated_at' => now(), + 'order' => 0, + ]); +} + +it('copies old-bucket media to the new bucket, rewrites urls and GCs the source', function () { + $media = makeOldBucketMedia(); + + $this->artisan('admin:MediaMoveStorageCloudToCloud', ['--force' => true]) + ->assertExitCode(0); + + // Copied to destination. + expect(Storage::disk('s3')->exists($media->media_path))->toBeTrue(); + // Removed from source (GC). + expect(Storage::disk('s3-old')->exists($media->media_path))->toBeFalse(); + + $media->refresh(); + expect(parse_url($media->cdn_url, PHP_URL_HOST))->toBe('cdneast.pixelfed.au'); + expect(parse_url($media->thumbnail_url, PHP_URL_HOST))->toBe('cdneast.pixelfed.au'); + expect(parse_url($media->optimized_url, PHP_URL_HOST))->toBe('cdneast.pixelfed.au'); +}); + +it('keeps the source objects with --keep-source', function () { + $media = makeOldBucketMedia(); + + $this->artisan('admin:MediaMoveStorageCloudToCloud', ['--force' => true, '--keep-source' => true]) + ->assertExitCode(0); + + expect(Storage::disk('s3')->exists($media->media_path))->toBeTrue(); + expect(Storage::disk('s3-old')->exists($media->media_path))->toBeTrue(); +}); + +it('makes no changes in dry-run', function () { + $media = makeOldBucketMedia(); + + $this->artisan('admin:MediaMoveStorageCloudToCloud', ['--force' => true, '--dry-run' => true]) + ->assertExitCode(0); + + expect(Storage::disk('s3')->exists($media->media_path))->toBeFalse(); + expect(parse_url($media->fresh()->cdn_url, PHP_URL_HOST))->toBe('cdn.pixelfed.au'); +}); + +it('skips media already pointing at the destination host', function () { + // cdn_url already on the destination host -> nothing to do. + $media = makeOldBucketMedia('https://cdneast.pixelfed.au'); + $before = $media->cdn_url; + + $this->artisan('admin:MediaMoveStorageCloudToCloud', ['--force' => true]) + ->assertExitCode(0); + + expect($media->fresh()->cdn_url)->toBe($before); + // Not copied (was skipped). + expect(Storage::disk('s3')->exists($media->media_path))->toBeFalse(); +}); + +it('errors when source and destination are the same disk', function () { + $this->artisan('admin:MediaMoveStorageCloudToCloud', ['--sourceDisk' => 's3', '--force' => true]) + ->assertExitCode(1); +}); From 68366c7dde2768709be812bf75f7deb9a86aa87f Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 29 Aug 2026 18:22:49 +0930 Subject: [PATCH 13/19] polish --- config/filesystems.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/filesystems.php b/config/filesystems.php index 7fbb0be58..b4cea06a8 100644 --- a/config/filesystems.php +++ b/config/filesystems.php @@ -99,8 +99,8 @@ return [ 'use_path_style_endpoint' => env('AWS_OLD_USE_PATH_STYLE_ENDPOINT', false), 'throw' => true, 'options' => [ - 'request_checksum_calculation' => env('AWS_REQUEST_CHECKSUM_CALCULATION', 'WHEN_SUPPORTED'), - 'response_checksum_validation' => env('AWS_RESPONSE_CHECKSUM_VALIDATION', 'WHEN_SUPPORTED'), + 'request_checksum_calculation' => env('AWS_OLD_REQUEST_CHECKSUM_CALCULATION', 'WHEN_SUPPORTED'), + 'response_checksum_validation' => env('AWS_OLD_RESPONSE_CHECKSUM_VALIDATION', 'WHEN_SUPPORTED'), ], ], From 0d01d5a96338b72c5265e80672dc579a4553545b Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 29 Aug 2026 18:27:38 +0930 Subject: [PATCH 14/19] Fix duplicate-key violation when importing remote media attachments Helpers::importNoteAttachment unconditionally inserted a new Media row per attachment, so re-importing a remote status (an Announce racing another inbox job, a re-fetch, or a duplicate url within one activity) hit the media_status_id_media_path_unique constraint and crashed the queue job with a 1062 UniqueConstraintViolationException, dropping the boost/import. Make createMediaAttachment idempotent on (status_id, media_path): skip when a row already exists, and catch the unique-constraint violation as a lost-race no-op, returning null so the caller skips re-dispatching storage. Adds regression tests (re-import no-op, distinct urls still stored, concurrent-insert returns null). --- app/Util/ActivityPub/Helpers.php | 30 ++++++- .../DuplicateMediaAttachmentTest.php | 90 +++++++++++++++++++ 2 files changed, 116 insertions(+), 4 deletions(-) create mode 100644 tests/Feature/Federation/DuplicateMediaAttachmentTest.php diff --git a/app/Util/ActivityPub/Helpers.php b/app/Util/ActivityPub/Helpers.php index 7d9926744..c10452c17 100644 --- a/app/Util/ActivityPub/Helpers.php +++ b/app/Util/ActivityPub/Helpers.php @@ -25,6 +25,7 @@ use App\Services\SanitizeService; use App\Services\UserFilterService; use App\Util\Media\License; use Carbon\Carbon; +use Illuminate\Database\UniqueConstraintViolationException; use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Validator; use Illuminate\Validation\Rule; @@ -1141,7 +1142,9 @@ class Helpers } $mediaModel = self::createMediaAttachment($media, $status, $key); - self::handleMediaStorage($mediaModel); + if ($mediaModel) { + self::handleMediaStorage($mediaModel); + } } $status->viewType(); @@ -1170,16 +1173,35 @@ class Helpers } /** - * Create media attachment record + * Create media attachment record. + * + * Idempotent on the (status_id, media_path) unique key: if a row already + * exists (e.g. a re-fetch, an Announce racing another inbox job, or a + * duplicate url within one activity's attachments) the existing row is + * returned instead of triggering a duplicate-key violation. + * + * @return Media|null the newly created model, or null when the attachment + * already existed (so the caller can skip re-storage) */ - public static function createMediaAttachment(array $media, Status $status, int $key): Media + public static function createMediaAttachment(array $media, Status $status, int $key): ?Media { + // Fast path: already imported for this status. + if (Media::whereStatusId($status->id)->whereMediaPath($media['url'])->exists()) { + return null; + } + $mediaModel = new Media; self::setBasicMediaAttributes($mediaModel, $media, $status, $key); self::setOptionalMediaAttributes($mediaModel, $media); - $mediaModel->save(); + try { + $mediaModel->save(); + } catch (UniqueConstraintViolationException $e) { + // Lost a race with a concurrent inbox job that inserted the same + // (status_id, media_path). Treat as already-imported. + return null; + } return $mediaModel; } diff --git a/tests/Feature/Federation/DuplicateMediaAttachmentTest.php b/tests/Feature/Federation/DuplicateMediaAttachmentTest.php new file mode 100644 index 000000000..a4d76f872 --- /dev/null +++ b/tests/Feature/Federation/DuplicateMediaAttachmentTest.php @@ -0,0 +1,90 @@ + 'Document', + 'mediaType' => 'image/jpeg', + 'url' => $url, + 'name' => 'alt text', + 'blurhash' => 'UREVf}R:E2WB~qNKWBs.XURkxZofD+n~oJR-', + 'width' => 768, + 'height' => 1024, + ]; +} + +it('does not create a duplicate media row for the same status and url', function () { + $user = User::factory()->create(); + $user->refresh(); + $status = Status::factory()->create(['profile_id' => $user->profile->id, 'type' => 'photo']); + + $url = 'https://files.mastodon.social/media_attachments/files/117/original/e9ab6f0314043b12.jpeg'; + $payload = attachmentPayload($url); + + $first = Helpers::createMediaAttachment($payload, $status, 0); + // Second call (simulating re-import / race) must not throw and must be a no-op. + $second = Helpers::createMediaAttachment($payload, $status, 0); + + expect($first)->not->toBeNull(); + expect($second)->toBeNull(); + expect(Media::whereStatusId($status->id)->whereMediaPath($url)->count())->toBe(1); +}); + +it('creates distinct rows for different urls on the same status', function () { + $user = User::factory()->create(); + $user->refresh(); + $status = Status::factory()->create(['profile_id' => $user->profile->id, 'type' => 'photo:album']); + + $a = Helpers::createMediaAttachment(attachmentPayload('https://files.mastodon.social/a.jpeg'), $status, 0); + $b = Helpers::createMediaAttachment(attachmentPayload('https://files.mastodon.social/b.jpeg'), $status, 1); + + expect($a)->not->toBeNull(); + expect($b)->not->toBeNull(); + expect(Media::whereStatusId($status->id)->count())->toBe(2); +}); + +it('returns null when the row was inserted concurrently after the existence check', function () { + $user = User::factory()->create(); + $user->refresh(); + $status = Status::factory()->create(['profile_id' => $user->profile->id, 'type' => 'photo']); + + $url = 'https://files.mastodon.social/race.jpeg'; + + // Pre-insert the row to simulate the concurrent winner. + Media::create([ + 'remote_media' => true, + 'status_id' => $status->id, + 'profile_id' => $status->profile_id, + 'media_path' => $url, + 'remote_url' => $url, + 'mime' => 'image/jpeg', + 'version' => 3, + 'order' => 1, + ]); + + // Should detect the existing row and return null without throwing. + $result = Helpers::createMediaAttachment(attachmentPayload($url), $status, 0); + + expect($result)->toBeNull(); + expect(Media::whereStatusId($status->id)->whereMediaPath($url)->count())->toBe(1); +}); From 16c7c5d2e36752425418ac698402f4a51a01e44c Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 29 Aug 2026 18:52:42 +0930 Subject: [PATCH 15/19] refactor: rename status debug commands to status: prefix Rename user:status, profile:status, and post:status console commands to status:user, status:profile, and status:post. Rename the command files and classes to match (StatusUser, StatusProfile, StatusPost) and update the cross-reference tip in StatusProfile. --- app/Console/Commands/{PostStatus.php => StatusPost.php} | 4 ++-- .../Commands/{ProfileStatus.php => StatusProfile.php} | 6 +++--- app/Console/Commands/{UserStatus.php => StatusUser.php} | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) rename app/Console/Commands/{PostStatus.php => StatusPost.php} (99%) rename app/Console/Commands/{ProfileStatus.php => StatusProfile.php} (98%) rename app/Console/Commands/{UserStatus.php => StatusUser.php} (99%) diff --git a/app/Console/Commands/PostStatus.php b/app/Console/Commands/StatusPost.php similarity index 99% rename from app/Console/Commands/PostStatus.php rename to app/Console/Commands/StatusPost.php index ddfce8164..f6cde4848 100644 --- a/app/Console/Commands/PostStatus.php +++ b/app/Console/Commands/StatusPost.php @@ -10,14 +10,14 @@ use Illuminate\Console\Command; use Illuminate\Support\Facades\Storage; use Illuminate\Support\Str; -class PostStatus extends Command +class StatusPost extends Command { /** * The name and signature of the console command. * * @var string */ - protected $signature = 'post:status {id : Status id, or a post URL like https://host/p/username/ID}'; + protected $signature = 'status:post {id : Status id, or a post URL like https://host/p/username/ID}'; /** * The console command description. diff --git a/app/Console/Commands/ProfileStatus.php b/app/Console/Commands/StatusProfile.php similarity index 98% rename from app/Console/Commands/ProfileStatus.php rename to app/Console/Commands/StatusProfile.php index 9322678c3..fd224583e 100644 --- a/app/Console/Commands/ProfileStatus.php +++ b/app/Console/Commands/StatusProfile.php @@ -8,14 +8,14 @@ use App\Models\User; use Illuminate\Console\Command; use Illuminate\Support\Str; -class ProfileStatus extends Command +class StatusProfile extends Command { /** * The name and signature of the console command. * * @var string */ - protected $signature = 'profile:status {id : Profile id, username, user@domain, @user@domain, webfinger, or remote_url}'; + protected $signature = 'status:profile {id : Profile id, username, user@domain, @user@domain, webfinger, or remote_url}'; /** * The console command description. @@ -212,7 +212,7 @@ class ProfileStatus extends Command ['last_active_at', $this->format($user->last_active_at)], ]; $this->table(['User Field', 'Value'], $rows); - $this->comment('Tip: run `user:status '.$user->username.'` for full auth diagnostics.'); + $this->comment('Tip: run `status:user '.$user->username.'` for full auth diagnostics.'); } protected function dumpInstance(Profile $profile): void diff --git a/app/Console/Commands/UserStatus.php b/app/Console/Commands/StatusUser.php similarity index 99% rename from app/Console/Commands/UserStatus.php rename to app/Console/Commands/StatusUser.php index 96702463a..9615fb32c 100644 --- a/app/Console/Commands/UserStatus.php +++ b/app/Console/Commands/StatusUser.php @@ -9,14 +9,14 @@ use Illuminate\Console\Command; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Schema; -class UserStatus extends Command +class StatusUser extends Command { /** * The name and signature of the console command. * * @var string */ - protected $signature = 'user:status {id : Username or numeric user id} + protected $signature = 'status:user {id : Username or numeric user id} {--logs=10 : Number of recent account log entries to show}'; /** From 1eae4bbd4304d3a59f1bfeadaff4daa2b8754e77 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 29 Aug 2026 19:02:50 +0930 Subject: [PATCH 16/19] refactor: organize Artisan commands into subfolders Group console commands into Admin, Dev, FixBugs, Install, Internal, and User subfolders (matching the earlier reorganization), and add a new Status subfolder for the status:user, status:profile, and status:post debug commands. Namespaces updated to match; command signatures and the total command count are unchanged. --- app/Console/Commands/{ => Admin}/AdminInviteCommand.php | 2 +- app/Console/Commands/{ => Admin}/BackupToCloud.php | 2 +- app/Console/Commands/{ => Admin}/BannedEmailCheck.php | 2 +- app/Console/Commands/{ => Admin}/CaptchaToggleCommand.php | 2 +- app/Console/Commands/{ => Admin}/CuratedOnboardingCommand.php | 2 +- app/Console/Commands/{ => Admin}/DeleteRemoteProfile.php | 2 +- app/Console/Commands/{ => Admin}/ImportCities.php | 2 +- app/Console/Commands/{ => Admin}/ImportEmojis.php | 2 +- app/Console/Commands/{ => Admin}/InstanceManager.php | 2 +- .../Commands/{ => Admin}/MediaMoveStorageCloudToCloud.php | 2 +- .../Commands/{ => Admin}/MediaMoveStorageCloudToLocal.php | 2 +- .../Commands/{ => Admin}/MediaMoveStorageLocalToCloud.php | 2 +- app/Console/Commands/{ => Admin}/MigrateLocalS3MediaURL.php | 2 +- app/Console/Commands/{ => Admin}/RegenerateThumbnails.php | 2 +- app/Console/Commands/{ => Admin}/SendUpdateActor.php | 2 +- app/Console/Commands/{ => Admin}/VideoThumbnail.php | 2 +- app/Console/Commands/{ => Dev}/ExportLanguages.php | 2 +- app/Console/Commands/{ => Dev}/Localization.php | 2 +- app/Console/Commands/{ => Dev}/SeedDevUsers.php | 2 +- app/Console/Commands/{ => Dev}/SeedFollows.php | 2 +- app/Console/Commands/{ => FixBugs}/AvatarDefaultMigration.php | 2 +- app/Console/Commands/{ => FixBugs}/AvatarStorage.php | 2 +- app/Console/Commands/{ => FixBugs}/AvatarStorageDeepClean.php | 2 +- app/Console/Commands/{ => FixBugs}/CatchUnoptimizedMedia.php | 2 +- .../Commands/{ => FixBugs}/FetchMissingMediaMimeType.php | 2 +- app/Console/Commands/{ => FixBugs}/FixDuplicateProfiles.php | 2 +- app/Console/Commands/{ => FixBugs}/FixHashtags.php | 2 +- app/Console/Commands/{ => FixBugs}/FixLikes.php | 2 +- app/Console/Commands/{ => FixBugs}/FixMediaDriver.php | 2 +- app/Console/Commands/{ => FixBugs}/FixMissingUserProfile.php | 2 +- app/Console/Commands/{ => FixBugs}/FixProfileCounts.php | 2 +- app/Console/Commands/{ => FixBugs}/FixUsernames.php | 2 +- app/Console/Commands/{ => FixBugs}/HashtagRelatedGenerate.php | 2 +- app/Console/Commands/{ => FixBugs}/MediaFix.php | 2 +- app/Console/Commands/{ => FixBugs}/StatusDedupe.php | 2 +- app/Console/Commands/{ => Install}/GenerateInstanceActor.php | 2 +- app/Console/Commands/{ => Install}/Installer.php | 2 +- app/Console/Commands/{ => Install}/UpdateCommand.php | 2 +- .../Commands/{ => Internal}/AccountPostCountStatUpdate.php | 2 +- .../Commands/{ => Internal}/CleanupExpiredAppRegistrations.php | 2 +- .../Commands/{ => Internal}/DatabaseSessionGarbageCollector.php | 2 +- app/Console/Commands/{ => Internal}/FailedJobGC.php | 2 +- .../Commands/{ => Internal}/HashtagCachedCountUpdate.php | 2 +- .../Commands/{ => Internal}/ImportRemoveDeletedAccounts.php | 2 +- .../Commands/{ => Internal}/ImportUploadCleanStorage.php | 2 +- .../Commands/{ => Internal}/ImportUploadGarbageCollection.php | 2 +- .../Commands/{ => Internal}/ImportUploadMediaToCloudStorage.php | 2 +- .../Commands/{ => Internal}/InstanceUpdateTotalLocalPosts.php | 2 +- app/Console/Commands/{ => Internal}/MediaGarbageCollector.php | 2 +- app/Console/Commands/{ => Internal}/NotificationEpochUpdate.php | 2 +- app/Console/Commands/{ => Internal}/PasswordResetGC.php | 2 +- app/Console/Commands/{ => Internal}/PushGatewayRefresh.php | 2 +- app/Console/Commands/{ => Internal}/SoftwareUpdateRefresh.php | 2 +- app/Console/Commands/{ => Internal}/StoryGC.php | 2 +- app/Console/Commands/{ => Internal}/TransformImports.php | 2 +- app/Console/Commands/{ => Internal}/WeeklyInstanceScan.php | 2 +- app/Console/Commands/{ => Status}/StatusPost.php | 2 +- app/Console/Commands/{ => Status}/StatusProfile.php | 2 +- app/Console/Commands/{ => Status}/StatusUser.php | 2 +- app/Console/Commands/{ => User}/AddUserDomainBlock.php | 2 +- app/Console/Commands/{ => User}/DeleteUserDomainBlock.php | 2 +- app/Console/Commands/{ => User}/ReclaimUsername.php | 2 +- app/Console/Commands/{ => User}/UserAccountDelete.php | 2 +- app/Console/Commands/{ => User}/UserAdmin.php | 2 +- app/Console/Commands/{ => User}/UserAvatarDelete.php | 2 +- app/Console/Commands/{ => User}/UserCheckPassword.php | 2 +- app/Console/Commands/{ => User}/UserCreate.php | 2 +- app/Console/Commands/{ => User}/UserDelete.php | 2 +- app/Console/Commands/{ => User}/UserRegistrationMagicLink.php | 2 +- app/Console/Commands/{ => User}/UserSetPassword.php | 2 +- app/Console/Commands/{ => User}/UserShow.php | 2 +- app/Console/Commands/{ => User}/UserSuspend.php | 2 +- app/Console/Commands/{ => User}/UserTable.php | 2 +- app/Console/Commands/{ => User}/UserToggle2FA.php | 2 +- app/Console/Commands/{ => User}/UserUnsuspend.php | 2 +- app/Console/Commands/{ => User}/UserVerifyEmail.php | 2 +- 76 files changed, 76 insertions(+), 76 deletions(-) rename app/Console/Commands/{ => Admin}/AdminInviteCommand.php (99%) rename app/Console/Commands/{ => Admin}/BackupToCloud.php (98%) rename app/Console/Commands/{ => Admin}/BannedEmailCheck.php (96%) rename app/Console/Commands/{ => Admin}/CaptchaToggleCommand.php (96%) rename app/Console/Commands/{ => Admin}/CuratedOnboardingCommand.php (99%) rename app/Console/Commands/{ => Admin}/DeleteRemoteProfile.php (97%) rename app/Console/Commands/{ => Admin}/ImportCities.php (99%) rename app/Console/Commands/{ => Admin}/ImportEmojis.php (98%) rename app/Console/Commands/{ => Admin}/InstanceManager.php (99%) rename app/Console/Commands/{ => Admin}/MediaMoveStorageCloudToCloud.php (99%) rename app/Console/Commands/{ => Admin}/MediaMoveStorageCloudToLocal.php (99%) rename app/Console/Commands/{ => Admin}/MediaMoveStorageLocalToCloud.php (99%) rename app/Console/Commands/{ => Admin}/MigrateLocalS3MediaURL.php (99%) rename app/Console/Commands/{ => Admin}/RegenerateThumbnails.php (96%) rename app/Console/Commands/{ => Admin}/SendUpdateActor.php (99%) rename app/Console/Commands/{ => Admin}/VideoThumbnail.php (96%) rename app/Console/Commands/{ => Dev}/ExportLanguages.php (98%) rename app/Console/Commands/{ => Dev}/Localization.php (98%) rename app/Console/Commands/{ => Dev}/SeedDevUsers.php (98%) rename app/Console/Commands/{ => Dev}/SeedFollows.php (97%) rename app/Console/Commands/{ => FixBugs}/AvatarDefaultMigration.php (98%) rename app/Console/Commands/{ => FixBugs}/AvatarStorage.php (99%) rename app/Console/Commands/{ => FixBugs}/AvatarStorageDeepClean.php (98%) rename app/Console/Commands/{ => FixBugs}/CatchUnoptimizedMedia.php (97%) rename app/Console/Commands/{ => FixBugs}/FetchMissingMediaMimeType.php (97%) rename app/Console/Commands/{ => FixBugs}/FixDuplicateProfiles.php (99%) rename app/Console/Commands/{ => FixBugs}/FixHashtags.php (98%) rename app/Console/Commands/{ => FixBugs}/FixLikes.php (97%) rename app/Console/Commands/{ => FixBugs}/FixMediaDriver.php (99%) rename app/Console/Commands/{ => FixBugs}/FixMissingUserProfile.php (99%) rename app/Console/Commands/{ => FixBugs}/FixProfileCounts.php (99%) rename app/Console/Commands/{ => FixBugs}/FixUsernames.php (99%) rename app/Console/Commands/{ => FixBugs}/HashtagRelatedGenerate.php (98%) rename app/Console/Commands/{ => FixBugs}/MediaFix.php (97%) rename app/Console/Commands/{ => FixBugs}/StatusDedupe.php (97%) rename app/Console/Commands/{ => Install}/GenerateInstanceActor.php (98%) rename app/Console/Commands/{ => Install}/Installer.php (99%) rename app/Console/Commands/{ => Install}/UpdateCommand.php (96%) rename app/Console/Commands/{ => Internal}/AccountPostCountStatUpdate.php (97%) rename app/Console/Commands/{ => Internal}/CleanupExpiredAppRegistrations.php (93%) rename app/Console/Commands/{ => Internal}/DatabaseSessionGarbageCollector.php (96%) rename app/Console/Commands/{ => Internal}/FailedJobGC.php (95%) rename app/Console/Commands/{ => Internal}/HashtagCachedCountUpdate.php (96%) rename app/Console/Commands/{ => Internal}/ImportRemoveDeletedAccounts.php (97%) rename app/Console/Commands/{ => Internal}/ImportUploadCleanStorage.php (95%) rename app/Console/Commands/{ => Internal}/ImportUploadGarbageCollection.php (96%) rename app/Console/Commands/{ => Internal}/ImportUploadMediaToCloudStorage.php (97%) rename app/Console/Commands/{ => Internal}/InstanceUpdateTotalLocalPosts.php (98%) rename app/Console/Commands/{ => Internal}/MediaGarbageCollector.php (96%) rename app/Console/Commands/{ => Internal}/NotificationEpochUpdate.php (93%) rename app/Console/Commands/{ => Internal}/PasswordResetGC.php (95%) rename app/Console/Commands/{ => Internal}/PushGatewayRefresh.php (97%) rename app/Console/Commands/{ => Internal}/SoftwareUpdateRefresh.php (95%) rename app/Console/Commands/{ => Internal}/StoryGC.php (97%) rename app/Console/Commands/{ => Internal}/TransformImports.php (99%) rename app/Console/Commands/{ => Internal}/WeeklyInstanceScan.php (96%) rename app/Console/Commands/{ => Status}/StatusPost.php (99%) rename app/Console/Commands/{ => Status}/StatusProfile.php (99%) rename app/Console/Commands/{ => Status}/StatusUser.php (99%) rename app/Console/Commands/{ => User}/AddUserDomainBlock.php (98%) rename app/Console/Commands/{ => User}/DeleteUserDomainBlock.php (98%) rename app/Console/Commands/{ => User}/ReclaimUsername.php (98%) rename app/Console/Commands/{ => User}/UserAccountDelete.php (99%) rename app/Console/Commands/{ => User}/UserAdmin.php (97%) rename app/Console/Commands/{ => User}/UserAvatarDelete.php (99%) rename app/Console/Commands/{ => User}/UserCheckPassword.php (99%) rename app/Console/Commands/{ => User}/UserCreate.php (98%) rename app/Console/Commands/{ => User}/UserDelete.php (99%) rename app/Console/Commands/{ => User}/UserRegistrationMagicLink.php (98%) rename app/Console/Commands/{ => User}/UserSetPassword.php (98%) rename app/Console/Commands/{ => User}/UserShow.php (97%) rename app/Console/Commands/{ => User}/UserSuspend.php (97%) rename app/Console/Commands/{ => User}/UserTable.php (96%) rename app/Console/Commands/{ => User}/UserToggle2FA.php (97%) rename app/Console/Commands/{ => User}/UserUnsuspend.php (97%) rename app/Console/Commands/{ => User}/UserVerifyEmail.php (97%) diff --git a/app/Console/Commands/AdminInviteCommand.php b/app/Console/Commands/Admin/AdminInviteCommand.php similarity index 99% rename from app/Console/Commands/AdminInviteCommand.php rename to app/Console/Commands/Admin/AdminInviteCommand.php index e2a6a1d47..6f094032a 100644 --- a/app/Console/Commands/AdminInviteCommand.php +++ b/app/Console/Commands/Admin/AdminInviteCommand.php @@ -1,6 +1,6 @@ Date: Sat, 29 Aug 2026 19:07:50 +0930 Subject: [PATCH 17/19] docs: add README for Artisan commands with listing and audit --- app/Console/Commands/README.md | 187 +++++++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 app/Console/Commands/README.md diff --git a/app/Console/Commands/README.md b/app/Console/Commands/README.md new file mode 100644 index 000000000..939339875 --- /dev/null +++ b/app/Console/Commands/README.md @@ -0,0 +1,187 @@ +# Artisan Commands + +This directory contains Pixelfed's custom Artisan console commands, grouped into +subfolders by purpose. Laravel auto-discovers every command in this tree, so the +subfolder is purely organizational — the command name is defined by each class's +`$signature`. + +Run any command with `php artisan `, and append `--help` to see its +full argument and option list. + +## Folder layout + +| Folder | Namespace | Purpose | +| --- | --- | --- | +| `Admin/` | `App\Console\Commands\Admin` | Instance administration and operator tooling | +| `Dev/` | `App\Console\Commands\Dev` | Local development and localization build helpers | +| `FixBugs/` | `App\Console\Commands\FixBugs` | One-off repair and data-fix utilities | +| `Install/` | `App\Console\Commands\Install` | Installation and upgrade | +| `Internal/` | `App\Console\Commands\Internal` | Scheduled/background maintenance (mostly run by the scheduler) | +| `Status/` | `App\Console\Commands\Status` | Read-only debug/diagnostic inspectors | +| `User/` | `App\Console\Commands\User` | User account management | +| `Concerns/` | `App\Console\Commands\Concerns` | Shared traits used by commands (not commands themselves) | + +--- + +## Admin + +| Command | Description | +| --- | --- | +| `admin:invite` | Create an invite link. | +| `backup:cloud` | Send backups to cloud storage. | +| `email:bancheck` | Check user emails against banned domains. | +| `app:captcha-toggle-command` | Show captcha status and optionally disable it. | +| `app:curated-onboarding` | Manage curated onboarding applications. | +| `app:delete-remote-profile` | Delete a remote profile. | +| `import:cities` | Import the cities dataset into the database. | +| `import:emojis` | Import custom emojis from a `tar.gz` archive (supports `--prefix`/`--suffix`). | +| `app:instance-manager` | Manage federated instances. | +| `admin:MediaMoveStorageCloudToCloud` | Cold-migrate media from an old S3 bucket to the current cloud bucket, verifying each copy. | +| `admin:MediaMoveStorageCloudToLocal` | Migrate cloud media back to local storage (download, verify, rewrite URLs, optionally delete cloud copy). | +| `admin:MediaMoveStorageLocalToCloud` | Migrate local media to cloud storage (upload, verify, rewrite URLs, delete local copy). | +| `admin:MigrateLocalS3MediaURL` | Rewrite stale local media cloud URLs from storage paths to the configured S3 host. Replaces the old `media:cloud-url-rewrite`. | +| `regenerate:thumbnails` | Regenerate image thumbnails for all image media. | +| `ap:update-actors` | Send Update Actor activities to known remote servers (`--force`). | +| `video:thumbnail` | Generate missing video thumbnails. | + +## Dev + +| Command | Description | +| --- | --- | +| `i18n:export` | Build and export the JS localization files. | +| `localization:generate` | Generate JSON files for all available localizations. | +| `seed:devusers` | Seed dev users (admin + regular) with random passwords. | +| `seed:follows` | Seed follow relationships for testing. | + +## FixBugs + +| Command | Description | +| --- | --- | +| `fix:avatars` | Replace old SVG identicon avatars with the default PNG avatar. | +| `avatar:storage` | Manage avatar storage. | +| `avatar:storage-deep-clean` | Clean up orphaned avatar storage. | +| `media:optimize` | Find and optimize media that has not yet been optimized. | +| `app:fetch-missing-media-mime-type` | Backfill missing MIME types on remote media by issuing HEAD requests. | +| `fix:profile:duplicates` | Fix duplicate profiles. | +| `fix:hashtags` | Fix hashtag records. | +| `fix:likes` | Recalculate like counts. | +| `media:fix-nonlocal-driver` | Repair filesystem records when `FILESYSTEM_DRIVER` is not set to local. | +| `app:fix-missing-user-profile` | Interactively create a missing profile for an affected user. | +| `admin:fixProfileCounts` | Resync a profile's cached counts (followers, following, statuses) from source tables; supports bulk `--all`/`--active`. | +| `fix:usernames` | Fix invalid usernames. | +| `app:hashtag-related-generate` | Generate related-hashtag data for a given tag. | +| `media:fix` | Repair media filter data (legacy, requires v0.10.8+). | +| `status:dedup` | Remove duplicate statuses from before the unique-URI migration. | + +## Install + +| Command | Description | +| --- | --- | +| `instance:actor` | Generate the instance actor. | +| `install` | CLI installer (`--dangerously-overwrite-env`, `--domain`, `--name`). | +| `update` | Run Pixelfed schema updates between versions. | + +## Internal + +These are primarily invoked by the scheduler (see `bootstrap/app.php`) rather than run by hand. + +| Command | Description | +| --- | --- | +| `app:account-post-count-stat-update` | Update post counts from recent activity. | +| `app:cleanup-expired-app-registrations` | Delete app registrations older than 90 days. | +| `gc:sessions` | Garbage-collect database sessions. | +| `gc:failedjobs` | Delete failed jobs older than one month. | +| `app:hashtag-cached-count-update` | Update cached hashtag counters (`--limit`). | +| `app:import-remove-deleted-accounts` | Remove import data belonging to deleted accounts. | +| `app:import-upload-clean-storage` | Delete import storage directories for non-active users. | +| `app:import-upload-garbage-collection` | Garbage-collect skipped Instagram import posts. | +| `app:import-upload-media-to-cloud-storage` | Migrate imported Instagram media to S3 (`--limit`). | +| `app:instance-update-total-local-posts` | Update the total local post count. | +| `media:gc` | Delete media uploads not attached to any active status. | +| `app:notification-epoch-update` | Update the notification epoch. | +| `gc:passwordreset` | Delete password reset tokens older than 24 hours. | +| `app:push-gateway-refresh` | Refresh push-notification gateway support. | +| `app:software-update-refresh` | Refresh latest software version data. | +| `story:gc` | Clear expired stories. | +| `app:transform-imports` | Transform completed imports into statuses. | +| `app:weekly-instance-scan` | Scan instance nodeinfo weekly. | + +## Status (debug/diagnostics) + +Read-only inspectors for troubleshooting. They do not modify data. + +| Command | Description | +| --- | --- | +| `status:post` | Show detailed metadata for a post and its media, including stored vs expected media URLs. | +| `status:profile` | Show detailed metadata for a local or remote profile (federation-aware). | +| `status:user` | Show detailed diagnostics for a user account (login & password reset), with recent account logs (`--logs`). | + +## User + +| Command | Description | +| --- | --- | +| `app:add-user-domain-block` | Apply a domain block for all users. | +| `app:delete-user-domain-block` | Remove a domain block for all users. | +| `app:reclaim-username` | Force-delete a user and profile to reclaim a username. | +| `app:user-account-delete` | Federate an account deletion (`--concurrency`, `--chunk`, `--attempts`, `--target`, `--dry-run`). | +| `user:admin` | Grant or remove admin privileges for a user. | +| `user:avatar-delete` | Delete a user avatar and reset to default (`--force`). | +| `user:checkpassword` | Read-only: verify a candidate password against the stored hash and diagnose login rejection. | +| `user:create` | Create a new user. | +| `user:delete` | Delete an account (`--force`). | +| `user:app-magic-link` | Get the app magic link for in-app registrations missing a confirmation email. | +| `user:setpassword` | Set/reset a user password (prompts securely, bcrypt). | +| `user:show` | Show user info. | +| `user:suspend` | Suspend a local user. | +| `user:table` | Display the latest users. | +| `user:2fa` | Disable two-factor authentication for a username. | +| `user:unsuspend` | Unsuspend a local user. | +| `user:verifyemail` | Verify a user's email address. | + +--- + +## Audit notes + +No commands were found to be **broken**: every `handle()` body is functional, +all referenced classes/config resolve (`App\Util\Media\Filter`, the media +`filter_class` column, the `s3-old` disk, and `config('import.instagram')` all +exist), and there are no duplicate command names. + +The items below are cleanup opportunities, not failures. Nothing here is deleted +automatically — these are recommendations for a maintainer to confirm. + +### Likely obsolete (legacy one-off migrations) + +These were written to repair specific historical data states and are unlikely to +be needed on a current install. Candidates for removal after confirming they are +no longer required: + +- `FixBugs/MediaFix.php` (`media:fix`) — repairs image-filter data and refuses to + run below v0.10.8. Image filters are a deprecated feature. +- `FixBugs/StatusDedupe.php` (`status:dedup`) — dedupes statuses created *before* + the unique-URI migration; not compatible with Postgres. +- `FixBugs/AvatarDefaultMigration.php` (`fix:avatars`) — replaces old SVG + identicon avatars, a long-since-removed avatar style. + +### Missing descriptions (metadata only) + +These work but still carry the scaffold default `"Command description"`, so they +read poorly in `php artisan list`. Worth filling in: + +- `Admin/CaptchaToggleCommand.php` (`app:captcha-toggle-command`) +- `FixBugs/FetchMissingMediaMimeType.php` (`app:fetch-missing-media-mime-type`) +- `FixBugs/FixMissingUserProfile.php` (`app:fix-missing-user-profile`) +- `FixBugs/HashtagRelatedGenerate.php` (`app:hashtag-related-generate`) +- `Internal/CleanupExpiredAppRegistrations.php` (`app:cleanup-expired-app-registrations`) +- `Internal/ImportRemoveDeletedAccounts.php` (`app:import-remove-deleted-accounts`) +- `Internal/ImportUploadCleanStorage.php` (`app:import-upload-clean-storage`) +- `Internal/ImportUploadGarbageCollection.php` (`app:import-upload-garbage-collection`) + +### Naming inconsistencies + +- The three media-move commands use PascalCase signatures + (`admin:MediaMoveStorageCloudToCloud`, `...CloudToLocal`, `...LocalToCloud`), + which breaks the kebab-case convention used elsewhere. Consider + `admin:media-move-storage-*`. +- Many commands use the generic scaffold prefix `app:` rather than a meaningful + namespace (`user:`, `media:`, `gc:`, etc.). Not harmful, but inconsistent. From cafec250fe5911844ca44869b35537fc833618ca Mon Sep 17 00:00:00 2001 From: Shlee Date: Sat, 29 Aug 2026 19:12:24 +0930 Subject: [PATCH 18/19] Update README.md --- app/Console/Commands/README.md | 48 ---------------------------------- 1 file changed, 48 deletions(-) diff --git a/app/Console/Commands/README.md b/app/Console/Commands/README.md index 939339875..5c84d55e1 100644 --- a/app/Console/Commands/README.md +++ b/app/Console/Commands/README.md @@ -137,51 +137,3 @@ Read-only inspectors for troubleshooting. They do not modify data. | `user:2fa` | Disable two-factor authentication for a username. | | `user:unsuspend` | Unsuspend a local user. | | `user:verifyemail` | Verify a user's email address. | - ---- - -## Audit notes - -No commands were found to be **broken**: every `handle()` body is functional, -all referenced classes/config resolve (`App\Util\Media\Filter`, the media -`filter_class` column, the `s3-old` disk, and `config('import.instagram')` all -exist), and there are no duplicate command names. - -The items below are cleanup opportunities, not failures. Nothing here is deleted -automatically — these are recommendations for a maintainer to confirm. - -### Likely obsolete (legacy one-off migrations) - -These were written to repair specific historical data states and are unlikely to -be needed on a current install. Candidates for removal after confirming they are -no longer required: - -- `FixBugs/MediaFix.php` (`media:fix`) — repairs image-filter data and refuses to - run below v0.10.8. Image filters are a deprecated feature. -- `FixBugs/StatusDedupe.php` (`status:dedup`) — dedupes statuses created *before* - the unique-URI migration; not compatible with Postgres. -- `FixBugs/AvatarDefaultMigration.php` (`fix:avatars`) — replaces old SVG - identicon avatars, a long-since-removed avatar style. - -### Missing descriptions (metadata only) - -These work but still carry the scaffold default `"Command description"`, so they -read poorly in `php artisan list`. Worth filling in: - -- `Admin/CaptchaToggleCommand.php` (`app:captcha-toggle-command`) -- `FixBugs/FetchMissingMediaMimeType.php` (`app:fetch-missing-media-mime-type`) -- `FixBugs/FixMissingUserProfile.php` (`app:fix-missing-user-profile`) -- `FixBugs/HashtagRelatedGenerate.php` (`app:hashtag-related-generate`) -- `Internal/CleanupExpiredAppRegistrations.php` (`app:cleanup-expired-app-registrations`) -- `Internal/ImportRemoveDeletedAccounts.php` (`app:import-remove-deleted-accounts`) -- `Internal/ImportUploadCleanStorage.php` (`app:import-upload-clean-storage`) -- `Internal/ImportUploadGarbageCollection.php` (`app:import-upload-garbage-collection`) - -### Naming inconsistencies - -- The three media-move commands use PascalCase signatures - (`admin:MediaMoveStorageCloudToCloud`, `...CloudToLocal`, `...LocalToCloud`), - which breaks the kebab-case convention used elsewhere. Consider - `admin:media-move-storage-*`. -- Many commands use the generic scaffold prefix `app:` rather than a meaningful - namespace (`user:`, `media:`, `gc:`, etc.). Not harmful, but inconsistent. From cd353a8305514377d8e55b3b3f7a026e1dc1988e Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 29 Aug 2026 19:17:57 +0930 Subject: [PATCH 19/19] refactor: move resolved one-off migrations to Deprecated/ status:dedup and fix:avatars address historical data states that can no longer occur (unique statuses.uri index since 2019; SVG identicon avatars no longer generated). Move both to a Deprecated/ folder and update the README audit accordingly. media:fix stays in FixBugs/ since image filters are still an active feature. --- .../AvatarDefaultMigration.php | 2 +- .../{FixBugs => Deprecated}/StatusDedupe.php | 2 +- app/Console/Commands/README.md | 43 +++++++++++++------ 3 files changed, 33 insertions(+), 14 deletions(-) rename app/Console/Commands/{FixBugs => Deprecated}/AvatarDefaultMigration.php (98%) rename app/Console/Commands/{FixBugs => Deprecated}/StatusDedupe.php (97%) diff --git a/app/Console/Commands/FixBugs/AvatarDefaultMigration.php b/app/Console/Commands/Deprecated/AvatarDefaultMigration.php similarity index 98% rename from app/Console/Commands/FixBugs/AvatarDefaultMigration.php rename to app/Console/Commands/Deprecated/AvatarDefaultMigration.php index a46de5505..772555af2 100644 --- a/app/Console/Commands/FixBugs/AvatarDefaultMigration.php +++ b/app/Console/Commands/Deprecated/AvatarDefaultMigration.php @@ -1,6 +1,6 @@