Merge pull request #7134 from shleeable/fix/storyfetch-ssrf

Route StoryFetch outbound requests through SSRF-hardened fetch service
pull/7143/head
Shlee 2 weeks ago committed by GitHub
commit 0fc121b685
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -4,6 +4,7 @@ namespace App\Jobs\StoryPipeline;
use App\Models\Story;
use App\Services\MediaPathService;
use App\Services\SecureMediaFetchService;
use App\Services\StoryIndexService;
use App\Services\StoryService;
use App\Util\ActivityPub\Helpers;
@ -13,14 +14,11 @@ use Exception;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Http\Client\ConnectionException;
use Illuminate\Http\Client\RequestException;
use Illuminate\Http\File;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Http;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
use Illuminate\Support\Facades\Validator;
@ -34,10 +32,6 @@ class StoryFetch implements ShouldQueue
private const MAX_DURATION = 300;
private const REQUEST_TIMEOUT = 30;
private const MAX_REDIRECTS = 3;
// Rate limiting
public $tries = 3;
@ -281,28 +275,21 @@ class StoryFetch implements ShouldQueue
];
try {
$response = Http::withHeaders($headers)
->timeout(self::REQUEST_TIMEOUT)
->connectTimeout(10)
->retry(2, 1000)
->withOptions([
'verify' => true,
'max_redirects' => self::MAX_REDIRECTS,
])
->get($url);
if (! $response->successful()) {
// Fetch the bearcap story JSON through the SSRF-hardened path so the
// request cannot be redirected to an internal address. The bearer
// token is forwarded only to the original host and stripped on any
// cross-origin redirect hop (see SecureMediaFetchService::request).
$body = SecureMediaFetchService::get($url, null, null, $headers);
if ($body === false) {
if (config('app.dev_log')) {
Log::warning('Story fetch failed', [
'url' => $url,
'status' => $response->status(),
]);
Log::warning('Story fetch failed', ['url' => $url]);
}
return null;
}
$payload = $response->json();
$payload = json_decode($body, true);
if (! is_array($payload)) {
if (config('app.dev_log')) {
@ -314,15 +301,6 @@ class StoryFetch implements ShouldQueue
return $payload;
} catch (RequestException|ConnectionException $e) {
if (config('app.dev_log')) {
Log::warning('HTTP request failed', [
'url' => $url,
'error' => $e->getMessage(),
]);
}
return null;
} catch (Exception $e) {
if (config('app.dev_log')) {
Log::error('Unexpected error in story fetch', [
@ -464,23 +442,12 @@ class StoryFetch implements ShouldQueue
}
try {
$contextOptions = [
'ssl' => [
'verify_peer' => true,
'verify_peername' => true,
'allow_self_signed' => false,
'SNI_enabled' => true,
],
'http' => [
'timeout' => self::REQUEST_TIMEOUT,
'max_redirects' => self::MAX_REDIRECTS,
'user_agent' => 'Pixelfed/'.config('pixelfed.version'),
],
];
$ctx = stream_context_create($contextOptions);
$data = $this->downloadWithSizeLimit($mediaUrl, $ctx);
// Fetch through the SSRF-hardened path: https-only, resolves + pins
// to public IPs (CURLOPT_RESOLVE), disables auto-redirects and
// re-validates every hop against private/reserved ranges, and caps
// the body size. Replaces the bare fopen() stream that followed
// redirects to arbitrary internal addresses without re-validation.
$data = SecureMediaFetchService::get($mediaUrl, $this->getMaxFileSizeBytes());
if (! $data) {
return null;
}
@ -533,48 +500,6 @@ class StoryFetch implements ShouldQueue
}
}
/**
* Download with size limit enforcement
*/
private function downloadWithSizeLimit(string $url, $context): ?string
{
$maxFileSizeBytes = $this->getMaxFileSizeBytes();
$handle = fopen($url, 'r', false, $context);
if (! $handle) {
if (config('app.dev_log')) {
Log::warning('Failed to open URL stream', ['url' => $url]);
}
return null;
}
$data = '';
$size = 0;
while (! feof($handle) && $size < $maxFileSizeBytes) {
$chunk = fread($handle, 8192);
if ($chunk === false) {
break;
}
$data .= $chunk;
$size += strlen($chunk);
}
fclose($handle);
if ($size >= $maxFileSizeBytes) {
if (config('app.dev_log')) {
Log::warning('File too large', ['size' => $size, 'limit' => $maxFileSizeBytes]);
}
return null;
}
return $data;
}
/**
* Validate downloaded file
*/

@ -50,10 +50,10 @@ class SecureMediaFetchService
*
* @return string|false
*/
public static function get(string $url, ?int $maxBytes = null, ?int $expectedLength = null)
public static function get(string $url, ?int $maxBytes = null, ?int $expectedLength = null, array $headers = [])
{
$maxBytes = $maxBytes ?? self::defaultMaxBytes();
$result = (new self)->request($url, 'get', $maxBytes, $expectedLength);
$result = (new self)->request($url, 'get', $maxBytes, $expectedLength, $headers);
if (! is_array($result)) {
return false;
@ -69,10 +69,17 @@ class SecureMediaFetchService
* @return array|false For 'head': ['length'=>int,'mime'=>string].
* For 'get': ['body'=>string,'length'=>int,'mime'=>string].
*/
protected function request(string $url, string $method, int $maxBytes, ?int $expectedLength = null)
protected function request(string $url, string $method, int $maxBytes, ?int $expectedLength = null, array $extraHeaders = [])
{
$currentUrl = $url;
// Host of the original request. Caller-supplied headers (e.g. an
// Authorization bearer token) are only sent to this host and are
// stripped on any cross-origin redirect hop, mirroring Guzzle's
// RedirectMiddleware credential-stripping behaviour.
$originHost = parse_url($url, PHP_URL_HOST);
$originHost = is_string($originHost) ? strtolower($originHost) : null;
for ($redirects = 0; $redirects <= self::MAX_REDIRECTS; $redirects++) {
$currentUrl = Helpers::validateUrl($currentUrl);
@ -88,6 +95,13 @@ class SecureMediaFetchService
return false;
}
// Only forward caller headers when the current hop is the same host
// as the original request; drop them across origins.
$headers = ['User-Agent' => self::userAgent()];
if (! empty($extraHeaders) && strtolower((string) $host) === $originHost) {
$headers = array_merge($extraHeaders, $headers);
}
// Resolve the host and reject if ANY resolved address is
// non-global. Fail-closed: empty means unresolved or private.
$ips = Helpers::resolvePublicIps($host);
@ -116,7 +130,7 @@ class SecureMediaFetchService
}
},
])
->withHeaders(['User-Agent' => self::userAgent()])
->withHeaders($headers)
->timeout(self::TIMEOUT)
->connectTimeout(self::CONNECT_TIMEOUT)
->{$method}($currentUrl);

@ -0,0 +1,99 @@
<?php
use App\Services\SecureMediaFetchService;
use Illuminate\Foundation\Testing\LazilyRefreshDatabase;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Http;
uses(LazilyRefreshDatabase::class);
/*
|--------------------------------------------------------------------------
| SecureMediaFetchService header passthrough + redirect safety
|--------------------------------------------------------------------------
|
| get() accepts caller headers (e.g. an Authorization bearer for bearcap
| fetches) but must forward them only to the original host and strip them on
| cross-origin redirects. Redirects to private/reserved addresses must be
| refused (never connected to).
|
*/
function seedPublicIp(string $host): void
{
Cache::put('helpers:url:public-ips:'.hash('xxh128', $host), ['203.0.113.50'], 3600);
}
beforeEach(function () {
Cache::flush();
});
it('forwards caller headers to the origin host', function () {
seedPublicIp('origin.example');
Http::fake([
'https://origin.example/story' => Http::response('{"ok":true}', 200, [
'Content-Type' => 'application/json',
]),
]);
$body = SecureMediaFetchService::get('https://origin.example/story', null, null, [
'Authorization' => 'Bearer secret-token',
]);
expect($body)->toBe('{"ok":true}');
Http::assertSent(function ($request) {
return $request->url() === 'https://origin.example/story'
&& $request->hasHeader('Authorization', 'Bearer secret-token');
});
});
it('strips the Authorization header on a cross-origin redirect', function () {
seedPublicIp('origin.example');
seedPublicIp('other.example');
Http::fake([
'https://origin.example/story' => Http::response('', 302, [
'Location' => 'https://other.example/story',
]),
'https://other.example/story' => Http::response('{"ok":true}', 200, [
'Content-Type' => 'application/json',
]),
]);
$body = SecureMediaFetchService::get('https://origin.example/story', null, null, [
'Authorization' => 'Bearer secret-token',
]);
expect($body)->toBe('{"ok":true}');
// The cross-origin hop must NOT carry the bearer token.
Http::assertSent(function ($request) {
if ($request->url() !== 'https://other.example/story') {
return false;
}
return ! $request->hasHeader('Authorization');
});
});
it('refuses to follow a redirect to a private address', function () {
seedPublicIp('origin.example');
Http::fake([
'https://origin.example/story' => Http::response('', 302, [
'Location' => 'http://169.254.169.254/latest/meta-data/',
]),
// If the service (incorrectly) followed, this would answer; it must not.
'169.254.169.254/*' => Http::response('SECRET', 200),
]);
$body = SecureMediaFetchService::get('https://origin.example/story');
expect($body)->toBeFalse();
Http::assertNotSent(function ($request) {
return str_contains($request->url(), '169.254.169.254');
});
});

@ -0,0 +1,74 @@
<?php
use App\Jobs\StoryPipeline\StoryFetch;
use Illuminate\Foundation\Testing\LazilyRefreshDatabase;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Http;
uses(LazilyRefreshDatabase::class);
/*
|--------------------------------------------------------------------------
| StoryFetch SSRF hardening
|--------------------------------------------------------------------------
|
| Both outbound fetches (bearcap story JSON and media download) route through
| SecureMediaFetchService, which refuses to follow redirects to private /
| reserved addresses. These tests exercise the payload-fetch sink directly.
|
*/
function seedPublicIpForStory(string $host): void
{
Cache::put('helpers:url:public-ips:'.hash('xxh128', $host), ['203.0.113.60'], 3600);
}
/**
* Invoke the private fetchStoryPayload() via reflection.
*/
function callFetchStoryPayload(string $url, string $token)
{
$job = new StoryFetch([]);
$ref = new ReflectionMethod($job, 'fetchStoryPayload');
$ref->setAccessible(true);
return $ref->invoke($job, $url, $token);
}
beforeEach(function () {
Cache::flush();
});
it('fetches the story payload over the hardened path', function () {
seedPublicIpForStory('peer.example');
Http::fake([
'https://peer.example/story' => Http::response('{"id":"https://peer.example/s/1"}', 200, [
'Content-Type' => 'application/json',
]),
]);
$payload = callFetchStoryPayload('https://peer.example/story', 'bearcap-token-1234567890');
expect($payload)->toBeArray()
->and($payload['id'])->toBe('https://peer.example/s/1');
});
it('refuses a payload fetch that redirects to a private address', function () {
seedPublicIpForStory('peer.example');
Http::fake([
'https://peer.example/story' => Http::response('', 302, [
'Location' => 'http://169.254.169.254/latest/meta-data/',
]),
'169.254.169.254/*' => Http::response('SECRET', 200),
]);
$payload = callFetchStoryPayload('https://peer.example/story', 'bearcap-token-1234567890');
expect($payload)->toBeNull();
Http::assertNotSent(function ($request) {
return str_contains($request->url(), '169.254.169.254');
});
});
Loading…
Cancel
Save