From 15da72dd949e672d3be6544962f42a90b277f5b4 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 20 Sep 2026 07:21:47 -0600 Subject: [PATCH] Refactor blurhash --- app/Services/InstanceService.php | 34 +-- app/Util/Blurhash/Color.php | 7 +- app/Util/Media/Blurhash.php | 239 +++++++++++++++---- app/Util/Media/Image.php | 37 +-- app/Util/Media/ImageDriverManager.php | 11 +- tests/Feature/MediaPipeline/BlurhashTest.php | 187 +++++++++++++++ 6 files changed, 414 insertions(+), 101 deletions(-) create mode 100644 tests/Feature/MediaPipeline/BlurhashTest.php diff --git a/app/Services/InstanceService.php b/app/Services/InstanceService.php index 2cad911a7..1d62a77c5 100644 --- a/app/Services/InstanceService.php +++ b/app/Services/InstanceService.php @@ -3,7 +3,7 @@ namespace App\Services; use App\Models\Instance; -use App\Util\Blurhash\Blurhash; +use App\Util\Media\Blurhash; use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Storage; @@ -131,32 +131,12 @@ class InstanceService $file = config_cache('app.banner_image') ?? url(Storage::url('public/headers/default.jpg')); - $image = imagecreatefromstring(file_get_contents($file)); - if (! $image) { - return 'UzJR]l{wHZRjM}R%XRkCH?X9xaWEjZj]kAjt'; - } - $width = imagesx($image); - $height = imagesy($image); - - $pixels = []; - for ($y = 0; $y < $height; $y++) { - $row = []; - for ($x = 0; $x < $width; $x++) { - $index = imagecolorat($image, $x, $y); - $colors = imagecolorsforindex($image, $index); - - $row[] = [$colors['red'], $colors['green'], $colors['blue']]; - } - $pixels[] = $row; - } - - // Free the allocated GdImage object from memory: - imagedestroy($image); - - $components_x = 4; - $components_y = 4; - $blurhash = Blurhash::encode($pixels, $components_x, $components_y); - if (strlen($blurhash) > 191) { + // Goes through the configured image driver (and a downscaled sample) + // rather than raw GD calls at full resolution, so this no longer + // fatals on hosts that run vips without ext-gd. + $contents = @file_get_contents($file); + $blurhash = $contents ? Blurhash::fromBinary($contents) : null; + if (! $blurhash) { return 'UzJR]l{wHZRjM}R%XRkCH?X9xaWEjZj]kAjt'; } diff --git a/app/Util/Blurhash/Color.php b/app/Util/Blurhash/Color.php index df7473d0d..25f7a2f19 100644 --- a/app/Util/Blurhash/Color.php +++ b/app/Util/Blurhash/Color.php @@ -17,8 +17,13 @@ final class Color { $normalized = max(0, min(1, $value)); - return ($normalized <= 0.0031308) + $result = ($normalized <= 0.0031308) ? (int) round($normalized * 12.92 * 255 + 0.5) : (int) round((1.055 * pow($normalized, 1 / 2.4) - 0.055) * 255 + 0.5); + + // The + 0.5 rounds a fully saturated channel up to 256, which does not fit + // in the 8 bits DC::encode() packs it into and carries into the next channel + // (pure white came out as R=257 G=1 B=0). Clamp like upstream php-blurhash. + return max(0, min($result, 255)); } } diff --git a/app/Util/Media/Blurhash.php b/app/Util/Media/Blurhash.php index 169bb9625..5871beaed 100644 --- a/app/Util/Media/Blurhash.php +++ b/app/Util/Media/Blurhash.php @@ -4,14 +4,48 @@ namespace App\Util\Media; use App\Models\Media; use App\Util\Blurhash\Blurhash as BlurhashEngine; +use GdImage; +use Imagick; +use Jcupitt\Vips\BandFormat; +use Jcupitt\Vips\Image as VipsImage; +use Jcupitt\Vips\Interpretation; +use RuntimeException; +use Throwable; class Blurhash { const DEFAULT_HASH = 'U4Rfzst8?bt7ogayj[j[~pfQ9Goe%Mj[WBay'; + // Long edge, in pixels, of the sample the hash is computed from. + // + // The output is a 4x4-component DCT, so sampling the source at full resolution + // buys almost nothing, while the per-pixel PHP arrays the encoder needs cost a + // few hundred bytes each: a 720x1280 frame measured at 224 MB peak, which is + // what used to kill workers on full resolution video thumbnails (pixelfed#2652). + // + // 128px measured as the point of diminishing returns: against the full + // resolution hash, mean per-channel deviation of the decoded 24x24 preview is + // ~7.5/255 at a 32px sample, ~4.5/255 at 64px, ~2.5/255 at 128px, and no better + // at 256px. At 128px a 1920x1080 frame samples 9,216 pixels instead of 2,073,600. + const SAMPLE_MAX = 128; + + const COMPONENTS_X = 4; + + const COMPONENTS_Y = 4; + + // Mime types of the media a thumbnail is hashed for. The thumbnail itself keeps + // the source format (png, jpg, webp), with avif/heic sources landing as jpg. + const SUPPORTED_MIMES = [ + 'image/png', + 'image/jpeg', + 'image/jpg', + 'image/webp', + 'video/mp4', + ]; + public static function generate(Media $media, $path = false): string { - if (! in_array($media->mime, ['image/png', 'image/jpeg', 'image/jpg', 'video/mp4'])) { + if (! in_array($media->mime, self::SUPPORTED_MIMES)) { return self::DEFAULT_HASH; } @@ -19,79 +53,182 @@ class Blurhash return self::DEFAULT_HASH; } - if ($path) { - $file = $path; - } else { - $localFs = config('filesystems.default') === 'local'; - $file = storage_path('app/'.$media->thumbnail_path); - } + $file = $path ?: storage_path('app/'.$media->thumbnail_path); if (! is_file($file)) { return self::DEFAULT_HASH; } - $image = imagecreatefromstring(file_get_contents($file)); - if (! $image) { + $contents = file_get_contents($file); + if ($contents === false || $contents === '') { return self::DEFAULT_HASH; } - $width = imagesx($image); - $height = imagesy($image); - // The loop below allocates one PHP array per pixel, which costs a few hundred - // bytes each once the hashtable is counted: a 720x1280 frame measured at 224 MB - // peak, i.e. ~255 bytes per pixel, which puts 1920x1080 near half a gigabyte - // (#2652 reports over a gigabyte on an older PHP). Image thumbnails are capped - // at 640x640 and run under Image::__construct()'s ini_set('memory_limit', - // '1024M'), so they stay inside it. Video thumbnails come straight out of - // FFmpeg at source resolution with no - // such raise, and blow memory_limit outright — a PHP fatal, which is not an - // \Exception, so it kills the worker instead of being caught (pixelfed#2652). - // - // The output is a 4x4-component DCT, so sampling the source at full resolution - // buys almost nothing. Downscaling first removes the ceiling for every caller - // rather than moving it, which is all raising memory_limit would have done. - // - // 128px on the long edge measured as the point of diminishing returns: against - // the full-resolution hash, mean per-channel deviation of the decoded 24x24 - // preview is ~7.5/255 at a 32px sample, ~4.5/255 at 64px, ~2.5/255 at 128px, - // and no better at 256px. At 128px a 1920x1080 frame samples 9,216 pixels - // instead of 2,073,600. - $sampleMax = 128; - if ($width > $sampleMax || $height > $sampleMax) { - $scale = $sampleMax / max($width, $height); - $sampleWidth = max(1, (int) round($width * $scale)); - $sampleHeight = max(1, (int) round($height * $scale)); - - $resized = imagescale($image, $sampleWidth, $sampleHeight); - if ($resized !== false) { - imagedestroy($image); - $image = $resized; - $width = $sampleWidth; - $height = $sampleHeight; + return self::fromBinary($contents) ?? self::DEFAULT_HASH; + } + + /** + * Hash raw image bytes, or return null when they cannot be decoded. + * + * Decoding goes through the configured Intervention driver instead of calling + * GD directly, for two reasons. A vips (or imagick) host is not required to + * ship ext-gd at all, and there the old imagecreatefromstring() call was an + * undefined function: an \Error, which slipped past every catch (\Exception) + * above it and failed the thumbnail job. And the driver that wrote the + * thumbnail is the one guaranteed to be able to read it back (webp included). + * + * GD stays as a fallback when it is installed, so a misconfigured driver + * degrades to the previous behaviour instead of to the default hash. + */ + public static function fromBinary(string $contents): ?string + { + foreach (self::candidateDrivers() as $driver) { + try { + $pixels = self::samplePixels($contents, $driver); + if (! $pixels) { + continue; + } + + $blurhash = BlurhashEngine::encode($pixels, self::COMPONENTS_X, self::COMPONENTS_Y); + + return strlen($blurhash) > 191 ? null : $blurhash; + } catch (Throwable $e) { + continue; + } + } + + return null; + } + + protected static function candidateDrivers(): array + { + $drivers = [config('image.driver', 'vips')]; + + if (function_exists('imagecreatefromstring')) { + $drivers[] = 'gd'; + } + + return array_values(array_unique($drivers)); + } + + /** + * Decode, downscale and flatten the image, then return it as rows of [r, g, b]. + */ + protected static function samplePixels(string $contents, string $driver): array + { + $image = ImageDriverManager::createImageManager([ + 'decodeAnimation' => false, + ], $driver)->decodeBinary($contents); + + $image = $image->scaleDown(self::SAMPLE_MAX, self::SAMPLE_MAX); + + // Transparent pixels have to be composited onto the background before they + // are read, because only RGB is sampled. GD happened to leave white behind + // fully transparent pixels after a resize, libvips leaves black, so without + // this every transparent PNG hashes to a dark placeholder under vips. + $image = $image->fillTransparentAreas(); + + $native = $image->core()->native(); + + return match (true) { + $native instanceof VipsImage => self::pixelsFromVips($native), + $native instanceof Imagick => self::pixelsFromImagick($native), + $native instanceof GdImage => self::pixelsFromGd($native), + default => throw new RuntimeException('Unsupported image driver for blurhash'), + }; + } + + protected static function pixelsFromVips(VipsImage $image): array + { + // The vips driver keeps the source colourspace (a CMYK jpg thumbnail stays + // CMYK), so normalise to 8-bit sRGB before reading the first three bands. + if ($image->interpretation !== Interpretation::SRGB) { + $image = $image->colourspace(Interpretation::SRGB); + } + + // Already opaque after fillTransparentAreas(), this only drops the alpha band. + if ($image->hasAlpha()) { + $image = $image->flatten(['background' => [255, 255, 255]]); + } + + if ($image->bands > 3) { + $image = $image->extract_band(0, ['n' => 3]); + } + + if ($image->format !== BandFormat::UCHAR) { + $image = $image->cast(BandFormat::UCHAR); + } + + $width = $image->width; + $height = $image->height; + $bands = $image->bands; + + return self::pixelsFromBytes($image->writeToMemory(), $width, $height, $bands); + } + + protected static function pixelsFromImagick(Imagick $image): array + { + if ($image->getImageColorspace() !== Imagick::COLORSPACE_SRGB) { + $image->transformImageColorspace(Imagick::COLORSPACE_SRGB); + } + + $width = $image->getImageWidth(); + $height = $image->getImageHeight(); + $values = $image->exportImagePixels(0, 0, $width, $height, 'RGB', Imagick::PIXEL_CHAR); + + $pixels = []; + $i = 0; + for ($y = 0; $y < $height; $y++) { + $row = []; + for ($x = 0; $x < $width; $x++) { + $row[] = [$values[$i], $values[$i + 1], $values[$i + 2]]; + $i += 3; } + $pixels[] = $row; } + return $pixels; + } + + protected static function pixelsFromGd(GdImage $image): array + { + $width = imagesx($image); + $height = imagesy($image); + $pixels = []; for ($y = 0; $y < $height; $y++) { $row = []; for ($x = 0; $x < $width; $x++) { - $index = imagecolorat($image, $x, $y); - $colors = imagecolorsforindex($image, $index); + $colors = imagecolorsforindex($image, imagecolorat($image, $x, $y)); $row[] = [$colors['red'], $colors['green'], $colors['blue']]; } $pixels[] = $row; } - imagedestroy($image); + return $pixels; + } - $components_x = 4; - $components_y = 4; - $blurhash = BlurhashEngine::encode($pixels, $components_x, $components_y); - if (strlen($blurhash) > 191) { - return self::DEFAULT_HASH; + protected static function pixelsFromBytes(string $bytes, int $width, int $height, int $bands): array + { + if ($width < 1 || $height < 1 || strlen($bytes) < $width * $height * $bands) { + throw new RuntimeException('Unexpected pixel buffer size for blurhash'); + } + + $pixels = []; + $i = 0; + for ($y = 0; $y < $height; $y++) { + $row = []; + for ($x = 0; $x < $width; $x++) { + $r = ord($bytes[$i]); + $row[] = $bands >= 3 + ? [$r, ord($bytes[$i + 1]), ord($bytes[$i + 2])] + : [$r, $r, $r]; + $i += $bands; + } + $pixels[] = $row; } - return $blurhash; + return $pixels; } } diff --git a/app/Util/Media/Image.php b/app/Util/Media/Image.php index c62d2386f..8cc6dbc8d 100644 --- a/app/Util/Media/Image.php +++ b/app/Util/Media/Image.php @@ -297,7 +297,7 @@ class Image $media->save(); if ($thumbnail) { - $this->generateBlurhash($media); + $this->generateBlurhash($media, $encoded->toString()); } if ($media->status_id) { @@ -354,27 +354,28 @@ class Image return ['path' => $basePath, 'png' => false]; } - protected function generateBlurhash($media) + /** + * Hash the thumbnail that was just encoded. The bytes are already in memory, + * so they are hashed directly instead of being read back from disk (or pulled + * back down from cloud storage into a temp file) a moment after being written. + */ + protected function generateBlurhash($media, ?string $contents = null) { try { - if ($this->defaultDisk === 'local') { - $thumbnailPath = storage_path('app/'.$media->thumbnail_path); - $blurhash = Blurhash::generate($media, $thumbnailPath); - } else { - $tempFile = tempnam(sys_get_temp_dir(), 'blurhash_'); - $contents = Storage::disk($this->defaultDisk)->get($media->thumbnail_path); - file_put_contents($tempFile, $contents); - - $blurhash = Blurhash::generate($media, $tempFile); - - unlink($tempFile); + if ($contents === null) { + $contents = $this->defaultDisk === 'local' + ? file_get_contents(storage_path('app/'.$media->thumbnail_path)) + : Storage::disk($this->defaultDisk)->get($media->thumbnail_path); } - if ($blurhash) { - $media->blurhash = $blurhash; - $media->save(); - } - } catch (\Exception $e) { + $blurhash = $contents ? Blurhash::fromBinary($contents) : null; + + $media->blurhash = $blurhash ?? Blurhash::DEFAULT_HASH; + $media->save(); + } catch (\Throwable $e) { + // \Throwable, not \Exception: the blurhash is decorative and must never + // fail the thumbnail job, which still has to mark the media as processed + // and dispatch ImageUpdate. if (config('app.dev_log')) { Log::info('Blurhash generation failed: '.$e->getMessage()); } diff --git a/app/Util/Media/ImageDriverManager.php b/app/Util/Media/ImageDriverManager.php index ebf82e603..d94b590d2 100644 --- a/app/Util/Media/ImageDriverManager.php +++ b/app/Util/Media/ImageDriverManager.php @@ -9,10 +9,12 @@ class ImageDriverManager { /** * Get the appropriate image driver class based on configuration. + * + * @param string|null $driver Driver name to resolve instead of the configured one */ - public static function getDriverClass(): string + public static function getDriverClass(?string $driver = null): string { - return match (config('image.driver')) { + return match ($driver ?? config('image.driver')) { 'gd' => Driver::class, 'imagick' => \Intervention\Image\Drivers\Imagick\Driver::class, 'vips' => \Intervention\Image\Drivers\Vips\Driver::class, @@ -24,15 +26,16 @@ class ImageDriverManager * Create a new ImageManager instance with the configured driver. * * @param array $options Additional options for ImageManager + * @param string|null $driver Driver name to use instead of the configured one */ - public static function createImageManager(array $options = []): ImageManager + public static function createImageManager(array $options = [], ?string $driver = null): ImageManager { $configOptions = config('image.options', []); $options = array_merge($configOptions, $options); return new ImageManager( - self::getDriverClass(), + self::getDriverClass($driver), autoOrientation: (bool) ($options['autoOrientation'] ?? true), decodeAnimation: (bool) ($options['decodeAnimation'] ?? true), backgroundColor: (string) ($options['backgroundColor'] ?? 'ffffff'), diff --git a/tests/Feature/MediaPipeline/BlurhashTest.php b/tests/Feature/MediaPipeline/BlurhashTest.php new file mode 100644 index 000000000..b8965417b --- /dev/null +++ b/tests/Feature/MediaPipeline/BlurhashTest.php @@ -0,0 +1,187 @@ +markTestSkipped('Image driver "'.config('image.driver').'" is not available: '.$e->getMessage()); + } +}); + +function blurhashFixture(int $width, int $height, ?string $fill = null, string $format = 'png', ?string $driver = null): string +{ + $image = ImageDriverManager::createImageManager([], $driver)->createImage($width, $height); + + if ($fill !== null) { + $image = $image->fill($fill); + } + + $encoder = $format === 'webp' ? new WebpEncoder(90) : new PngEncoder; + + return $image->encode($encoder)->toString(); +} + +function blurhashAverageColor(string $hash): array +{ + $value = Base83::decode(substr($hash, 2, 4)); + + return [$value >> 16, ($value >> 8) & 255, $value & 255]; +} + +it('hashes a solid colour image to that colour', function () { + $hash = Blurhash::fromBinary(blurhashFixture(64, 48, '3366cc')); + + expect($hash)->toBeString()->toHaveLength(36); + + [$r, $g, $b] = blurhashAverageColor($hash); + expect($r)->toBeBetween(48, 54); + expect($g)->toBeBetween(99, 105); + expect($b)->toBeBetween(201, 207); +}); + +it('composites transparent areas onto the background instead of hashing them as black', function () { + $hash = Blurhash::fromBinary(blurhashFixture(64, 48)); + + foreach (blurhashAverageColor($hash) as $channel) { + expect($channel)->toBeGreaterThanOrEqual(250); + } +}); + +it('does not overflow a fully saturated channel into its neighbour', function () { + expect(Color::tosRGB(1.0))->toBe(255); + + $hash = Blurhash::fromBinary(blurhashFixture(32, 32, 'ffffff')); + + expect(blurhashAverageColor($hash))->toBe([255, 255, 255]); +}); + +it('hashes webp thumbnails instead of returning the default hash', function () { + try { + $webp = blurhashFixture(64, 64, 'cc3333', 'webp'); + } catch (Throwable $e) { + test()->markTestSkipped('Configured image driver cannot encode webp: '.$e->getMessage()); + } + + $file = tempnam(sys_get_temp_dir(), 'blurhash_test_'); + file_put_contents($file, $webp); + + $media = new Media; + $media->mime = 'image/webp'; + $media->thumbnail_path = 'public/m/photo_thumb.webp'; + + try { + $hash = Blurhash::generate($media, $file); + } finally { + @unlink($file); + } + + expect($hash)->not->toBe(Blurhash::DEFAULT_HASH); + expect(blurhashAverageColor($hash)[0])->toBeGreaterThan(190); +}); + +it('returns the default hash for bytes that cannot be decoded', function () { + expect(Blurhash::fromBinary('definitely not an image'))->toBeNull(); + + $file = tempnam(sys_get_temp_dir(), 'blurhash_test_'); + file_put_contents($file, 'definitely not an image'); + + $media = new Media; + $media->mime = 'image/jpeg'; + $media->thumbnail_path = 'public/m/photo_thumb.jpg'; + + try { + expect(Blurhash::generate($media, $file))->toBe(Blurhash::DEFAULT_HASH); + } finally { + @unlink($file); + } +}); + +it('falls back to gd when the configured driver is unavailable', function () { + if (! function_exists('imagecreatefromstring')) { + test()->markTestSkipped('ext-gd is not installed.'); + } + + $unavailable = collect(['imagick', 'vips'])->first(function ($driver) { + try { + ImageDriverManager::createImageManager([], $driver); + + return false; + } catch (Throwable $e) { + return true; + } + }); + + if (! $unavailable) { + test()->markTestSkipped('Every image driver is available, nothing to fall back from.'); + } + + $png = blurhashFixture(64, 48, '3366cc', 'png', 'gd'); + + Config::set('image.driver', $unavailable); + + expect(Blurhash::fromBinary($png))->toBeString()->toHaveLength(36); +}); + +it('stores a blurhash when a thumbnail is generated', function () { + Config::set('filesystems.default', 's3'); + Config::set('pixelfed.optimize_image', false); + Storage::fake('s3', ['url' => 'https://cdn.test']); + + $user = User::factory()->create(); + $user->refresh(); + $pid = $user->profile->id; + + $mediaPath = 'public/m/_v2/'.$pid.'/ee/ff/photo.png'; + $pngBytes = blurhashFixture(800, 600, '3366cc'); + Storage::disk('s3')->put($mediaPath, $pngBytes); + + $media = Media::create([ + 'profile_id' => $pid, + 'user_id' => $user->id, + 'media_path' => $mediaPath, + 'mime' => 'image/png', + 'size' => strlen($pngBytes), + 'remote_media' => false, + 'order' => 0, + ]); + + (new Image)->resizeThumbnail($media); + $media->refresh(); + + expect($media->thumbnail_path)->toBe('public/m/_v2/'.$pid.'/ee/ff/photo_thumb.png'); + expect($media->blurhash)->toBeString()->not->toBe(Blurhash::DEFAULT_HASH); + + [$r, $g, $b] = blurhashAverageColor($media->blurhash); + expect($r)->toBeBetween(48, 54); + expect($g)->toBeBetween(99, 105); + expect($b)->toBeBetween(201, 207); +});