Refactor blurhash

pull/7394/head
Daniel Supernault 3 days ago
parent 6660f2b18b
commit 15da72dd94
No known key found for this signature in database
GPG Key ID: 23740873EE6F76A1

@ -3,7 +3,7 @@
namespace App\Services;
use App\Models\Instance;
use App\Util\Blurhash\Blurhash;
use App\Util\Media\Blurhash;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Storage;
@ -131,32 +131,12 @@ class InstanceService
$file = config_cache('app.banner_image') ?? url(Storage::url('public/headers/default.jpg'));
$image = imagecreatefromstring(file_get_contents($file));
if (! $image) {
return 'UzJR]l{wHZRjM}R%XRkCH?X9xaWEjZj]kAjt';
}
$width = imagesx($image);
$height = imagesy($image);
$pixels = [];
for ($y = 0; $y < $height; $y++) {
$row = [];
for ($x = 0; $x < $width; $x++) {
$index = imagecolorat($image, $x, $y);
$colors = imagecolorsforindex($image, $index);
$row[] = [$colors['red'], $colors['green'], $colors['blue']];
}
$pixels[] = $row;
}
// Free the allocated GdImage object from memory:
imagedestroy($image);
$components_x = 4;
$components_y = 4;
$blurhash = Blurhash::encode($pixels, $components_x, $components_y);
if (strlen($blurhash) > 191) {
// Goes through the configured image driver (and a downscaled sample)
// rather than raw GD calls at full resolution, so this no longer
// fatals on hosts that run vips without ext-gd.
$contents = @file_get_contents($file);
$blurhash = $contents ? Blurhash::fromBinary($contents) : null;
if (! $blurhash) {
return 'UzJR]l{wHZRjM}R%XRkCH?X9xaWEjZj]kAjt';
}

@ -17,8 +17,13 @@ final class Color
{
$normalized = max(0, min(1, $value));
return ($normalized <= 0.0031308)
$result = ($normalized <= 0.0031308)
? (int) round($normalized * 12.92 * 255 + 0.5)
: (int) round((1.055 * pow($normalized, 1 / 2.4) - 0.055) * 255 + 0.5);
// The + 0.5 rounds a fully saturated channel up to 256, which does not fit
// in the 8 bits DC::encode() packs it into and carries into the next channel
// (pure white came out as R=257 G=1 B=0). Clamp like upstream php-blurhash.
return max(0, min($result, 255));
}
}

@ -4,14 +4,48 @@ namespace App\Util\Media;
use App\Models\Media;
use App\Util\Blurhash\Blurhash as BlurhashEngine;
use GdImage;
use Imagick;
use Jcupitt\Vips\BandFormat;
use Jcupitt\Vips\Image as VipsImage;
use Jcupitt\Vips\Interpretation;
use RuntimeException;
use Throwable;
class Blurhash
{
const DEFAULT_HASH = 'U4Rfzst8?bt7ogayj[j[~pfQ9Goe%Mj[WBay';
// Long edge, in pixels, of the sample the hash is computed from.
//
// The output is a 4x4-component DCT, so sampling the source at full resolution
// buys almost nothing, while the per-pixel PHP arrays the encoder needs cost a
// few hundred bytes each: a 720x1280 frame measured at 224 MB peak, which is
// what used to kill workers on full resolution video thumbnails (pixelfed#2652).
//
// 128px measured as the point of diminishing returns: against the full
// resolution hash, mean per-channel deviation of the decoded 24x24 preview is
// ~7.5/255 at a 32px sample, ~4.5/255 at 64px, ~2.5/255 at 128px, and no better
// at 256px. At 128px a 1920x1080 frame samples 9,216 pixels instead of 2,073,600.
const SAMPLE_MAX = 128;
const COMPONENTS_X = 4;
const COMPONENTS_Y = 4;
// Mime types of the media a thumbnail is hashed for. The thumbnail itself keeps
// the source format (png, jpg, webp), with avif/heic sources landing as jpg.
const SUPPORTED_MIMES = [
'image/png',
'image/jpeg',
'image/jpg',
'image/webp',
'video/mp4',
];
public static function generate(Media $media, $path = false): string
{
if (! in_array($media->mime, ['image/png', 'image/jpeg', 'image/jpg', 'video/mp4'])) {
if (! in_array($media->mime, self::SUPPORTED_MIMES)) {
return self::DEFAULT_HASH;
}
@ -19,79 +53,182 @@ class Blurhash
return self::DEFAULT_HASH;
}
if ($path) {
$file = $path;
} else {
$localFs = config('filesystems.default') === 'local';
$file = storage_path('app/'.$media->thumbnail_path);
}
$file = $path ?: storage_path('app/'.$media->thumbnail_path);
if (! is_file($file)) {
return self::DEFAULT_HASH;
}
$image = imagecreatefromstring(file_get_contents($file));
if (! $image) {
$contents = file_get_contents($file);
if ($contents === false || $contents === '') {
return self::DEFAULT_HASH;
}
$width = imagesx($image);
$height = imagesy($image);
// The loop below allocates one PHP array per pixel, which costs a few hundred
// bytes each once the hashtable is counted: a 720x1280 frame measured at 224 MB
// peak, i.e. ~255 bytes per pixel, which puts 1920x1080 near half a gigabyte
// (#2652 reports over a gigabyte on an older PHP). Image thumbnails are capped
// at 640x640 and run under Image::__construct()'s ini_set('memory_limit',
// '1024M'), so they stay inside it. Video thumbnails come straight out of
// FFmpeg at source resolution with no
// such raise, and blow memory_limit outright — a PHP fatal, which is not an
// \Exception, so it kills the worker instead of being caught (pixelfed#2652).
//
// The output is a 4x4-component DCT, so sampling the source at full resolution
// buys almost nothing. Downscaling first removes the ceiling for every caller
// rather than moving it, which is all raising memory_limit would have done.
//
// 128px on the long edge measured as the point of diminishing returns: against
// the full-resolution hash, mean per-channel deviation of the decoded 24x24
// preview is ~7.5/255 at a 32px sample, ~4.5/255 at 64px, ~2.5/255 at 128px,
// and no better at 256px. At 128px a 1920x1080 frame samples 9,216 pixels
// instead of 2,073,600.
$sampleMax = 128;
if ($width > $sampleMax || $height > $sampleMax) {
$scale = $sampleMax / max($width, $height);
$sampleWidth = max(1, (int) round($width * $scale));
$sampleHeight = max(1, (int) round($height * $scale));
$resized = imagescale($image, $sampleWidth, $sampleHeight);
if ($resized !== false) {
imagedestroy($image);
$image = $resized;
$width = $sampleWidth;
$height = $sampleHeight;
return self::fromBinary($contents) ?? self::DEFAULT_HASH;
}
/**
* Hash raw image bytes, or return null when they cannot be decoded.
*
* Decoding goes through the configured Intervention driver instead of calling
* GD directly, for two reasons. A vips (or imagick) host is not required to
* ship ext-gd at all, and there the old imagecreatefromstring() call was an
* undefined function: an \Error, which slipped past every catch (\Exception)
* above it and failed the thumbnail job. And the driver that wrote the
* thumbnail is the one guaranteed to be able to read it back (webp included).
*
* GD stays as a fallback when it is installed, so a misconfigured driver
* degrades to the previous behaviour instead of to the default hash.
*/
public static function fromBinary(string $contents): ?string
{
foreach (self::candidateDrivers() as $driver) {
try {
$pixels = self::samplePixels($contents, $driver);
if (! $pixels) {
continue;
}
$blurhash = BlurhashEngine::encode($pixels, self::COMPONENTS_X, self::COMPONENTS_Y);
return strlen($blurhash) > 191 ? null : $blurhash;
} catch (Throwable $e) {
continue;
}
}
return null;
}
protected static function candidateDrivers(): array
{
$drivers = [config('image.driver', 'vips')];
if (function_exists('imagecreatefromstring')) {
$drivers[] = 'gd';
}
return array_values(array_unique($drivers));
}
/**
* Decode, downscale and flatten the image, then return it as rows of [r, g, b].
*/
protected static function samplePixels(string $contents, string $driver): array
{
$image = ImageDriverManager::createImageManager([
'decodeAnimation' => false,
], $driver)->decodeBinary($contents);
$image = $image->scaleDown(self::SAMPLE_MAX, self::SAMPLE_MAX);
// Transparent pixels have to be composited onto the background before they
// are read, because only RGB is sampled. GD happened to leave white behind
// fully transparent pixels after a resize, libvips leaves black, so without
// this every transparent PNG hashes to a dark placeholder under vips.
$image = $image->fillTransparentAreas();
$native = $image->core()->native();
return match (true) {
$native instanceof VipsImage => self::pixelsFromVips($native),
$native instanceof Imagick => self::pixelsFromImagick($native),
$native instanceof GdImage => self::pixelsFromGd($native),
default => throw new RuntimeException('Unsupported image driver for blurhash'),
};
}
protected static function pixelsFromVips(VipsImage $image): array
{
// The vips driver keeps the source colourspace (a CMYK jpg thumbnail stays
// CMYK), so normalise to 8-bit sRGB before reading the first three bands.
if ($image->interpretation !== Interpretation::SRGB) {
$image = $image->colourspace(Interpretation::SRGB);
}
// Already opaque after fillTransparentAreas(), this only drops the alpha band.
if ($image->hasAlpha()) {
$image = $image->flatten(['background' => [255, 255, 255]]);
}
if ($image->bands > 3) {
$image = $image->extract_band(0, ['n' => 3]);
}
if ($image->format !== BandFormat::UCHAR) {
$image = $image->cast(BandFormat::UCHAR);
}
$width = $image->width;
$height = $image->height;
$bands = $image->bands;
return self::pixelsFromBytes($image->writeToMemory(), $width, $height, $bands);
}
protected static function pixelsFromImagick(Imagick $image): array
{
if ($image->getImageColorspace() !== Imagick::COLORSPACE_SRGB) {
$image->transformImageColorspace(Imagick::COLORSPACE_SRGB);
}
$width = $image->getImageWidth();
$height = $image->getImageHeight();
$values = $image->exportImagePixels(0, 0, $width, $height, 'RGB', Imagick::PIXEL_CHAR);
$pixels = [];
$i = 0;
for ($y = 0; $y < $height; $y++) {
$row = [];
for ($x = 0; $x < $width; $x++) {
$row[] = [$values[$i], $values[$i + 1], $values[$i + 2]];
$i += 3;
}
$pixels[] = $row;
}
return $pixels;
}
protected static function pixelsFromGd(GdImage $image): array
{
$width = imagesx($image);
$height = imagesy($image);
$pixels = [];
for ($y = 0; $y < $height; $y++) {
$row = [];
for ($x = 0; $x < $width; $x++) {
$index = imagecolorat($image, $x, $y);
$colors = imagecolorsforindex($image, $index);
$colors = imagecolorsforindex($image, imagecolorat($image, $x, $y));
$row[] = [$colors['red'], $colors['green'], $colors['blue']];
}
$pixels[] = $row;
}
imagedestroy($image);
return $pixels;
}
$components_x = 4;
$components_y = 4;
$blurhash = BlurhashEngine::encode($pixels, $components_x, $components_y);
if (strlen($blurhash) > 191) {
return self::DEFAULT_HASH;
protected static function pixelsFromBytes(string $bytes, int $width, int $height, int $bands): array
{
if ($width < 1 || $height < 1 || strlen($bytes) < $width * $height * $bands) {
throw new RuntimeException('Unexpected pixel buffer size for blurhash');
}
$pixels = [];
$i = 0;
for ($y = 0; $y < $height; $y++) {
$row = [];
for ($x = 0; $x < $width; $x++) {
$r = ord($bytes[$i]);
$row[] = $bands >= 3
? [$r, ord($bytes[$i + 1]), ord($bytes[$i + 2])]
: [$r, $r, $r];
$i += $bands;
}
$pixels[] = $row;
}
return $blurhash;
return $pixels;
}
}

@ -297,7 +297,7 @@ class Image
$media->save();
if ($thumbnail) {
$this->generateBlurhash($media);
$this->generateBlurhash($media, $encoded->toString());
}
if ($media->status_id) {
@ -354,27 +354,28 @@ class Image
return ['path' => $basePath, 'png' => false];
}
protected function generateBlurhash($media)
/**
* Hash the thumbnail that was just encoded. The bytes are already in memory,
* so they are hashed directly instead of being read back from disk (or pulled
* back down from cloud storage into a temp file) a moment after being written.
*/
protected function generateBlurhash($media, ?string $contents = null)
{
try {
if ($this->defaultDisk === 'local') {
$thumbnailPath = storage_path('app/'.$media->thumbnail_path);
$blurhash = Blurhash::generate($media, $thumbnailPath);
} else {
$tempFile = tempnam(sys_get_temp_dir(), 'blurhash_');
$contents = Storage::disk($this->defaultDisk)->get($media->thumbnail_path);
file_put_contents($tempFile, $contents);
$blurhash = Blurhash::generate($media, $tempFile);
unlink($tempFile);
if ($contents === null) {
$contents = $this->defaultDisk === 'local'
? file_get_contents(storage_path('app/'.$media->thumbnail_path))
: Storage::disk($this->defaultDisk)->get($media->thumbnail_path);
}
if ($blurhash) {
$media->blurhash = $blurhash;
$media->save();
}
} catch (\Exception $e) {
$blurhash = $contents ? Blurhash::fromBinary($contents) : null;
$media->blurhash = $blurhash ?? Blurhash::DEFAULT_HASH;
$media->save();
} catch (\Throwable $e) {
// \Throwable, not \Exception: the blurhash is decorative and must never
// fail the thumbnail job, which still has to mark the media as processed
// and dispatch ImageUpdate.
if (config('app.dev_log')) {
Log::info('Blurhash generation failed: '.$e->getMessage());
}

@ -9,10 +9,12 @@ class ImageDriverManager
{
/**
* Get the appropriate image driver class based on configuration.
*
* @param string|null $driver Driver name to resolve instead of the configured one
*/
public static function getDriverClass(): string
public static function getDriverClass(?string $driver = null): string
{
return match (config('image.driver')) {
return match ($driver ?? config('image.driver')) {
'gd' => Driver::class,
'imagick' => \Intervention\Image\Drivers\Imagick\Driver::class,
'vips' => \Intervention\Image\Drivers\Vips\Driver::class,
@ -24,15 +26,16 @@ class ImageDriverManager
* Create a new ImageManager instance with the configured driver.
*
* @param array $options Additional options for ImageManager
* @param string|null $driver Driver name to use instead of the configured one
*/
public static function createImageManager(array $options = []): ImageManager
public static function createImageManager(array $options = [], ?string $driver = null): ImageManager
{
$configOptions = config('image.options', []);
$options = array_merge($configOptions, $options);
return new ImageManager(
self::getDriverClass(),
self::getDriverClass($driver),
autoOrientation: (bool) ($options['autoOrientation'] ?? true),
decodeAnimation: (bool) ($options['decodeAnimation'] ?? true),
backgroundColor: (string) ($options['backgroundColor'] ?? 'ffffff'),

@ -0,0 +1,187 @@
<?php
use App\Models\Media;
use App\Models\User;
use App\Util\Blurhash\Base83;
use App\Util\Blurhash\Color;
use App\Util\Media\Blurhash;
use App\Util\Media\Image;
use App\Util\Media\ImageDriverManager;
use Illuminate\Foundation\Testing\LazilyRefreshDatabase;
use Illuminate\Support\Facades\Config;
use Illuminate\Support\Facades\Storage;
use Intervention\Image\Encoders\PngEncoder;
use Intervention\Image\Encoders\WebpEncoder;
uses(LazilyRefreshDatabase::class);
/*
|--------------------------------------------------------------------------
| Blurhash generation
|--------------------------------------------------------------------------
|
| The hasher used to call GD directly no matter which image driver was
| configured. Under vips that meant dark placeholders for transparent PNGs
| (libvips leaves black behind transparent pixels, the hasher ignored alpha),
| the default hash for every webp, and an uncaught \Error that failed the
| thumbnail job on hosts without ext-gd. It now decodes through the
| configured driver and flattens transparency before sampling.
|
*/
beforeEach(function () {
try {
ImageDriverManager::createImageManager();
} catch (Throwable $e) {
test()->markTestSkipped('Image driver "'.config('image.driver').'" is not available: '.$e->getMessage());
}
});
function blurhashFixture(int $width, int $height, ?string $fill = null, string $format = 'png', ?string $driver = null): string
{
$image = ImageDriverManager::createImageManager([], $driver)->createImage($width, $height);
if ($fill !== null) {
$image = $image->fill($fill);
}
$encoder = $format === 'webp' ? new WebpEncoder(90) : new PngEncoder;
return $image->encode($encoder)->toString();
}
function blurhashAverageColor(string $hash): array
{
$value = Base83::decode(substr($hash, 2, 4));
return [$value >> 16, ($value >> 8) & 255, $value & 255];
}
it('hashes a solid colour image to that colour', function () {
$hash = Blurhash::fromBinary(blurhashFixture(64, 48, '3366cc'));
expect($hash)->toBeString()->toHaveLength(36);
[$r, $g, $b] = blurhashAverageColor($hash);
expect($r)->toBeBetween(48, 54);
expect($g)->toBeBetween(99, 105);
expect($b)->toBeBetween(201, 207);
});
it('composites transparent areas onto the background instead of hashing them as black', function () {
$hash = Blurhash::fromBinary(blurhashFixture(64, 48));
foreach (blurhashAverageColor($hash) as $channel) {
expect($channel)->toBeGreaterThanOrEqual(250);
}
});
it('does not overflow a fully saturated channel into its neighbour', function () {
expect(Color::tosRGB(1.0))->toBe(255);
$hash = Blurhash::fromBinary(blurhashFixture(32, 32, 'ffffff'));
expect(blurhashAverageColor($hash))->toBe([255, 255, 255]);
});
it('hashes webp thumbnails instead of returning the default hash', function () {
try {
$webp = blurhashFixture(64, 64, 'cc3333', 'webp');
} catch (Throwable $e) {
test()->markTestSkipped('Configured image driver cannot encode webp: '.$e->getMessage());
}
$file = tempnam(sys_get_temp_dir(), 'blurhash_test_');
file_put_contents($file, $webp);
$media = new Media;
$media->mime = 'image/webp';
$media->thumbnail_path = 'public/m/photo_thumb.webp';
try {
$hash = Blurhash::generate($media, $file);
} finally {
@unlink($file);
}
expect($hash)->not->toBe(Blurhash::DEFAULT_HASH);
expect(blurhashAverageColor($hash)[0])->toBeGreaterThan(190);
});
it('returns the default hash for bytes that cannot be decoded', function () {
expect(Blurhash::fromBinary('definitely not an image'))->toBeNull();
$file = tempnam(sys_get_temp_dir(), 'blurhash_test_');
file_put_contents($file, 'definitely not an image');
$media = new Media;
$media->mime = 'image/jpeg';
$media->thumbnail_path = 'public/m/photo_thumb.jpg';
try {
expect(Blurhash::generate($media, $file))->toBe(Blurhash::DEFAULT_HASH);
} finally {
@unlink($file);
}
});
it('falls back to gd when the configured driver is unavailable', function () {
if (! function_exists('imagecreatefromstring')) {
test()->markTestSkipped('ext-gd is not installed.');
}
$unavailable = collect(['imagick', 'vips'])->first(function ($driver) {
try {
ImageDriverManager::createImageManager([], $driver);
return false;
} catch (Throwable $e) {
return true;
}
});
if (! $unavailable) {
test()->markTestSkipped('Every image driver is available, nothing to fall back from.');
}
$png = blurhashFixture(64, 48, '3366cc', 'png', 'gd');
Config::set('image.driver', $unavailable);
expect(Blurhash::fromBinary($png))->toBeString()->toHaveLength(36);
});
it('stores a blurhash when a thumbnail is generated', function () {
Config::set('filesystems.default', 's3');
Config::set('pixelfed.optimize_image', false);
Storage::fake('s3', ['url' => 'https://cdn.test']);
$user = User::factory()->create();
$user->refresh();
$pid = $user->profile->id;
$mediaPath = 'public/m/_v2/'.$pid.'/ee/ff/photo.png';
$pngBytes = blurhashFixture(800, 600, '3366cc');
Storage::disk('s3')->put($mediaPath, $pngBytes);
$media = Media::create([
'profile_id' => $pid,
'user_id' => $user->id,
'media_path' => $mediaPath,
'mime' => 'image/png',
'size' => strlen($pngBytes),
'remote_media' => false,
'order' => 0,
]);
(new Image)->resizeThumbnail($media);
$media->refresh();
expect($media->thumbnail_path)->toBe('public/m/_v2/'.$pid.'/ee/ff/photo_thumb.png');
expect($media->blurhash)->toBeString()->not->toBe(Blurhash::DEFAULT_HASH);
[$r, $g, $b] = blurhashAverageColor($media->blurhash);
expect($r)->toBeBetween(48, 54);
expect($g)->toBeBetween(99, 105);
expect($b)->toBeBetween(201, 207);
});
Loading…
Cancel
Save