Merge pull request #7394 from pixelfed/staging

Refactor blurhash
pull/7395/head^2
dansup 3 days ago committed by GitHub
commit 8744fe7b76
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -3,7 +3,7 @@
namespace App\Services; namespace App\Services;
use App\Models\Instance; use App\Models\Instance;
use App\Util\Blurhash\Blurhash; use App\Util\Media\Blurhash;
use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Storage; use Illuminate\Support\Facades\Storage;
@ -131,32 +131,12 @@ class InstanceService
$file = config_cache('app.banner_image') ?? url(Storage::url('public/headers/default.jpg')); $file = config_cache('app.banner_image') ?? url(Storage::url('public/headers/default.jpg'));
$image = imagecreatefromstring(file_get_contents($file)); // Goes through the configured image driver (and a downscaled sample)
if (! $image) { // rather than raw GD calls at full resolution, so this no longer
return 'UzJR]l{wHZRjM}R%XRkCH?X9xaWEjZj]kAjt'; // fatals on hosts that run vips without ext-gd.
} $contents = @file_get_contents($file);
$width = imagesx($image); $blurhash = $contents ? Blurhash::fromBinary($contents) : null;
$height = imagesy($image); if (! $blurhash) {
$pixels = [];
for ($y = 0; $y < $height; $y++) {
$row = [];
for ($x = 0; $x < $width; $x++) {
$index = imagecolorat($image, $x, $y);
$colors = imagecolorsforindex($image, $index);
$row[] = [$colors['red'], $colors['green'], $colors['blue']];
}
$pixels[] = $row;
}
// Free the allocated GdImage object from memory:
imagedestroy($image);
$components_x = 4;
$components_y = 4;
$blurhash = Blurhash::encode($pixels, $components_x, $components_y);
if (strlen($blurhash) > 191) {
return 'UzJR]l{wHZRjM}R%XRkCH?X9xaWEjZj]kAjt'; return 'UzJR]l{wHZRjM}R%XRkCH?X9xaWEjZj]kAjt';
} }

@ -17,8 +17,13 @@ final class Color
{ {
$normalized = max(0, min(1, $value)); $normalized = max(0, min(1, $value));
return ($normalized <= 0.0031308) $result = ($normalized <= 0.0031308)
? (int) round($normalized * 12.92 * 255 + 0.5) ? (int) round($normalized * 12.92 * 255 + 0.5)
: (int) round((1.055 * pow($normalized, 1 / 2.4) - 0.055) * 255 + 0.5); : (int) round((1.055 * pow($normalized, 1 / 2.4) - 0.055) * 255 + 0.5);
// The + 0.5 rounds a fully saturated channel up to 256, which does not fit
// in the 8 bits DC::encode() packs it into and carries into the next channel
// (pure white came out as R=257 G=1 B=0). Clamp like upstream php-blurhash.
return max(0, min($result, 255));
} }
} }

@ -4,14 +4,48 @@ namespace App\Util\Media;
use App\Models\Media; use App\Models\Media;
use App\Util\Blurhash\Blurhash as BlurhashEngine; use App\Util\Blurhash\Blurhash as BlurhashEngine;
use GdImage;
use Imagick;
use Jcupitt\Vips\BandFormat;
use Jcupitt\Vips\Image as VipsImage;
use Jcupitt\Vips\Interpretation;
use RuntimeException;
use Throwable;
class Blurhash class Blurhash
{ {
const DEFAULT_HASH = 'U4Rfzst8?bt7ogayj[j[~pfQ9Goe%Mj[WBay'; const DEFAULT_HASH = 'U4Rfzst8?bt7ogayj[j[~pfQ9Goe%Mj[WBay';
// Long edge, in pixels, of the sample the hash is computed from.
//
// The output is a 4x4-component DCT, so sampling the source at full resolution
// buys almost nothing, while the per-pixel PHP arrays the encoder needs cost a
// few hundred bytes each: a 720x1280 frame measured at 224 MB peak, which is
// what used to kill workers on full resolution video thumbnails (pixelfed#2652).
//
// 128px measured as the point of diminishing returns: against the full
// resolution hash, mean per-channel deviation of the decoded 24x24 preview is
// ~7.5/255 at a 32px sample, ~4.5/255 at 64px, ~2.5/255 at 128px, and no better
// at 256px. At 128px a 1920x1080 frame samples 9,216 pixels instead of 2,073,600.
const SAMPLE_MAX = 128;
const COMPONENTS_X = 4;
const COMPONENTS_Y = 4;
// Mime types of the media a thumbnail is hashed for. The thumbnail itself keeps
// the source format (png, jpg, webp), with avif/heic sources landing as jpg.
const SUPPORTED_MIMES = [
'image/png',
'image/jpeg',
'image/jpg',
'image/webp',
'video/mp4',
];
public static function generate(Media $media, $path = false): string public static function generate(Media $media, $path = false): string
{ {
if (! in_array($media->mime, ['image/png', 'image/jpeg', 'image/jpg', 'video/mp4'])) { if (! in_array($media->mime, self::SUPPORTED_MIMES)) {
return self::DEFAULT_HASH; return self::DEFAULT_HASH;
} }
@ -19,79 +53,182 @@ class Blurhash
return self::DEFAULT_HASH; return self::DEFAULT_HASH;
} }
if ($path) { $file = $path ?: storage_path('app/'.$media->thumbnail_path);
$file = $path;
} else {
$localFs = config('filesystems.default') === 'local';
$file = storage_path('app/'.$media->thumbnail_path);
}
if (! is_file($file)) { if (! is_file($file)) {
return self::DEFAULT_HASH; return self::DEFAULT_HASH;
} }
$image = imagecreatefromstring(file_get_contents($file)); $contents = file_get_contents($file);
if (! $image) { if ($contents === false || $contents === '') {
return self::DEFAULT_HASH; return self::DEFAULT_HASH;
} }
$width = imagesx($image);
$height = imagesy($image);
// The loop below allocates one PHP array per pixel, which costs a few hundred return self::fromBinary($contents) ?? self::DEFAULT_HASH;
// bytes each once the hashtable is counted: a 720x1280 frame measured at 224 MB }
// peak, i.e. ~255 bytes per pixel, which puts 1920x1080 near half a gigabyte
// (#2652 reports over a gigabyte on an older PHP). Image thumbnails are capped /**
// at 640x640 and run under Image::__construct()'s ini_set('memory_limit', * Hash raw image bytes, or return null when they cannot be decoded.
// '1024M'), so they stay inside it. Video thumbnails come straight out of *
// FFmpeg at source resolution with no * Decoding goes through the configured Intervention driver instead of calling
// such raise, and blow memory_limit outright — a PHP fatal, which is not an * GD directly, for two reasons. A vips (or imagick) host is not required to
// \Exception, so it kills the worker instead of being caught (pixelfed#2652). * ship ext-gd at all, and there the old imagecreatefromstring() call was an
// * undefined function: an \Error, which slipped past every catch (\Exception)
// The output is a 4x4-component DCT, so sampling the source at full resolution * above it and failed the thumbnail job. And the driver that wrote the
// buys almost nothing. Downscaling first removes the ceiling for every caller * thumbnail is the one guaranteed to be able to read it back (webp included).
// rather than moving it, which is all raising memory_limit would have done. *
// * GD stays as a fallback when it is installed, so a misconfigured driver
// 128px on the long edge measured as the point of diminishing returns: against * degrades to the previous behaviour instead of to the default hash.
// the full-resolution hash, mean per-channel deviation of the decoded 24x24 */
// preview is ~7.5/255 at a 32px sample, ~4.5/255 at 64px, ~2.5/255 at 128px, public static function fromBinary(string $contents): ?string
// and no better at 256px. At 128px a 1920x1080 frame samples 9,216 pixels {
// instead of 2,073,600. foreach (self::candidateDrivers() as $driver) {
$sampleMax = 128; try {
if ($width > $sampleMax || $height > $sampleMax) { $pixels = self::samplePixels($contents, $driver);
$scale = $sampleMax / max($width, $height); if (! $pixels) {
$sampleWidth = max(1, (int) round($width * $scale)); continue;
$sampleHeight = max(1, (int) round($height * $scale)); }
$resized = imagescale($image, $sampleWidth, $sampleHeight); $blurhash = BlurhashEngine::encode($pixels, self::COMPONENTS_X, self::COMPONENTS_Y);
if ($resized !== false) {
imagedestroy($image); return strlen($blurhash) > 191 ? null : $blurhash;
$image = $resized; } catch (Throwable $e) {
$width = $sampleWidth; continue;
$height = $sampleHeight; }
}
return null;
}
protected static function candidateDrivers(): array
{
$drivers = [config('image.driver', 'vips')];
if (function_exists('imagecreatefromstring')) {
$drivers[] = 'gd';
}
return array_values(array_unique($drivers));
}
/**
* Decode, downscale and flatten the image, then return it as rows of [r, g, b].
*/
protected static function samplePixels(string $contents, string $driver): array
{
$image = ImageDriverManager::createImageManager([
'decodeAnimation' => false,
], $driver)->decodeBinary($contents);
$image = $image->scaleDown(self::SAMPLE_MAX, self::SAMPLE_MAX);
// Transparent pixels have to be composited onto the background before they
// are read, because only RGB is sampled. GD happened to leave white behind
// fully transparent pixels after a resize, libvips leaves black, so without
// this every transparent PNG hashes to a dark placeholder under vips.
$image = $image->fillTransparentAreas();
$native = $image->core()->native();
return match (true) {
$native instanceof VipsImage => self::pixelsFromVips($native),
$native instanceof Imagick => self::pixelsFromImagick($native),
$native instanceof GdImage => self::pixelsFromGd($native),
default => throw new RuntimeException('Unsupported image driver for blurhash'),
};
}
protected static function pixelsFromVips(VipsImage $image): array
{
// The vips driver keeps the source colourspace (a CMYK jpg thumbnail stays
// CMYK), so normalise to 8-bit sRGB before reading the first three bands.
if ($image->interpretation !== Interpretation::SRGB) {
$image = $image->colourspace(Interpretation::SRGB);
}
// Already opaque after fillTransparentAreas(), this only drops the alpha band.
if ($image->hasAlpha()) {
$image = $image->flatten(['background' => [255, 255, 255]]);
}
if ($image->bands > 3) {
$image = $image->extract_band(0, ['n' => 3]);
}
if ($image->format !== BandFormat::UCHAR) {
$image = $image->cast(BandFormat::UCHAR);
}
$width = $image->width;
$height = $image->height;
$bands = $image->bands;
return self::pixelsFromBytes($image->writeToMemory(), $width, $height, $bands);
}
protected static function pixelsFromImagick(Imagick $image): array
{
if ($image->getImageColorspace() !== Imagick::COLORSPACE_SRGB) {
$image->transformImageColorspace(Imagick::COLORSPACE_SRGB);
}
$width = $image->getImageWidth();
$height = $image->getImageHeight();
$values = $image->exportImagePixels(0, 0, $width, $height, 'RGB', Imagick::PIXEL_CHAR);
$pixels = [];
$i = 0;
for ($y = 0; $y < $height; $y++) {
$row = [];
for ($x = 0; $x < $width; $x++) {
$row[] = [$values[$i], $values[$i + 1], $values[$i + 2]];
$i += 3;
} }
$pixels[] = $row;
} }
return $pixels;
}
protected static function pixelsFromGd(GdImage $image): array
{
$width = imagesx($image);
$height = imagesy($image);
$pixels = []; $pixels = [];
for ($y = 0; $y < $height; $y++) { for ($y = 0; $y < $height; $y++) {
$row = []; $row = [];
for ($x = 0; $x < $width; $x++) { for ($x = 0; $x < $width; $x++) {
$index = imagecolorat($image, $x, $y); $colors = imagecolorsforindex($image, imagecolorat($image, $x, $y));
$colors = imagecolorsforindex($image, $index);
$row[] = [$colors['red'], $colors['green'], $colors['blue']]; $row[] = [$colors['red'], $colors['green'], $colors['blue']];
} }
$pixels[] = $row; $pixels[] = $row;
} }
imagedestroy($image); return $pixels;
}
$components_x = 4; protected static function pixelsFromBytes(string $bytes, int $width, int $height, int $bands): array
$components_y = 4; {
$blurhash = BlurhashEngine::encode($pixels, $components_x, $components_y); if ($width < 1 || $height < 1 || strlen($bytes) < $width * $height * $bands) {
if (strlen($blurhash) > 191) { throw new RuntimeException('Unexpected pixel buffer size for blurhash');
return self::DEFAULT_HASH; }
$pixels = [];
$i = 0;
for ($y = 0; $y < $height; $y++) {
$row = [];
for ($x = 0; $x < $width; $x++) {
$r = ord($bytes[$i]);
$row[] = $bands >= 3
? [$r, ord($bytes[$i + 1]), ord($bytes[$i + 2])]
: [$r, $r, $r];
$i += $bands;
}
$pixels[] = $row;
} }
return $blurhash; return $pixels;
} }
} }

@ -297,7 +297,7 @@ class Image
$media->save(); $media->save();
if ($thumbnail) { if ($thumbnail) {
$this->generateBlurhash($media); $this->generateBlurhash($media, $encoded->toString());
} }
if ($media->status_id) { if ($media->status_id) {
@ -354,27 +354,28 @@ class Image
return ['path' => $basePath, 'png' => false]; return ['path' => $basePath, 'png' => false];
} }
protected function generateBlurhash($media) /**
* Hash the thumbnail that was just encoded. The bytes are already in memory,
* so they are hashed directly instead of being read back from disk (or pulled
* back down from cloud storage into a temp file) a moment after being written.
*/
protected function generateBlurhash($media, ?string $contents = null)
{ {
try { try {
if ($this->defaultDisk === 'local') { if ($contents === null) {
$thumbnailPath = storage_path('app/'.$media->thumbnail_path); $contents = $this->defaultDisk === 'local'
$blurhash = Blurhash::generate($media, $thumbnailPath); ? file_get_contents(storage_path('app/'.$media->thumbnail_path))
} else { : Storage::disk($this->defaultDisk)->get($media->thumbnail_path);
$tempFile = tempnam(sys_get_temp_dir(), 'blurhash_');
$contents = Storage::disk($this->defaultDisk)->get($media->thumbnail_path);
file_put_contents($tempFile, $contents);
$blurhash = Blurhash::generate($media, $tempFile);
unlink($tempFile);
} }
if ($blurhash) { $blurhash = $contents ? Blurhash::fromBinary($contents) : null;
$media->blurhash = $blurhash;
$media->save(); $media->blurhash = $blurhash ?? Blurhash::DEFAULT_HASH;
} $media->save();
} catch (\Exception $e) { } catch (\Throwable $e) {
// \Throwable, not \Exception: the blurhash is decorative and must never
// fail the thumbnail job, which still has to mark the media as processed
// and dispatch ImageUpdate.
if (config('app.dev_log')) { if (config('app.dev_log')) {
Log::info('Blurhash generation failed: '.$e->getMessage()); Log::info('Blurhash generation failed: '.$e->getMessage());
} }

@ -9,10 +9,12 @@ class ImageDriverManager
{ {
/** /**
* Get the appropriate image driver class based on configuration. * Get the appropriate image driver class based on configuration.
*
* @param string|null $driver Driver name to resolve instead of the configured one
*/ */
public static function getDriverClass(): string public static function getDriverClass(?string $driver = null): string
{ {
return match (config('image.driver')) { return match ($driver ?? config('image.driver')) {
'gd' => Driver::class, 'gd' => Driver::class,
'imagick' => \Intervention\Image\Drivers\Imagick\Driver::class, 'imagick' => \Intervention\Image\Drivers\Imagick\Driver::class,
'vips' => \Intervention\Image\Drivers\Vips\Driver::class, 'vips' => \Intervention\Image\Drivers\Vips\Driver::class,
@ -24,15 +26,16 @@ class ImageDriverManager
* Create a new ImageManager instance with the configured driver. * Create a new ImageManager instance with the configured driver.
* *
* @param array $options Additional options for ImageManager * @param array $options Additional options for ImageManager
* @param string|null $driver Driver name to use instead of the configured one
*/ */
public static function createImageManager(array $options = []): ImageManager public static function createImageManager(array $options = [], ?string $driver = null): ImageManager
{ {
$configOptions = config('image.options', []); $configOptions = config('image.options', []);
$options = array_merge($configOptions, $options); $options = array_merge($configOptions, $options);
return new ImageManager( return new ImageManager(
self::getDriverClass(), self::getDriverClass($driver),
autoOrientation: (bool) ($options['autoOrientation'] ?? true), autoOrientation: (bool) ($options['autoOrientation'] ?? true),
decodeAnimation: (bool) ($options['decodeAnimation'] ?? true), decodeAnimation: (bool) ($options['decodeAnimation'] ?? true),
backgroundColor: (string) ($options['backgroundColor'] ?? 'ffffff'), backgroundColor: (string) ($options['backgroundColor'] ?? 'ffffff'),

@ -0,0 +1,187 @@
<?php
use App\Models\Media;
use App\Models\User;
use App\Util\Blurhash\Base83;
use App\Util\Blurhash\Color;
use App\Util\Media\Blurhash;
use App\Util\Media\Image;
use App\Util\Media\ImageDriverManager;
use Illuminate\Foundation\Testing\LazilyRefreshDatabase;
use Illuminate\Support\Facades\Config;
use Illuminate\Support\Facades\Storage;
use Intervention\Image\Encoders\PngEncoder;
use Intervention\Image\Encoders\WebpEncoder;
uses(LazilyRefreshDatabase::class);
/*
|--------------------------------------------------------------------------
| Blurhash generation
|--------------------------------------------------------------------------
|
| The hasher used to call GD directly no matter which image driver was
| configured. Under vips that meant dark placeholders for transparent PNGs
| (libvips leaves black behind transparent pixels, the hasher ignored alpha),
| the default hash for every webp, and an uncaught \Error that failed the
| thumbnail job on hosts without ext-gd. It now decodes through the
| configured driver and flattens transparency before sampling.
|
*/
beforeEach(function () {
try {
ImageDriverManager::createImageManager();
} catch (Throwable $e) {
test()->markTestSkipped('Image driver "'.config('image.driver').'" is not available: '.$e->getMessage());
}
});
function blurhashFixture(int $width, int $height, ?string $fill = null, string $format = 'png', ?string $driver = null): string
{
$image = ImageDriverManager::createImageManager([], $driver)->createImage($width, $height);
if ($fill !== null) {
$image = $image->fill($fill);
}
$encoder = $format === 'webp' ? new WebpEncoder(90) : new PngEncoder;
return $image->encode($encoder)->toString();
}
function blurhashAverageColor(string $hash): array
{
$value = Base83::decode(substr($hash, 2, 4));
return [$value >> 16, ($value >> 8) & 255, $value & 255];
}
it('hashes a solid colour image to that colour', function () {
$hash = Blurhash::fromBinary(blurhashFixture(64, 48, '3366cc'));
expect($hash)->toBeString()->toHaveLength(36);
[$r, $g, $b] = blurhashAverageColor($hash);
expect($r)->toBeBetween(48, 54);
expect($g)->toBeBetween(99, 105);
expect($b)->toBeBetween(201, 207);
});
it('composites transparent areas onto the background instead of hashing them as black', function () {
$hash = Blurhash::fromBinary(blurhashFixture(64, 48));
foreach (blurhashAverageColor($hash) as $channel) {
expect($channel)->toBeGreaterThanOrEqual(250);
}
});
it('does not overflow a fully saturated channel into its neighbour', function () {
expect(Color::tosRGB(1.0))->toBe(255);
$hash = Blurhash::fromBinary(blurhashFixture(32, 32, 'ffffff'));
expect(blurhashAverageColor($hash))->toBe([255, 255, 255]);
});
it('hashes webp thumbnails instead of returning the default hash', function () {
try {
$webp = blurhashFixture(64, 64, 'cc3333', 'webp');
} catch (Throwable $e) {
test()->markTestSkipped('Configured image driver cannot encode webp: '.$e->getMessage());
}
$file = tempnam(sys_get_temp_dir(), 'blurhash_test_');
file_put_contents($file, $webp);
$media = new Media;
$media->mime = 'image/webp';
$media->thumbnail_path = 'public/m/photo_thumb.webp';
try {
$hash = Blurhash::generate($media, $file);
} finally {
@unlink($file);
}
expect($hash)->not->toBe(Blurhash::DEFAULT_HASH);
expect(blurhashAverageColor($hash)[0])->toBeGreaterThan(190);
});
it('returns the default hash for bytes that cannot be decoded', function () {
expect(Blurhash::fromBinary('definitely not an image'))->toBeNull();
$file = tempnam(sys_get_temp_dir(), 'blurhash_test_');
file_put_contents($file, 'definitely not an image');
$media = new Media;
$media->mime = 'image/jpeg';
$media->thumbnail_path = 'public/m/photo_thumb.jpg';
try {
expect(Blurhash::generate($media, $file))->toBe(Blurhash::DEFAULT_HASH);
} finally {
@unlink($file);
}
});
it('falls back to gd when the configured driver is unavailable', function () {
if (! function_exists('imagecreatefromstring')) {
test()->markTestSkipped('ext-gd is not installed.');
}
$unavailable = collect(['imagick', 'vips'])->first(function ($driver) {
try {
ImageDriverManager::createImageManager([], $driver);
return false;
} catch (Throwable $e) {
return true;
}
});
if (! $unavailable) {
test()->markTestSkipped('Every image driver is available, nothing to fall back from.');
}
$png = blurhashFixture(64, 48, '3366cc', 'png', 'gd');
Config::set('image.driver', $unavailable);
expect(Blurhash::fromBinary($png))->toBeString()->toHaveLength(36);
});
it('stores a blurhash when a thumbnail is generated', function () {
Config::set('filesystems.default', 's3');
Config::set('pixelfed.optimize_image', false);
Storage::fake('s3', ['url' => 'https://cdn.test']);
$user = User::factory()->create();
$user->refresh();
$pid = $user->profile->id;
$mediaPath = 'public/m/_v2/'.$pid.'/ee/ff/photo.png';
$pngBytes = blurhashFixture(800, 600, '3366cc');
Storage::disk('s3')->put($mediaPath, $pngBytes);
$media = Media::create([
'profile_id' => $pid,
'user_id' => $user->id,
'media_path' => $mediaPath,
'mime' => 'image/png',
'size' => strlen($pngBytes),
'remote_media' => false,
'order' => 0,
]);
(new Image)->resizeThumbnail($media);
$media->refresh();
expect($media->thumbnail_path)->toBe('public/m/_v2/'.$pid.'/ee/ff/photo_thumb.png');
expect($media->blurhash)->toBeString()->not->toBe(Blurhash::DEFAULT_HASH);
[$r, $g, $b] = blurhashAverageColor($media->blurhash);
expect($r)->toBeBetween(48, 54);
expect($g)->toBeBetween(99, 105);
expect($b)->toBeBetween(201, 207);
});
Loading…
Cancel
Save