aiden
/
pixelfed
mirror of https://github.com/pixelfed/pixelfed
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update media storage pipeline, improve support for non-local filesystems

Browse Source
pull/6178/head
Daniel Supernault 1 year ago
parent 0e2685ceff
commit 2e719bd008
No known key found for this signature in database
GPG Key ID: 23740873EE6F76A1
7 changed files with 324 additions and 124 deletions
Show Stats Download Patch File Download Diff File

35
app/Jobs/ImageOptimizePipeline/ImageOptimize.php
Unescape Escape View File

@ -8,6 +8,7 @@ use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Storage;
class ImageOptimize implements ShouldQueue
{
@ -40,20 +41,32 @@ class ImageOptimize implements ShouldQueue
public function handle()
{
$media = $this->media;
if(!$media) {
if (! $media) {
return;
}
$path = storage_path('app/'.$media->media_path);
if (!is_file($path) || $media->skip_optimize) {
return;
$localFs = config('filesystems.default') === 'local';
if ($localFs) {
$path = storage_path('app/'.$media->media_path);
if (! is_file($path) || $media->skip_optimize) {
return;
}
} else {
$disk = Storage::disk(config('filesystems.default'));
if (! $disk->exists($media->media_path) || $media->skip_optimize) {
return;
}
}
if((bool) config_cache('pixelfed.optimize_image') == false) {
ImageThumbnail::dispatch($media)->onQueue('mmo');
return;
} else {
ImageResize::dispatch($media)->onQueue('mmo');
return;
}
if ((bool) config_cache('pixelfed.optimize_image') == false) {
ImageThumbnail::dispatch($media)->onQueue('mmo');
return;
} else {
ImageResize::dispatch($media)->onQueue('mmo');
return;
}
}
}

38
app/Jobs/ImageOptimizePipeline/ImageResize.php
Unescape Escape View File

@ -10,6 +10,7 @@ use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Log;
use Storage;
class ImageResize implements ShouldQueue
{
@ -23,7 +24,7 @@ class ImageResize implements ShouldQueue
* @var bool
*/
public $deleteWhenMissingModels = true;
/**
* Create a new job instance.
*
@ -42,24 +43,37 @@ class ImageResize implements ShouldQueue
public function handle()
{
$media = $this->media;
if(!$media) {
if (! $media) {
return;
}
$path = storage_path('app/'.$media->media_path);
if (!is_file($path) || $media->skip_optimize) {
Log::info('Tried to optimize media that does not exist or is not readable. ' . $path);
return;
$localFs = config('filesystems.default') === 'local';
if ($localFs) {
$path = storage_path('app/'.$media->media_path);
if (! is_file($path) || $media->skip_optimize) {
return;
}
} else {
$disk = Storage::disk(config('filesystems.default'));
if (! $disk->exists($media->media_path) || $media->skip_optimize) {
return;
}
}
if((bool) config_cache('pixelfed.optimize_image') === false) {
ImageThumbnail::dispatch($media)->onQueue('mmo');
return;
if ((bool) config_cache('pixelfed.optimize_image') === false) {
ImageThumbnail::dispatch($media)->onQueue('mmo');
return;
}
try {
$img = new Image();
$img = new Image;
$img->resizeImage($media);
} catch (Exception $e) {
Log::error($e);
} catch (\Exception $e) {
if (config('app.dev_log')) {
Log::error('Image resize failed: '.$e->getMessage());
}
}
ImageThumbnail::dispatch($media)->onQueue('mmo');

29
app/Jobs/ImageOptimizePipeline/ImageThumbnail.php
Unescape Escape View File

@ -10,6 +10,8 @@ use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Log;
use Storage;
class ImageThumbnail implements ShouldQueue
{
@ -23,7 +25,7 @@ class ImageThumbnail implements ShouldQueue
* @var bool
*/
public $deleteWhenMissingModels = true;
/**
* Create a new job instance.
*
@ -42,18 +44,31 @@ class ImageThumbnail implements ShouldQueue
public function handle()
{
$media = $this->media;
if(!$media) {
if (! $media) {
return;
}
$path = storage_path('app/'.$media->media_path);
if (!is_file($path)) {
return;
$localFs = config('filesystems.default') === 'local';
if ($localFs) {
$path = storage_path('app/'.$media->media_path);
if (! is_file($path)) {
return;
}
} else {
$disk = Storage::disk(config('filesystems.default'));
if (! $disk->exists($media->media_path)) {
return;
}
}
try {
$img = new Image();
$img = new Image;
$img->resizeThumbnail($media);
} catch (Exception $e) {
} catch (\Exception $e) {
if (config('app.dev_log')) {
Log::error('Thumbnail generation failed: '.$e->getMessage());
}
}
$media->processed_at = Carbon::now();

133
app/Jobs/ImageOptimizePipeline/ImageUpdate.php
Unescape Escape View File

@ -2,17 +2,17 @@
namespace App\Jobs\ImageOptimizePipeline;
use Storage;
use App\Jobs\MediaPipeline\MediaStoragePipeline;
use App\Media;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Str;
use ImageOptimizer;
use Illuminate\Http\File;
use App\Services\MediaPathService;
use App\Jobs\MediaPipeline\MediaStoragePipeline;
use Log;
use Storage;
class ImageUpdate implements ShouldQueue
{
@ -24,7 +24,7 @@ class ImageUpdate implements ShouldQueue
'image/jpeg',
'image/png',
'image/webp',
'image/avif'
'image/avif',
];
/**
@ -52,35 +52,124 @@ class ImageUpdate implements ShouldQueue
public function handle()
{
$media = $this->media;
if(!$media) {
if (! $media) {
return;
}
$path = storage_path('app/'.$media->media_path);
$thumb = storage_path('app/'.$media->thumbnail_path);
if (!is_file($path)) {
return;
$disk = Storage::disk(config('filesystems.default'));
$localFs = config('filesystems.default') === 'local';
$mediaPath = $media->media_path;
$fileSize = 0;
if ($localFs) {
$path = storage_path('app/'.$media->media_path);
$thumbPath = storage_path('app/'.$media->thumbnail_path);
if (! is_file($path)) {
return;
}
$mediaPath = $path;
} else {
if (! $disk->exists($media->media_path)) {
return;
}
}
if((bool) config_cache('pixelfed.optimize_image')) {
if ((bool) config_cache('pixelfed.optimize_image') && $localFs) {
if (in_array($media->mime, $this->protectedMimes) == true) {
ImageOptimizer::optimize($thumb);
if(!$media->skip_optimize) {
ImageOptimizer::optimize($path);
try {
$thumbPath = storage_path('app/'.$media->thumbnail_path);
if (file_exists($thumbPath)) {
ImageOptimizer::optimize($thumbPath);
}
if (! $media->skip_optimize) {
$mediaPath = storage_path('app/'.$media->media_path);
ImageOptimizer::optimize($mediaPath);
}
} catch (\Exception $e) {
if (config('app.dev_log')) {
Log::error('Image optimization failed: '.$e->getMessage());
}
}
}
} elseif ((bool) config_cache('pixelfed.optimize_image') && ! $localFs) {
if (in_array($media->mime, $this->protectedMimes) == true) {
$this->optimizeRemoteImages($media, $disk);
}
}
if (!is_file($path) || !is_file($thumb)) {
return;
try {
$photo_size = $this->getFileSize($media->media_path);
$thumb_size = $media->thumbnail_path ? $this->getFileSize($media->thumbnail_path) : 0;
$total = ($photo_size + $thumb_size);
$media->size = $total;
$media->save();
} catch (\Exception $e) {
if (config('app.dev_log')) {
Log::error('Failed to calculate media sizes: '.$e->getMessage());
}
}
$photo_size = filesize($path);
$thumb_size = filesize($thumb);
$total = ($photo_size + $thumb_size);
$media->size = $total;
$media->save();
MediaStoragePipeline::dispatch($media);
}
protected function getFileSize($path)
{
$disk = Storage::disk(config('filesystems.default'));
$localFs = config('filesystems.default') === 'local';
if (! $path || empty($path)) {
return 0;
}
if ($localFs) {
return filesize(storage_path('app/'.$path)) ?? 0;
} else {
return $disk->size($path) ?? 0;
}
}
/**
* Optimize images stored on remote storage (S3, etc)
*/
protected function optimizeRemoteImages($media, $disk)
{
try {
$tempDir = sys_get_temp_dir().'/pixelfed_optimize_'.Str::random(18);
mkdir($tempDir, 0755, true);
if ($media->thumbnail_path) {
$tempThumb = $tempDir.'/thumb_'.basename($media->thumbnail_path);
$thumbContents = $disk->get($media->thumbnail_path);
file_put_contents($tempThumb, $thumbContents);
ImageOptimizer::optimize($tempThumb);
$disk->put($media->thumbnail_path, file_get_contents($tempThumb));
unlink($tempThumb);
}
if (! $media->skip_optimize) {
$tempMedia = $tempDir.'/media_'.basename($media->media_path);
$mediaContents = $disk->get($media->media_path);
file_put_contents($tempMedia, $mediaContents);
ImageOptimizer::optimize($tempMedia);
$disk->put($media->media_path, file_get_contents($tempMedia));
unlink($tempMedia);
}
rmdir($tempDir);
} catch (\Exception $e) {
if (isset($tempDir) && is_dir($tempDir)) {
array_map('unlink', glob($tempDir.'/*'));
rmdir($tempDir);
}
if (config('app.dev_log')) {
Log::error('Remote image optimization failed: '.$e->getMessage());
}
}
}
}

5
app/Services/MediaStorageService.php
Unescape Escape View File

@ -18,10 +18,9 @@ class MediaStorageService
{
public static function store(Media $media)
{
if ((bool) config_cache('pixelfed.cloud_storage') == true) {
if ((bool) config_cache('pixelfed.cloud_storage') == true && config('filesystems.default') === 'local') {
(new self)->cloudStore($media);
}
}
public static function move(Media $media)
@ -30,7 +29,7 @@ class MediaStorageService
return;
}
if ((bool) config_cache('pixelfed.cloud_storage') == true) {
if ((bool) config_cache('pixelfed.cloud_storage') == true && config('filesystems.default') === 'local') {
return (new self)->cloudMove($media);
}

31
app/Util/Media/Blurhash.php
Unescape Escape View File

@ -2,40 +2,45 @@
namespace App\Util\Media;
use App\Util\Blurhash\Blurhash as BlurhashEngine;
use App\Media;
use App\Util\Blurhash\Blurhash as BlurhashEngine;
class Blurhash {
class Blurhash
{
const DEFAULT_HASH = 'U4Rfzst8?bt7ogayj[j[~pfQ9Goe%Mj[WBay';
public static function generate(Media $media)
public static function generate(Media $media, $path = false)
{
if(!in_array($media->mime, ['image/png', 'image/jpeg', 'image/jpg', 'video/mp4'])) {
if (! in_array($media->mime, ['image/png', 'image/jpeg', 'image/jpg', 'video/mp4'])) {
return self::DEFAULT_HASH;
}
if($media->thumbnail_path == null) {
if ($media->thumbnail_path == null) {
return self::DEFAULT_HASH;
}
$file = storage_path('app/' . $media->thumbnail_path);
if ($path) {
$file = $path;
} else {
$localFs = config('filesystems.default') === 'local';
$file = storage_path('app/'.$media->thumbnail_path);
}
if(!is_file($file)) {
if (! is_file($file)) {
return self::DEFAULT_HASH;
}
$image = imagecreatefromstring(file_get_contents($file));
if(!$image) {
if (! $image) {
return self::DEFAULT_HASH;
}
$width = imagesx($image);
$height = imagesy($image);
$pixels = [];
for ($y = 0; $y < $height; ++$y) {
for ($y = 0; $y < $height; $y++) {
$row = [];
for ($x = 0; $x < $width; ++$x) {
for ($x = 0; $x < $width; $x++) {
$index = imagecolorat($image, $x, $y);
$colors = imagecolorsforindex($image, $index);
@ -49,10 +54,10 @@ class Blurhash {
$components_x = 4;
$components_y = 4;
$blurhash = BlurhashEngine::encode($pixels, $components_x, $components_y);
if(strlen($blurhash) > 191) {
if (strlen($blurhash) > 191) {
return self::DEFAULT_HASH;
}
return $blurhash;
}
}

177
app/Util/Media/Image.php
Unescape Escape View File

@ -3,22 +3,27 @@
namespace App\Util\Media;
use App\Media;
use Intervention\Image\ImageManager;
use App\Services\StatusService;
use Cache;
use Intervention\Image\Encoders\JpegEncoder;
use Intervention\Image\Encoders\WebpEncoder;
use Intervention\Image\Encoders\AvifEncoder;
use Intervention\Image\Encoders\PngEncoder;
use Cache, Log, Storage;
use App\Util\Media\Blurhash;
use App\Services\StatusService;
use Intervention\Image\Encoders\WebpEncoder;
use Intervention\Image\ImageManager;
use Log;
use Storage;
class Image
{
public $square;
public $landscape;
public $portrait;
public $thumbnail;
public $orientation;
public $acceptedMimes = [
'image/png',
'image/jpeg',
@ -30,6 +35,8 @@ class Image
protected $imageManager;
protected $defaultDisk;
public function __construct()
{
ini_set('memory_limit', config('pixelfed.memory_limit', '1024M'));
@ -38,12 +45,14 @@ class Image
$this->landscape = $this->orientations()['landscape'];
$this->portrait = $this->orientations()['portrait'];
$this->thumbnail = [
'width' => 640,
'width' => 640,
'height' => 640,
];
$this->orientation = null;
$driver = match(config('image.driver')) {
$this->defaultDisk = config('filesystems.default');
$driver = match (config('image.driver')) {
'imagick' => \Intervention\Image\Drivers\Imagick\Driver::class,
'vips' => \Intervention\Image\Drivers\Vips\Driver::class,
default => \Intervention\Image\Drivers\Gd\Driver::class
@ -62,15 +71,15 @@ class Image
{
return [
'square' => [
'width' => 1080,
'width' => 1080,
'height' => 1080,
],
'landscape' => [
'width' => 1920,
'width' => 1920,
'height' => 1080,
],
'portrait' => [
'width' => 1080,
'width' => 1080,
'height' => 1350,
],
];
@ -80,7 +89,7 @@ class Image
{
if ($isThumbnail) {
return [
'dimensions' => $this->thumbnail,
'dimensions' => $this->thumbnail,
'orientation' => 'thumbnail',
];
}
@ -91,7 +100,7 @@ class Image
$this->orientation = $orientation;
return [
'dimensions' => $this->orientations()[$orientation],
'dimensions' => $this->orientations()[$orientation],
'orientation' => $orientation,
'width_original' => $width,
'height_original' => $height,
@ -121,48 +130,68 @@ class Image
public function handleImageTransform(Media $media, $thumbnail = false)
{
$path = $media->media_path;
$file = storage_path('app/'.$path);
if (!in_array($media->mime, $this->acceptedMimes)) {
$localFs = config('filesystems.default') === 'local';
if (! in_array($media->mime, $this->acceptedMimes)) {
return;
}
try {
$fileInfo = pathinfo($file);
$fileContents = null;
$tempFile = null;
if ($this->defaultDisk === 'local') {
$filePath = storage_path('app/'.$path);
$fileContents = file_get_contents($filePath);
} else {
$fileContents = Storage::disk($this->defaultDisk)->get($path);
}
$fileInfo = pathinfo($path);
$extension = strtolower($fileInfo['extension'] ?? 'jpg');
$outputExtension = $extension;
$metadata = null;
if (!$thumbnail && config('media.exif.database', false) == true) {
if (! $thumbnail && config('media.exif.database', false) == true) {
try {
$exif = @exif_read_data($file);
if ($this->defaultDisk !== 'local') {
$tempFile = tempnam(sys_get_temp_dir(), 'exif_');
file_put_contents($tempFile, $fileContents);
$exifPath = $tempFile;
} else {
$exifPath = storage_path('app/'.$path);
}
$exif = @exif_read_data($exifPath);
if ($exif) {
$meta = [];
$keys = [
"FileName",
"FileSize",
"FileType",
"Make",
"Model",
"MimeType",
"ColorSpace",
"ExifVersion",
"Orientation",
"UserComment",
"XResolution",
"YResolution",
"FileDateTime",
"SectionsFound",
"ExifImageWidth",
"ResolutionUnit",
"ExifImageLength",
"FlashPixVersion",
"Exif_IFD_Pointer",
"YCbCrPositioning",
"ComponentsConfiguration",
"ExposureTime",
"FNumber",
"ISOSpeedRatings",
"ShutterSpeedValue"
'FileName',
'FileSize',
'FileType',
'Make',
'Model',
'MimeType',
'ColorSpace',
'ExifVersion',
'Orientation',
'UserComment',
'XResolution',
'YResolution',
'FileDateTime',
'SectionsFound',
'ExifImageWidth',
'ResolutionUnit',
'ExifImageLength',
'FlashPixVersion',
'Exif_IFD_Pointer',
'YCbCrPositioning',
'ComponentsConfiguration',
'ExposureTime',
'FNumber',
'ISOSpeedRatings',
'ShutterSpeedValue',
];
foreach ($exif as $k => $v) {
if (in_array($k, $keys)) {
@ -171,12 +200,22 @@ class Image
}
$media->metadata = json_encode($meta);
}
if ($tempFile && file_exists($tempFile)) {
unlink($tempFile);
$tempFile = null;
}
} catch (\Exception $e) {
Log::info('EXIF extraction failed: ' . $e->getMessage());
if ($tempFile && file_exists($tempFile)) {
unlink($tempFile);
}
if (config('app.dev_log')) {
Log::info('EXIF extraction failed: '.$e->getMessage());
}
}
}
$img = $this->imageManager->read($file);
$img = $this->imageManager->read($fileContents);
$ratio = $this->getAspect($img->width(), $img->height(), $thumbnail);
$aspect = $ratio['dimensions'];
@ -209,7 +248,7 @@ class Image
$outputExtension = 'jpg';
break;
case 'png':
$encoder = new PngEncoder();
$encoder = new PngEncoder;
$outputExtension = 'png';
break;
case 'webp':
@ -230,11 +269,17 @@ class Image
}
$converted = $this->setBaseName($path, $thumbnail, $outputExtension);
$newPath = storage_path('app/'.$converted['path']);
$encoded = $encoder->encode($img);
file_put_contents($newPath, $encoded->toString());
if ($localFs) {
$newPath = storage_path('app/'.$converted['path']);
file_put_contents($newPath, $encoded->toString());
} else {
Storage::disk($this->defaultDisk)->put(
$converted['path'],
$encoded->toString()
);
}
if ($thumbnail == true) {
$media->thumbnail_path = $converted['path'];
@ -244,7 +289,7 @@ class Image
$media->height = $img->height();
$media->orientation = $orientation;
$media->media_path = $converted['path'];
$media->mime = 'image/' . $outputExtension;
$media->mime = 'image/'.$outputExtension;
}
$media->save();
@ -253,7 +298,7 @@ class Image
$this->generateBlurhash($media);
}
if($media->status_id) {
if ($media->status_id) {
Cache::forget('status:transformer:media:attachments:'.$media->status_id);
Cache::forget('status:thumb:'.$media->status_id);
StatusService::del($media->status_id);
@ -262,7 +307,9 @@ class Image
} catch (\Exception $e) {
$media->processed_at = now();
$media->save();
Log::info('MediaResizeException: ' . $e->getMessage() . ' | Could not process media id: ' . $media->id);
if (config('app.dev_log')) {
Log::info('MediaResizeException: '.$e->getMessage().' | Could not process media id: '.$media->id);
}
}
}
@ -277,10 +324,28 @@ class Image
protected function generateBlurhash($media)
{
$blurhash = Blurhash::generate($media);
if ($blurhash) {
$media->blurhash = $blurhash;
$media->save();
try {
if ($this->defaultDisk === 'local') {
$thumbnailPath = storage_path('app/'.$media->thumbnail_path);
$blurhash = Blurhash::generate($media, $thumbnailPath);
} else {
$tempFile = tempnam(sys_get_temp_dir(), 'blurhash_');
$contents = Storage::disk($this->defaultDisk)->get($media->thumbnail_path);
file_put_contents($tempFile, $contents);
$blurhash = Blurhash::generate($media, $tempFile);
unlink($tempFile);
}
if ($blurhash) {
$media->blurhash = $blurhash;
$media->save();
}
} catch (\Exception $e) {
if (config('app.dev_log')) {
Log::info('Blurhash generation failed: '.$e->getMessage());
}
}
}
}

Write Preview
Loading…
Cancel
Save