diff --git a/CHANGELOG.md b/CHANGELOG.md index 49b906cfe..20e331ae7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - Update Status storage, add SanitizerService to fix spacing in html stripped content ([3686c9212](https://github.com/pixelfed/pixelfed/commit/3686c9212)) - Update app config, add description and rule env variables ([0980519a9](https://github.com/pixelfed/pixelfed/commit/0980519a9)) - Update InstanceService, fix total post count when config_cache is disabled ([f0bc9d66e](https://github.com/pixelfed/pixelfed/commit/f0bc9d66e)) +- Update media storage pipeline, improve support for non-local filesystems ([2e719bd00](https://github.com/pixelfed/pixelfed/commit/2e719bd00)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.12.6 (2025-09-03)](https://github.com/pixelfed/pixelfed/compare/v0.12.6...dev) diff --git a/app/Jobs/ImageOptimizePipeline/ImageOptimize.php b/app/Jobs/ImageOptimizePipeline/ImageOptimize.php index 28685d1f7..573a87f90 100644 --- a/app/Jobs/ImageOptimizePipeline/ImageOptimize.php +++ b/app/Jobs/ImageOptimizePipeline/ImageOptimize.php @@ -8,6 +8,7 @@ use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; +use Storage; class ImageOptimize implements ShouldQueue { @@ -40,20 +41,32 @@ class ImageOptimize implements ShouldQueue public function handle() { $media = $this->media; - if(!$media) { + if (! $media) { return; } - $path = storage_path('app/'.$media->media_path); - if (!is_file($path) || $media->skip_optimize) { - return; + + $localFs = config('filesystems.default') === 'local'; + + if ($localFs) { + $path = storage_path('app/'.$media->media_path); + if (! is_file($path) || $media->skip_optimize) { + return; + } + } else { + $disk = Storage::disk(config('filesystems.default')); + if (! $disk->exists($media->media_path) || $media->skip_optimize) { + return; + } } - if((bool) config_cache('pixelfed.optimize_image') == false) { - ImageThumbnail::dispatch($media)->onQueue('mmo'); - return; - } else { - ImageResize::dispatch($media)->onQueue('mmo'); - return; - } + if ((bool) config_cache('pixelfed.optimize_image') == false) { + ImageThumbnail::dispatch($media)->onQueue('mmo'); + + return; + } else { + ImageResize::dispatch($media)->onQueue('mmo'); + + return; + } } } diff --git a/app/Jobs/ImageOptimizePipeline/ImageResize.php b/app/Jobs/ImageOptimizePipeline/ImageResize.php index 2aa51a532..d5cd53264 100644 --- a/app/Jobs/ImageOptimizePipeline/ImageResize.php +++ b/app/Jobs/ImageOptimizePipeline/ImageResize.php @@ -10,6 +10,7 @@ use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; use Log; +use Storage; class ImageResize implements ShouldQueue { @@ -23,7 +24,7 @@ class ImageResize implements ShouldQueue * @var bool */ public $deleteWhenMissingModels = true; - + /** * Create a new job instance. * @@ -42,24 +43,37 @@ class ImageResize implements ShouldQueue public function handle() { $media = $this->media; - if(!$media) { + if (! $media) { return; } - $path = storage_path('app/'.$media->media_path); - if (!is_file($path) || $media->skip_optimize) { - Log::info('Tried to optimize media that does not exist or is not readable. ' . $path); - return; + + $localFs = config('filesystems.default') === 'local'; + + if ($localFs) { + $path = storage_path('app/'.$media->media_path); + if (! is_file($path) || $media->skip_optimize) { + return; + } + } else { + $disk = Storage::disk(config('filesystems.default')); + if (! $disk->exists($media->media_path) || $media->skip_optimize) { + return; + } } - if((bool) config_cache('pixelfed.optimize_image') === false) { - ImageThumbnail::dispatch($media)->onQueue('mmo'); - return; + if ((bool) config_cache('pixelfed.optimize_image') === false) { + ImageThumbnail::dispatch($media)->onQueue('mmo'); + + return; } + try { - $img = new Image(); + $img = new Image; $img->resizeImage($media); - } catch (Exception $e) { - Log::error($e); + } catch (\Exception $e) { + if (config('app.dev_log')) { + Log::error('Image resize failed: '.$e->getMessage()); + } } ImageThumbnail::dispatch($media)->onQueue('mmo'); diff --git a/app/Jobs/ImageOptimizePipeline/ImageThumbnail.php b/app/Jobs/ImageOptimizePipeline/ImageThumbnail.php index a96beb331..d8cbf7621 100644 --- a/app/Jobs/ImageOptimizePipeline/ImageThumbnail.php +++ b/app/Jobs/ImageOptimizePipeline/ImageThumbnail.php @@ -10,6 +10,8 @@ use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; +use Log; +use Storage; class ImageThumbnail implements ShouldQueue { @@ -23,7 +25,7 @@ class ImageThumbnail implements ShouldQueue * @var bool */ public $deleteWhenMissingModels = true; - + /** * Create a new job instance. * @@ -42,18 +44,31 @@ class ImageThumbnail implements ShouldQueue public function handle() { $media = $this->media; - if(!$media) { + if (! $media) { return; } - $path = storage_path('app/'.$media->media_path); - if (!is_file($path)) { - return; + + $localFs = config('filesystems.default') === 'local'; + + if ($localFs) { + $path = storage_path('app/'.$media->media_path); + if (! is_file($path)) { + return; + } + } else { + $disk = Storage::disk(config('filesystems.default')); + if (! $disk->exists($media->media_path)) { + return; + } } try { - $img = new Image(); + $img = new Image; $img->resizeThumbnail($media); - } catch (Exception $e) { + } catch (\Exception $e) { + if (config('app.dev_log')) { + Log::error('Thumbnail generation failed: '.$e->getMessage()); + } } $media->processed_at = Carbon::now(); diff --git a/app/Jobs/ImageOptimizePipeline/ImageUpdate.php b/app/Jobs/ImageOptimizePipeline/ImageUpdate.php index e59741eda..2c74f8d6d 100644 --- a/app/Jobs/ImageOptimizePipeline/ImageUpdate.php +++ b/app/Jobs/ImageOptimizePipeline/ImageUpdate.php @@ -2,17 +2,17 @@ namespace App\Jobs\ImageOptimizePipeline; -use Storage; +use App\Jobs\MediaPipeline\MediaStoragePipeline; use App\Media; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; +use Illuminate\Support\Str; use ImageOptimizer; -use Illuminate\Http\File; -use App\Services\MediaPathService; -use App\Jobs\MediaPipeline\MediaStoragePipeline; +use Log; +use Storage; class ImageUpdate implements ShouldQueue { @@ -24,7 +24,7 @@ class ImageUpdate implements ShouldQueue 'image/jpeg', 'image/png', 'image/webp', - 'image/avif' + 'image/avif', ]; /** @@ -52,35 +52,124 @@ class ImageUpdate implements ShouldQueue public function handle() { $media = $this->media; - if(!$media) { + if (! $media) { return; } - $path = storage_path('app/'.$media->media_path); - $thumb = storage_path('app/'.$media->thumbnail_path); - if (!is_file($path)) { - return; + $disk = Storage::disk(config('filesystems.default')); + $localFs = config('filesystems.default') === 'local'; + $mediaPath = $media->media_path; + $fileSize = 0; + + if ($localFs) { + $path = storage_path('app/'.$media->media_path); + $thumbPath = storage_path('app/'.$media->thumbnail_path); + if (! is_file($path)) { + return; + } + $mediaPath = $path; + } else { + if (! $disk->exists($media->media_path)) { + return; + } } - if((bool) config_cache('pixelfed.optimize_image')) { + if ((bool) config_cache('pixelfed.optimize_image') && $localFs) { if (in_array($media->mime, $this->protectedMimes) == true) { - ImageOptimizer::optimize($thumb); - if(!$media->skip_optimize) { - ImageOptimizer::optimize($path); + try { + $thumbPath = storage_path('app/'.$media->thumbnail_path); + if (file_exists($thumbPath)) { + ImageOptimizer::optimize($thumbPath); + } + + if (! $media->skip_optimize) { + $mediaPath = storage_path('app/'.$media->media_path); + ImageOptimizer::optimize($mediaPath); + } + } catch (\Exception $e) { + if (config('app.dev_log')) { + Log::error('Image optimization failed: '.$e->getMessage()); + } } } + } elseif ((bool) config_cache('pixelfed.optimize_image') && ! $localFs) { + if (in_array($media->mime, $this->protectedMimes) == true) { + $this->optimizeRemoteImages($media, $disk); + } } - if (!is_file($path) || !is_file($thumb)) { - return; + try { + $photo_size = $this->getFileSize($media->media_path); + $thumb_size = $media->thumbnail_path ? $this->getFileSize($media->thumbnail_path) : 0; + $total = ($photo_size + $thumb_size); + $media->size = $total; + $media->save(); + } catch (\Exception $e) { + if (config('app.dev_log')) { + Log::error('Failed to calculate media sizes: '.$e->getMessage()); + } } - $photo_size = filesize($path); - $thumb_size = filesize($thumb); - $total = ($photo_size + $thumb_size); - $media->size = $total; - $media->save(); - MediaStoragePipeline::dispatch($media); } + + protected function getFileSize($path) + { + $disk = Storage::disk(config('filesystems.default')); + $localFs = config('filesystems.default') === 'local'; + + if (! $path || empty($path)) { + return 0; + } + + if ($localFs) { + return filesize(storage_path('app/'.$path)) ?? 0; + } else { + return $disk->size($path) ?? 0; + } + } + + /** + * Optimize images stored on remote storage (S3, etc) + */ + protected function optimizeRemoteImages($media, $disk) + { + try { + $tempDir = sys_get_temp_dir().'/pixelfed_optimize_'.Str::random(18); + mkdir($tempDir, 0755, true); + + if ($media->thumbnail_path) { + $tempThumb = $tempDir.'/thumb_'.basename($media->thumbnail_path); + $thumbContents = $disk->get($media->thumbnail_path); + file_put_contents($tempThumb, $thumbContents); + + ImageOptimizer::optimize($tempThumb); + + $disk->put($media->thumbnail_path, file_get_contents($tempThumb)); + unlink($tempThumb); + } + + if (! $media->skip_optimize) { + $tempMedia = $tempDir.'/media_'.basename($media->media_path); + $mediaContents = $disk->get($media->media_path); + file_put_contents($tempMedia, $mediaContents); + + ImageOptimizer::optimize($tempMedia); + + $disk->put($media->media_path, file_get_contents($tempMedia)); + unlink($tempMedia); + } + + rmdir($tempDir); + + } catch (\Exception $e) { + if (isset($tempDir) && is_dir($tempDir)) { + array_map('unlink', glob($tempDir.'/*')); + rmdir($tempDir); + } + if (config('app.dev_log')) { + Log::error('Remote image optimization failed: '.$e->getMessage()); + } + } + } } diff --git a/app/Services/MediaStorageService.php b/app/Services/MediaStorageService.php index 05c0a460c..3b5884811 100644 --- a/app/Services/MediaStorageService.php +++ b/app/Services/MediaStorageService.php @@ -18,10 +18,9 @@ class MediaStorageService { public static function store(Media $media) { - if ((bool) config_cache('pixelfed.cloud_storage') == true) { + if ((bool) config_cache('pixelfed.cloud_storage') == true && config('filesystems.default') === 'local') { (new self)->cloudStore($media); } - } public static function move(Media $media) @@ -30,7 +29,7 @@ class MediaStorageService return; } - if ((bool) config_cache('pixelfed.cloud_storage') == true) { + if ((bool) config_cache('pixelfed.cloud_storage') == true && config('filesystems.default') === 'local') { return (new self)->cloudMove($media); } diff --git a/app/Util/Media/Blurhash.php b/app/Util/Media/Blurhash.php index 037c2d70a..f627550af 100644 --- a/app/Util/Media/Blurhash.php +++ b/app/Util/Media/Blurhash.php @@ -2,40 +2,45 @@ namespace App\Util\Media; -use App\Util\Blurhash\Blurhash as BlurhashEngine; use App\Media; +use App\Util\Blurhash\Blurhash as BlurhashEngine; -class Blurhash { - +class Blurhash +{ const DEFAULT_HASH = 'U4Rfzst8?bt7ogayj[j[~pfQ9Goe%Mj[WBay'; - public static function generate(Media $media) + public static function generate(Media $media, $path = false) { - if(!in_array($media->mime, ['image/png', 'image/jpeg', 'image/jpg', 'video/mp4'])) { + if (! in_array($media->mime, ['image/png', 'image/jpeg', 'image/jpg', 'video/mp4'])) { return self::DEFAULT_HASH; } - if($media->thumbnail_path == null) { + if ($media->thumbnail_path == null) { return self::DEFAULT_HASH; } - $file = storage_path('app/' . $media->thumbnail_path); + if ($path) { + $file = $path; + } else { + $localFs = config('filesystems.default') === 'local'; + $file = storage_path('app/'.$media->thumbnail_path); + } - if(!is_file($file)) { + if (! is_file($file)) { return self::DEFAULT_HASH; } $image = imagecreatefromstring(file_get_contents($file)); - if(!$image) { + if (! $image) { return self::DEFAULT_HASH; } $width = imagesx($image); $height = imagesy($image); $pixels = []; - for ($y = 0; $y < $height; ++$y) { + for ($y = 0; $y < $height; $y++) { $row = []; - for ($x = 0; $x < $width; ++$x) { + for ($x = 0; $x < $width; $x++) { $index = imagecolorat($image, $x, $y); $colors = imagecolorsforindex($image, $index); @@ -49,10 +54,10 @@ class Blurhash { $components_x = 4; $components_y = 4; $blurhash = BlurhashEngine::encode($pixels, $components_x, $components_y); - if(strlen($blurhash) > 191) { + if (strlen($blurhash) > 191) { return self::DEFAULT_HASH; } + return $blurhash; } - } diff --git a/app/Util/Media/Image.php b/app/Util/Media/Image.php index 1bde79f06..e8941c035 100644 --- a/app/Util/Media/Image.php +++ b/app/Util/Media/Image.php @@ -3,22 +3,27 @@ namespace App\Util\Media; use App\Media; -use Intervention\Image\ImageManager; +use App\Services\StatusService; +use Cache; use Intervention\Image\Encoders\JpegEncoder; -use Intervention\Image\Encoders\WebpEncoder; -use Intervention\Image\Encoders\AvifEncoder; use Intervention\Image\Encoders\PngEncoder; -use Cache, Log, Storage; -use App\Util\Media\Blurhash; -use App\Services\StatusService; +use Intervention\Image\Encoders\WebpEncoder; +use Intervention\Image\ImageManager; +use Log; +use Storage; class Image { public $square; + public $landscape; + public $portrait; + public $thumbnail; + public $orientation; + public $acceptedMimes = [ 'image/png', 'image/jpeg', @@ -30,6 +35,8 @@ class Image protected $imageManager; + protected $defaultDisk; + public function __construct() { ini_set('memory_limit', config('pixelfed.memory_limit', '1024M')); @@ -38,12 +45,14 @@ class Image $this->landscape = $this->orientations()['landscape']; $this->portrait = $this->orientations()['portrait']; $this->thumbnail = [ - 'width' => 640, + 'width' => 640, 'height' => 640, ]; $this->orientation = null; - $driver = match(config('image.driver')) { + $this->defaultDisk = config('filesystems.default'); + + $driver = match (config('image.driver')) { 'imagick' => \Intervention\Image\Drivers\Imagick\Driver::class, 'vips' => \Intervention\Image\Drivers\Vips\Driver::class, default => \Intervention\Image\Drivers\Gd\Driver::class @@ -62,15 +71,15 @@ class Image { return [ 'square' => [ - 'width' => 1080, + 'width' => 1080, 'height' => 1080, ], 'landscape' => [ - 'width' => 1920, + 'width' => 1920, 'height' => 1080, ], 'portrait' => [ - 'width' => 1080, + 'width' => 1080, 'height' => 1350, ], ]; @@ -80,7 +89,7 @@ class Image { if ($isThumbnail) { return [ - 'dimensions' => $this->thumbnail, + 'dimensions' => $this->thumbnail, 'orientation' => 'thumbnail', ]; } @@ -91,7 +100,7 @@ class Image $this->orientation = $orientation; return [ - 'dimensions' => $this->orientations()[$orientation], + 'dimensions' => $this->orientations()[$orientation], 'orientation' => $orientation, 'width_original' => $width, 'height_original' => $height, @@ -121,48 +130,68 @@ class Image public function handleImageTransform(Media $media, $thumbnail = false) { $path = $media->media_path; - $file = storage_path('app/'.$path); - if (!in_array($media->mime, $this->acceptedMimes)) { + $localFs = config('filesystems.default') === 'local'; + + if (! in_array($media->mime, $this->acceptedMimes)) { return; } try { - $fileInfo = pathinfo($file); + $fileContents = null; + $tempFile = null; + + if ($this->defaultDisk === 'local') { + $filePath = storage_path('app/'.$path); + $fileContents = file_get_contents($filePath); + } else { + $fileContents = Storage::disk($this->defaultDisk)->get($path); + } + + $fileInfo = pathinfo($path); $extension = strtolower($fileInfo['extension'] ?? 'jpg'); $outputExtension = $extension; $metadata = null; - if (!$thumbnail && config('media.exif.database', false) == true) { + if (! $thumbnail && config('media.exif.database', false) == true) { try { - $exif = @exif_read_data($file); + if ($this->defaultDisk !== 'local') { + $tempFile = tempnam(sys_get_temp_dir(), 'exif_'); + file_put_contents($tempFile, $fileContents); + $exifPath = $tempFile; + } else { + $exifPath = storage_path('app/'.$path); + } + + $exif = @exif_read_data($exifPath); + if ($exif) { $meta = []; $keys = [ - "FileName", - "FileSize", - "FileType", - "Make", - "Model", - "MimeType", - "ColorSpace", - "ExifVersion", - "Orientation", - "UserComment", - "XResolution", - "YResolution", - "FileDateTime", - "SectionsFound", - "ExifImageWidth", - "ResolutionUnit", - "ExifImageLength", - "FlashPixVersion", - "Exif_IFD_Pointer", - "YCbCrPositioning", - "ComponentsConfiguration", - "ExposureTime", - "FNumber", - "ISOSpeedRatings", - "ShutterSpeedValue" + 'FileName', + 'FileSize', + 'FileType', + 'Make', + 'Model', + 'MimeType', + 'ColorSpace', + 'ExifVersion', + 'Orientation', + 'UserComment', + 'XResolution', + 'YResolution', + 'FileDateTime', + 'SectionsFound', + 'ExifImageWidth', + 'ResolutionUnit', + 'ExifImageLength', + 'FlashPixVersion', + 'Exif_IFD_Pointer', + 'YCbCrPositioning', + 'ComponentsConfiguration', + 'ExposureTime', + 'FNumber', + 'ISOSpeedRatings', + 'ShutterSpeedValue', ]; foreach ($exif as $k => $v) { if (in_array($k, $keys)) { @@ -171,12 +200,22 @@ class Image } $media->metadata = json_encode($meta); } + + if ($tempFile && file_exists($tempFile)) { + unlink($tempFile); + $tempFile = null; + } } catch (\Exception $e) { - Log::info('EXIF extraction failed: ' . $e->getMessage()); + if ($tempFile && file_exists($tempFile)) { + unlink($tempFile); + } + if (config('app.dev_log')) { + Log::info('EXIF extraction failed: '.$e->getMessage()); + } } } - $img = $this->imageManager->read($file); + $img = $this->imageManager->read($fileContents); $ratio = $this->getAspect($img->width(), $img->height(), $thumbnail); $aspect = $ratio['dimensions']; @@ -209,7 +248,7 @@ class Image $outputExtension = 'jpg'; break; case 'png': - $encoder = new PngEncoder(); + $encoder = new PngEncoder; $outputExtension = 'png'; break; case 'webp': @@ -230,11 +269,17 @@ class Image } $converted = $this->setBaseName($path, $thumbnail, $outputExtension); - $newPath = storage_path('app/'.$converted['path']); - $encoded = $encoder->encode($img); - file_put_contents($newPath, $encoded->toString()); + if ($localFs) { + $newPath = storage_path('app/'.$converted['path']); + file_put_contents($newPath, $encoded->toString()); + } else { + Storage::disk($this->defaultDisk)->put( + $converted['path'], + $encoded->toString() + ); + } if ($thumbnail == true) { $media->thumbnail_path = $converted['path']; @@ -244,7 +289,7 @@ class Image $media->height = $img->height(); $media->orientation = $orientation; $media->media_path = $converted['path']; - $media->mime = 'image/' . $outputExtension; + $media->mime = 'image/'.$outputExtension; } $media->save(); @@ -253,7 +298,7 @@ class Image $this->generateBlurhash($media); } - if($media->status_id) { + if ($media->status_id) { Cache::forget('status:transformer:media:attachments:'.$media->status_id); Cache::forget('status:thumb:'.$media->status_id); StatusService::del($media->status_id); @@ -262,7 +307,9 @@ class Image } catch (\Exception $e) { $media->processed_at = now(); $media->save(); - Log::info('MediaResizeException: ' . $e->getMessage() . ' | Could not process media id: ' . $media->id); + if (config('app.dev_log')) { + Log::info('MediaResizeException: '.$e->getMessage().' | Could not process media id: '.$media->id); + } } } @@ -277,10 +324,28 @@ class Image protected function generateBlurhash($media) { - $blurhash = Blurhash::generate($media); - if ($blurhash) { - $media->blurhash = $blurhash; - $media->save(); + try { + if ($this->defaultDisk === 'local') { + $thumbnailPath = storage_path('app/'.$media->thumbnail_path); + $blurhash = Blurhash::generate($media, $thumbnailPath); + } else { + $tempFile = tempnam(sys_get_temp_dir(), 'blurhash_'); + $contents = Storage::disk($this->defaultDisk)->get($media->thumbnail_path); + file_put_contents($tempFile, $contents); + + $blurhash = Blurhash::generate($media, $tempFile); + + unlink($tempFile); + } + + if ($blurhash) { + $media->blurhash = $blurhash; + $media->save(); + } + } catch (\Exception $e) { + if (config('app.dev_log')) { + Log::info('Blurhash generation failed: '.$e->getMessage()); + } } } } diff --git a/resources/views/atom/user.blade.php b/resources/views/atom/user.blade.php index badb1f8de..9f15211f3 100644 --- a/resources/views/atom/user.blade.php +++ b/resources/views/atom/user.blade.php @@ -12,6 +12,9 @@ {{$profile['url']}} + {{$profile['avatar']}} + {{$profile['avatar']}} +