mirror of https://github.com/pixelfed/pixelfed
daniel
1 year ago
committed by
GitHub
commit
4050055e5e
@ -0,0 +1,54 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use Illuminate\Console\Command;
|
||||
use App\Models\ImportPost;
|
||||
use App\Jobs\ImportPipeline\ImportMediaToCloudPipeline;
|
||||
use function Laravel\Prompts\progress;
|
||||
|
||||
class ImportUploadMediaToCloudStorage extends Command
|
||||
{
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'app:import-upload-media-to-cloud-storage {--limit=500}';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Migrate media imported from Instagram to S3 cloud storage.';
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
*/
|
||||
public function handle()
|
||||
{
|
||||
if(
|
||||
(bool) config('import.instagram.storage.cloud.enabled') === false ||
|
||||
(bool) config_cache('pixelfed.cloud_storage') === false
|
||||
) {
|
||||
$this->error('Aborted. Cloud storage is not enabled for IG imports.');
|
||||
return;
|
||||
}
|
||||
|
||||
$limit = $this->option('limit');
|
||||
|
||||
$progress = progress(label: 'Migrating import media', steps: $limit);
|
||||
|
||||
$progress->start();
|
||||
|
||||
$posts = ImportPost::whereUploadedToS3(false)->take($limit)->get();
|
||||
|
||||
foreach($posts as $post) {
|
||||
ImportMediaToCloudPipeline::dispatch($post)->onQueue('low');
|
||||
$progress->advance();
|
||||
}
|
||||
|
||||
$progress->finish();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,129 @@
|
||||
<?php
|
||||
|
||||
namespace App\Jobs\ImportPipeline;
|
||||
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldBeUnique;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Queue\Middleware\WithoutOverlapping;
|
||||
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
|
||||
use App\Models\ImportPost;
|
||||
use App\Media;
|
||||
use App\Services\MediaStorageService;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use App\Jobs\VideoPipeline\VideoThumbnailToCloudPipeline;
|
||||
|
||||
class ImportMediaToCloudPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
protected $importPost;
|
||||
|
||||
public $timeout = 900;
|
||||
public $tries = 3;
|
||||
public $maxExceptions = 1;
|
||||
public $failOnTimeout = true;
|
||||
|
||||
/**
|
||||
* The number of seconds after which the job's unique lock will be released.
|
||||
*
|
||||
* @var int
|
||||
*/
|
||||
public $uniqueFor = 3600;
|
||||
|
||||
/**
|
||||
* Get the unique ID for the job.
|
||||
*/
|
||||
public function uniqueId(): string
|
||||
{
|
||||
return 'import-media-to-cloud-pipeline:ip-id:' . $this->importPost->id;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the middleware the job should pass through.
|
||||
*
|
||||
* @return array<int, object>
|
||||
*/
|
||||
public function middleware(): array
|
||||
{
|
||||
return [(new WithoutOverlapping("import-media-to-cloud-pipeline:ip-id:{$this->importPost->id}"))->shared()->dontRelease()];
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete the job if its models no longer exist.
|
||||
*
|
||||
* @var bool
|
||||
*/
|
||||
public $deleteWhenMissingModels = true;
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
*/
|
||||
public function __construct(ImportPost $importPost)
|
||||
{
|
||||
$this->importPost = $importPost;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the job.
|
||||
*/
|
||||
public function handle(): void
|
||||
{
|
||||
$ip = $this->importPost;
|
||||
|
||||
if(
|
||||
$ip->status_id === null ||
|
||||
$ip->uploaded_to_s3 === true ||
|
||||
(bool) config_cache('pixelfed.cloud_storage') === false) {
|
||||
return;
|
||||
}
|
||||
|
||||
$media = Media::whereStatusId($ip->status_id)->get();
|
||||
|
||||
if(!$media || !$media->count()) {
|
||||
$importPost = ImportPost::find($ip->id);
|
||||
$importPost->uploaded_to_s3 = true;
|
||||
$importPost->save();
|
||||
return;
|
||||
}
|
||||
|
||||
foreach($media as $mediaPart) {
|
||||
$this->handleMedia($mediaPart);
|
||||
}
|
||||
}
|
||||
|
||||
protected function handleMedia($media)
|
||||
{
|
||||
$ip = $this->importPost;
|
||||
|
||||
$importPost = ImportPost::find($ip->id);
|
||||
|
||||
if(!$importPost) {
|
||||
return;
|
||||
}
|
||||
|
||||
$res = MediaStorageService::move($media);
|
||||
|
||||
$importPost->uploaded_to_s3 = true;
|
||||
$importPost->save();
|
||||
|
||||
if(!$res) {
|
||||
return;
|
||||
}
|
||||
|
||||
if($res === 'invalid file') {
|
||||
return;
|
||||
}
|
||||
|
||||
if($res === 'success') {
|
||||
if($media->mime === 'video/mp4') {
|
||||
VideoThumbnailToCloudPipeline::dispatch($media)->onQueue('low');
|
||||
} else {
|
||||
Storage::disk('local')->delete($media->media_path);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,147 @@
|
||||
<?php
|
||||
|
||||
namespace App\Jobs\VideoPipeline;
|
||||
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Queue\Middleware\WithoutOverlapping;
|
||||
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
|
||||
use Illuminate\Http\File;
|
||||
use Cache;
|
||||
use FFMpeg;
|
||||
use Storage;
|
||||
use App\Media;
|
||||
use App\Jobs\MediaPipeline\MediaStoragePipeline;
|
||||
use App\Util\Media\Blurhash;
|
||||
use App\Services\MediaService;
|
||||
use App\Services\StatusService;
|
||||
use App\Services\ResilientMediaStorageService;
|
||||
|
||||
class VideoThumbnailToCloudPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
protected $media;
|
||||
|
||||
public $timeout = 900;
|
||||
public $tries = 3;
|
||||
public $maxExceptions = 1;
|
||||
public $failOnTimeout = true;
|
||||
public $deleteWhenMissingModels = true;
|
||||
|
||||
/**
|
||||
* The number of seconds after which the job's unique lock will be released.
|
||||
*
|
||||
* @var int
|
||||
*/
|
||||
public $uniqueFor = 3600;
|
||||
|
||||
/**
|
||||
* Get the unique ID for the job.
|
||||
*/
|
||||
public function uniqueId(): string
|
||||
{
|
||||
return 'media:video-thumb-to-cloud:id-' . $this->media->id;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the middleware the job should pass through.
|
||||
*
|
||||
* @return array<int, object>
|
||||
*/
|
||||
public function middleware(): array
|
||||
{
|
||||
return [(new WithoutOverlapping("media:video-thumb-to-cloud:id-{$this->media->id}"))->shared()->dontRelease()];
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
*/
|
||||
public function __construct(Media $media)
|
||||
{
|
||||
$this->media = $media;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the job.
|
||||
*/
|
||||
public function handle(): void
|
||||
{
|
||||
if((bool) config_cache('pixelfed.cloud_storage') === false) {
|
||||
return;
|
||||
}
|
||||
|
||||
$media = $this->media;
|
||||
|
||||
if($media->mime != 'video/mp4') {
|
||||
return;
|
||||
}
|
||||
|
||||
if($media->profile_id === null || $media->status_id === null) {
|
||||
return;
|
||||
}
|
||||
|
||||
if($media->thumbnail_url) {
|
||||
return;
|
||||
}
|
||||
|
||||
$base = $media->media_path;
|
||||
$path = explode('/', $base);
|
||||
$name = last($path);
|
||||
|
||||
try {
|
||||
$t = explode('.', $name);
|
||||
$t = $t[0].'_thumb.jpeg';
|
||||
$i = count($path) - 1;
|
||||
$path[$i] = $t;
|
||||
$save = implode('/', $path);
|
||||
$video = FFMpeg::open($base)
|
||||
->getFrameFromSeconds(1)
|
||||
->export()
|
||||
->toDisk('local')
|
||||
->save($save);
|
||||
|
||||
if(!$save) {
|
||||
return;
|
||||
}
|
||||
|
||||
$media->thumbnail_path = $save;
|
||||
$p = explode('/', $media->media_path);
|
||||
array_pop($p);
|
||||
$pt = explode('/', $save);
|
||||
$thumbname = array_pop($pt);
|
||||
$storagePath = implode('/', $p);
|
||||
$thumb = storage_path('app/' . $save);
|
||||
$thumbUrl = ResilientMediaStorageService::store($storagePath, $thumb, $thumbname);
|
||||
$media->thumbnail_url = $thumbUrl;
|
||||
$media->save();
|
||||
|
||||
$blurhash = Blurhash::generate($media);
|
||||
if($blurhash) {
|
||||
$media->blurhash = $blurhash;
|
||||
$media->save();
|
||||
}
|
||||
|
||||
if(str_starts_with($save, 'public/m/_v2/') && str_ends_with($save, '.jpeg')) {
|
||||
Storage::delete($save);
|
||||
}
|
||||
|
||||
if(str_starts_with($media->media_path, 'public/m/_v2/') && str_ends_with($media->media_path, '.mp4')) {
|
||||
Storage::disk('local')->delete($media->media_path);
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
}
|
||||
|
||||
if($media->status_id) {
|
||||
Cache::forget('status:transformer:media:attachments:' . $media->status_id);
|
||||
MediaService::del($media->status_id);
|
||||
Cache::forget('status:thumb:nsfw0' . $media->status_id);
|
||||
Cache::forget('status:thumb:nsfw1' . $media->status_id);
|
||||
Cache::forget('pf:services:sh:id:' . $media->status_id);
|
||||
StatusService::del($media->status_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue