mirror of https://github.com/pixelfed/pixelfed
Add media storage migration commands (local<->cloud) with integrated GC
Add admin:MediaMoveStorageLocalToCloud and admin:MediaMoveStorageCloudToLocal: - Copy media (+thumbnail) between local and cloud disks, verify by size (and sha256 against original_sha256 when present) before deleting the source. - Integrated GC: delete the verified source copy (local on upload, cloud on download), set version=4 / reset to 3, and bust MediaService/StatusService caches. --keep-local / --keep-cloud opt out. - Manage PF_ENABLE_CLOUD in .env AND the live runtime + config cache so new uploads route to the correct backend mid-migration on a hot server. Uses the installer's atomic .env writer (shared ManagesMediaStorageEnv trait). - --limit / --dry-run / --force. Replaces media:migrate2cloud (CloudMediaMigrate) and media:s3gc (MediaS3GarbageCollector); scheduler now runs MediaMoveStorageLocalToCloud hourly for straggler upload + GC. Keeps media:fix-nonlocal-driver. Adds feature tests (download+GC, --keep-cloud, dry-run, env-flag flip both directions, unknown-disk guard).pull/6932/head
Your Name
4 weeks ago
parent
34d6fb31f9
commit
6ff9ffbbb8
@ -1,102 +0,0 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Models\Media;
|
||||
use App\Services\MediaStorageService;
|
||||
use App\Util\Lexer\PrettyNumber;
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Symfony\Component\HttpFoundation\File\Exception\FileNotFoundException;
|
||||
use Symfony\Component\HttpKernel\Exception\NotFoundHttpException;
|
||||
|
||||
class CloudMediaMigrate extends Command
|
||||
{
|
||||
public $totalSize = 0;
|
||||
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'media:migrate2cloud {--limit=200} {--huge}';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Move older media to cloud storage';
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
*
|
||||
* @return int
|
||||
*/
|
||||
public function handle()
|
||||
{
|
||||
$enabled = (bool) config_cache('pixelfed.cloud_storage');
|
||||
if (! $enabled) {
|
||||
$this->error('Cloud storage not enabled. Exiting...');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (! $this->confirm('Are you sure you want to proceed?')) {
|
||||
return;
|
||||
}
|
||||
|
||||
$limit = $this->option('limit');
|
||||
$hugeMode = $this->option('huge');
|
||||
|
||||
if ($limit > 500 && ! $hugeMode) {
|
||||
$this->error('Max limit exceeded, use a limit lower than 500 or run again with the --huge flag');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$bar = $this->output->createProgressBar($limit);
|
||||
$bar->start();
|
||||
|
||||
Media::whereNot('version', '4')
|
||||
->where('created_at', '<', now()->subDays(2))
|
||||
->whereRemoteMedia(false)
|
||||
->whereNotNull(['status_id', 'profile_id'])
|
||||
->whereNull(['cdn_url', 'replicated_at'])
|
||||
->orderByDesc('size')
|
||||
->take($limit)
|
||||
->get()
|
||||
->each(function ($media) use ($bar) {
|
||||
if (Storage::disk('local')->exists($media->media_path)) {
|
||||
$this->totalSize = $this->totalSize + $media->size;
|
||||
try {
|
||||
MediaStorageService::store($media);
|
||||
} catch (FileNotFoundException $e) {
|
||||
$this->error('Error migrating media '.$media->id.' to cloud storage: '.$e->getMessage());
|
||||
|
||||
return;
|
||||
} catch (NotFoundHttpException $e) {
|
||||
$this->error('Error migrating media '.$media->id.' to cloud storage: '.$e->getMessage());
|
||||
|
||||
return;
|
||||
} catch (\Exception $e) {
|
||||
$this->error('Error migrating media '.$media->id.' to cloud storage: '.$e->getMessage());
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
$bar->advance();
|
||||
});
|
||||
|
||||
$bar->finish();
|
||||
$this->line(' ');
|
||||
$this->info('Finished!');
|
||||
if ($this->totalSize) {
|
||||
$this->info('Uploaded '.PrettyNumber::size($this->totalSize).' of media to cloud storage!');
|
||||
$this->line(' ');
|
||||
$this->info('These files are still stored locally, and will be automatically removed.');
|
||||
}
|
||||
|
||||
return Command::SUCCESS;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,122 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands\Concerns;
|
||||
|
||||
use App\Services\ConfigCacheService;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
|
||||
/**
|
||||
* Shared helpers for the media storage migration commands.
|
||||
*
|
||||
* Handles reading/writing the .env storage flags atomically (reusing the
|
||||
* installer's vetted approach) and applying the change to the live runtime
|
||||
* so that new uploads route to the correct backend during a migration on a
|
||||
* hot (running) server, without requiring a restart.
|
||||
*/
|
||||
trait ManagesMediaStorageEnv
|
||||
{
|
||||
/**
|
||||
* Read the raw current value of an .env key (unquoted), or null.
|
||||
*/
|
||||
protected function readEnvValue(string $key): ?string
|
||||
{
|
||||
$envPath = app()->environmentFilePath();
|
||||
if (! is_file($envPath)) {
|
||||
return null;
|
||||
}
|
||||
$payload = file_get_contents($envPath);
|
||||
if ($payload === false) {
|
||||
return null;
|
||||
}
|
||||
if (! preg_match("/^{$key}=([^\r\n]*)/m", $payload, $m)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return trim($m[1], " \t\"'");
|
||||
}
|
||||
|
||||
/**
|
||||
* Set an .env key + the live runtime config + config-cache entry so the
|
||||
* change takes effect immediately on a running server.
|
||||
*
|
||||
* @param string $configKey dotted config key kept in sync (e.g. 'pixelfed.cloud_storage')
|
||||
* @param mixed $configValue the typed runtime value (e.g. true/false)
|
||||
*/
|
||||
protected function setStorageEnv(string $envKey, string $envValue, string $configKey, $configValue): void
|
||||
{
|
||||
// 1. Persist to .env atomically (survives restarts).
|
||||
$this->updateEnvFile($envKey, $envValue);
|
||||
|
||||
// 2. Update the live runtime config for the current process.
|
||||
config([$configKey => $configValue]);
|
||||
|
||||
// 3. Update the DB-backed config cache so other workers/requests
|
||||
// reading via config_cache() see the new value (hot server).
|
||||
try {
|
||||
ConfigCacheService::put($configKey, $configValue);
|
||||
} catch (\Throwable $e) {
|
||||
$this->warn('Could not update config cache for '.$configKey.': '.$e->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The configured cloud disk host (used to sanity check cloud config).
|
||||
*/
|
||||
protected function cloudHost(): ?string
|
||||
{
|
||||
try {
|
||||
$url = Storage::disk(config('filesystems.cloud'))->url('probe');
|
||||
$host = parse_url($url, PHP_URL_HOST);
|
||||
|
||||
return $host ?: null;
|
||||
} catch (\Throwable $e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Atomic .env writer (adapted from Installer) ---------------------
|
||||
|
||||
protected function updateEnvFile($key, $value): void
|
||||
{
|
||||
$envPath = app()->environmentFilePath();
|
||||
$payload = file_get_contents($envPath);
|
||||
|
||||
$value = str_replace(['\\', '"', "\n", "\r"], ['\\\\', '\\"', '\\n', '\\r'], $value);
|
||||
|
||||
if (($existing = $this->existingEnv($key, $payload)) !== false) {
|
||||
$payload = str_replace("{$key}={$existing}", "{$key}=\"{$value}\"", $payload);
|
||||
} else {
|
||||
$payload = $payload."\n{$key}=\"{$value}\"\n";
|
||||
}
|
||||
|
||||
$this->storeEnv($payload);
|
||||
}
|
||||
|
||||
protected function existingEnv($needle, $haystack)
|
||||
{
|
||||
preg_match("/^{$needle}=[^\r\n]*/m", $haystack, $matches);
|
||||
if ($matches && count($matches)) {
|
||||
return substr($matches[0], strlen($needle) + 1);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
protected function storeEnv($payload): void
|
||||
{
|
||||
$envPath = app()->environmentFilePath();
|
||||
$tempPath = $envPath.'.tmp';
|
||||
|
||||
$file = fopen($tempPath, 'w');
|
||||
if ($file === false) {
|
||||
throw new \RuntimeException("Cannot write to {$tempPath}");
|
||||
}
|
||||
fwrite($file, $payload);
|
||||
fclose($file);
|
||||
|
||||
if (! rename($tempPath, $envPath)) {
|
||||
@unlink($tempPath);
|
||||
throw new \RuntimeException('Cannot update .env file');
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,227 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Console\Commands\Concerns\ManagesMediaStorageEnv;
|
||||
use App\Models\Media;
|
||||
use App\Services\MediaService;
|
||||
use App\Services\StatusService;
|
||||
use App\Util\Lexer\PrettyNumber;
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Illuminate\Support\Str;
|
||||
|
||||
class MediaMoveStorageCloudToLocal extends Command
|
||||
{
|
||||
use ManagesMediaStorageEnv;
|
||||
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'admin:MediaMoveStorageCloudToLocal
|
||||
{--limit=500 : Max media rows to process this run}
|
||||
{--dry-run : Report what would happen without copying or writing}
|
||||
{--keep-cloud : Do not delete the cloud copy after verifying the local file}
|
||||
{--force : Skip confirmation prompts}';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Migrate cloud media back to local storage: download, verify (size/sha256), update URLs, then optionally delete the cloud copy. Ensures new uploads stay local during the migration.';
|
||||
|
||||
protected int $movedBytes = 0;
|
||||
|
||||
public function handle()
|
||||
{
|
||||
$localDisk = Storage::disk('local');
|
||||
$cloudDisk = Storage::disk(config('filesystems.cloud'));
|
||||
|
||||
// --- Ensure new uploads stay LOCAL during the migration -----------
|
||||
$envCloud = $this->readEnvValue('PF_ENABLE_CLOUD');
|
||||
$cloudEnabled = filter_var($envCloud, FILTER_VALIDATE_BOOLEAN);
|
||||
|
||||
if ($cloudEnabled) {
|
||||
$this->warn('PF_ENABLE_CLOUD is currently true.');
|
||||
$this->line('New uploads would keep landing on CLOUD storage during this migration.');
|
||||
if ($this->option('dry-run')) {
|
||||
$this->line('[dry-run] Would set PF_ENABLE_CLOUD=false (.env + runtime + config cache).');
|
||||
} elseif ($this->option('force') || $this->confirm('Set PF_ENABLE_CLOUD=false now so new uploads stay local?', true)) {
|
||||
$this->setStorageEnv('PF_ENABLE_CLOUD', 'false', 'pixelfed.cloud_storage', false);
|
||||
$this->info('PF_ENABLE_CLOUD set to false (.env + live runtime + config cache).');
|
||||
} else {
|
||||
$this->error('Aborting: refusing to migrate to local while new uploads go to cloud.');
|
||||
|
||||
return 1;
|
||||
}
|
||||
} else {
|
||||
$this->info('PF_ENABLE_CLOUD is already false; new uploads stay local. ✓');
|
||||
}
|
||||
|
||||
$this->newLine();
|
||||
if (! $this->option('dry-run') && ! $this->option('force')) {
|
||||
if (! $this->confirm('Begin migrating cloud media to local?', true)) {
|
||||
$this->comment('Aborted.');
|
||||
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
$limit = (int) $this->option('limit');
|
||||
$moved = 0;
|
||||
$skipped = 0;
|
||||
$failed = 0;
|
||||
|
||||
// Candidates: non-remote media that has a cloud copy (cdn_url set).
|
||||
$query = Media::whereRemoteMedia(false)
|
||||
->whereNotNull(['media_path', 'cdn_url'])
|
||||
->orderByDesc('id')
|
||||
->limit($limit);
|
||||
|
||||
$bar = $this->output->createProgressBar($query->count());
|
||||
$bar->start();
|
||||
|
||||
foreach ($query->get() as $media) {
|
||||
$result = $this->migrateOne($media, $localDisk, $cloudDisk);
|
||||
match ($result) {
|
||||
'moved' => $moved++,
|
||||
'skipped' => $skipped++,
|
||||
default => $failed++,
|
||||
};
|
||||
$bar->advance();
|
||||
}
|
||||
|
||||
$bar->finish();
|
||||
$this->newLine(2);
|
||||
$this->info(($this->option('dry-run') ? '[dry-run] ' : '').'Done. moved='.$moved.' skipped='.$skipped.' failed='.$failed.'.');
|
||||
if ($this->movedBytes) {
|
||||
$this->info('Transferred '.PrettyNumber::size($this->movedBytes).' back to local storage.');
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return string one of moved|skipped|failed
|
||||
*/
|
||||
protected function migrateOne(Media $media, $localDisk, $cloudDisk): string
|
||||
{
|
||||
if (Str::startsWith((string) $media->media_path, 'http')) {
|
||||
return 'skipped';
|
||||
}
|
||||
|
||||
// Must exist on cloud to pull down.
|
||||
if (! $cloudDisk->exists($media->media_path)) {
|
||||
// Already local-only? just clear the cloud url fields.
|
||||
if ($localDisk->exists($media->media_path)) {
|
||||
if (! $this->option('dry-run')) {
|
||||
$this->clearCloudFields($media);
|
||||
$media->save();
|
||||
}
|
||||
|
||||
return 'skipped';
|
||||
}
|
||||
|
||||
return 'failed';
|
||||
}
|
||||
|
||||
if ($this->option('dry-run')) {
|
||||
return 'moved';
|
||||
}
|
||||
|
||||
try {
|
||||
$this->copyToLocal($media->media_path, $localDisk, $cloudDisk);
|
||||
if ($media->thumbnail_path && $cloudDisk->exists($media->thumbnail_path)) {
|
||||
$this->copyToLocal($media->thumbnail_path, $localDisk, $cloudDisk);
|
||||
}
|
||||
|
||||
if (! $this->verify($media->media_path, $localDisk, $cloudDisk, $media->original_sha256)) {
|
||||
$this->warn(PHP_EOL.'Verify failed for media '.$media->id.' ('.$media->media_path.'); left cloud copy intact.');
|
||||
|
||||
return 'failed';
|
||||
}
|
||||
|
||||
// Point URLs back at local storage.
|
||||
$this->clearCloudFields($media);
|
||||
|
||||
// Integrated GC: delete the verified cloud copy unless --keep-cloud.
|
||||
if (! $this->option('keep-cloud')) {
|
||||
$cloudDisk->delete($media->media_path);
|
||||
if ($media->thumbnail_path && $cloudDisk->exists($media->thumbnail_path)) {
|
||||
$cloudDisk->delete($media->thumbnail_path);
|
||||
}
|
||||
}
|
||||
|
||||
$media->save();
|
||||
$this->movedBytes += (int) $media->size;
|
||||
|
||||
if ($media->status_id) {
|
||||
MediaService::del($media->status_id);
|
||||
StatusService::del($media->status_id, false);
|
||||
}
|
||||
|
||||
return 'moved';
|
||||
} catch (\Throwable $e) {
|
||||
$this->warn(PHP_EOL.'Error migrating media '.$media->id.': '.$e->getMessage());
|
||||
|
||||
return 'failed';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset a media row to local-served state.
|
||||
*/
|
||||
protected function clearCloudFields(Media $media): void
|
||||
{
|
||||
$media->cdn_url = null;
|
||||
$media->optimized_url = null;
|
||||
$media->thumbnail_url = null;
|
||||
$media->replicated_at = null;
|
||||
// version 4 meant "local deleted, cloud only"; reset so the file is
|
||||
// treated as locally present again.
|
||||
if ($media->version === '4' || $media->version === 4) {
|
||||
$media->version = 3;
|
||||
}
|
||||
}
|
||||
|
||||
protected function copyToLocal(string $path, $localDisk, $cloudDisk): void
|
||||
{
|
||||
$stream = $cloudDisk->readStream($path);
|
||||
if ($stream === false || $stream === null) {
|
||||
throw new \RuntimeException('Could not open cloud stream for '.$path);
|
||||
}
|
||||
$localDisk->writeStream($path, $stream);
|
||||
if (is_resource($stream)) {
|
||||
fclose($stream);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify the local copy matches the cloud source by size, and by sha256
|
||||
* against the stored original checksum when available. Fails closed.
|
||||
*/
|
||||
protected function verify(string $path, $localDisk, $cloudDisk, ?string $expectedSha = null): bool
|
||||
{
|
||||
if (! $localDisk->exists($path)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$localSize = $localDisk->size($path);
|
||||
$cloudSize = $cloudDisk->size($path);
|
||||
if ($localSize === false || $cloudSize === false || $localSize !== $cloudSize) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if ($expectedSha) {
|
||||
$localSha = @hash_file('sha256', $localDisk->path($path));
|
||||
if ($localSha && ! hash_equals($expectedSha, $localSha)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,239 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Console\Commands\Concerns\ManagesMediaStorageEnv;
|
||||
use App\Models\Media;
|
||||
use App\Services\MediaService;
|
||||
use App\Services\ResilientMediaStorageService;
|
||||
use App\Services\StatusService;
|
||||
use App\Util\Lexer\PrettyNumber;
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Illuminate\Support\Str;
|
||||
|
||||
class MediaMoveStorageLocalToCloud extends Command
|
||||
{
|
||||
use ManagesMediaStorageEnv;
|
||||
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'admin:MediaMoveStorageLocalToCloud
|
||||
{--limit=500 : Max media rows to process this run}
|
||||
{--dry-run : Report what would happen without copying or writing}
|
||||
{--keep-local : Do not delete local files after verifying the cloud copy}
|
||||
{--force : Skip confirmation prompts}';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Migrate local media to cloud storage: copy up, verify (size/sha256), update URLs, then delete the local copy. Ensures new uploads go to cloud during the migration.';
|
||||
|
||||
protected int $movedBytes = 0;
|
||||
|
||||
public function handle()
|
||||
{
|
||||
try {
|
||||
$localDisk = Storage::disk('local');
|
||||
$cloudDisk = Storage::disk(config('filesystems.cloud'));
|
||||
} catch (\Throwable $e) {
|
||||
$this->error('Cloud disk ('.config('filesystems.cloud').') could not be resolved: '.$e->getMessage());
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (! $this->cloudHost()) {
|
||||
$this->error('Cloud disk ('.config('filesystems.cloud').') is not configured (no resolvable URL).');
|
||||
$this->line('Set AWS_URL / AWS_* in your .env before migrating to cloud.');
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
// --- Ensure new uploads route to cloud during the migration --------
|
||||
$envCloud = $this->readEnvValue('PF_ENABLE_CLOUD');
|
||||
$cloudEnabled = filter_var($envCloud, FILTER_VALIDATE_BOOLEAN);
|
||||
|
||||
if (! $cloudEnabled) {
|
||||
$this->warn('PF_ENABLE_CLOUD is currently "'.($envCloud ?? 'unset').'".');
|
||||
$this->line('New uploads would keep landing on LOCAL storage during this migration.');
|
||||
if ($this->option('dry-run')) {
|
||||
$this->line('[dry-run] Would set PF_ENABLE_CLOUD=true (.env + runtime + config cache).');
|
||||
} elseif ($this->option('force') || $this->confirm('Set PF_ENABLE_CLOUD=true now so new uploads go to cloud?', true)) {
|
||||
$this->setStorageEnv('PF_ENABLE_CLOUD', 'true', 'pixelfed.cloud_storage', true);
|
||||
$this->info('PF_ENABLE_CLOUD set to true (.env + live runtime + config cache).');
|
||||
} else {
|
||||
$this->error('Aborting: refusing to migrate to cloud while new uploads stay local.');
|
||||
|
||||
return 1;
|
||||
}
|
||||
} else {
|
||||
$this->info('PF_ENABLE_CLOUD is already true; new uploads route to cloud. ✓');
|
||||
}
|
||||
|
||||
$this->newLine();
|
||||
if (! $this->option('dry-run') && ! $this->option('force')) {
|
||||
if (! $this->confirm('Begin migrating local media to cloud?', true)) {
|
||||
$this->comment('Aborted.');
|
||||
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
$limit = (int) $this->option('limit');
|
||||
$moved = 0;
|
||||
$skipped = 0;
|
||||
$failed = 0;
|
||||
|
||||
// Candidates: local, non-remote media not yet replicated to cloud.
|
||||
$query = Media::whereRemoteMedia(false)
|
||||
->whereNotNull('media_path')
|
||||
->where(function ($q) {
|
||||
$q->whereNull('cdn_url')->orWhereNull('replicated_at')->orWhereNot('version', '4');
|
||||
})
|
||||
->orderByDesc('id')
|
||||
->limit($limit);
|
||||
|
||||
$bar = $this->output->createProgressBar($query->count());
|
||||
$bar->start();
|
||||
|
||||
foreach ($query->get() as $media) {
|
||||
$result = $this->migrateOne($media, $localDisk, $cloudDisk);
|
||||
match ($result) {
|
||||
'moved' => $moved++,
|
||||
'skipped' => $skipped++,
|
||||
default => $failed++,
|
||||
};
|
||||
$bar->advance();
|
||||
}
|
||||
|
||||
$bar->finish();
|
||||
$this->newLine(2);
|
||||
$this->info(($this->option('dry-run') ? '[dry-run] ' : '').'Done. moved='.$moved.' skipped='.$skipped.' failed='.$failed.'.');
|
||||
if ($this->movedBytes) {
|
||||
$this->info('Transferred '.PrettyNumber::size($this->movedBytes).' to cloud storage.');
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return string one of moved|skipped|failed
|
||||
*/
|
||||
protected function migrateOne(Media $media, $localDisk, $cloudDisk): string
|
||||
{
|
||||
if (Str::startsWith((string) $media->media_path, 'http')) {
|
||||
return 'skipped';
|
||||
}
|
||||
|
||||
// Nothing to do if the local file is gone.
|
||||
if (! $localDisk->exists($media->media_path)) {
|
||||
// Already on cloud only? mark version and move on.
|
||||
if ($cloudDisk->exists($media->media_path)) {
|
||||
if (! $this->option('dry-run') && $media->version !== '4') {
|
||||
$media->version = 4;
|
||||
$media->save();
|
||||
}
|
||||
|
||||
return 'skipped';
|
||||
}
|
||||
|
||||
return 'skipped';
|
||||
}
|
||||
|
||||
if ($this->option('dry-run')) {
|
||||
return 'moved';
|
||||
}
|
||||
|
||||
try {
|
||||
// Copy the primary file (and thumbnail) to cloud.
|
||||
$this->copyToCloud($media->media_path, $localDisk, $cloudDisk);
|
||||
if ($media->thumbnail_path && $localDisk->exists($media->thumbnail_path)) {
|
||||
$this->copyToCloud($media->thumbnail_path, $localDisk, $cloudDisk);
|
||||
}
|
||||
|
||||
// Verify the primary file before touching anything else.
|
||||
if (! $this->verify($media->media_path, $localDisk, $cloudDisk, $media->original_sha256)) {
|
||||
$this->warn(PHP_EOL.'Verify failed for media '.$media->id.' ('.$media->media_path.'); left local copy intact.');
|
||||
|
||||
return 'failed';
|
||||
}
|
||||
|
||||
// Update URL fields to the cloud disk.
|
||||
$media->cdn_url = $cloudDisk->url($media->media_path);
|
||||
$media->optimized_url = $media->cdn_url;
|
||||
if ($media->thumbnail_path && $cloudDisk->exists($media->thumbnail_path)) {
|
||||
$media->thumbnail_url = $cloudDisk->url($media->thumbnail_path);
|
||||
}
|
||||
$media->replicated_at = now();
|
||||
|
||||
// Integrated GC: delete the verified local copy unless --keep-local.
|
||||
if (! $this->option('keep-local')) {
|
||||
$localDisk->delete($media->media_path);
|
||||
if ($media->thumbnail_path && $localDisk->exists($media->thumbnail_path)) {
|
||||
$localDisk->delete($media->thumbnail_path);
|
||||
}
|
||||
$media->version = 4;
|
||||
}
|
||||
|
||||
$media->save();
|
||||
$this->movedBytes += (int) $media->size;
|
||||
|
||||
if ($media->status_id) {
|
||||
MediaService::del($media->status_id);
|
||||
StatusService::del($media->status_id, false);
|
||||
}
|
||||
|
||||
return 'moved';
|
||||
} catch (\Throwable $e) {
|
||||
$this->warn(PHP_EOL.'Error migrating media '.$media->id.': '.$e->getMessage());
|
||||
|
||||
return 'failed';
|
||||
}
|
||||
}
|
||||
|
||||
protected function copyToCloud(string $path, $localDisk, $cloudDisk): void
|
||||
{
|
||||
$p = explode('/', $path);
|
||||
$name = array_pop($p);
|
||||
$storagePath = implode('/', $p);
|
||||
|
||||
// Reuse the resilient uploader (handles alt disks + retries).
|
||||
ResilientMediaStorageService::store($storagePath, $localDisk->path($path), $name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify the cloud copy matches the local source by size, and by sha256
|
||||
* when a checksum is available/cheap. Fails closed.
|
||||
*/
|
||||
protected function verify(string $path, $localDisk, $cloudDisk, ?string $expectedSha = null): bool
|
||||
{
|
||||
if (! $cloudDisk->exists($path)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$localSize = $localDisk->size($path);
|
||||
$cloudSize = $cloudDisk->size($path);
|
||||
if ($localSize === false || $cloudSize === false || $localSize !== $cloudSize) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// If we already have the original checksum, verify the local file still
|
||||
// matches it (so we never delete a locally-corrupted-but-uploaded file
|
||||
// without noticing). Cloud content hashing would require a full
|
||||
// download, which we avoid for large media; size parity + known sha
|
||||
// is a strong signal.
|
||||
if ($expectedSha) {
|
||||
$localSha = @hash_file('sha256', $localDisk->path($path));
|
||||
if ($localSha && ! hash_equals($expectedSha, $localSha)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@ -1,204 +0,0 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Models\Media;
|
||||
use App\Services\MediaService;
|
||||
use App\Services\StatusService;
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Symfony\Component\HttpFoundation\File\Exception\FileNotFoundException;
|
||||
use Symfony\Component\HttpKernel\Exception\NotFoundHttpException;
|
||||
|
||||
class MediaS3GarbageCollector extends Command
|
||||
{
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'media:s3gc {--limit=200} {--huge} {--log-errors}';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Delete (local) media uploads that exist on S3';
|
||||
|
||||
/**
|
||||
* Create a new command instance.
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function __construct()
|
||||
{
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
*
|
||||
* @return int
|
||||
*/
|
||||
public function handle()
|
||||
{
|
||||
$enabled = (bool) config_cache('pixelfed.cloud_storage');
|
||||
if (! $enabled) {
|
||||
$this->error('Cloud storage not enabled. Exiting...');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$deleteEnabled = config('media.delete_local_after_cloud');
|
||||
if (! $deleteEnabled) {
|
||||
$this->error('Delete local storage after cloud upload is not enabled');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$limit = $this->option('limit');
|
||||
$hugeMode = $this->option('huge');
|
||||
$log = $this->option('log-errors');
|
||||
|
||||
if ($limit > 2000 && ! $hugeMode) {
|
||||
$this->error('Limit exceeded, please use a limit under 2000 or run again with the --huge flag');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$minId = Media::orderByDesc('id')->where('created_at', '<', now()->subHours(12))->first();
|
||||
|
||||
if (! $minId) {
|
||||
return;
|
||||
} else {
|
||||
$minId = $minId->id;
|
||||
}
|
||||
|
||||
return $hugeMode ?
|
||||
$this->hugeMode($minId, $limit, $log) :
|
||||
$this->regularMode($minId, $limit, $log);
|
||||
}
|
||||
|
||||
protected function regularMode($minId, $limit, $log)
|
||||
{
|
||||
$gc = Media::whereRemoteMedia(false)
|
||||
->whereNotNull(['status_id', 'cdn_url', 'replicated_at'])
|
||||
->whereNot('version', '4')
|
||||
->where('id', '<', $minId)
|
||||
->inRandomOrder()
|
||||
->take($limit)
|
||||
->get();
|
||||
|
||||
$totalSize = 0;
|
||||
$bar = $this->output->createProgressBar($gc->count());
|
||||
$bar->start();
|
||||
$cloudDisk = Storage::disk(config('filesystems.cloud'));
|
||||
$localDisk = Storage::disk('local');
|
||||
|
||||
foreach ($gc as $media) {
|
||||
try {
|
||||
if (
|
||||
$cloudDisk->exists($media->media_path)
|
||||
) {
|
||||
if ($localDisk->exists($media->media_path)) {
|
||||
$localDisk->delete($media->media_path);
|
||||
$media->version = 4;
|
||||
$media->save();
|
||||
$totalSize = $totalSize + $media->size;
|
||||
MediaService::del($media->status_id);
|
||||
StatusService::del($media->status_id, false);
|
||||
if ($localDisk->exists($media->thumbnail_path)) {
|
||||
$localDisk->delete($media->thumbnail_path);
|
||||
}
|
||||
} else {
|
||||
$media->version = 4;
|
||||
$media->save();
|
||||
}
|
||||
} else {
|
||||
if ($log) {
|
||||
Log::channel('media')->info('[GC] Local media not properly persisted to cloud storage', ['media_id' => $media->id]);
|
||||
}
|
||||
}
|
||||
$bar->advance();
|
||||
} catch (FileNotFoundException $e) {
|
||||
$bar->advance();
|
||||
|
||||
continue;
|
||||
} catch (NotFoundHttpException $e) {
|
||||
$bar->advance();
|
||||
|
||||
continue;
|
||||
} catch (\Exception $e) {
|
||||
$bar->advance();
|
||||
|
||||
continue;
|
||||
}
|
||||
}
|
||||
$bar->finish();
|
||||
$this->line(' ');
|
||||
$this->info('Finished!');
|
||||
if ($totalSize) {
|
||||
$this->info('Cleared '.$totalSize.' bytes of media from local disk!');
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
protected function hugeMode($minId, $limit, $log)
|
||||
{
|
||||
$cloudDisk = Storage::disk(config('filesystems.cloud'));
|
||||
$localDisk = Storage::disk('local');
|
||||
|
||||
$bar = $this->output->createProgressBar($limit);
|
||||
$bar->start();
|
||||
|
||||
Media::whereRemoteMedia(false)
|
||||
->whereNotNull(['status_id', 'cdn_url', 'replicated_at'])
|
||||
->whereNot('version', '4')
|
||||
->where('id', '<', $minId)
|
||||
->chunk(50, function ($medias) use ($cloudDisk, $localDisk, $bar, $log) {
|
||||
foreach ($medias as $media) {
|
||||
try {
|
||||
if ($cloudDisk->exists($media->media_path)) {
|
||||
if ($localDisk->exists($media->media_path)) {
|
||||
$localDisk->delete($media->media_path);
|
||||
$media->version = 4;
|
||||
$media->save();
|
||||
MediaService::del($media->status_id);
|
||||
StatusService::del($media->status_id, false);
|
||||
if ($localDisk->exists($media->thumbnail_path)) {
|
||||
$localDisk->delete($media->thumbnail_path);
|
||||
}
|
||||
} else {
|
||||
$media->version = 4;
|
||||
$media->save();
|
||||
}
|
||||
} else {
|
||||
if ($log) {
|
||||
Log::channel('media')->info('[GC] Local media not properly persisted to cloud storage', ['media_id' => $media->id]);
|
||||
}
|
||||
}
|
||||
$bar->advance();
|
||||
} catch (FileNotFoundException $e) {
|
||||
$bar->advance();
|
||||
|
||||
continue;
|
||||
} catch (NotFoundHttpException $e) {
|
||||
$bar->advance();
|
||||
|
||||
continue;
|
||||
} catch (\Exception $e) {
|
||||
$bar->advance();
|
||||
|
||||
continue;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
$bar->finish();
|
||||
$this->line(' ');
|
||||
$this->info('Finished!');
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,150 @@
|
||||
<?php
|
||||
|
||||
use App\Models\Media;
|
||||
use App\Models\Status;
|
||||
use App\Models\User;
|
||||
use Illuminate\Foundation\Testing\LazilyRefreshDatabase;
|
||||
use Illuminate\Support\Facades\Config;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
|
||||
uses(LazilyRefreshDatabase::class);
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Media storage migration commands
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| admin:MediaMoveStorageLocalToCloud / admin:MediaMoveStorageCloudToLocal
|
||||
| Move media between local and cloud disks, verify by size/sha256, GC the
|
||||
| source, and manage the PF_ENABLE_CLOUD .env flag for hot migrations.
|
||||
|
|
||||
*/
|
||||
|
||||
beforeEach(function () {
|
||||
Config::set('filesystems.cloud', 's3');
|
||||
Storage::fake('local');
|
||||
Storage::fake('s3', ['url' => 'https://cdn.test']);
|
||||
|
||||
// Use a throwaway env file so the command's env edits don't touch the
|
||||
// real one. App::useEnvironmentPath expects a directory; the app resolves
|
||||
// the environment-specific filename (e.g. .env.testing) itself.
|
||||
$this->originalEnvPath = app()->environmentPath();
|
||||
$dir = sys_get_temp_dir().'/pf-env-test-'.uniqid();
|
||||
mkdir($dir);
|
||||
app()->useEnvironmentPath($dir);
|
||||
file_put_contents(app()->environmentFilePath(), "APP_KEY=base64:test\nPF_ENABLE_CLOUD=false\n");
|
||||
});
|
||||
|
||||
afterEach(function () {
|
||||
// Restore the real environment path so we don't leak into other test files.
|
||||
if (isset($this->originalEnvPath)) {
|
||||
app()->useEnvironmentPath($this->originalEnvPath);
|
||||
}
|
||||
});
|
||||
|
||||
function makeCloudMedia(): Media
|
||||
{
|
||||
$user = User::factory()->create();
|
||||
$user->refresh();
|
||||
$pid = $user->profile->id;
|
||||
$status = Status::factory()->create(['profile_id' => $pid, 'type' => 'photo']);
|
||||
|
||||
$path = 'public/m/_v2/'.$pid.'/aa/bb/file.jpg';
|
||||
$thumb = 'public/m/_v2/'.$pid.'/aa/bb/file_thumb.jpeg';
|
||||
|
||||
// Put the files on the cloud disk only.
|
||||
Storage::disk('s3')->put($path, 'PRIMARY-BYTES-1234567890');
|
||||
Storage::disk('s3')->put($thumb, 'THUMB-BYTES');
|
||||
|
||||
return Media::create([
|
||||
'status_id' => $status->id,
|
||||
'profile_id' => $pid,
|
||||
'user_id' => $user->id,
|
||||
'media_path' => $path,
|
||||
'thumbnail_path' => $thumb,
|
||||
'cdn_url' => Storage::disk('s3')->url($path),
|
||||
'thumbnail_url' => Storage::disk('s3')->url($thumb),
|
||||
'optimized_url' => Storage::disk('s3')->url($path),
|
||||
'mime' => 'image/jpeg',
|
||||
'size' => strlen('PRIMARY-BYTES-1234567890'),
|
||||
'remote_media' => false,
|
||||
'version' => 4,
|
||||
'replicated_at' => now(),
|
||||
'order' => 0,
|
||||
]);
|
||||
}
|
||||
|
||||
describe('admin:MediaMoveStorageCloudToLocal', function () {
|
||||
it('downloads cloud media to local, clears cloud urls and deletes the cloud copy', function () {
|
||||
$media = makeCloudMedia();
|
||||
|
||||
$this->artisan('admin:MediaMoveStorageCloudToLocal', ['--force' => true])
|
||||
->assertExitCode(0);
|
||||
|
||||
// File is now on local disk.
|
||||
expect(Storage::disk('local')->exists($media->media_path))->toBeTrue();
|
||||
// Cloud copy removed (GC), thumbnail too.
|
||||
expect(Storage::disk('s3')->exists($media->media_path))->toBeFalse();
|
||||
|
||||
$media->refresh();
|
||||
expect($media->cdn_url)->toBeNull();
|
||||
expect($media->optimized_url)->toBeNull();
|
||||
expect($media->thumbnail_url)->toBeNull();
|
||||
expect($media->replicated_at)->toBeNull();
|
||||
expect((string) $media->version)->toBe('3');
|
||||
});
|
||||
|
||||
it('keeps the cloud copy with --keep-cloud', function () {
|
||||
$media = makeCloudMedia();
|
||||
|
||||
$this->artisan('admin:MediaMoveStorageCloudToLocal', ['--force' => true, '--keep-cloud' => true])
|
||||
->assertExitCode(0);
|
||||
|
||||
expect(Storage::disk('local')->exists($media->media_path))->toBeTrue();
|
||||
expect(Storage::disk('s3')->exists($media->media_path))->toBeTrue();
|
||||
});
|
||||
|
||||
it('does not modify anything in dry-run', function () {
|
||||
$media = makeCloudMedia();
|
||||
|
||||
$this->artisan('admin:MediaMoveStorageCloudToLocal', ['--force' => true, '--dry-run' => true])
|
||||
->assertExitCode(0);
|
||||
|
||||
expect(Storage::disk('local')->exists($media->media_path))->toBeFalse();
|
||||
expect($media->fresh()->cdn_url)->not->toBeNull();
|
||||
});
|
||||
|
||||
it('sets PF_ENABLE_CLOUD=false in .env and runtime when cloud is enabled', function () {
|
||||
// Start with cloud enabled.
|
||||
file_put_contents(app()->environmentFilePath(), "APP_KEY=base64:test\nPF_ENABLE_CLOUD=true\n");
|
||||
Config::set('pixelfed.cloud_storage', true);
|
||||
makeCloudMedia();
|
||||
|
||||
$this->artisan('admin:MediaMoveStorageCloudToLocal', ['--force' => true])
|
||||
->assertExitCode(0);
|
||||
|
||||
expect(file_get_contents(app()->environmentFilePath()))->toContain('PF_ENABLE_CLOUD="false"');
|
||||
expect(config('pixelfed.cloud_storage'))->toBeFalse();
|
||||
});
|
||||
});
|
||||
|
||||
describe('admin:MediaMoveStorageLocalToCloud', function () {
|
||||
it('requires a configured cloud disk', function () {
|
||||
// Fake s3 disk has no url() host resolvable? Storage::fake provides a
|
||||
// url, so instead point cloud at a disk that throws.
|
||||
Config::set('filesystems.cloud', 'does-not-exist');
|
||||
|
||||
$this->artisan('admin:MediaMoveStorageLocalToCloud', ['--force' => true])
|
||||
->assertExitCode(1);
|
||||
});
|
||||
|
||||
it('flips PF_ENABLE_CLOUD to true before migrating (dry-run reports it)', function () {
|
||||
// cloud currently false in the temp .env from beforeEach.
|
||||
$this->artisan('admin:MediaMoveStorageLocalToCloud', ['--dry-run' => true])
|
||||
->expectsOutputToContain('PF_ENABLE_CLOUD')
|
||||
->assertExitCode(0);
|
||||
|
||||
// dry-run must not write the .env.
|
||||
expect(file_get_contents(app()->environmentFilePath()))->toContain('PF_ENABLE_CLOUD=false');
|
||||
});
|
||||
});
|
||||
Loading…
Reference in New Issue