mirror of https://github.com/pixelfed/pixelfed
feat: storage:maintenance command + in-flow cleanup of emptied dirs
Replace the remcache GC (GarbageCollectorRemcache / gc:remcache) with a
broader storage:maintenance command that sweeps stale remcache temp files and
recursively removes the random empty directories accumulated under the media,
story, avatar and import trees (--hours/--only/--except/--dry-run), scheduled
daily.
Fix the root causes so flows clean up after themselves rather than relying on
the sweep:
- MediaDeletePipeline removes its own emptied m/_v2 leaf dir
- AvatarOptimize logs the previously-swallowed exception, still cleans up the
old avatar on failure, and removes the old file's now-empty splayed dir
- AvatarController::deleteAvatar removes the emptied splayed dir
- StoryExpire/StoryDelete remove the story's own emptied leaf dir
- TransformImports removes imports/{userId} once its files are moved out
- StoryFetch cleans up its remcache temp file in a finally block
pull/6987/head
Your Name
3 weeks ago
parent
5600b75c00
commit
400f00c5e1
@ -1,103 +0,0 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands\Internal;
|
||||
|
||||
use Illuminate\Console\Command;
|
||||
|
||||
class GarbageCollectorRemcache extends Command
|
||||
{
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'gc:remcache {--hours=24 : Delete remcache files older than this many hours} {--dry-run : Report what would be deleted without deleting}';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Delete stale temporary files left in storage/app/remcache/';
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
*/
|
||||
public function handle(): int
|
||||
{
|
||||
$hours = (int) $this->option('hours');
|
||||
|
||||
if ($hours < 1) {
|
||||
$this->error('The --hours value must be at least 1.');
|
||||
|
||||
return self::FAILURE;
|
||||
}
|
||||
|
||||
$dryRun = (bool) $this->option('dry-run');
|
||||
$dir = storage_path('app/remcache');
|
||||
|
||||
if (! is_dir($dir)) {
|
||||
$this->info('remcache directory does not exist, nothing to do.');
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
$cutoff = now()->subHours($hours)->getTimestamp();
|
||||
$deleted = 0;
|
||||
$reclaimed = 0;
|
||||
|
||||
foreach (new \FilesystemIterator($dir, \FilesystemIterator::SKIP_DOTS) as $file) {
|
||||
if (! $file->isFile()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Preserve dotfiles such as the directory's .gitignore.
|
||||
if (str_starts_with($file->getFilename(), '.')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($file->getMTime() >= $cutoff) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$size = $file->getSize();
|
||||
$path = $file->getPathname();
|
||||
|
||||
if ($dryRun) {
|
||||
$this->line('[dry-run] would delete: '.$file->getFilename());
|
||||
$deleted++;
|
||||
$reclaimed += $size;
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if (@unlink($path)) {
|
||||
$deleted++;
|
||||
$reclaimed += $size;
|
||||
}
|
||||
}
|
||||
|
||||
$verb = $dryRun ? 'Would delete' : 'Deleted';
|
||||
$this->info(sprintf('%s %d file(s), %s.', $verb, $deleted, $this->humanBytes($reclaimed)));
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
private function humanBytes(int $bytes): string
|
||||
{
|
||||
if ($bytes < 1024) {
|
||||
return $bytes.' B';
|
||||
}
|
||||
|
||||
$units = ['KB', 'MB', 'GB', 'TB'];
|
||||
$value = $bytes / 1024;
|
||||
$i = 0;
|
||||
|
||||
while ($value >= 1024 && $i < count($units) - 1) {
|
||||
$value /= 1024;
|
||||
$i++;
|
||||
}
|
||||
|
||||
return sprintf('%.2f %s', $value, $units[$i]);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,246 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands\Internal;
|
||||
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Contracts\Filesystem\Filesystem;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
|
||||
class StorageMaintenance extends Command
|
||||
{
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'storage:maintenance
|
||||
{--hours=24 : Delete remcache files older than this many hours}
|
||||
{--only= : Comma-separated tasks to run (remcache,empty-dirs). Default: all}
|
||||
{--except= : Comma-separated tasks to skip}
|
||||
{--dry-run : Report what would be removed without deleting}';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Reclaim leftover storage: sweep stale remcache temp files and prune empty directories left behind by the media, story, avatar and import flows.';
|
||||
|
||||
/**
|
||||
* The empty-directory trees swept by the "empty-dirs" task, keyed by the
|
||||
* disk-relative root that is itself preserved. All live on the local disk.
|
||||
*
|
||||
* @var list<string>
|
||||
*/
|
||||
protected array $emptyDirRoots = [
|
||||
'public/m/_v2',
|
||||
'public/_esm.t3',
|
||||
'public/avatars',
|
||||
'story_archives',
|
||||
'imports',
|
||||
];
|
||||
|
||||
protected bool $dryRun = false;
|
||||
|
||||
public function handle(): int
|
||||
{
|
||||
$this->dryRun = (bool) $this->option('dry-run');
|
||||
|
||||
$tasks = $this->resolveTasks();
|
||||
|
||||
if (empty($tasks)) {
|
||||
$this->error('No tasks to run. Valid tasks: remcache, empty-dirs.');
|
||||
|
||||
return self::FAILURE;
|
||||
}
|
||||
|
||||
if (in_array('remcache', $tasks, true)) {
|
||||
$result = $this->sweepRemcache();
|
||||
if ($result !== self::SUCCESS) {
|
||||
return $result;
|
||||
}
|
||||
}
|
||||
|
||||
if (in_array('empty-dirs', $tasks, true)) {
|
||||
$this->pruneEmptyDirectories();
|
||||
}
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine which tasks to run from --only / --except.
|
||||
*
|
||||
* @return list<string>
|
||||
*/
|
||||
protected function resolveTasks(): array
|
||||
{
|
||||
$all = ['remcache', 'empty-dirs'];
|
||||
|
||||
$only = $this->parseList($this->option('only'));
|
||||
$except = $this->parseList($this->option('except'));
|
||||
|
||||
$tasks = $only ? array_values(array_intersect($all, $only)) : $all;
|
||||
|
||||
return array_values(array_diff($tasks, $except));
|
||||
}
|
||||
|
||||
/**
|
||||
* @return list<string>
|
||||
*/
|
||||
protected function parseList(?string $value): array
|
||||
{
|
||||
if (! $value) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return collect(explode(',', $value))
|
||||
->map(fn ($v) => trim($v))
|
||||
->filter()
|
||||
->unique()
|
||||
->values()
|
||||
->all();
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete stale temporary files left in storage/app/remcache/. The media
|
||||
* fetchers now clean up their own temp files (try/finally in
|
||||
* MediaStorageService); this is the backstop for any writer that misses it.
|
||||
*/
|
||||
protected function sweepRemcache(): int
|
||||
{
|
||||
$hours = (int) $this->option('hours');
|
||||
|
||||
if ($hours < 1) {
|
||||
$this->error('The --hours value must be at least 1.');
|
||||
|
||||
return self::FAILURE;
|
||||
}
|
||||
|
||||
$dir = storage_path('app/remcache');
|
||||
|
||||
if (! is_dir($dir)) {
|
||||
$this->info('remcache: directory does not exist, nothing to do.');
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
$cutoff = now()->subHours($hours)->getTimestamp();
|
||||
$deleted = 0;
|
||||
$reclaimed = 0;
|
||||
|
||||
foreach (new \FilesystemIterator($dir, \FilesystemIterator::SKIP_DOTS) as $file) {
|
||||
if (! $file->isFile()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Preserve dotfiles such as the directory's .gitignore.
|
||||
if (str_starts_with($file->getFilename(), '.')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($file->getMTime() >= $cutoff) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$size = $file->getSize();
|
||||
$path = $file->getPathname();
|
||||
|
||||
if ($this->dryRun) {
|
||||
$this->line('[dry-run] remcache: would delete '.$file->getFilename());
|
||||
$deleted++;
|
||||
$reclaimed += $size;
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if (@unlink($path)) {
|
||||
$deleted++;
|
||||
$reclaimed += $size;
|
||||
}
|
||||
}
|
||||
|
||||
$verb = $this->dryRun ? 'would delete' : 'deleted';
|
||||
$this->info(sprintf('remcache: %s %d file(s), %s.', $verb, $deleted, $this->humanBytes($reclaimed)));
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sweep each managed local tree and remove the random empty directories
|
||||
* accumulated over time. This is the safety net: the per-flow jobs clean
|
||||
* up their own directories, but older data (and any missed edge case) can
|
||||
* still leave empty folders behind, so we reclaim them here.
|
||||
*/
|
||||
protected function pruneEmptyDirectories(): void
|
||||
{
|
||||
$disk = Storage::disk('local');
|
||||
$total = 0;
|
||||
|
||||
foreach ($this->emptyDirRoots as $root) {
|
||||
if (! $disk->directoryExists($root)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$removed = $this->pruneEmptyDirectoriesUnder($disk, $root);
|
||||
|
||||
$verb = $this->dryRun ? 'would remove' : 'removed';
|
||||
$this->info(sprintf('empty-dirs: %s %s %d empty dir(s).', $root, $verb, $removed));
|
||||
$total += $removed;
|
||||
}
|
||||
|
||||
$verb = $this->dryRun ? 'would remove' : 'removed';
|
||||
$this->info(sprintf('empty-dirs: %s %d empty dir(s) in total.', $verb, $total));
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively remove every empty directory under $root (bottom-up), leaving
|
||||
* $root itself in place. A directory is treated as empty when it holds no
|
||||
* files at any depth, so a branch of only-empty subdirectories collapses.
|
||||
*/
|
||||
protected function pruneEmptyDirectoriesUnder(Filesystem $disk, string $root): int
|
||||
{
|
||||
$directories = $disk->allDirectories($root);
|
||||
|
||||
// Deepest first so children are evaluated before their parents.
|
||||
usort($directories, fn ($a, $b) => substr_count($b, '/') <=> substr_count($a, '/'));
|
||||
|
||||
$removed = 0;
|
||||
|
||||
foreach ($directories as $directory) {
|
||||
if (count($disk->allFiles($directory)) !== 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($this->dryRun) {
|
||||
$this->line('[dry-run] empty-dirs: would remove '.$directory);
|
||||
$removed++;
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
$disk->deleteDirectory($directory);
|
||||
$removed++;
|
||||
}
|
||||
|
||||
return $removed;
|
||||
}
|
||||
|
||||
private function humanBytes(int $bytes): string
|
||||
{
|
||||
if ($bytes < 1024) {
|
||||
return $bytes.' B';
|
||||
}
|
||||
|
||||
$units = ['KB', 'MB', 'GB', 'TB'];
|
||||
$value = $bytes / 1024;
|
||||
$i = 0;
|
||||
|
||||
while ($value >= 1024 && $i < count($units) - 1) {
|
||||
$value /= 1024;
|
||||
$i++;
|
||||
}
|
||||
|
||||
return sprintf('%.2f %s', $value, $units[$i]);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,111 @@
|
||||
<?php
|
||||
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| storage:maintenance
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Sweeps stale remcache temp files and prunes empty directories left behind
|
||||
| by the media/story/avatar/import storage flows.
|
||||
|
|
||||
*/
|
||||
|
||||
beforeEach(function () {
|
||||
Storage::fake('local');
|
||||
|
||||
// remcache lives under the real storage_path, not the faked disk root, so
|
||||
// isolate a temp remcache dir for the age-based sweep assertions.
|
||||
$this->remcacheDir = storage_path('app/remcache');
|
||||
if (! is_dir($this->remcacheDir)) {
|
||||
mkdir($this->remcacheDir, 0755, true);
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(function () {
|
||||
// Clean up any temp files this test created in the real remcache dir.
|
||||
foreach (glob($this->remcacheDir.'/{,.}pf-maint-test-*', GLOB_BRACE) ?: [] as $f) {
|
||||
@unlink($f);
|
||||
}
|
||||
});
|
||||
|
||||
it('deletes stale remcache files older than the cutoff and preserves fresh ones and dotfiles', function () {
|
||||
$stale = $this->remcacheDir.'/pf-maint-test-stale.tmp';
|
||||
$fresh = $this->remcacheDir.'/pf-maint-test-fresh.tmp';
|
||||
// A stale dotfile (like the directory's .gitignore) must be preserved.
|
||||
$dot = $this->remcacheDir.'/.pf-maint-test-keep';
|
||||
|
||||
file_put_contents($stale, 'old');
|
||||
file_put_contents($fresh, 'new');
|
||||
file_put_contents($dot, 'keep');
|
||||
touch($stale, now()->subHours(48)->getTimestamp());
|
||||
touch($dot, now()->subHours(48)->getTimestamp());
|
||||
|
||||
$this->artisan('storage:maintenance', ['--only' => 'remcache', '--hours' => 24])
|
||||
->assertExitCode(0);
|
||||
|
||||
expect(is_file($stale))->toBeFalse();
|
||||
expect(is_file($fresh))->toBeTrue();
|
||||
// Dotfiles (e.g. .gitignore) are always preserved.
|
||||
expect(is_file($dot))->toBeTrue();
|
||||
});
|
||||
|
||||
it('dry-run reports remcache deletions without removing files', function () {
|
||||
$stale = $this->remcacheDir.'/pf-maint-test-stale.tmp';
|
||||
file_put_contents($stale, 'old');
|
||||
touch($stale, now()->subHours(48)->getTimestamp());
|
||||
|
||||
$this->artisan('storage:maintenance', ['--only' => 'remcache', '--hours' => 24, '--dry-run' => true])
|
||||
->assertExitCode(0);
|
||||
|
||||
expect(is_file($stale))->toBeTrue();
|
||||
});
|
||||
|
||||
it('prunes empty directories across managed trees while keeping live files', function () {
|
||||
$disk = Storage::disk('local');
|
||||
|
||||
// Empty leftovers.
|
||||
$disk->makeDirectory('public/m/_v2/9/month/rand');
|
||||
$disk->makeDirectory('public/_esm.t3/m/u/leaf');
|
||||
$disk->makeDirectory('public/avatars/000/111/222');
|
||||
$disk->makeDirectory('imports/42');
|
||||
$disk->makeDirectory('story_archives/7/202601');
|
||||
|
||||
// Live files that must survive.
|
||||
$disk->put('public/m/_v2/9/month/keep/live.jpg', 'x');
|
||||
$disk->put('imports/99/live.jpg', 'y');
|
||||
|
||||
$this->artisan('storage:maintenance', ['--only' => 'empty-dirs'])
|
||||
->assertExitCode(0);
|
||||
|
||||
expect($disk->directoryExists('public/m/_v2/9/month/rand'))->toBeFalse();
|
||||
expect($disk->directoryExists('public/_esm.t3/m'))->toBeFalse();
|
||||
expect($disk->directoryExists('public/avatars/000'))->toBeFalse();
|
||||
expect($disk->directoryExists('imports/42'))->toBeFalse();
|
||||
expect($disk->directoryExists('story_archives/7'))->toBeFalse();
|
||||
|
||||
// Roots and live branches preserved.
|
||||
expect($disk->exists('public/m/_v2/9/month/keep/live.jpg'))->toBeTrue();
|
||||
expect($disk->exists('imports/99/live.jpg'))->toBeTrue();
|
||||
});
|
||||
|
||||
it('dry-run reports empty-dir removals without deleting them', function () {
|
||||
$disk = Storage::disk('local');
|
||||
$disk->makeDirectory('public/m/_v2/9/month/rand');
|
||||
|
||||
$this->artisan('storage:maintenance', ['--only' => 'empty-dirs', '--dry-run' => true])
|
||||
->assertExitCode(0);
|
||||
|
||||
expect($disk->directoryExists('public/m/_v2/9/month/rand'))->toBeTrue();
|
||||
});
|
||||
|
||||
it('rejects an --hours value below 1', function () {
|
||||
$this->artisan('storage:maintenance', ['--only' => 'remcache', '--hours' => 0])
|
||||
->assertExitCode(1);
|
||||
});
|
||||
|
||||
it('errors when only/except leave no tasks to run', function () {
|
||||
$this->artisan('storage:maintenance', ['--except' => 'remcache,empty-dirs'])
|
||||
->assertExitCode(1);
|
||||
});
|
||||
@ -0,0 +1,82 @@
|
||||
<?php
|
||||
|
||||
use App\Jobs\MediaPipeline\MediaDeletePipeline;
|
||||
use App\Models\Media;
|
||||
use App\Models\User;
|
||||
use Illuminate\Foundation\Testing\LazilyRefreshDatabase;
|
||||
use Illuminate\Support\Facades\Config;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
|
||||
uses(LazilyRefreshDatabase::class);
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| MediaDeletePipeline — in-flow leaf directory cleanup
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| A media delete removes its own files AND the leaf directory it emptied
|
||||
| (public/m/_v2/{pid}/{month}/{random}), so the flow does not leave empty
|
||||
| folders behind for the scheduled sweep to find.
|
||||
|
|
||||
*/
|
||||
|
||||
beforeEach(function () {
|
||||
Config::set('filesystems.local', 'local');
|
||||
Config::set('pixelfed.cloud_storage', false);
|
||||
Storage::fake('local');
|
||||
});
|
||||
|
||||
function makeOrphanLocalMedia(): Media
|
||||
{
|
||||
$user = User::factory()->create();
|
||||
$user->refresh();
|
||||
$pid = $user->profile->id;
|
||||
|
||||
$leaf = 'public/m/_v2/'.$pid.'/aa-bb/rndrndrndrnd';
|
||||
$path = $leaf.'/file.jpg';
|
||||
$thumb = $leaf.'/file_thumb.jpeg';
|
||||
|
||||
Storage::disk('local')->put($path, 'PRIMARY');
|
||||
Storage::disk('local')->put($thumb, 'THUMB');
|
||||
|
||||
return Media::create([
|
||||
'status_id' => null,
|
||||
'profile_id' => $pid,
|
||||
'user_id' => $user->id,
|
||||
'media_path' => $path,
|
||||
'thumbnail_path' => $thumb,
|
||||
'mime' => 'image/jpeg',
|
||||
'size' => 7,
|
||||
'remote_media' => false,
|
||||
'order' => 0,
|
||||
]);
|
||||
}
|
||||
|
||||
it('deletes the media files and removes its now-empty leaf directory', function () {
|
||||
$media = makeOrphanLocalMedia();
|
||||
$leaf = implode('/', array_slice(explode('/', $media->media_path), 0, -1));
|
||||
$disk = Storage::disk('local');
|
||||
|
||||
expect($disk->exists($media->media_path))->toBeTrue();
|
||||
|
||||
(new MediaDeletePipeline($media))->handle();
|
||||
|
||||
expect($disk->exists($media->media_path))->toBeFalse();
|
||||
expect($disk->exists($media->thumbnail_path))->toBeFalse();
|
||||
expect($disk->directoryExists($leaf))->toBeFalse();
|
||||
});
|
||||
|
||||
it('leaves the leaf directory in place when another file still lives there', function () {
|
||||
$media = makeOrphanLocalMedia();
|
||||
$leaf = implode('/', array_slice(explode('/', $media->media_path), 0, -1));
|
||||
$disk = Storage::disk('local');
|
||||
|
||||
// A sibling file that this media does not own.
|
||||
$disk->put($leaf.'/sibling.jpg', 'KEEP');
|
||||
|
||||
(new MediaDeletePipeline($media))->handle();
|
||||
|
||||
expect($disk->exists($media->media_path))->toBeFalse();
|
||||
expect($disk->directoryExists($leaf))->toBeTrue();
|
||||
expect($disk->exists($leaf.'/sibling.jpg'))->toBeTrue();
|
||||
});
|
||||
Loading…
Reference in New Issue