diff --git a/app/Console/Commands/Internal/RemcacheGarbageCollector.php b/app/Console/Commands/Internal/RemcacheGarbageCollector.php new file mode 100644 index 000000000..f08fa71df --- /dev/null +++ b/app/Console/Commands/Internal/RemcacheGarbageCollector.php @@ -0,0 +1,103 @@ +option('hours'); + + if ($hours < 1) { + $this->error('The --hours value must be at least 1.'); + + return self::FAILURE; + } + + $dryRun = (bool) $this->option('dry-run'); + $dir = storage_path('app/remcache'); + + if (! is_dir($dir)) { + $this->info('remcache directory does not exist, nothing to do.'); + + return self::SUCCESS; + } + + $cutoff = now()->subHours($hours)->getTimestamp(); + $deleted = 0; + $reclaimed = 0; + + foreach (new \FilesystemIterator($dir, \FilesystemIterator::SKIP_DOTS) as $file) { + if (! $file->isFile()) { + continue; + } + + // Preserve dotfiles such as the directory's .gitignore. + if (str_starts_with($file->getFilename(), '.')) { + continue; + } + + if ($file->getMTime() >= $cutoff) { + continue; + } + + $size = $file->getSize(); + $path = $file->getPathname(); + + if ($dryRun) { + $this->line('[dry-run] would delete: '.$file->getFilename()); + $deleted++; + $reclaimed += $size; + + continue; + } + + if (@unlink($path)) { + $deleted++; + $reclaimed += $size; + } + } + + $verb = $dryRun ? 'Would delete' : 'Deleted'; + $this->info(sprintf('%s %d file(s), %s.', $verb, $deleted, $this->humanBytes($reclaimed))); + + return self::SUCCESS; + } + + private function humanBytes(int $bytes): string + { + if ($bytes < 1024) { + return $bytes.' B'; + } + + $units = ['KB', 'MB', 'GB', 'TB']; + $value = $bytes / 1024; + $i = 0; + + while ($value >= 1024 && $i < count($units) - 1) { + $value /= 1024; + $i++; + } + + return sprintf('%.2f %s', $value, $units[$i]); + } +} diff --git a/app/Services/MediaStorageService.php b/app/Services/MediaStorageService.php index d52063e3e..9be04ea01 100644 --- a/app/Services/MediaStorageService.php +++ b/app/Services/MediaStorageService.php @@ -185,23 +185,28 @@ class MediaStorageService return; } file_put_contents($tmpName, $data); - $hash = hash_file('sha256', $tmpName); - $disk = Storage::disk(config('filesystems.cloud')); - $file = $disk->putFileAs($base, new File($tmpName), $path, 'public'); - $permalink = $disk->url($file); + try { + $hash = hash_file('sha256', $tmpName); - $media->media_path = $file; - $media->cdn_url = $permalink; - $media->original_sha256 = $hash; - $media->replicated_at = now(); - $media->save(); + $disk = Storage::disk(config('filesystems.cloud')); + $file = $disk->putFileAs($base, new File($tmpName), $path, 'public'); + $permalink = $disk->url($file); - if ($media->status_id) { - Cache::forget('status:transformer:media:attachments:'.$media->status_id); - } + $media->media_path = $file; + $media->cdn_url = $permalink; + $media->original_sha256 = $hash; + $media->replicated_at = now(); + $media->save(); - unlink($tmpName); + if ($media->status_id) { + Cache::forget('status:transformer:media:attachments:'.$media->status_id); + } + } finally { + if (is_file($tmpName)) { + @unlink($tmpName); + } + } } protected function fetchAvatar($avatar, $local = false, $skipRecentCheck = false) @@ -265,33 +270,36 @@ class MediaStorageService } file_put_contents($tmpName, $data); - $mimeCheck = Storage::mimeType('remcache/'.$tmpPath); + try { + $mimeCheck = Storage::mimeType('remcache/'.$tmpPath); - if (! $mimeCheck || ! in_array($mimeCheck, ['image/png', 'image/jpeg', 'image/jpg'])) { - $avatar->last_fetched_at = now(); - $avatar->save(); - unlink($tmpName); + if (! $mimeCheck || ! in_array($mimeCheck, ['image/png', 'image/jpeg', 'image/jpg'])) { + $avatar->last_fetched_at = now(); + $avatar->save(); - return; - } - - $disk = Storage::disk($driver); - $file = $disk->putFileAs($base, new File($tmpName), $path, 'public'); - $permalink = $disk->url($file); + return; + } - $avatar->media_path = $base.'/'.$path; - $avatar->is_remote = true; - $avatar->cdn_url = $local ? config('app.url').$permalink : $permalink; - $avatar->size = $head['length']; - $avatar->change_count = $avatar->change_count + 1; - $avatar->last_fetched_at = now(); - $avatar->save(); + $disk = Storage::disk($driver); + $file = $disk->putFileAs($base, new File($tmpName), $path, 'public'); + $permalink = $disk->url($file); - Cache::forget('avatar:'.$avatar->profile_id); - AccountService::del($avatar->profile_id); - AvatarStorageCleanup::dispatch($avatar)->onQueue($queue)->delay(now()->addMinutes(random_int(3, 15))); + $avatar->media_path = $base.'/'.$path; + $avatar->is_remote = true; + $avatar->cdn_url = $local ? config('app.url').$permalink : $permalink; + $avatar->size = $head['length']; + $avatar->change_count = $avatar->change_count + 1; + $avatar->last_fetched_at = now(); + $avatar->save(); - unlink($tmpName); + Cache::forget('avatar:'.$avatar->profile_id); + AccountService::del($avatar->profile_id); + AvatarStorageCleanup::dispatch($avatar)->onQueue($queue)->delay(now()->addMinutes(random_int(3, 15))); + } finally { + if (is_file($tmpName)) { + @unlink($tmpName); + } + } } public static function delete(Media $media, $confirm = false) diff --git a/bootstrap/app.php b/bootstrap/app.php index 9d4f06c0b..dfeca931c 100644 --- a/bootstrap/app.php +++ b/bootstrap/app.php @@ -138,6 +138,7 @@ return Application::configure(basePath: dirname(__DIR__)) $schedule->command('gc:failedjobs')->dailyAt(3)->onOneServer(); $schedule->command('gc:passwordreset')->dailyAt('09:41')->onOneServer(); $schedule->command('gc:sessions')->twiceDaily(13, 23)->onOneServer(); + $schedule->command('gc:remcache')->dailyAt('04:15')->onOneServer(); $schedule->command('app:weekly-instance-scan')->weeklyOn(2, '4:20')->onOneServer(); $schedule->command('app:cleanup-expired-app-registrations')->dailyAt(1)->onOneServer(); $schedule->command('passport:purge')->everyFourHours(20)->onOneServer();