aiden
/
pixelfed
mirror of https://github.com/pixelfed/pixelfed
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add NetworkTimelineService cache

Browse Source
pull/3532/head
Daniel Supernault 3 years ago
parent 9c17def4a0
commit 1310d95cdb
No known key found for this signature in database
GPG Key ID: 0DEF1C662C9033F7
4 changed files with 199 additions and 53 deletions
Show Stats Download Patch File Download Diff File

140
app/Http/Controllers/PublicApiController.php
Unescape Escape View File

@ -32,6 +32,7 @@ use App\Services\{
LikeService,
PublicTimelineService,
ProfileService,
NetworkTimelineService,
ReblogService,
RelationshipService,
StatusService,
@ -608,59 +609,92 @@ class PublicApiController extends Controller
$filtered = $user ? UserFilterService::filters($user->profile_id) : [];
if($min || $max) {
$dir = $min ? '>' : '<';
$id = $min ?? $max;
$timeline = Status::select(
'id',
'uri',
'type',
'scope',
'created_at',
)
->where('id', $dir, $id)
->whereNull(['in_reply_to_id', 'reblog_of_id'])
->whereNotIn('profile_id', $filtered)
->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
->whereNotNull('uri')
->whereScope('public')
->where('id', '>', $amin)
->orderBy('created_at', 'desc')
->limit($limit)
->get()
->map(function($s) use ($user) {
$status = StatusService::get($s->id);
$status['favourited'] = (bool) LikeService::liked($user->profile_id, $s->id);
$status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $s->id);
$status['reblogged'] = (bool) ReblogService::get($user->profile_id, $s->id);
return $status;
});
$res = $timeline->toArray();
} else {
$timeline = Status::select(
'id',
'uri',
'type',
'scope',
'created_at',
)
->whereNull(['in_reply_to_id', 'reblog_of_id'])
->whereNotIn('profile_id', $filtered)
->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
->whereNotNull('uri')
->whereScope('public')
->where('id', '>', $amin)
->orderBy('created_at', 'desc')
->limit($limit)
->get()
->map(function($s) use ($user) {
$status = StatusService::get($s->id);
$status['favourited'] = (bool) LikeService::liked($user->profile_id, $s->id);
$status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $s->id);
$status['reblogged'] = (bool) ReblogService::get($user->profile_id, $s->id);
return $status;
});
$res = $timeline->toArray();
if(config('instance.timeline.network.cached') == false) {
if($min || $max) {
$dir = $min ? '>' : '<';
$id = $min ?? $max;
$timeline = Status::select(
'id',
'uri',
'type',
'scope',
'created_at',
)
->where('id', $dir, $id)
->whereNull(['in_reply_to_id', 'reblog_of_id'])
->whereNotIn('profile_id', $filtered)
->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
->whereNotNull('uri')
->whereScope('public')
->where('id', '>', $amin)
->orderBy('created_at', 'desc')
->limit($limit)
->get()
->map(function($s) use ($user) {
$status = StatusService::get($s->id);
$status['favourited'] = (bool) LikeService::liked($user->profile_id, $s->id);
$status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $s->id);
$status['reblogged'] = (bool) ReblogService::get($user->profile_id, $s->id);
return $status;
});
$res = $timeline->toArray();
} else {
$timeline = Status::select(
'id',
'uri',
'type',
'scope',
'created_at',
)
->whereNull(['in_reply_to_id', 'reblog_of_id'])
->whereNotIn('profile_id', $filtered)
->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
->whereNotNull('uri')
->whereScope('public')
->where('id', '>', $amin)
->orderBy('created_at', 'desc')
->limit($limit)
->get()
->map(function($s) use ($user) {
$status = StatusService::get($s->id);
$status['favourited'] = (bool) LikeService::liked($user->profile_id, $s->id);
$status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $s->id);
$status['reblogged'] = (bool) ReblogService::get($user->profile_id, $s->id);
return $status;
});
$res = $timeline->toArray();
}
} else {
Cache::remember('api:v1:timelines:network:cache_check', 10368000, function() {
if(NetworkTimelineService::count() == 0) {
NetworkTimelineService::warmCache(true, 400);
}
});
if ($max) {
$feed = NetworkTimelineService::getRankedMaxId($max, $limit);
} else if ($min) {
$feed = NetworkTimelineService::getRankedMinId($min, $limit);
} else {
$feed = NetworkTimelineService::get(0, $limit);
}
$res = collect($feed)
->map(function($k) use($user) {
$status = StatusService::get($k);
if($status && isset($status['account']) && $user) {
$status['favourited'] = (bool) LikeService::liked($user->profile_id, $k);
$status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $k);
$status['reblogged'] = (bool) ReblogService::get($user->profile_id, $k);
$status['relationship'] = RelationshipService::get($user->profile_id, $status['account']['id']);
}
return $status;
})
->filter(function($s) use($filtered) {
return $s && isset($s['account']) && in_array($s['account']['id'], $filtered) == false;
})
->values()
->toArray();
}
return response()->json($res);

95
app/Services/NetworkTimelineService.php
Unescape Escape View File

@ -0,0 +1,95 @@
<?php
namespace App\Services;
use Illuminate\Support\Facades\Redis;
use App\{
Profile,
Status,
UserFilter
};
class NetworkTimelineService
{
const CACHE_KEY = 'pf:services:timeline:network';
public static function get($start = 0, $stop = 10)
{
if($stop > 100) {
$stop = 100;
}
return Redis::zrevrange(self::CACHE_KEY, $start, $stop);
}
public static function getRankedMaxId($start = null, $limit = 10)
{
if(!$start) {
return [];
}
return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY, $start, '-inf', [
'withscores' => true,
'limit' => [1, $limit]
]));
}
public static function getRankedMinId($end = null, $limit = 10)
{
if(!$end) {
return [];
}
return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY, '+inf', $end, [
'withscores' => true,
'limit' => [0, $limit]
]));
}
public static function add($val)
{
if(self::count() > config('instance.timeline.network.cache_dropoff')) {
if(config('database.redis.client') === 'phpredis') {
Redis::zpopmin(self::CACHE_KEY);
}
}
return Redis::zadd(self::CACHE_KEY, $val, $val);
}
public static function rem($val)
{
return Redis::zrem(self::CACHE_KEY, $val);
}
public static function del($val)
{
return self::rem($val);
}
public static function count()
{
return Redis::zcard(self::CACHE_KEY);
}
public static function warmCache($force = false, $limit = 100)
{
if(self::count() == 0 || $force == true) {
Redis::del(self::CACHE_KEY);
$ids = Status::whereNotNull('uri')
->whereScope('public')
->whereNull('in_reply_to_id')
->whereNull('reblog_of_id')
->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
->where('created_at', '>', now()->subHours(config('instance.timeline.network.max_hours_old')))
->orderByDesc('created_at')
->limit($limit)
->pluck('id');
foreach($ids as $id) {
self::add($id);
}
return 1;
}
return 0;
}
}

11
app/Util/ActivityPub/Helpers.php
Unescape Escape View File

@ -32,6 +32,7 @@ use App\Services\CustomEmojiService;
use App\Services\InstanceService;
use App\Services\MediaPathService;
use App\Services\MediaStorageService;
use App\Services\NetworkTimelineService;
use App\Jobs\MediaPipeline\MediaStoragePipeline;
use App\Jobs\AvatarPipeline\RemoteAvatarFetch;
use App\Util\Media\License;
@ -490,6 +491,16 @@ class Helpers {
if(isset($activity['tag']) && is_array($activity['tag']) && !empty($activity['tag'])) {
StatusTagsPipeline::dispatch($activity, $status);
}
if( config('instance.timeline.network.cached') &&
$status->in_reply_to_id === null &&
$status->reblog_of_id === null &&
in_array($status->type, ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album']) &&
$status->created_at->gt(now()->subHours(config('instance.timeline.network.max_hours_old'))
) {
NetworkTimelineService::add($status->id);
}
return $status;
});
}

6
config/instance.php
Unescape Escape View File

@ -24,6 +24,12 @@ return [
'timeline' => [
'local' => [
'is_public' => env('INSTANCE_PUBLIC_LOCAL_TIMELINE', false)
],
'network' => [
'cached' => env('PF_NETWORK_TIMELINE') ? env('INSTANCE_NETWORK_TIMELINE_CACHED', false) : false,
'cache_dropoff' => env('INSTANCE_NETWORK_TIMELINE_CACHE_DROPOFF', 100),
'max_hours_old' => env('INSTANCE_NETWORK_TIMELINE_CACHE_MAX_HOUR_INGEST', 6)
]
],

Write Preview
Loading…
Cancel
Save