Implement backfill requests

app/controllers/api/fasp/data_sharing/v0/backfill_requests_controller.rb

@@ -1,4 +1,26 @@
 # frozen_string_literal: true
 
-class Api::Fasp::DataSharing::V0::BackfillRequestsController < ApplicationController
+class Api::Fasp::DataSharing::V0::BackfillRequestsController < Api::Fasp::BaseController
+  def create
+    backfill_request = current_provider.fasp_backfill_requests.new(backfill_request_params)
+
+    respond_to do |format|
+      format.json do
+        if backfill_request.save
+          render json: { backfillRequest: { id: backfill_request.id } }, status: 201
+        else
+          head 422
+        end
+      end
+    end
+  end
+
+  private
+
+  def backfill_request_params
+    params
+      .permit(:category, :maxCount)
+      .to_unsafe_h
+      .transform_keys { |k| k.to_s.underscore }
+  end
 end

app/controllers/api/fasp/data_sharing/v0/continuations_controller.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class Api::Fasp::DataSharing::V0::ContinuationsController < Api::Fasp::BaseController
+  def create
+    backfill_request = current_provider.fasp_backfill_requests.find(params[:backfill_request_id])
+    Fasp::BackfillWorker.perform_async(backfill_request.id)
+
+    head 204
+  end
+end

app/models/fasp.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 module Fasp
+  DATA_CATEGORIES = %w(account content).freeze
+
   def self.table_name_prefix
     'fasp_'
   end

app/models/fasp/backfill_request.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: fasp_backfill_requests
+#
+#  id               :bigint(8)        not null, primary key
+#  category         :string           not null
+#  cursor           :string
+#  fulfilled        :boolean          default(FALSE), not null
+#  max_count        :integer          default(100), not null
+#  created_at       :datetime         not null
+#  updated_at       :datetime         not null
+#  fasp_provider_id :bigint(8)        not null
+#
+class Fasp::BackfillRequest < ApplicationRecord
+  belongs_to :fasp_provider, class_name: 'Fasp::Provider'
+
+  validates :category, presence: true, inclusion: Fasp::DATA_CATEGORIES
+  validates :max_count, presence: true,
+                        numericality: { only_integer: true }
+
+  after_commit :queue_fulfillment_job, on: :create
+
+  def next_objects
+    @next_objects ||= base_scope.to_a
+  end
+
+  def next_uris
+    next_objects.map { |o| ActivityPub::TagManager.instance.uri_for(o) }
+  end
+
+  def more_objects_available?
+    return false if next_objects.empty?
+
+    base_scope.where(id: ...(next_objects.last.id)).any?
+  end
+
+  def advance!
+    if more_objects_available?
+      update!(cursor: next_objects.last.id)
+    else
+      update!(fulfilled: true)
+    end
+  end
+
+  private
+
+  def base_scope
+    result = category_scope.limit(max_count).order(id: :desc)
+    result = result.where(id: ...cursor) if cursor.present?
+    result
+  end
+
+  def category_scope
+    case category
+    when 'account'
+      Account.discoverable.without_instance_actor
+    when 'content'
+      Status.indexable
+    end
+  end
+
+  def queue_fulfillment_job
+    Fasp::BackfillWorker.perform_async(id)
+  end
+end

app/models/fasp/provider.rb

@@ -22,6 +22,7 @@
 class Fasp::Provider < ApplicationRecord
   include DebugConcern
 
+  has_many :fasp_backfill_requests, inverse_of: :fasp_provider, class_name: 'Fasp::BackfillRequest', dependent: :delete_all
   has_many :fasp_debug_callbacks, inverse_of: :fasp_provider, class_name: 'Fasp::DebugCallback', dependent: :delete_all
   has_many :fasp_subscriptions, inverse_of: :fasp_provider, class_name: 'Fasp::Subscription', dependent: :delete_all
 

app/models/fasp/subscription.rb

@@ -17,12 +17,11 @@
 #  fasp_provider_id    :bigint(8)        not null
 #
 class Fasp::Subscription < ApplicationRecord
-  CATEGORIES = %w(account content).freeze
   TYPES = %w(lifecycle trends).freeze
 
   belongs_to :fasp_provider, class_name: 'Fasp::Provider'
 
-  validates :category, presence: true, inclusion: CATEGORIES
+  validates :category, presence: true, inclusion: Fasp::DATA_CATEGORIES
   validates :subscription_type, presence: true,
                                 inclusion: TYPES
 

app/workers/fasp/backfill_worker.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+class Fasp::BackfillWorker
+  include Sidekiq::Worker
+
+  sidekiq_options queue: 'fasp', retry: 5
+
+  def perform(backfill_request_id)
+    backfill_request = Fasp::BackfillRequest.find(backfill_request_id)
+
+    announce(backfill_request)
+
+    backfill_request.advance!
+  rescue ActiveRecord::RecordNotFound
+    # ignore missing backfill requests
+  end
+
+  private
+
+  def announce(backfill_request)
+    Fasp::Request.new(backfill_request.fasp_provider).post('/data_sharing/v0/announcements', body: {
+      source: {
+        backfillRequest: {
+          id: backfill_request.id.to_s,
+        },
+      },
+      category: backfill_request.category,
+      objectUris: backfill_request.next_uris,
+      moreObjectsAvailable: backfill_request.more_objects_available?,
+    })
+  end
+end

config/routes/fasp.rb

@@ -12,7 +12,9 @@ namespace :api, format: false do
 
     namespace :data_sharing do
       namespace :v0 do
-        resources :backfill_requests, only: [:create]
+        resources :backfill_requests, only: [:create] do
+          resource :continuation, only: [:create]
+        end
 
         resources :event_subscriptions, only: [:create, :destroy]
       end

db/migrate/20250103131909_create_fasp_backfill_requests.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class CreateFaspBackfillRequests < ActiveRecord::Migration[7.2]
+  def change
+    create_table :fasp_backfill_requests do |t|
+      t.string :category, null: false
+      t.integer :max_count, null: false, default: 100
+      t.string :cursor
+      t.boolean :fulfilled, null: false, default: false
+      t.references :fasp_provider, null: false, foreign_key: true
+
+      t.timestamps
+    end
+  end
+end

db/schema.rb

@@ -444,6 +444,17 @@ ActiveRecord::Schema[8.0].define(version: 2025_01_29_144813) do
     t.index ["domain"], name: "index_email_domain_blocks_on_domain", unique: true
   end
 
+  create_table "fasp_backfill_requests", force: :cascade do |t|
+    t.string "category", null: false
+    t.integer "max_count", default: 100, null: false
+    t.string "cursor"
+    t.boolean "fulfilled", default: false, null: false
+    t.bigint "fasp_provider_id", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["fasp_provider_id"], name: "index_fasp_backfill_requests_on_fasp_provider_id"
+  end
+
   create_table "fasp_debug_callbacks", force: :cascade do |t|
     t.bigint "fasp_provider_id", null: false
     t.string "ip"
@@ -1324,6 +1335,7 @@ ActiveRecord::Schema[8.0].define(version: 2025_01_29_144813) do
   add_foreign_key "custom_filter_statuses", "statuses", on_delete: :cascade
   add_foreign_key "custom_filters", "accounts", on_delete: :cascade
   add_foreign_key "email_domain_blocks", "email_domain_blocks", column: "parent_id", on_delete: :cascade
+  add_foreign_key "fasp_backfill_requests", "fasp_providers"
   add_foreign_key "fasp_debug_callbacks", "fasp_providers"
   add_foreign_key "fasp_subscriptions", "fasp_providers"
   add_foreign_key "favourites", "accounts", name: "fk_5eb6c2b873", on_delete: :cascade

spec/fabricators/fasp/backfill_request_fabricator.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+Fabricator('Fasp::BackfillRequest') do
+  category      'MyString'
+  max_count     1
+  cursor        'MyString'
+  fulfilled     false
+  fasp_provider nil
+end

spec/models/fasp/backfill_request_spec.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe Fasp::BackfillRequest do
+  pending "add some examples to (or delete) #{__FILE__}"
+end

spec/requests/api/fasp/data_sharing/v0/continuations_spec.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe 'Api::Fasp::DataSharing::V0::Continuations' do
+  describe 'GET /index' do
+    pending "add some examples (or delete) #{__FILE__}"
+  end
+end