Use remote storage for autocomplete binaries

assets/js/autocomplete/__tests__/context.ts

@@ -34,6 +34,7 @@ export class TestContext {
 
   constructor(fakeAutocompleteResponse: Response) {
     this.fakeAutocompleteResponse = fakeAutocompleteResponse;
+    window.booru.autocompleteFileUrl = '/autocomplete/compiled';
 
     vi.useFakeTimers().setSystemTime(0);
     fetchMock.enableMocks();
@@ -179,7 +180,7 @@ export class TestContext {
     await vi.runAllTimersAsync();
   }
 
-  snapPequests() {
+  snapRequests() {
     const snapshot = vi.mocked(fetch).mock.calls.map(([input]) => {
       const request = input as unknown as Request;
       const meta: Record<string, unknown> = {};
@@ -283,10 +284,10 @@ export const autocompleteTest = test.extend<{ ctx: TestContext }>({
     // Initialize the lazy autocomplete index cache
     await ctx.focusInput();
 
-    expect(ctx.snapPequests()).toMatchInlineSnapshot(`
+    expect(ctx.snapRequests()).toMatchInlineSnapshot(`
       [
         {
-          "dest": "GET http://localhost:3000/autocomplete/compiled?vsn=2&key=1970-0-1",
+          "dest": "GET http://localhost:3000/autocomplete/compiled",
           "meta": {
             "cache": "force-cache",
             "credentials": "omit",

assets/js/autocomplete/client.ts

@@ -59,11 +59,7 @@ export class AutocompleteClient {
    * Issues a GET request to fetch the compiled autocomplete index.
    */
   async getCompiledAutocomplete(): Promise<ArrayBuffer> {
-    const now = new Date();
-    const key = `${now.getUTCFullYear()}-${now.getUTCMonth()}-${now.getUTCDate()}`;
-
-    const response = await this.http.fetch(`/autocomplete/compiled`, {
-      query: { vsn: '2', key },
+    const response = await this.http.fetch(window.booru.autocompleteFileUrl, {
       credentials: 'omit',
       cache: 'force-cache',
     });

assets/js/booru.ts

@@ -81,6 +81,10 @@ export interface BooruObject {
    * Indicates whether sensitive staff-only info should be hidden or not.
    */
   hideStaffTools: boolean;
+  /**
+   * URL to the local autocomplete binary
+   */
+  autocompleteFileUrl: string;
   /**
    * List of image IDs in the current gallery.
    */

config/runtime.exs

@@ -11,6 +11,8 @@ config :bcrypt_elixir,
   log_rounds: String.to_integer(System.get_env("BCRYPT_ROUNDS", "12"))
 
 config :philomena,
+  autocomplete_file_root: System.fetch_env!("AUTOCOMPLETE_FILE_ROOT"),
+  autocomplete_url_root: System.fetch_env!("AUTOCOMPLETE_URL_ROOT"),
   anonymous_name_salt: System.fetch_env!("ANONYMOUS_NAME_SALT"),
   hcaptcha_secret_key: System.fetch_env!("HCAPTCHA_SECRET_KEY"),
   hcaptcha_site_key: System.fetch_env!("HCAPTCHA_SITE_KEY"),

docker-compose.yml

@@ -31,11 +31,13 @@ services:
       - PASSWORD_PEPPER=dn2e0EpZrvBLoxUM3gfQveBhjf0bG/6/bYhrOyq3L3hV9hdo/bimJ+irbDWsuXLP
       - TUMBLR_API_KEY=fuiKNFp9vQFvjLNvx4sUwti4Yb5yGutBN4Xh10LXZhhRKjWlV4
       - OTP_SECRET_KEY=Wn7O/8DD+qxL0X4X7bvT90wOkVGcA90bIHww4twR03Ci//zq7PnMw8ypqyyT/b/C
+      - AUTOCOMPLETE_FILE_ROOT=autocomplete
       - ADVERT_FILE_ROOT=adverts
       - AVATAR_FILE_ROOT=avatars
       - BADGE_FILE_ROOT=badges
       - IMAGE_FILE_ROOT=images
       - TAG_FILE_ROOT=tags
+      - AUTOCOMPLETE_URL_ROOT=/autocomplete-bin
       - AVATAR_URL_ROOT=/avatars
       - ADVERT_URL_ROOT=/spns
       - IMAGE_URL_ROOT=/img

docker/app/run-development

@@ -17,7 +17,7 @@ background() {
     step mix run -e 'Philomena.Release.verify_artist_links()' > /dev/null
     step mix run -e 'Philomena.Release.update_stats()' > /dev/null
     step mix run -e 'Philomena.Release.clean_moderation_logs()' > /dev/null
-    step mix run -e 'Philomena.Release.generate_autocomplete()' > /dev/null
+    step mix run -e 'Philomena.Release.generate_and_prune_autocomplete()' > /dev/null
     step mix run -e 'Philomena.Release.clean_tags()' > /dev/null
 
     sleep 300

docker/production/.env-example

@@ -6,6 +6,7 @@ TUMBLR_API_KEY=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
 PROXY_HOST=http://tinyproxy:3128
 DATABASE_URL=ecto://philomena@postgres/philomena
 IMAGE_URL_ROOT=https://philomena-cdn.example/img
+AUTOCOMPLETE_URL_ROOT=https://philomena-cdn.example/autocomplete
 AVATAR_URL_ROOT=https://philomena-cdn.example/avatars
 ADVERT_URL_ROOT=https://philomena-cdn.example/spns
 BADGE_URL_ROOT=https://philomena-cdn.example/badges
@@ -33,6 +34,7 @@ REDIS_HOST=valkey
 OPENSEARCH_URL=http://opensearch:9200
 MEDIAPROC_ADDR=mediaproc:1500
 
+AUTOCOMPLETE_FILE_ROOT=autocomplete
 ADVERT_FILE_ROOT=adverts
 AVATAR_FILE_ROOT=avatars
 BADGE_FILE_ROOT=badges

docker/production/run-cron-daily

@@ -3,5 +3,5 @@ set -e
 
 export RELEASE_NODE=philomena_daily
 
-philomena eval 'Philomena.Release.generate_autocomplete()'
+philomena eval 'Philomena.Release.generate_and_prune_autocomplete()'
 philomena eval 'Philomena.Release.clean_tags()'

docker/production/web/Caddyfile

@@ -146,6 +146,7 @@
 	@imgview path_regexp ^/img/view/(.+)/([0-9]+).*\.([A-Za-z0-9]+)$
 	@img path_regexp ^/img/(.+)$
 	@spns path_regexp ^/spns/(.+)$
+	@autocomplete path_regexp ^/autocomplete/(.+)$
 	@avatars path_regexp ^/avatars/(.+)$
 	@badges path_regexp ^/badges/(.+)$
 	@tags path_regexp ^/tags/(.+)$
@@ -155,6 +156,7 @@
 	import s3path @imgdownload /images/{re.imgdownload.1}/{re.imgdownload.2}/full.{re.imgdownload.3}
 	import s3path @imgview /images/{re.imgview.1}/{re.imgview.2}/full.{re.imgview.3}
 	import s3path @img /images/{re.img.1}
+	import s3path @autocomplete /autocomplete/{re.autocomplete.1}
 	import s3path @avatars /avatars/{re.avatars.1}
 	import s3path @spns /adverts/{re.spns.1}
 	import s3path @badges /badges/{re.badges.1}

docker/web/config/Caddyfile

@@ -23,13 +23,15 @@
 	@imgview path_regexp ^/img/view/(.+)/([0-9]+).*\.([A-Za-z0-9]+)$
 	@img path_regexp ^/img/(.+)$
 	@spns path_regexp ^/spns/(.+)$
+	@autocomplete path_regexp ^/autocomplete-bin/(.+)$
 	@avatars path_regexp ^/avatars/(.+)$
 	@badges path_regexp ^/badge-img/(.+)$
 	@tags path_regexp ^/tag-img/(.+)$
 
 	import s3path @imgdownload /images/{re.imgdownload.1}/{re.imgdownload.2}/full.{re.imgdownload.3}
 	import s3path @imgview /images/{re.imgview.1}/{re.imgview.2}/full.{re.imgview.3}
 	import s3path @img /images/{re.img.1}
+	import s3path @autocomplete /autocomplete/{re.autocomplete.1}
 	import s3path @avatars /avatars/{re.avatars.1}
 	import s3path @spns /adverts/{re.spns.1}
 	import s3path @badges /badges/{re.badges.1}

lib/philomena/autocomplete.ex

@@ -12,9 +12,10 @@ defmodule Philomena.Autocomplete do
 
   alias Philomena.Autocomplete.Autocomplete
   alias Philomena.Autocomplete.Generator
+  alias Philomena.Autocomplete.Uploader
 
   @doc """
-  Gets the current local autocompletion binary.
+  Gets the current local autocompletion information.
 
   Returns nil if the binary is not currently generated.
 
@@ -35,20 +36,52 @@ defmodule Philomena.Autocomplete do
   end
 
   @doc """
-  Creates a new local autocompletion binary, replacing any which currently exist.
+  Creates a new local autocompletion binary and prunes existing binaries.
+  """
+  def generate_and_prune_autocomplete! do
+    generate_autocomplete!()
+    prune_autocomplete!()
+  end
+
+  @doc """
+  Creates a new local autocompletion binary.
   """
   def generate_autocomplete! do
-    ac_file = Generator.generate()
+    path = generate_autocomplete_file!()
 
-    # Insert the autocomplete binary
-    new_ac =
-      %Autocomplete{}
-      |> Autocomplete.changeset(%{content: ac_file})
-      |> Repo.insert!()
+    %Autocomplete{}
+    |> Uploader.prepare_upload(path)
+    |> Repo.insert!()
+    |> Uploader.persist_upload()
+  end
 
-    # Remove anything older
+  @doc """
+  Removes old autocomplete binaries.
+  """
+  def prune_autocomplete! do
     Autocomplete
-    |> where([ac], ac.created_at < ^new_ac.created_at)
-    |> Repo.delete_all()
+    |> where([ac], ac.created_at < ago(1, "week"))
+    |> Repo.all()
+    |> Enum.each(&delete_autocomplete/1)
+  end
+
+  defp delete_autocomplete(%Autocomplete{} = autocomplete) do
+    if autocomplete.file do
+      Uploader.unpersist_upload(autocomplete)
+
+      Autocomplete
+      |> where([ac], ac.file == ^autocomplete.file)
+      |> Repo.delete_all()
+    end
+  end
+
+  # sobelow_skip ["Traversal.FileModule"]
+  defp generate_autocomplete_file! do
+    content = Generator.generate()
+    file = Briefly.create!()
+
+    File.write!(file, content)
+
+    file
   end
 end

lib/philomena/autocomplete/autocomplete.ex

@@ -6,14 +6,16 @@ defmodule Philomena.Autocomplete.Autocomplete do
 
   @primary_key false
   schema "autocomplete" do
-    field :content, :binary
+    field :file, :string
+    field :uploaded_file, :string, virtual: true
+    field :removed_file, :string, virtual: true
     timestamps(inserted_at: :created_at, updated_at: false, type: :utc_datetime)
   end
 
   @doc false
   def changeset(autocomplete, attrs) do
     autocomplete
-    |> cast(attrs, [:content])
-    |> validate_required([:content])
+    |> cast(attrs, [:file, :uploaded_file, :removed_file])
+    |> validate_required([:file])
   end
 end

lib/philomena/autocomplete/uploader.ex

@@ -0,0 +1,37 @@
+defmodule Philomena.Autocomplete.Uploader do
+  @moduledoc """
+  Upload callback logic for Autocomplete.
+  """
+
+  alias Philomena.Autocomplete.Autocomplete
+  alias PhilomenaMedia.Filename
+  alias PhilomenaMedia.Uploader
+
+  @field_name "file"
+
+  def prepare_upload(autocomplete, path) do
+    storage_key = Filename.build("bin")
+
+    Uploader.prepare_upload(
+      autocomplete,
+      @field_name,
+      storage_key,
+      path,
+      &Autocomplete.changeset/2
+    )
+  end
+
+  def persist_upload(autocomplete) do
+    Uploader.persist_upload(autocomplete, autocomplete_file_root(), @field_name)
+  end
+
+  def unpersist_upload(autocomplete) do
+    autocomplete
+    |> Map.put(:removed_file, autocomplete.file)
+    |> Uploader.unpersist_old_upload(autocomplete_file_root(), @field_name)
+  end
+
+  defp autocomplete_file_root do
+    Application.get_env(:philomena, :autocomplete_file_root)
+  end
+end

lib/philomena/release.ex

@@ -44,9 +44,9 @@ defmodule Philomena.Release do
     Philomena.ModerationLogs.cleanup!()
   end
 
-  def generate_autocomplete do
+  def generate_and_prune_autocomplete do
     start_app()
-    Philomena.Autocomplete.generate_autocomplete!()
+    Philomena.Autocomplete.generate_and_prune_autocomplete!()
   end
 
   def clean_tags do