DEV: Add Upload to IntermediateDB (#29780)

Gemfile

@@ -284,6 +284,9 @@ group :migrations, optional: true do
 
   # CLI
   gem "ruby-progressbar"
+
+  # non-cryptographic hashing algorithm for generating placeholder IDs
+  gem "digest-xxhash"
 end
 
 gem "dry-initializer", "~> 3.1"

Gemfile.lock

@@ -124,6 +124,7 @@ GEM
     diff-lcs (1.5.1)
     diffy (3.4.3)
     digest (3.1.1)
+    digest-xxhash (0.2.9)
     discourse-fonts (0.0.14)
     discourse-seed-fu (2.3.12)
       activerecord (>= 3.1)
@@ -630,6 +631,7 @@ DEPENDENCIES
   csv
   diffy
   digest
+  digest-xxhash
   discourse-fonts
   discourse-seed-fu
   discourse_dev_assets

migrations/db/intermediate_db_schema/003-uploads.sql

@@ -0,0 +1,12 @@
+CREATE TABLE uploads
+(
+    id          TEXT NOT NULL PRIMARY KEY,
+    filename    TEXT NOT NULL,
+    path        TEXT,
+    data        BLOB,
+    url         TEXT,
+    type        TEXT,
+    description TEXT,
+    origin      TEXT,
+    user_id     NUMERIC
+);

migrations/db/intermediate_db_schema/100-base-schema.sql

@@ -33,5 +33,5 @@ CREATE TABLE users
     suspended_at              DATETIME,
     suspended_till            DATETIME,
     title                     TEXT,
-    uploaded_avatar_id        INTEGER
-);
+    uploaded_avatar_id        TEXT
+);

migrations/lib/common/id.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "digest/xxhash"
+
+module Migrations
+  module ID
+    def self.hash(value)
+      Digest::XXH3_128bits.base64digest(value)
+    end
+
+    def self.build(part1, part2, *others)
+      [part1, part2, *others].join("-")
+    end
+  end
+end

migrations/lib/database/intermediate_db/upload.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Migrations::Database::IntermediateDB
+  module Upload
+    SQL = <<~SQL
+      INSERT OR IGNORE INTO uploads (
+          id,
+          filename,
+          path,
+          data,
+          url,
+          type,
+          description,
+          origin,
+          user_id
+      )
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+    SQL
+
+    class << self
+      def create_for_file!(
+        path:,
+        filename: nil,
+        type: nil,
+        description: nil,
+        origin: nil,
+        user_id: nil
+      )
+        create!(
+          id: ::Migrations::ID.hash(path),
+          filename: filename || File.basename(path),
+          path:,
+          type:,
+          description:,
+          origin:,
+          user_id:,
+        )
+      end
+
+      def create_for_url!(url:, filename:, type: nil, description: nil, origin: nil, user_id: nil)
+        create!(
+          id: ::Migrations::ID.hash(url),
+          filename:,
+          url:,
+          type:,
+          description:,
+          origin:,
+          user_id:,
+        )
+      end
+
+      def create_for_data!(data:, filename:, type: nil, description: nil, origin: nil, user_id: nil)
+        create!(
+          id: ::Migrations::ID.hash(data),
+          filename:,
+          data: ::Migrations::Database.to_blob(data),
+          type:,
+          description:,
+          origin:,
+          user_id:,
+        )
+      end
+
+      private
+
+      def create!(
+        id:,
+        filename:,
+        path: nil,
+        data: nil,
+        url: nil,
+        type: nil,
+        description: nil,
+        origin: nil,
+        user_id: nil
+      )
+        ::Migrations::Database::IntermediateDB.insert(
+          SQL,
+          id,
+          filename,
+          path,
+          data,
+          url,
+          type,
+          description,
+          origin,
+          user_id,
+        )
+      end
+    end
+  end
+end

migrations/lib/migrations.rb

@@ -45,7 +45,12 @@ module Migrations
     loader.log! if ENV["DEBUG"]
 
     loader.inflector.inflect(
-      { "cli" => "CLI", "intermediate_db" => "IntermediateDB", "uploads_db" => "UploadsDB" },
+      {
+        "cli" => "CLI",
+        "id" => "ID",
+        "intermediate_db" => "IntermediateDB",
+        "uploads_db" => "UploadsDB",
+      },
     )
 
     loader.push_dir(File.join(::Migrations.root_path, "lib"), namespace: ::Migrations)