Save scratch assets to bucket

config/initializers/mime_types.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+Mime::Type.register 'audio/vnd.wav', :wav

lib/scratch_asset_importer.rb

@@ -2,51 +2,106 @@
 
 require 'ruby-progressbar'
 require 'stringio'
+require 'aws-sdk-s3'
 
 class ScratchAssetImporter
-  def self.import(...)
-    new(...).import
+  class << self
+    def import_all(asset_names, asset_base_url)
+      bar = ProgressBar.create(format: '%t: |%B| %c of %C %E', total: asset_names.count) if show_progress?
+
+      asset_names.each do |asset_name|
+        bar.increment if show_progress?
+        new(asset_name, asset_base_url).import
+      end
+    end
+
+    private
+
+    def show_progress?
+      !Rails.env.test?
+    end
   end
 
-  attr_reader :asset_base_url, :asset_names
+  attr_reader :asset_base_url, :asset_name
 
   ASSET_FETCHING_DELAY = 0.2
 
-  def initialize(asset_names, asset_base_url)
-    @asset_names = asset_names
+  def initialize(asset_name, asset_base_url)
+    @asset_name = asset_name
     @asset_base_url = asset_base_url
   end
 
   def import
-    bar = ProgressBar.create(format: '%t: |%B| %c of %C %E', total: asset_names.count) if show_progress?
+    create_scratch_asset
+    save_to_editor_asset_bucket
+  rescue Faraday::Error => e
+    Rails.logger.error("Failed to import asset #{asset_name}: #{e.message}")
+  end
 
-    asset_names.each do |asset_name|
-      bar.increment if show_progress?
-      import_asset(asset_name)
+  def asset
+    @asset ||= begin
+      sleep(ASSET_FETCHING_DELAY)
+      connection.get("#{asset_name}/get/")
     end
   end
 
-  private
-
-  def import_asset(asset_name)
+  def create_scratch_asset
     return if ScratchAsset.global_assets.exists?(filename: asset_name)
 
-    sleep(ASSET_FETCHING_DELAY)
-    asset = connection.get("#{asset_name}/get/")
+    io = StringIO.new(asset.body)
+
     ScratchAsset.create!(filename: asset_name, project_id: nil, uploaded_user_id: nil)
                 .file
-                .attach(io: StringIO.new(asset.body), filename: asset_name)
-  rescue StandardError => e
-    Rails.logger.error("Failed to import asset #{asset_name}: #{e.message}")
+                .attach(io:, filename: asset_name)
+  end
+
+  def save_to_editor_asset_bucket
+    return unless save_to_editor_asset_bucket?
+
+    body = StringIO.new(asset.body)
+
+    s3_client.put_object(
+      bucket: ENV.fetch('EDITOR_ASSETS_BUCKET'),
+      key: asset_key,
+      body:,
+      content_type: asset_content_type,
+      cache_control: 'public, max-age=604800'
+    )
+  end
+
+  def save_to_editor_asset_bucket?
+    return false unless ENV['EDITOR_ASSETS_BUCKET']
+
+    s3_client.head_object(bucket: ENV.fetch('EDITOR_ASSETS_BUCKET'), key: asset_key)
+    false
+  rescue Aws::S3::Errors::NotFound
+    true
+  end
+
+  def asset_key
+    "internalapi/asset/#{asset_name}/get/"
+  end
+
+  def asset_content_type
+    extension = File.extname(asset_name).delete('.')
+    mime_type = Mime::Type.lookup_by_extension(extension)
+    raise "Unknown content type for extension: #{extension}" unless mime_type
+
+    mime_type.to_s
+  end
+
+  def s3_client
+    @s3_client ||= Aws::S3::Client.new(
+      access_key_id: ENV.fetch('EDITOR_ASSETS_ACCESS_KEY_ID'),
+      secret_access_key: ENV.fetch('EDITOR_ASSETS_SECRET_ACCESS_KEY'),
+      endpoint: ENV.fetch('EDITOR_ASSETS_ENDPOINT'),
+      region: 'auto'
+    )
   end
 
   def connection
     @connection ||= Faraday.new(url: asset_base_url) do |faraday|
       faraday.response :raise_error
     end
   end
-
-  def show_progress?
-    !Rails.env.test?
-  end
 end

lib/scratch_config_importer.rb

@@ -16,7 +16,7 @@ def import
     config = connection.get.body
     asset_config = JSON.parse(config, symbolize_names: true)
     asset_names = extract_asset_names(asset_config)
-    ScratchAssetImporter.import(asset_names, asset_base_url)
+    ScratchAssetImporter.import_all(asset_names, asset_base_url)
   end
 
   def connection

lib/tasks/scratch_assets.rake

@@ -3,7 +3,7 @@
 require_relative 'seeds_helper'
 
 namespace :scratch_assets do
-  desc 'Import scratch assets'
+  desc 'Import scratch assets, see https://github.com/RaspberryPiFoundation/digital-editor-issues/issues/1229#issuecomment-4144077069 for usage'
   task import_all: %i[import_backdrops import_costumes import_sounds import_sprites]
 
   task import_backdrops: :environment do

spec/lib/scratch_asset_importer_spec.rb

@@ -4,12 +4,12 @@
 require 'scratch_asset_importer'
 
 RSpec.describe ScratchAssetImporter do
-  describe '.import' do
+  describe '.import_all' do
     it 'imports assets from the config' do
       image = Rails.root.join('spec/fixtures/files/test_image_1.png').read
       stub_request(:get, 'https://example.net/internalapi/asset/123abc.png/get/').to_return(status: 200, body: image)
 
-      described_class.import(['123abc.png'], 'https://example.net/internalapi/asset/')
+      described_class.import_all(['123abc.png'], 'https://example.net/internalapi/asset/')
 
       scratch_asset = ScratchAsset.find_by(filename: '123abc.png')
       expect(scratch_asset).to be_present
@@ -21,7 +21,7 @@
       create(:scratch_asset, :with_file, filename: '123abc.png')
 
       expect do
-        described_class.import(['123abc.png'], 'https://example.net/internalapi/asset/')
+        described_class.import_all(['123abc.png'], 'https://example.net/internalapi/asset/')
       end.not_to change(ScratchAsset, :count)
     end
 
@@ -31,7 +31,7 @@
       stub_request(:get, 'https://example.net/internalapi/asset/123abc.png/get/').to_return(status: 200, body: image)
       stub_request(:get, 'https://example.net/internalapi/asset/456xyz.png/get/').to_return(status: 200, body: image)
 
-      described_class.import(['123abc.png', '456xyz.png'], 'https://example.net/internalapi/asset/')
+      described_class.import_all(['123abc.png', '456xyz.png'], 'https://example.net/internalapi/asset/')
       expect(ScratchAsset.find_by(filename: '123abc.png')).to be_present
       expect(ScratchAsset.find_by(filename: '456xyz.png')).to be_present
     end
@@ -45,7 +45,7 @@
       stub_request(:get, 'https://example.net/internalapi/asset/123abc.png/get/').to_return(status: 200, body: image)
 
       expect do
-        described_class.import(['123abc.png'], 'https://example.net/internalapi/asset/')
+        described_class.import_all(['123abc.png'], 'https://example.net/internalapi/asset/')
       end.to change { ScratchAsset.global_assets.where(filename: '123abc.png').count }.by(1)
     end
 
@@ -55,9 +55,67 @@
       stub_request(:get, 'https://example.net/internalapi/asset/123abc.png/get/').to_return(status: 500, body: 'error')
       stub_request(:get, 'https://example.net/internalapi/asset/456xyz.png/get/').to_return(status: 200, body: image)
 
-      described_class.import(['123abc.png', '456xyz.png'], 'https://example.net/internalapi/asset/')
+      described_class.import_all(['123abc.png', '456xyz.png'], 'https://example.net/internalapi/asset/')
       expect(ScratchAsset.find_by(filename: '123abc.png')).not_to be_present
       expect(ScratchAsset.find_by(filename: '456xyz.png')).to be_present
     end
+
+    describe 'syncing to editor asset bucket' do
+      let(:s3_client) { instance_double(Aws::S3::Client) }
+
+      around do |example|
+        editor_asset_env_vars = {
+          EDITOR_ASSETS_BUCKET: 'test-bucket',
+          EDITOR_ASSETS_ACCESS_KEY_ID: 'test-access-key-id',
+          EDITOR_ASSETS_SECRET_ACCESS_KEY: 'test-secret-access-key',
+          EDITOR_ASSETS_ENDPOINT: 'https://r2.example.com'
+        }
+        ClimateControl.modify(editor_asset_env_vars) do
+          example.run
+        end
+      end
+
+      before do
+        allow(Aws::S3::Client).to receive(:new).and_return(s3_client)
+        allow(s3_client).to receive(:head_object).and_raise(Aws::S3::Errors::NotFound.new(nil, nil))
+        allow(s3_client).to receive(:put_object)
+      end
+
+      it 'saves asset to editor asset bucket if it does not exist' do
+        image = Rails.root.join('spec/fixtures/files/test_image_1.png').read
+        stub_request(:get, 'https://example.net/internalapi/asset/123abc.png/get/').to_return(status: 200, body: image)
+
+        described_class.import_all(['123abc.png'], 'https://example.net/internalapi/asset/')
+
+        expect(s3_client).to have_received(:put_object).with(
+          bucket: 'test-bucket',
+          key: 'internalapi/asset/123abc.png/get/',
+          body: instance_of(StringIO),
+          content_type: 'image/png',
+          cache_control: 'public, max-age=604800'
+        )
+      end
+
+      it 'does not save asset to editor asset bucket if it already exists' do
+        image = Rails.root.join('spec/fixtures/files/test_image_1.png').read
+        stub_request(:get, 'https://example.net/internalapi/asset/123abc.png/get/').to_return(status: 200, body: image)
+
+        allow(s3_client).to receive(:head_object).with(bucket: 'test-bucket', key: 'internalapi/asset/123abc.png/get/').and_return(true)
+
+        described_class.import_all(['123abc.png'], 'https://example.net/internalapi/asset/')
+        expect(s3_client).not_to have_received(:put_object)
+      end
+
+      it 'raises if asset is an unknown mimetype' do
+        image = Rails.root.join('spec/fixtures/files/unknown_mime_type.xyz').read
+        stub_request(:get, 'https://example.net/internalapi/asset/unknown_mime_type.xyz/get/').to_return(status: 200, body: image)
+
+        expect do
+          described_class.import_all(['unknown_mime_type.xyz'], 'https://example.net/internalapi/asset/')
+        end.to raise_error('Unknown content type for extension: xyz')
+
+        expect(s3_client).not_to have_received(:put_object)
+      end
+    end
   end
 end

spec/lib/scratch_config_importer_spec.rb

@@ -5,7 +5,7 @@
 
 RSpec.describe ScratchConfigImporter do
   before do
-    allow(ScratchAssetImporter).to receive(:import)
+    allow(ScratchAssetImporter).to receive(:import_all)
   end
 
   describe '.import' do
@@ -15,7 +15,7 @@
 
       described_class.import('https://example.com/config/backdrops.json', 'https://example.net/internalapi/asset/')
 
-      expect(ScratchAssetImporter).to have_received(:import).with(['123abc.png'], 'https://example.net/internalapi/asset/')
+      expect(ScratchAssetImporter).to have_received(:import_all).with(['123abc.png'], 'https://example.net/internalapi/asset/')
     end
 
     it 'handles assets nested under sounds and costumes' do
@@ -33,7 +33,7 @@
 
       described_class.import('https://example.com/config/sprites.json', 'https://example.net/internalapi/asset/')
 
-      expect(ScratchAssetImporter).to have_received(:import).with(['123abc.png', '456xyz.png'], 'https://example.net/internalapi/asset/')
+      expect(ScratchAssetImporter).to have_received(:import_all).with(['123abc.png', '456xyz.png'], 'https://example.net/internalapi/asset/')
     end
   end
 end