Fix BC-10008678752: tame self-hosted import disk demands

app/jobs/account/data_import_job.rb

@@ -2,7 +2,8 @@ class Account::DataImportJob < ApplicationJob
   include ActiveJob::Continuable
 
   queue_as :backend
-  discard_on Account::DataTransfer::RecordSet::IntegrityError, ZipFile::InvalidFileError
+  discard_on Account::DataTransfer::RecordSet::IntegrityError, ZipFile::InvalidFileError,
+    Account::Import::InsufficientSpaceError
 
   def perform(import)
     step :check do |step|

app/models/account/import.rb

@@ -1,4 +1,10 @@
+require "shellwords"
+
 class Account::Import < ApplicationRecord
+  class InsufficientSpaceError < StandardError; end
+
+  REQUIRED_SPACE_FACTOR = 2
+
   broadcasts_refreshes
 
   belongs_to :account
@@ -7,7 +13,7 @@ class Account::Import < ApplicationRecord
   has_one_attached :file, dependent: :purge_later
 
   enum :status, %w[ pending processing completed failed ].index_by(&:itself), default: :pending
-  enum :failure_reason, %w[ conflict invalid_export ].index_by(&:itself), prefix: :failed_due_to, scopes: false
+  enum :failure_reason, %w[ conflict invalid_export insufficient_space ].index_by(&:itself), prefix: :failed_due_to, scopes: false
 
   scope :expired, -> { where(completed_at: ...24.hours.ago).or(where(status: :failed, created_at: ...7.days.ago)) }
 
@@ -21,6 +27,7 @@ def process_later
 
   def check(start: nil, callback: nil)
     processing!
+    ensure_sufficient_space
 
     ZipFile.read_from(file.blob) do |zip|
       Account::DataTransfer::Manifest.new(account).each_record_set(start: start) do |record_set, last_id|
@@ -33,6 +40,9 @@ def check(start: nil, callback: nil)
   rescue Account::DataTransfer::RecordSet::IntegrityError, ZipFile::InvalidFileError => e
     mark_as_failed(:invalid_export)
     raise e
+  rescue InsufficientSpaceError => e
+    mark_as_failed(:insufficient_space)
+    raise e
   rescue => e
     mark_as_failed
     raise e
@@ -41,6 +51,8 @@ def check(start: nil, callback: nil)
   def process(start: nil, callback: nil)
     processing!
 
+    ensure_sufficient_space if start.nil?
+
     ZipFile.read_from(file.blob) do |zip|
       Account::DataTransfer::Manifest.new(account).each_record_set(start: start) do |record_set, last_id|
         record_set.import(from: zip, start: last_id, callback: callback)
@@ -58,6 +70,9 @@ def process(start: nil, callback: nil)
   rescue Account::DataTransfer::RecordSet::IntegrityError, ZipFile::InvalidFileError => e
     mark_as_failed(:invalid_export)
     raise e
+  rescue InsufficientSpaceError => e
+    mark_as_failed(:insufficient_space)
+    raise e
   rescue => e
     mark_as_failed
     raise e
@@ -69,6 +84,31 @@ def cleanup
   end
 
   private
+    def ensure_sufficient_space
+      return unless path = ZipFile.path_on_disk(file.blob)
+
+      required = file.blob.byte_size * REQUIRED_SPACE_FACTOR
+      available = available_space(path)
+
+      if available && available < required
+        raise InsufficientSpaceError, "import needs ~#{human_size(required)} free, found #{human_size(available)}"
+      end
+    end
+
+    def human_size(bytes)
+      ActiveSupport::NumberHelper.number_to_human_size(bytes)
+    end
+
+    def available_space(path)
+      fields = `df -Pk #{Shellwords.escape(path)} 2>/dev/null`.lines.last.to_s.split
+      capacity_index = fields.index { |field| field.match?(/\A\d+%\z/) }
+
+      if capacity_index
+        available_kilobytes = Integer(fields[capacity_index - 1], exception: false)
+        available_kilobytes&.kilobytes
+      end
+    end
+
     def mark_completed
       update!(status: :completed, completed_at: Time.current)
       ImportMailer.completed(identity, account).deliver_later

app/models/zip_file.rb

@@ -26,6 +26,10 @@ def read_from(blob)
       end
     end
 
+    def path_on_disk(blob)
+      blob.service.path_for(blob.key) if blob.service.respond_to?(:path_for)
+    end
+
     private
       def s3_service?(service)
         # The S3 service doesn't get loaded in development unless it's used
@@ -91,9 +95,16 @@ def read_from_s3(blob)
       end
 
       def read_from_disk(blob)
-        blob.open do |file|
-          reader = Reader.new(file)
-          yield reader
+        if path = path_on_disk(blob)
+          File.open(path, "rb") do |file|
+            reader = Reader.new(file)
+            yield reader
+          end
+        else
+          blob.open do |file|
+            reader = Reader.new(file)
+            yield reader
+          end
         end
       end
   end

app/views/account/imports/show.html.erb

@@ -22,6 +22,8 @@
         <div>The account you’re trying to import already exists. Make sure you’re importing a <%= Fizzy.saas? ? "self-hosted" : "fizzy.do" %> account.</div>
       <% elsif @import.failed_due_to_invalid_export? %>
         <div>The .zip file you uploaded doesn’t look like a Fizzy account export.</div>
+      <% elsif @import.failed_due_to_insufficient_space? %>
+        <div>There wasn’t enough storage space to process your import.</div>
       <% else %>
         <div>This may be due to corrupted export data or a conflict with existing data. Please try again with a fresh export.</div>
       <% end %>

app/views/mailers/import_mailer/failed.html.erb

@@ -4,6 +4,8 @@
   <p>It looks like the account you are trying to import already exists.</p>
 <% elsif @import.failed_due_to_invalid_export? %>
   <p>The ZIP file isn't a Fizzy account export.</p>
+<% elsif @import.failed_due_to_insufficient_space? %>
+  <p>There wasn't enough storage space to process your import.</p>
 <% else %>
   <p>This may be due to corrupted export data or a conflict with existing data. Please try again with a fresh export, or reach out for help if the problem persists.</p>
 <% end %>

app/views/mailers/import_mailer/failed.text.erb

@@ -4,6 +4,8 @@ Unfortunately, we couldn't import your Fizzy account.
 It looks like the account you are trying to import already exists.
 <% elsif @import.failed_due_to_invalid_export? -%>
 The ZIP file isn't a Fizzy account export.
+<% elsif @import.failed_due_to_insufficient_space? -%>
+There wasn't enough storage space to process your import.
 <% else -%>
 This may be due to corrupted export data or a conflict with existing data. Please try again with a fresh export, or reach out for help if the problem persists.
 <% end -%>

docs/docker-deployment.md

@@ -153,6 +153,17 @@ This is for convenience: typically when you self-host you'll be running a single
 
 If you do want to allow multiple accounts to be created in your instance, set `MULTI_TENANT=true`
 
+## Importing an existing Fizzy account
+
+You can move an account between Fizzy instances by exporting it on the old instance and uploading the export zip to the new one during signup.
+
+Imports need free space: at least twice the export file's size, beyond the export itself, since the imported attachments roughly mirror the zip's contents. If there isn't enough, the import fails fast with "There wasn't enough storage space to process your import." — free up space (or grow the volume) and try again. If free space can't be determined, the check is skipped and the import proceeds.
+
+For very large exports:
+
+- Browser uploads pass through Thruster, which drops slow uploads after its read timeout (a 502 before the import ever starts). Raise `THRUSTER_HTTP_READ_TIMEOUT` (seconds) and recreate the container so the setting takes effect.
+- `script/import-account` runs the import directly on the server from a zip already on disk, bypassing the browser upload entirely — handy for multi-gigabyte exports.
+
 ## Example
 
 Here's an example of a `docker-compose.yml` that you could use to run Fizzy via `docker compose up`

test/mailers/import_mailer_test.rb

@@ -47,4 +47,15 @@ class ImportMailerTest < ActionMailer::TestCase
 
     assert_match "isn't a Fizzy account export", email.body.encoded
   end
+
+  test "failed with insufficient_space reason" do
+    import = Account::Import.create!(account: Current.account, identity: identities(:david), status: :failed, failure_reason: :insufficient_space)
+    email = ImportMailer.failed(import)
+
+    assert_emails 1 do
+      email.deliver_now
+    end
+
+    assert_match "enough storage space", email.body.encoded
+  end
 end

test/models/account/import_test.rb

@@ -220,7 +220,66 @@ class Account::ImportTest < ActiveSupport::TestCase
     export_tempfile&.unlink
   end
 
+  test "check fails fast with a clear reason when free disk space is insufficient" do
+    import = import_with_attached_zip
+    import.stubs(:available_space).returns(import.file.blob.byte_size)
+
+    error = assert_raises(Account::Import::InsufficientSpaceError) { import.check }
+    assert_match(/import needs ~.+ free, found/, error.message)
+    assert import.reload.failed_due_to_insufficient_space?
+  end
+
+  test "check proceeds when free disk space cannot be determined" do
+    import = import_with_attached_zip
+    import.stubs(:available_space).returns(nil)
+
+    assert_raises(ZipFile::InvalidFileError) { import.check }
+    assert import.reload.failed_due_to_invalid_export?
+  end
+
+  test "process fails fast with a clear reason when free disk space is insufficient" do
+    import = import_with_attached_zip
+    import.stubs(:available_space).returns(import.file.blob.byte_size)
+
+    assert_raises(Account::Import::InsufficientSpaceError) { import.process }
+    assert import.reload.failed_due_to_insufficient_space?
+  end
+
+  test "resumed process skips the disk space preflight" do
+    import = import_with_attached_zip
+    import.stubs(:available_space).returns(import.file.blob.byte_size)
+
+    assert_raises(ZipFile::InvalidFileError) { import.process(start: [ "Board", nil ]) }
+    assert import.reload.failed_due_to_invalid_export?
+  end
+
+  test "available_space is indeterminate when df output is unparseable" do
+    import = import_with_attached_zip
+    import.stubs(:`).returns("Filesystem 1024-blocks Used Available Capacity Mounted on\n")
+
+    assert_nil import.send(:available_space, "/tmp")
+  end
+
+  test "available_space parses df output whose filesystem name contains spaces" do
+    import = import_with_attached_zip
+    import.stubs(:`).returns(<<~DF)
+      Filesystem 1024-blocks Used Available Capacity Mounted on
+      map auto home 1000000 250000 750000 25% /System/Volumes/Data/home
+    DF
+
+    assert_equal 750_000 * 1024, import.send(:available_space, "/tmp")
+  end
+
   private
+    def import_with_attached_zip
+      account = Account.create!(name: "Disk Check")
+      import = Account::Import.create!(account: account, identity: identities(:david))
+      Current.set(account: account) do
+        import.file.attach(io: StringIO.new("not actually a zip"), filename: "export.zip", content_type: "application/zip")
+      end
+      import
+    end
+
     def account_digest(account)
       {
         name: account.name,