Improved importer script

With this change we have multiple options for updating existing records,
one of that is the existing body update, the other is reimporting the
date from the mbox file.

Author: dutow

app/services/email_ingestor.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class EmailIngestor
-  def ingest_raw(raw_message, fallback_threading: false, trust_date: false)
+  def ingest_raw(raw_message, fallback_threading: false, trust_date: false, update_existing: [])
     m = Mail.new(raw_message)
 
     message_id = clean_reference(m.message_id)
@@ -12,7 +12,7 @@ def ingest_raw(raw_message, fallback_threading: false, trust_date: false)
     body = normalize_body(extract_body(m))
     existing_message = Message.find_by_message_id(message_id)
     if existing_message
-      existing_message.update_columns(body: body)
+      update_existing_message(existing_message, body: body, sent_at: sent_at, update_existing: update_existing)
       return existing_message
     end
 
@@ -64,6 +64,25 @@ def ingest_raw(raw_message, fallback_threading: false, trust_date: false)
 
   private
 
+  def update_existing_message(message, body:, sent_at:, update_existing:)
+    return if update_existing.empty?
+
+    updates = {}
+    updates[:body] = body if update_existing.include?(:body)
+
+    if update_existing.include?(:date) && sent_at && message.created_at != sent_at
+      updates[:created_at] = sent_at
+
+      # Update topic date if this is the first message
+      topic = message.topic
+      if topic && topic.messages.order(:created_at).first&.id == message.id
+        topic.update_columns(created_at: sent_at)
+      end
+    end
+
+    message.update_columns(updates) if updates.any?
+  end
+
   def build_from_aliases(m, sent_at)
     if m.from.nil? || m.from[0].nil?
       name = m[:from].to_s.strip

lib/import_options.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'optparse'
+
+module ImportOptions
+  def self.parse!(argv = ARGV)
+    options = { update_existing: [] }
+
+    OptionParser.new do |opts|
+      opts.banner = "Usage: #{$PROGRAM_NAME} [options] /path/to/mbox [...]"
+
+      opts.on('--update-body', 'Update body of existing messages') do
+        options[:update_existing] |= [:body]
+      end
+      opts.on('--update-date', 'Update date of existing messages') do
+        options[:update_existing] |= [:date]
+      end
+    end.parse!(argv)
+
+    options
+  end
+end

script/mbox_import.rb

@@ -1,7 +1,10 @@
 require_relative "../config/environment"
 require_relative "../app/services/email_ingestor"
+require_relative "../lib/import_options"
 
-def create_users fields, created_at, limit = 0 
+options = ImportOptions.parse!
+
+def create_users fields, created_at, limit = 0
   return [] unless fields
 
   # Some old mboxes have header fields without parsed addresses
@@ -103,11 +106,12 @@ def sanitize_email_date(mail_date, mail_date_header, message_id)
   sanitized_date
 end
 
-def parse_message(message)
-  msg = EmailIngestor.new.ingest_raw(message, fallback_threading: true)
+def parse_message(message, update_existing:)
+  msg = EmailIngestor.new.ingest_raw(message, fallback_threading: true, update_existing: update_existing)
   puts "Processing #{msg&.message_id || '(duplicate or invalid)'}"
 end
 
+update_existing = options[:update_existing]
 message = ""
 
 ARGV.each do |fn|
@@ -121,7 +125,7 @@ def parse_message(message)
       # all new messages refer to lists.postgresql.org, but not old emails
       # And we can't simply check for From, as it also matches inline attachments containing git diffs
       if (line.match(/^From [^@]+@[a-z\d\-]+(\.[a-z\d\-]+)*\.[a-z]+/i))
-        parse_message(message) unless message.empty?
+        parse_message(message, update_existing: update_existing) unless message.empty?
         message = ""
       else
         message << line

script/mbox_single_import.rb

@@ -1,8 +1,11 @@
 require_relative "../config/environment"
 require_relative "../app/services/email_ingestor"
+require_relative "../lib/import_options"
+
+options = ImportOptions.parse!
 
 if ARGV.length != 2
-  puts "Usage: #{$PROGRAM_NAME} /path/to/mbox <message-id>"
+  puts "Usage: #{$PROGRAM_NAME} [options] /path/to/mbox <message-id>"
   exit 1
 end
 
@@ -21,14 +24,14 @@ def normalize_message_id(message)
   ''
 end
 
-def process_message(message, target_id)
+def process_message(message, target_id, update_existing:)
   return false if message.empty?
 
   message_id = normalize_message_id(message)
   return false if message_id.empty?
   return false unless message_id == target_id
 
-  msg = EmailIngestor.new.ingest_raw(message, fallback_threading: true)
+  msg = EmailIngestor.new.ingest_raw(message, fallback_threading: true, update_existing: update_existing)
   if msg
     puts "Reimported #{msg.message_id}"
   else
@@ -37,6 +40,7 @@ def process_message(message, target_id)
   true
 end
 
+update_existing = options[:update_existing]
 found = false
 message = ""
 
@@ -47,7 +51,7 @@ def process_message(message, target_id)
     line = line.encode("utf-8", :invalid => :replace)
 
     if line.match(/^From [^@]+@[a-z\d\-]+(\.[a-z\d\-]+)*\.[a-z]+/i)
-      if process_message(message, target_id)
+      if process_message(message, target_id, update_existing: update_existing)
         found = true
         break
       end
@@ -59,7 +63,7 @@ def process_message(message, target_id)
 end
 
 if !found
-  found = process_message(message, target_id)
+  found = process_message(message, target_id, update_existing: update_existing)
 end
 
 puts "Message not found in #{mbox_path}" unless found