(#920) Make envelope graphs archive downloadable

app/api/entities/envelope_download.rb

@@ -2,24 +2,24 @@ module API
   module Entities
     # Presenter for EnvelopeDownload
     class EnvelopeDownload < Grape::Entity
-      expose :display_status, as: :status,
-                              documentation: { type: 'string', desc: 'Status of download' }
-
       expose :enqueued_at,
              documentation: { type: 'string', desc: 'When the download was enqueued' },
              if: ->(object) { object.pending? }
 
       expose :finished_at,
              documentation: { type: 'string', desc: 'When the download finished' },
-             if: ->(object) { object.finished? }
+             if: ->(object) { object.failed? || object.finished? }
 
       expose :started_at,
              documentation: { type: 'string', desc: 'When the download started' },
              if: ->(object) { object.in_progress? }
 
+      expose :status,
+             documentation: { type: 'string', desc: 'Status of download' }
+
       expose :url,
              documentation: { type: 'string', desc: 'AWS S3 URL' },
-             if: ->(object) { object.finished? }
+             if: ->(object) { object.failed? || object.finished? }
     end
   end
 end

app/api/v1/envelopes.rb

@@ -73,8 +73,8 @@ class Envelopes < MountableAPI
               authenticate!
               authorize Envelope, :index?
 
-              @envelope_download = current_community.envelope_download ||
-                                   current_community.create_envelope_download!
+              downloads = current_community.envelope_downloads.envelope
+              @envelope_download = downloads.last || downloads.create!
             end
 
             desc 'Returns the envelope download'

app/api/v1/graph.rb

@@ -25,6 +25,32 @@ class Graph < MountableAPI
         end
 
         resource :graph do
+          resources :download do
+            before do
+              authenticate!
+              authorize Envelope, :index?
+
+              downloads = current_community.envelope_downloads.graph
+              @envelope_download = downloads.last || downloads.create!
+            end
+
+            desc 'Returns the envelope download'
+            get do
+              present @envelope_download, with: API::Entities::EnvelopeDownload
+            end
+
+            desc 'Starts an envelope download'
+            post do
+              @envelope_download.update!(
+                enqueued_at: Time.current,
+                status: :pending
+              )
+
+              DownloadEnvelopesJob.perform_later(@envelope_download.id)
+              present @envelope_download, with: API::Entities::EnvelopeDownload
+            end
+          end
+
           namespace do
             desc 'Return a resource. ' \
                  'If the resource is part of a graph, the entire graph is returned.'

app/models/envelope_community.rb

@@ -6,8 +6,8 @@ class EnvelopeCommunity < ActiveRecord::Base
   include AttributeNormalizer
 
   has_one :envelope_community_config
-  has_one :envelope_download
   has_many :envelopes
+  has_many :envelope_downloads
   has_many :envelope_resources, through: :envelopes
   has_many :indexed_envelope_resources
   has_many :versions, class_name: 'EnvelopeVersion'

app/models/envelope_download.rb

@@ -1,17 +1,20 @@
 # Stores the status and AWS S3 URL of an asynchronously performed envelope download
 class EnvelopeDownload < ActiveRecord::Base
+  self.inheritance_column = nil
+
   belongs_to :envelope_community
   has_many :envelopes, -> { not_deleted }, through: :envelope_community
 
   enum :status, {
+    failed: 'failed',
     finished: 'finished',
     in_progress: 'in_progress',
     pending: 'pending'
   }
 
-  def display_status
-    return 'failed' if internal_error_message?
+  enum :type, { envelope: 'envelope', graph: 'graph' }
 
-    status
+  def with_error?
+    internal_error_message?
   end
 end

app/services/download_envelopes.rb

@@ -1,100 +1,31 @@
-# Dumps an envelope community's envelopes into a ZIP archive and uploads it to S3
-class DownloadEnvelopes # rubocop:todo Metrics/ClassLength
-  attr_reader :envelope_download, :updated_at
+require 'envelope_dumps/envelope_builder'
+require 'envelope_dumps/graph_builder'
 
-  delegate :envelope_community, to: :envelope_download
+# Builds an envelope community's download according to its type
+class DownloadEnvelopes
+  attr_reader :envelope_download, :last_dumped_at
 
   def initialize(envelope_download)
     @envelope_download = envelope_download
-    @updated_at = envelope_download.started_at
+    @last_dumped_at = envelope_download.started_at unless envelope_download.with_error?
   end
 
   def self.call(envelope_download:)
     new(envelope_download).run
   end
 
-  def bucket
-    ENV.fetch('ENVELOPE_DOWNLOADS_BUCKET')
-  end
-
-  def create_or_update_entries
-    FileUtils.mkdir_p(dirname)
-
-    log('Adding recently published envelopes into the dump')
-
-    published_envelopes.find_each do |envelope|
-      File.write(
-        File.join(dirname, "#{envelope.envelope_ceterms_ctid}.json"),
-        API::Entities::Envelope.represent(envelope).to_json
-      )
-    end
-  end
-
-  def dirname
-    @dirname ||= [
-      envelope_community.name,
-      Time.current.to_i,
-      SecureRandom.hex
-    ].join('_')
-  end
-
-  def download_file # rubocop:todo Metrics/AbcSize
-    return unless envelope_download.url?
-
-    log("Downloading the existing dump from #{envelope_download.url}")
-
-    File.open(filename, 'wb') do |file|
-      URI.parse(envelope_download.url).open do |data|
-        file.write(data.read)
+  def builder
+    builder_class =
+      case envelope_download.type
+      when 'envelope'
+        EnvelopeDumps::EnvelopeBuilder
+      when 'graph'
+        EnvelopeDumps::GraphBuilder
+      else
+        raise "No dump builder is defined for `#{envelope_download.type}`"
       end
-    end
-
-    log("Unarchiving the downloaded dump into #{dirname}")
-    system("unzip -qq #{filename} -d #{dirname}", exception: true)
-  rescue StandardError => e
-    Airbrake.notify(e)
-  end
-
-  def destroy_envelope_events
-    @deleted_envelope_ctids = envelope_community
-                              .versions
-                              .where(event: 'destroy')
-                              .where('created_at >= ?', updated_at)
-  end
-
-  def filename
-    @filename ||= "#{dirname}.zip"
-  end
-
-  def log(message)
-    MR.logger.info(message)
-  end
-
-  def published_envelopes
-    @published_envelopes = begin
-      envelopes = envelope_community
-                  .envelopes
-                  .not_deleted
-                  .includes(:envelope_community, :organization, :publishing_organization)
-
-      envelopes.where!('updated_at >= ?', updated_at) if updated_at
-      envelopes
-    end
-  end
 
-  def region
-    ENV.fetch('AWS_REGION')
-  end
-
-  def remove_entries
-    log('Removing recently deleted envelopes from the dump')
-
-    destroy_envelope_events.select(:id, :envelope_ceterms_ctid).find_each do |event|
-      FileUtils.remove_file(
-        File.join(dirname, "#{event.envelope_ceterms_ctid}.json"),
-        true
-      )
-    end
+    builder_class.new(envelope_download, last_dumped_at)
   end
 
   def run # rubocop:todo Metrics/AbcSize, Metrics/MethodLength
@@ -106,44 +37,15 @@ def run # rubocop:todo Metrics/AbcSize, Metrics/MethodLength
     )
 
     envelope_download.with_lock do
-      if up_to_date?
-        log('The dump is up to date.')
-        return
-      end
-
-      download_file
-      create_or_update_entries
-      remove_entries
-      envelope_download.url = upload_file
+      envelope_download.status = :finished
+      envelope_download.url = builder.run
     rescue StandardError => e
       Airbrake.notify(e)
       envelope_download&.internal_error_backtrace = e.backtrace
       envelope_download&.internal_error_message = e.message
+      envelope_download.status = :failed
     ensure
-      log('Deleting intermediate files.')
-      FileUtils.rm_rf(dirname)
-      FileUtils.rm_f(filename)
-      envelope_download.update!(finished_at: Time.current, status: :finished)
-      log('Finished.')
+      envelope_download.update!(finished_at: Time.current)
     end
   end
-
-  def up_to_date?
-    published_envelopes.none? && destroy_envelope_events.none?
-  end
-
-  def upload_file
-    log('Archiving the updated dump.')
-
-    system(
-      "find #{dirname} -type f -print | zip -FSjqq #{filename} -@",
-      exception: true
-    )
-
-    log('Uploading the updated dump to S3.')
-
-    object = Aws::S3::Resource.new(region:).bucket(bucket).object(filename)
-    object.upload_file(filename)
-    object.public_url
-  end
 end