diff options
author | Levi Bard <taktaktaktaktaktaktaktaktaktak@gmail.com> | 2022-05-16 09:29:01 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-05-16 09:29:01 +0200 |
commit | 94e98864e39c010635e839fea984f2b4893bef1a (patch) | |
tree | 7a24465db019e89b4fe079a58918cf00c9acbf58 /app | |
parent | dc350be6f5a9f59385e6fbc7a06b5b9c0d57aec8 (diff) |
Allow import/export of instance-level domain blocks/allows (#1754)
* Allow import/export of instance-level domain blocks/allows.
Fixes #15095
* Pacify circleci
* Address simple code review feedback
* Add headers to exported CSV
* Extract common import/export functionality to
AdminExportControllerConcern
* Add additional fields to instance-blocked domain export
* Address review feedback
* Split instance domain block/allow import/export into separate pages/controllers
* Address code review feedback
* Pacify DeepSource
* Work around Paperclip::HasAttachmentFile for Rails 6
* Fix deprecated API warning in export tests
* Remove after_commit workaround
Diffstat (limited to 'app')
-rw-r--r-- | app/controllers/admin/export_domain_allows_controller.rb | 60 | ||||
-rw-r--r-- | app/controllers/admin/export_domain_blocks_controller.rb | 68 | ||||
-rw-r--r-- | app/controllers/concerns/admin_export_controller_concern.rb | 39 | ||||
-rw-r--r-- | app/models/admin/import.rb | 29 | ||||
-rw-r--r-- | app/models/domain_allow.rb | 4 | ||||
-rw-r--r-- | app/validators/admin_import_validator.rb | 19 | ||||
-rw-r--r-- | app/views/admin/export_domain_allows/new.html.haml | 10 | ||||
-rw-r--r-- | app/views/admin/export_domain_blocks/new.html.haml | 10 | ||||
-rw-r--r-- | app/views/admin/instances/index.html.haml | 4 |
9 files changed, 243 insertions, 0 deletions
diff --git a/app/controllers/admin/export_domain_allows_controller.rb b/app/controllers/admin/export_domain_allows_controller.rb new file mode 100644 index 00000000000..eb2955ac380 --- /dev/null +++ b/app/controllers/admin/export_domain_allows_controller.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +require 'csv' + +module Admin + class ExportDomainAllowsController < BaseController + include AdminExportControllerConcern + + before_action :set_dummy_import!, only: [:new] + + ROWS_PROCESSING_LIMIT = 20_000 + + def new + authorize :domain_allow, :create? + end + + def export + authorize :instance, :index? + send_export_file + end + + def import + authorize :domain_allow, :create? + begin + @import = Admin::Import.new(import_params) + parse_import_data!(export_headers) + + @data.take(ROWS_PROCESSING_LIMIT).each do |row| + domain = row['#domain'].strip + next if DomainAllow.allowed?(domain) + + domain_allow = DomainAllow.new(domain: domain) + log_action :create, domain_allow if domain_allow.save + end + flash[:notice] = I18n.t('admin.domain_allows.created_msg') + rescue ActionController::ParameterMissing + flash[:error] = I18n.t('admin.export_domain_allows.no_file') + end + redirect_to admin_instances_path + end + + private + + def export_filename + 'domain_allows.csv' + end + + def export_headers + %w(#domain) + end + + def export_data + CSV.generate(headers: export_headers, write_headers: true) do |content| + DomainAllow.allowed_domains.each do |instance| + content << [instance.domain] + end + end + end + end +end diff --git a/app/controllers/admin/export_domain_blocks_controller.rb b/app/controllers/admin/export_domain_blocks_controller.rb new file mode 100644 index 00000000000..0ad5b92b505 --- /dev/null +++ b/app/controllers/admin/export_domain_blocks_controller.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +require 'csv' + +module Admin + class ExportDomainBlocksController < BaseController + include AdminExportControllerConcern + + before_action :set_dummy_import!, only: [:new] + + ROWS_PROCESSING_LIMIT = 20_000 + + def new + authorize :domain_block, :create? + end + + def export + authorize :instance, :index? + send_export_file + end + + def import + authorize :domain_block, :create? + begin + @import = Admin::Import.new(import_params) + parse_import_data!(export_headers) + + @data.take(ROWS_PROCESSING_LIMIT).each do |row| + domain = row['#domain'].strip + next if DomainBlock.rule_for(domain).present? + + domain_block = DomainBlock.new(domain: domain, + severity: row['#severity'].strip, + reject_media: row['#reject_media'].strip, + reject_reports: row['#reject_reports'].strip, + public_comment: row['#public_comment'].strip, + obfuscate: row['#obfuscate'].strip) + if domain_block.save + DomainBlockWorker.perform_async(domain_block.id) + log_action :create, domain_block + end + end + flash[:notice] = I18n.t('admin.domain_blocks.created_msg') + rescue ActionController::ParameterMissing + flash[:error] = I18n.t('admin.export_domain_blocks.no_file') + end + redirect_to admin_instances_path(limited: '1') + end + + private + + def export_filename + 'domain_blocks.csv' + end + + def export_headers + %w(#domain #severity #reject_media #reject_reports #public_comment #obfuscate) + end + + def export_data + CSV.generate(headers: export_headers, write_headers: true) do |content| + DomainBlock.with_user_facing_limitations.each do |instance| + content << [instance.domain, instance.severity, instance.reject_media, instance.reject_reports, instance.public_comment, instance.obfuscate] + end + end + end + end +end diff --git a/app/controllers/concerns/admin_export_controller_concern.rb b/app/controllers/concerns/admin_export_controller_concern.rb new file mode 100644 index 00000000000..013915d025c --- /dev/null +++ b/app/controllers/concerns/admin_export_controller_concern.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module AdminExportControllerConcern + extend ActiveSupport::Concern + + private + + def send_export_file + respond_to do |format| + format.csv { send_data export_data, filename: export_filename } + end + end + + def export_data + raise 'Override in controller' + end + + def export_filename + raise 'Override in controller' + end + + def set_dummy_import! + @import = Admin::Import.new + end + + def import_params + params.require(:admin_import).permit(:data) + end + + def import_data + Paperclip.io_adapters.for(@import.data).read + end + + def parse_import_data!(default_headers) + data = CSV.parse(import_data, headers: true) + data = CSV.parse(import_data, headers: default_headers) unless data.headers&.first&.strip&.include?(default_headers[0]) + @data = data.reject(&:blank?) + end +end diff --git a/app/models/admin/import.rb b/app/models/admin/import.rb new file mode 100644 index 00000000000..c305be237a2 --- /dev/null +++ b/app/models/admin/import.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +# A non-activerecord helper class for csv upload +class Admin::Import + extend ActiveModel::Callbacks + include ActiveModel::Model + include Paperclip::Glue + + FILE_TYPES = %w(text/plain text/csv application/csv).freeze + + # Paperclip required callbacks + define_model_callbacks :save, only: [:after] + define_model_callbacks :destroy, only: [:before, :after] + + attr_accessor :data_file_name, :data_content_type + + has_attached_file :data + validates_attachment_content_type :data, content_type: FILE_TYPES + validates_attachment_presence :data + validates_with AdminImportValidator, on: :create + + def save + run_callbacks :save + end + + def destroy + run_callbacks :destroy + end +end diff --git a/app/models/domain_allow.rb b/app/models/domain_allow.rb index 4b0a89c184c..2e14fce25e0 100644 --- a/app/models/domain_allow.rb +++ b/app/models/domain_allow.rb @@ -23,6 +23,10 @@ class DomainAllow < ApplicationRecord !rule_for(domain).nil? end + def allowed_domains + select(:domain) + end + def rule_for(domain) return if domain.blank? diff --git a/app/validators/admin_import_validator.rb b/app/validators/admin_import_validator.rb new file mode 100644 index 00000000000..338ceb3a782 --- /dev/null +++ b/app/validators/admin_import_validator.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class AdminImportValidator < ActiveModel::Validator + FIRST_HEADER = '#domain' + + def validate(import) + return if import.type.blank? || import.data.blank? + + # We parse because newlines could be part of individual rows. This + # runs on create so we should be reading the local file here before + # it is uploaded to object storage or moved anywhere... + csv_data = CSV.parse(import.data.queued_for_write[:original].read) + + row_count = csv_data.size + row_count -= 1 if csv_data.first&.first == FIRST_HEADER + + import.errors.add(:data, I18n.t('imports.errors.over_rows_processing_limit', count: Admin::DomainBlocksController::ROWS_PROCESSING_LIMIT)) if row_count > Admin::DomainBlocksController::ROWS_PROCESSING_LIMIT + end +end diff --git a/app/views/admin/export_domain_allows/new.html.haml b/app/views/admin/export_domain_allows/new.html.haml new file mode 100644 index 00000000000..dc0cf8c52a1 --- /dev/null +++ b/app/views/admin/export_domain_allows/new.html.haml @@ -0,0 +1,10 @@ +- content_for :page_title do + = t('.title') + += simple_form_for @import, url: import_admin_export_domain_allows_path, html: { multipart: true } do |f| + .fields-row + .fields-group.fields-row__column.fields-row__column-6 + = f.input :data, wrapper: :with_block_label, hint: t('simple_form.hints.imports.data'), as: :file + + .actions + = f.button :button, t('imports.upload'), type: :submit diff --git a/app/views/admin/export_domain_blocks/new.html.haml b/app/views/admin/export_domain_blocks/new.html.haml new file mode 100644 index 00000000000..0291aeed776 --- /dev/null +++ b/app/views/admin/export_domain_blocks/new.html.haml @@ -0,0 +1,10 @@ +- content_for :page_title do + = t('.title') + += simple_form_for @import, url: import_admin_export_domain_blocks_path, html: { multipart: true } do |f| + .fields-row + .fields-group.fields-row__column.fields-row__column-6 + = f.input :data, wrapper: :with_block_label, hint: t('simple_form.hints.imports.data'), as: :file + + .actions + = f.button :button, t('imports.upload'), type: :submit diff --git a/app/views/admin/instances/index.html.haml b/app/views/admin/instances/index.html.haml index ee1b3d0ce7e..abb2d8c0e54 100644 --- a/app/views/admin/instances/index.html.haml +++ b/app/views/admin/instances/index.html.haml @@ -4,8 +4,12 @@ - content_for :heading_actions do - if whitelist_mode? = link_to t('admin.domain_allows.add_new'), new_admin_domain_allow_path, class: 'button', id: 'add-instance-button' + = link_to t('admin.domain_allows.export'), export_admin_export_domain_allows_path(format: :csv), class: 'button' + = link_to t('admin.domain_allows.import'), new_admin_export_domain_allow_path, class: 'button' - else = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path, class: 'button', id: 'add-instance-button' + = link_to t('admin.domain_blocks.export'), export_admin_export_domain_blocks_path(format: :csv), class: 'button' + = link_to t('admin.domain_blocks.import'), new_admin_export_domain_block_path, class: 'button' .filters .filter-subset |