From 1b7be798180cfeee85c56ebe98bb33cdb7c1445d Mon Sep 17 00:00:00 2001
From: Caitlin <caitlinhenry.ss@gmail.com>
Date: Wed, 14 Feb 2024 15:47:13 -0500
Subject: [PATCH 01/31] add mapping and mapping field models

---
 app/models/mapping.rb                         | 16 ++++++++++++++
 app/models/mapping_field.rb                   | 17 +++++++++++++++
 .../20240214201727_create_mapping_fields.rb   | 12 +++++++++++
 db/migrate/20240214201737_create_mappings.rb  | 11 ++++++++++
 db/schema.rb                                  | 21 ++++++++++++++++++-
 5 files changed, 76 insertions(+), 1 deletion(-)
 create mode 100644 app/models/mapping.rb
 create mode 100644 app/models/mapping_field.rb
 create mode 100644 db/migrate/20240214201727_create_mapping_fields.rb
 create mode 100644 db/migrate/20240214201737_create_mappings.rb

diff --git a/app/models/mapping.rb b/app/models/mapping.rb
new file mode 100644
index 000000000..5706c8949
--- /dev/null
+++ b/app/models/mapping.rb
@@ -0,0 +1,16 @@
+class Mapping < ApplicationRecord
+  # -- Relationships --------------------------------------------------------
+  has_many :mapping_fields, dependent: :destroy
+
+  # -- Callbacks ------------------------------------------------------------
+
+  # -- Validations ----------------------------------------------------------
+  validates :component, presence: true
+  validates :source, presence: true
+
+  # -- Scopes ---------------------------------------------------------------
+
+  # -- Class Methods --------------------------------------------------------
+
+  # -- Instance Methods -----------------------------------------------------
+end
diff --git a/app/models/mapping_field.rb b/app/models/mapping_field.rb
new file mode 100644
index 000000000..e9bda2709
--- /dev/null
+++ b/app/models/mapping_field.rb
@@ -0,0 +1,17 @@
+class MappingField < ApplicationRecord
+  # -- Relationships --------------------------------------------------------
+  belongs_to :mapping
+
+  # -- Callbacks ------------------------------------------------------------
+  #
+  # -- Validations ----------------------------------------------------------
+  validates :content, presence: true
+  validates :destination_field, presence: true, uniqueness: { scope: [:mapping_id, :source_field] }
+  validates :source_field, presence: true
+
+  # -- Scopes ---------------------------------------------------------------
+
+  # -- Class Methods --------------------------------------------------------
+
+  # -- Instance Methods -----------------------------------------------------
+end
diff --git a/db/migrate/20240214201727_create_mapping_fields.rb b/db/migrate/20240214201727_create_mapping_fields.rb
new file mode 100644
index 000000000..577c5b9d0
--- /dev/null
+++ b/db/migrate/20240214201727_create_mapping_fields.rb
@@ -0,0 +1,12 @@
+class CreateMappingFields < ActiveRecord::Migration[7.0]
+  def change
+    create_table :mapping_fields do |t|
+      t.references :mapping, null: false, foreign_key: true
+      t.string :source_field
+      t.string :destination_field
+      t.text :content
+
+      t.timestamps
+    end
+  end
+end
diff --git a/db/migrate/20240214201737_create_mappings.rb b/db/migrate/20240214201737_create_mappings.rb
new file mode 100644
index 000000000..09fef563a
--- /dev/null
+++ b/db/migrate/20240214201737_create_mappings.rb
@@ -0,0 +1,11 @@
+class CreateMappings < ActiveRecord::Migration[7.0]
+  def change
+    create_table :mappings do |t|
+      t.string :component
+      t.string :source
+      t.string :destination
+
+      t.timestamps
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 3d2eafce5..8fdfe0362 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2023_09_21_144728) do
+ActiveRecord::Schema[7.0].define(version: 2024_02_14_201737) do
   create_table "active_storage_attachments", force: :cascade do |t|
     t.string "name", null: false
     t.string "record_type", null: false
@@ -131,6 +131,24 @@
     t.datetime "updated_at", precision: nil, null: false
   end
 
+  create_table "mapping_fields", force: :cascade do |t|
+    t.integer "mapping_id", null: false
+    t.string "source_field"
+    t.string "destination_field"
+    t.text "content"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["mapping_id"], name: "index_mapping_fields_on_mapping_id"
+  end
+
+  create_table "mappings", force: :cascade do |t|
+    t.string "component"
+    t.string "source"
+    t.string "destination"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
   create_table "nodes", force: :cascade do |t|
     t.integer "type_id"
     t.string "label"
@@ -225,6 +243,7 @@
   add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id"
   add_foreign_key "boards", "nodes", on_delete: :cascade
   add_foreign_key "comments", "users", on_delete: :nullify
+  add_foreign_key "mapping_fields", "mappings"
   add_foreign_key "notifications", "users", column: "actor_id", on_delete: :cascade
   add_foreign_key "notifications", "users", column: "recipient_id", on_delete: :cascade
   add_foreign_key "subscriptions", "users"

From b1b252af84815bc44193f7adf8b8e1792bbe3b55 Mon Sep 17 00:00:00 2001
From: Caitlin <caitlinhenry.ss@gmail.com>
Date: Thu, 15 Feb 2024 09:46:21 -0500
Subject: [PATCH 02/31] add destination uniqueness validator

---
 app/models/mapping.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/models/mapping.rb b/app/models/mapping.rb
index 5706c8949..10841dce4 100644
--- a/app/models/mapping.rb
+++ b/app/models/mapping.rb
@@ -5,6 +5,7 @@ class Mapping < ApplicationRecord
   # -- Callbacks ------------------------------------------------------------
 
   # -- Validations ----------------------------------------------------------
+  validates :destination, uniqueness: { scope: [:component, :source] }
   validates :component, presence: true
   validates :source, presence: true
 

From c2a3f25db6e245d448321982316109a3debe2632 Mon Sep 17 00:00:00 2001
From: Caitlin <caitlinhenry.ss@gmail.com>
Date: Thu, 15 Feb 2024 10:51:29 -0500
Subject: [PATCH 03/31] add model specs for mapping and mapping_fields

---
 spec/factories/mapping_fields.rb  |  8 ++++++++
 spec/factories/mappings.rb        | 13 +++++++++++++
 spec/models/mapping_field_spec.rb | 14 ++++++++++++++
 spec/models/mapping_spec.rb       | 10 ++++++++++
 4 files changed, 45 insertions(+)
 create mode 100644 spec/factories/mapping_fields.rb
 create mode 100644 spec/factories/mappings.rb
 create mode 100644 spec/models/mapping_field_spec.rb
 create mode 100644 spec/models/mapping_spec.rb

diff --git a/spec/factories/mapping_fields.rb b/spec/factories/mapping_fields.rb
new file mode 100644
index 000000000..be68c174f
--- /dev/null
+++ b/spec/factories/mapping_fields.rb
@@ -0,0 +1,8 @@
+FactoryBot.define do
+  factory :mapping_field do
+    mapping { create(:mapping) }
+    destination_field { 'title' }
+    source_field { 'title' }
+    content { 'test content' }
+  end
+end
diff --git a/spec/factories/mappings.rb b/spec/factories/mappings.rb
new file mode 100644
index 000000000..ba21287f8
--- /dev/null
+++ b/spec/factories/mappings.rb
@@ -0,0 +1,13 @@
+FactoryBot.define do
+  factory :mapping do
+    component { 'qualys' }
+    sequence(:source) { |n| "source_#{n}" }
+    sequence(:destination) { |n| "rtp_#{n}" }
+
+    trait :export_integration do
+      component { 'jira' }
+      sequence(:source) { |n| "rtp_#{n}" }
+      sequence(:destination) { |n| "project_1_issuetype_#{n}" }
+    end
+  end
+end
diff --git a/spec/models/mapping_field_spec.rb b/spec/models/mapping_field_spec.rb
new file mode 100644
index 000000000..624970ef3
--- /dev/null
+++ b/spec/models/mapping_field_spec.rb
@@ -0,0 +1,14 @@
+require 'rails_helper'
+
+describe MappingField do
+  subject { create(:mapping_field) }
+
+  it { should belong_to(:mapping) }
+
+  it { should validate_presence_of(:content) }
+
+  it { should validate_presence_of(:destination_field) }
+  it { should validate_presence_of(:source_field) }
+
+  it { should validate_uniqueness_of(:destination_field).scoped_to([:mapping_id, :source_field]) }
+end
diff --git a/spec/models/mapping_spec.rb b/spec/models/mapping_spec.rb
new file mode 100644
index 000000000..1765830b1
--- /dev/null
+++ b/spec/models/mapping_spec.rb
@@ -0,0 +1,10 @@
+require 'rails_helper'
+
+describe Mapping do
+  it { should have_many(:mapping_fields) }
+
+  it { should validate_presence_of(:component) }
+  it { should validate_presence_of(:source) }
+
+  it { should validate_uniqueness_of(:destination).scoped_to([:component, :source]) }
+end

From c81ba24bc0c0fe80061c0e977411d13eb587e591 Mon Sep 17 00:00:00 2001
From: Caitlin <caitlinhenry.ss@gmail.com>
Date: Thu, 15 Feb 2024 11:03:23 -0500
Subject: [PATCH 04/31] backport mappings migrations

---
 ...737_create_mappings.rb => 20230717084536_create_mappings.rb} | 2 +-
 ...apping_fields.rb => 20230717103559_create_mapping_fields.rb} | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename db/migrate/{20240214201737_create_mappings.rb => 20230717084536_create_mappings.rb} (75%)
 rename db/migrate/{20240214201727_create_mapping_fields.rb => 20230717103559_create_mapping_fields.rb} (80%)

diff --git a/db/migrate/20240214201737_create_mappings.rb b/db/migrate/20230717084536_create_mappings.rb
similarity index 75%
rename from db/migrate/20240214201737_create_mappings.rb
rename to db/migrate/20230717084536_create_mappings.rb
index 09fef563a..ec20b86ef 100644
--- a/db/migrate/20240214201737_create_mappings.rb
+++ b/db/migrate/20230717084536_create_mappings.rb
@@ -1,4 +1,4 @@
-class CreateMappings < ActiveRecord::Migration[7.0]
+class CreateMappings < ActiveRecord::Migration[6.1]
   def change
     create_table :mappings do |t|
       t.string :component
diff --git a/db/migrate/20240214201727_create_mapping_fields.rb b/db/migrate/20230717103559_create_mapping_fields.rb
similarity index 80%
rename from db/migrate/20240214201727_create_mapping_fields.rb
rename to db/migrate/20230717103559_create_mapping_fields.rb
index 577c5b9d0..06a7d9b03 100644
--- a/db/migrate/20240214201727_create_mapping_fields.rb
+++ b/db/migrate/20230717103559_create_mapping_fields.rb
@@ -1,4 +1,4 @@
-class CreateMappingFields < ActiveRecord::Migration[7.0]
+class CreateMappingFields < ActiveRecord::Migration[6.1]
   def change
     create_table :mapping_fields do |t|
       t.references :mapping, null: false, foreign_key: true

From 2402bf51e5c6cdce22e8b082adaaf9ee1ba859e1 Mon Sep 17 00:00:00 2001
From: Caitlin <caitlinhenry.ss@gmail.com>
Date: Tue, 20 Feb 2024 13:33:26 -0500
Subject: [PATCH 05/31] validate mapping destination case-insensitively

---
 app/models/mapping.rb       | 2 +-
 spec/models/mapping_spec.rb | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/models/mapping.rb b/app/models/mapping.rb
index 10841dce4..01207df41 100644
--- a/app/models/mapping.rb
+++ b/app/models/mapping.rb
@@ -5,7 +5,7 @@ class Mapping < ApplicationRecord
   # -- Callbacks ------------------------------------------------------------
 
   # -- Validations ----------------------------------------------------------
-  validates :destination, uniqueness: { scope: [:component, :source] }
+  validates :destination, uniqueness: { scope: [:component, :source], case_sensitive: false }
   validates :component, presence: true
   validates :source, presence: true
 
diff --git a/spec/models/mapping_spec.rb b/spec/models/mapping_spec.rb
index 1765830b1..e2d73ce13 100644
--- a/spec/models/mapping_spec.rb
+++ b/spec/models/mapping_spec.rb
@@ -6,5 +6,5 @@
   it { should validate_presence_of(:component) }
   it { should validate_presence_of(:source) }
 
-  it { should validate_uniqueness_of(:destination).scoped_to([:component, :source]) }
+  it { should validate_uniqueness_of(:destination).scoped_to([:component, :source]).case_insensitive }
 end

From f7fec99eefc862f583e9d04f00a5c53035b661d7 Mon Sep 17 00:00:00 2001
From: Caitlin <caitlinhenry.ss@gmail.com>
Date: Tue, 20 Feb 2024 13:42:04 -0500
Subject: [PATCH 06/31] validate mapping field destination_field
 case-insensitively

---
 app/models/mapping.rb             | 3 ++-
 app/models/mapping_field.rb       | 4 +++-
 spec/models/mapping_field_spec.rb | 2 +-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/app/models/mapping.rb b/app/models/mapping.rb
index 01207df41..821126932 100644
--- a/app/models/mapping.rb
+++ b/app/models/mapping.rb
@@ -5,7 +5,8 @@ class Mapping < ApplicationRecord
   # -- Callbacks ------------------------------------------------------------
 
   # -- Validations ----------------------------------------------------------
-  validates :destination, uniqueness: { scope: [:component, :source], case_sensitive: false }
+  validates :destination,
+    uniqueness: { scope: [:component, :source], case_sensitive: false }
   validates :component, presence: true
   validates :source, presence: true
 
diff --git a/app/models/mapping_field.rb b/app/models/mapping_field.rb
index e9bda2709..2761b3a70 100644
--- a/app/models/mapping_field.rb
+++ b/app/models/mapping_field.rb
@@ -6,7 +6,9 @@ class MappingField < ApplicationRecord
   #
   # -- Validations ----------------------------------------------------------
   validates :content, presence: true
-  validates :destination_field, presence: true, uniqueness: { scope: [:mapping_id, :source_field] }
+  validates :destination_field,
+    presence: true,
+    uniqueness: { scope: [:mapping_id, :source_field], case_sensitive: false }
   validates :source_field, presence: true
 
   # -- Scopes ---------------------------------------------------------------
diff --git a/spec/models/mapping_field_spec.rb b/spec/models/mapping_field_spec.rb
index 624970ef3..72ffc5061 100644
--- a/spec/models/mapping_field_spec.rb
+++ b/spec/models/mapping_field_spec.rb
@@ -10,5 +10,5 @@
   it { should validate_presence_of(:destination_field) }
   it { should validate_presence_of(:source_field) }
 
-  it { should validate_uniqueness_of(:destination_field).scoped_to([:mapping_id, :source_field]) }
+  it { should validate_uniqueness_of(:destination_field).scoped_to([:mapping_id, :source_field]).case_insensitive }
 end

From 2a1f37708d2cecf77eddf97215f7c612ac7ed907 Mon Sep 17 00:00:00 2001
From: Caitlin <caitlinhenry.ss@gmail.com>
Date: Mon, 26 Feb 2024 14:56:22 -0500
Subject: [PATCH 07/31] create mappingmigrationservice and call it from
 migration file

---
 app/services/mapping_migration_service.rb     | 89 +++++++++++++++++++
 ...226193757_migrate_templates_to_mappings.rb |  5 ++
 2 files changed, 94 insertions(+)
 create mode 100644 app/services/mapping_migration_service.rb
 create mode 100644 db/migrate/20240226193757_migrate_templates_to_mappings.rb

diff --git a/app/services/mapping_migration_service.rb b/app/services/mapping_migration_service.rb
new file mode 100644
index 000000000..4e85ad9ad
--- /dev/null
+++ b/app/services/mapping_migration_service.rb
@@ -0,0 +1,89 @@
+class MappingMigrationService
+  attr_reader :integration_name, :rtp_id, :template_file, :templates_dir, :upload_integrations
+
+  def initialize
+    @templates_dir = Configuration.paths_templates_plugins
+    @upload_integrations = Dradis::Plugins::with_feature(:upload)
+  end
+
+  def call
+    upload_integrations.each do |integration|
+      @integration_name = integration.plugin_name.to_s
+      # for each file, create a mapping for that file name&plugin_name
+      # combination, for each RTP in the instance (or nil for CE)
+      template_files.each do |template_file|
+        @template_file = template_file
+        if defined? ReportTemplateProperties
+          ReportTemplateProperties.all.each do |rtp|
+            @rtp_id = rtp.id
+            migrate
+          end
+        else
+          migrate
+        end
+        # delete the .template files after migrating them to the db
+        File.delete(template_file)
+      end
+    end
+  end
+
+  private
+
+  def create_mapping
+    mapping_source = File.basename(template_file, '.template')
+    destination = rtp_id ? "rtp_#{rtp_id}" : nil
+
+    Mapping.find_or_create_by!(
+      component: integration_name,
+      source: mapping_source,
+      destination: destination
+    )
+  end
+
+  def create_mapping_field(mapping, field_title)
+    mapping.mapping_fields.find_or_create_by!(
+      source_field: @source_field,
+      destination_field: field_title,
+      content: @updated_content
+    )
+  end
+
+  def migrate
+    ActiveRecord::Base.transaction do
+      mapping = create_mapping
+
+      template_fields.each do |field_title, field_content|
+        # set source_field by taking the first match to the existing %% syntax
+        source_field = field_content.match(/%(?<field>\S*?)%/)
+        @source_field =
+          if source_field && !source_field['field'].empty?
+            source_field['field']
+          else
+            'custom text'
+          end
+        @updated_content = update_syntax(field_content)
+
+        # create a mapping field for each field in the .template file
+        create_mapping_field(mapping, field_title)
+      end
+    end
+  end
+
+  def template_fields
+    template_content = File.open(template_file).read
+    FieldParser.source_to_fields(template_content)
+  end
+
+  def template_files
+    plugin_templates_dir = File.join(templates_dir, integration_name)
+    Dir["#{plugin_templates_dir}/*.template"]
+  end
+
+  def update_syntax(field_content)
+    # turn the %% syntax into the new
+    # '{{ <integration>[was-issue.title] }}' format
+    field_content.gsub(/%(\S*?)%/) do |content|
+      "{{ #{integration_name}[#{content[1..-2]}] }}"
+    end
+  end
+end
diff --git a/db/migrate/20240226193757_migrate_templates_to_mappings.rb b/db/migrate/20240226193757_migrate_templates_to_mappings.rb
new file mode 100644
index 000000000..504c79989
--- /dev/null
+++ b/db/migrate/20240226193757_migrate_templates_to_mappings.rb
@@ -0,0 +1,5 @@
+class MigrateTemplatesToMappings < ActiveRecord::Migration[7.0]
+  def change
+    MappingMigrationService.new.call
+  end
+end

From c716a6c3def21deeeb193824d374652631429a74 Mon Sep 17 00:00:00 2001
From: Caitlin <caitlinhenry.ss@gmail.com>
Date: Mon, 26 Feb 2024 15:01:26 -0500
Subject: [PATCH 08/31] separate pro migration from ce migration

---
 app/services/mapping_migration_service.rb | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/app/services/mapping_migration_service.rb b/app/services/mapping_migration_service.rb
index 4e85ad9ad..cce17bc33 100644
--- a/app/services/mapping_migration_service.rb
+++ b/app/services/mapping_migration_service.rb
@@ -9,15 +9,12 @@ def initialize
   def call
     upload_integrations.each do |integration|
       @integration_name = integration.plugin_name.to_s
-      # for each file, create a mapping for that file name&plugin_name
-      # combination, for each RTP in the instance (or nil for CE)
+
       template_files.each do |template_file|
         @template_file = template_file
-        if defined? ReportTemplateProperties
-          ReportTemplateProperties.all.each do |rtp|
-            @rtp_id = rtp.id
-            migrate
-          end
+
+        if defined?(Dradis::Pro)
+          migrate_pro
         else
           migrate
         end
@@ -49,6 +46,7 @@ def create_mapping_field(mapping, field_title)
   end
 
   def migrate
+    # for each file, create a mapping for the uploader&plugin_name combination
     ActiveRecord::Base.transaction do
       mapping = create_mapping
 
@@ -69,6 +67,9 @@ def migrate
     end
   end
 
+  def migrate_pro
+  end
+
   def template_fields
     template_content = File.open(template_file).read
     FieldParser.source_to_fields(template_content)

From c8bcdb4482754d390d9db9d92752bf9f50b41e63 Mon Sep 17 00:00:00 2001
From: Caitlin <caitlinhenry.ss@gmail.com>
Date: Mon, 26 Feb 2024 16:32:43 -0500
Subject: [PATCH 09/31] add specs

---
 .../files/templates/plugins/evidence.template |  5 +++
 .../mapping_migration_service_spec.rb         | 41 +++++++++++++++++++
 2 files changed, 46 insertions(+)
 create mode 100644 spec/fixtures/files/templates/plugins/evidence.template
 create mode 100644 spec/services/mapping_migration_service_spec.rb

diff --git a/spec/fixtures/files/templates/plugins/evidence.template b/spec/fixtures/files/templates/plugins/evidence.template
new file mode 100644
index 000000000..3e09d23bd
--- /dev/null
+++ b/spec/fixtures/files/templates/plugins/evidence.template
@@ -0,0 +1,5 @@
+#[TestField]#
+%evidence.test_field%
+
+#[Custom]#
+Custom text
diff --git a/spec/services/mapping_migration_service_spec.rb b/spec/services/mapping_migration_service_spec.rb
new file mode 100644
index 000000000..6fb6c712b
--- /dev/null
+++ b/spec/services/mapping_migration_service_spec.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe MappingMigrationService do
+  describe '.call' do
+    subject(:migrate_templates) { described_class.new.call }
+
+    before do
+      @templates_dir = Rails.root.join('spec/fixtures/files/templates/plugins/')
+      templates_path = Pathname.new(@templates_dir)
+      FileUtils.mkdir_p(templates_path)
+      allow(Configuration).to receive(:paths_templates_plugins).and_return(templates_path)
+      FileUtils.mkdir_p(templates_path.join('qualys'))
+      FileUtils.cp(templates_path.join('evidence.template') , templates_path.join('qualys/evidence.template'))
+    end
+
+    after do
+      FileUtils.rm_r(Rails.root.join('spec/fixtures/files/templates/plugins/qualys'))
+    end
+    it 'creates mappings and associated mapping fields' do
+      migrate_templates
+
+      if defined?(Dradis::Pro)
+      else
+        expect(Mapping.last.destination).to eq(nil)
+      end
+      expect(Mapping.last.source).to eq('evidence')
+      expect(Mapping.last.mapping_fields.last.source_field).to eq('custom text')
+      expect(Mapping.last.mapping_fields.last.destination_field).to eq('Custom')
+      expect(Mapping.last.mapping_fields.first.destination_field).to eq('TestField')
+      expect(Mapping.last.mapping_fields.first.content).to eq('{{ qualys[evidence.test_field] }}')
+    end
+
+    it 'deletes .template files after migrating them to mappings' do
+      expect(File.exist?(@templates_dir.join('qualys/evidence.template'))).to be true
+      migrate_templates
+      expect(File.exist?(@templates_dir.join('qualys/evidence.template'))).to be false
+    end
+  end
+end

From 6a901423796f13ecbf4c317261d8c54dc30e7699 Mon Sep 17 00:00:00 2001
From: Caitlin <caitlinhenry.ss@gmail.com>
Date: Thu, 29 Feb 2024 10:45:43 -0500
Subject: [PATCH 10/31] remove separate pro method and update migration to use
 up/down

---
 app/services/mapping_migration_service.rb     | 35 +++++++++----------
 ...226193757_migrate_templates_to_mappings.rb |  5 ++-
 2 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/app/services/mapping_migration_service.rb b/app/services/mapping_migration_service.rb
index cce17bc33..6799304fa 100644
--- a/app/services/mapping_migration_service.rb
+++ b/app/services/mapping_migration_service.rb
@@ -1,25 +1,24 @@
 class MappingMigrationService
-  attr_reader :integration_name, :rtp_id, :template_file, :templates_dir, :upload_integrations
-
-  def initialize
-    @templates_dir = Configuration.paths_templates_plugins
-    @upload_integrations = Dradis::Plugins::with_feature(:upload)
-  end
+  LEGACY_TEMPLATE_REGEX = /%(\S*?)%/
+  attr_reader :integration_name, :rtp_id, :template_file
 
   def call
+    upload_integrations = Dradis::Plugins::with_feature(:upload)
+
     upload_integrations.each do |integration|
       @integration_name = integration.plugin_name.to_s
 
       template_files.each do |template_file|
         @template_file = template_file
 
-        if defined?(Dradis::Pro)
-          migrate_pro
-        else
+        rtp_ids = defined?(Dradis::Pro) ? ReportTemplateProperties.ids : [nil]
+        rtp_ids.each do |rtp_id|
+          @rtp_id = rtp_id
           migrate
+
+          # delete the .template files after migrating them to the db
+          File.delete(template_file)
         end
-        # delete the .template files after migrating them to the db
-        File.delete(template_file)
       end
     end
   end
@@ -52,10 +51,10 @@ def migrate
 
       template_fields.each do |field_title, field_content|
         # set source_field by taking the first match to the existing %% syntax
-        source_field = field_content.match(/%(?<field>\S*?)%/)
+        source_field = field_content.match(LEGACY_TEMPLATE_REGEX)
         @source_field =
-          if source_field && !source_field['field'].empty?
-            source_field['field']
+          if source_field && !source_field[1].empty?
+            source_field[1]
           else
             'custom text'
           end
@@ -67,15 +66,13 @@ def migrate
     end
   end
 
-  def migrate_pro
-  end
-
   def template_fields
-    template_content = File.open(template_file).read
+    template_content = File.read(template_file)
     FieldParser.source_to_fields(template_content)
   end
 
   def template_files
+    templates_dir = Configuration.paths_templates_plugins
     plugin_templates_dir = File.join(templates_dir, integration_name)
     Dir["#{plugin_templates_dir}/*.template"]
   end
@@ -83,7 +80,7 @@ def template_files
   def update_syntax(field_content)
     # turn the %% syntax into the new
     # '{{ <integration>[was-issue.title] }}' format
-    field_content.gsub(/%(\S*?)%/) do |content|
+    field_content.gsub(LEGACY_TEMPLATE_REGEX) do |content|
       "{{ #{integration_name}[#{content[1..-2]}] }}"
     end
   end
diff --git a/db/migrate/20240226193757_migrate_templates_to_mappings.rb b/db/migrate/20240226193757_migrate_templates_to_mappings.rb
index 504c79989..35641ead7 100644
--- a/db/migrate/20240226193757_migrate_templates_to_mappings.rb
+++ b/db/migrate/20240226193757_migrate_templates_to_mappings.rb
@@ -1,5 +1,8 @@
 class MigrateTemplatesToMappings < ActiveRecord::Migration[7.0]
-  def change
+  def up
     MappingMigrationService.new.call
   end
+
+  def down
+  end
 end

From c5f1eb75b86bd0a6ecf5eef56ce6918aa27a29c0 Mon Sep 17 00:00:00 2001
From: Caitlin <caitlinhenry.ss@gmail.com>
Date: Thu, 29 Feb 2024 12:26:37 -0500
Subject: [PATCH 11/31] rename template file instead of deleting it

---
 app/services/mapping_migration_service.rb | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/app/services/mapping_migration_service.rb b/app/services/mapping_migration_service.rb
index 6799304fa..4b8e139e1 100644
--- a/app/services/mapping_migration_service.rb
+++ b/app/services/mapping_migration_service.rb
@@ -14,10 +14,8 @@ def call
         rtp_ids = defined?(Dradis::Pro) ? ReportTemplateProperties.ids : [nil]
         rtp_ids.each do |rtp_id|
           @rtp_id = rtp_id
-          migrate
-
-          # delete the .template files after migrating them to the db
-          File.delete(template_file)
+          migrate(rtp_id)
+          File.rename template_file, "#{template_file}.legacy"
         end
       end
     end

From cf19a175ae69902c20a7b04e0f0d43b2d3e22464 Mon Sep 17 00:00:00 2001
From: Caitlin <caitlinhenry.ss@gmail.com>
Date: Thu, 29 Feb 2024 12:28:04 -0500
Subject: [PATCH 12/31] update specs

---
 app/services/mapping_migration_service.rb       | 2 +-
 db/schema.rb                                    | 2 +-
 spec/services/mapping_migration_service_spec.rb | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/app/services/mapping_migration_service.rb b/app/services/mapping_migration_service.rb
index 4b8e139e1..e11e2ea67 100644
--- a/app/services/mapping_migration_service.rb
+++ b/app/services/mapping_migration_service.rb
@@ -14,7 +14,7 @@ def call
         rtp_ids = defined?(Dradis::Pro) ? ReportTemplateProperties.ids : [nil]
         rtp_ids.each do |rtp_id|
           @rtp_id = rtp_id
-          migrate(rtp_id)
+          migrate
           File.rename template_file, "#{template_file}.legacy"
         end
       end
diff --git a/db/schema.rb b/db/schema.rb
index 8fdfe0362..2710c5d35 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2024_02_14_201737) do
+ActiveRecord::Schema[7.0].define(version: 2024_02_26_193757) do
   create_table "active_storage_attachments", force: :cascade do |t|
     t.string "name", null: false
     t.string "record_type", null: false
diff --git a/spec/services/mapping_migration_service_spec.rb b/spec/services/mapping_migration_service_spec.rb
index 6fb6c712b..127d242c3 100644
--- a/spec/services/mapping_migration_service_spec.rb
+++ b/spec/services/mapping_migration_service_spec.rb
@@ -32,10 +32,10 @@
       expect(Mapping.last.mapping_fields.first.content).to eq('{{ qualys[evidence.test_field] }}')
     end
 
-    it 'deletes .template files after migrating them to mappings' do
+    it 'renames .template files after migrating them to mappings' do
       expect(File.exist?(@templates_dir.join('qualys/evidence.template'))).to be true
       migrate_templates
-      expect(File.exist?(@templates_dir.join('qualys/evidence.template'))).to be false
+      expect(File.exist?(@templates_dir.join('qualys/evidence.template.old'))).to be false
     end
   end
 end

From 7d20aff95b5cd5dc05e9387eeb3b0ced85df0aa4 Mon Sep 17 00:00:00 2001
From: Caitlin <caitlinhenry.ss@gmail.com>
Date: Mon, 4 Mar 2024 18:01:18 -0500
Subject: [PATCH 13/31] use hard-coded source mapping for burp and qualys to
 update template naming

---
 app/services/mapping_migration_service.rb | 103 +++++++++++++++-------
 1 file changed, 71 insertions(+), 32 deletions(-)

diff --git a/app/services/mapping_migration_service.rb b/app/services/mapping_migration_service.rb
index e11e2ea67..84dd9093e 100644
--- a/app/services/mapping_migration_service.rb
+++ b/app/services/mapping_migration_service.rb
@@ -1,21 +1,22 @@
 class MappingMigrationService
   LEGACY_TEMPLATE_REGEX = /%(\S*?)%/
-  attr_reader :integration_name, :rtp_id, :template_file
+  attr_reader :integration_name, :rtp_id
 
   def call
     upload_integrations = Dradis::Plugins::with_feature(:upload)
+    upload_integrations = upload_integrations - [
+      Dradis::Plugins::Projects::Engine, Dradis::Plugins::CSV::Engine
+    ]
 
     upload_integrations.each do |integration|
       @integration_name = integration.plugin_name.to_s
 
-      template_files.each do |template_file|
-        @template_file = template_file
-
-        rtp_ids = defined?(Dradis::Pro) ? ReportTemplateProperties.ids : [nil]
-        rtp_ids.each do |rtp_id|
-          @rtp_id = rtp_id
-          migrate
-          File.rename template_file, "#{template_file}.legacy"
+      if integration.uploaders.count > 1
+        migrate_multiple_uploaders(integration_name)
+      else
+        template_files.each do |template_file|
+          mapping_source = File.basename(template_file, '.template')
+          migrate(template_file, mapping_source)
         end
       end
     end
@@ -23,8 +24,7 @@ def call
 
   private
 
-  def create_mapping
-    mapping_source = File.basename(template_file, '.template')
+  def create_mapping(template_file, mapping_source)
     destination = rtp_id ? "rtp_#{rtp_id}" : nil
 
     Mapping.find_or_create_by!(
@@ -42,36 +42,75 @@ def create_mapping_field(mapping, field_title)
     )
   end
 
-  def migrate
-    # for each file, create a mapping for the uploader&plugin_name combination
-    ActiveRecord::Base.transaction do
-      mapping = create_mapping
-
-      template_fields.each do |field_title, field_content|
-        # set source_field by taking the first match to the existing %% syntax
-        source_field = field_content.match(LEGACY_TEMPLATE_REGEX)
-        @source_field =
-          if source_field && !source_field[1].empty?
-            source_field[1]
-          else
-            'custom text'
-          end
-        @updated_content = update_syntax(field_content)
-
-        # create a mapping field for each field in the .template file
-        create_mapping_field(mapping, field_title)
+  def migrate_multiple_uploaders(integration_name)
+    source_mapping = self.send("#{integration_name}_source_mapping")
+    integration_templates_dir = File.join(@templates_dir, integration_name)
+
+    source_mapping.each do |source_field, legacy_name|
+      template_file = Dir["#{integration_templates_dir}/#{legacy_name}.template*"]
+      if template_file.any? { |file| File.exist?(file) }
+        migrate(template_file[0], source_field)
+      end
+    end
+  end
+
+  def burp_source_mapping
+    # create a mapping for each legacy template type:
+    {
+      'html_evidence' => 'html_evidence',
+      'html_issue' => 'issue',
+      'xml_evidence' => 'evidence',
+      'xml_issue' => 'issue'
+    }
+  end
+
+  def qualys_source_mapping
+    # create a mapping for each legacy template type:
+    {
+      'asset_evidence' => 'asset-evidence',
+      'asset_issue' => 'asset-issue',
+      'vuln_evidence' => 'evidence',
+      'vuln_element' => 'element',
+      'was_evidence' => 'was-evidence',
+      'was_issue' => 'was-issue'
+    }
+  end
+
+  def migrate(template_file, mapping_source)
+    rtp_ids = defined?(Dradis::Pro) ? ReportTemplateProperties.ids : [nil]
+    rtp_ids.each do |rtp_id|
+      @rtp_id = rtp_id
+      # for each file, create a mapping for the uploader&plugin_name combination
+      ActiveRecord::Base.transaction do
+        mapping = create_mapping(template_file, mapping_source)
+        template_fields = template_fields(template_file)
+        template_fields.each do |field_title, field_content|
+          # set source_field by taking the first match to the existing %% syntax
+          source_field = field_content.match(LEGACY_TEMPLATE_REGEX)
+          @source_field =
+            if source_field && !source_field[1].empty?
+              source_field[1]
+            else
+              'custom text'
+            end
+          @updated_content = update_syntax(field_content)
+
+          # create a mapping field for each field in the .template file
+          create_mapping_field(mapping, field_title)
+        end
       end
     end
+    File.rename template_file, "#{template_file}.legacy"
   end
 
-  def template_fields
+  def template_fields(template_file)
     template_content = File.read(template_file)
     FieldParser.source_to_fields(template_content)
   end
 
   def template_files
-    templates_dir = Configuration.paths_templates_plugins
-    plugin_templates_dir = File.join(templates_dir, integration_name)
+    @templates_dir = Configuration.paths_templates_plugins
+    plugin_templates_dir = File.join(@templates_dir, integration_name)
     Dir["#{plugin_templates_dir}/*.template"]
   end
 

From 6bf751553d7614531ffa14d8f996eace18e484b2 Mon Sep 17 00:00:00 2001
From: Caitlin <caitlinhenry.ss@gmail.com>
Date: Tue, 5 Mar 2024 10:41:44 -0500
Subject: [PATCH 14/31] move legacy_mapping_reference out of framework and into
 integration

---
 Gemfile                                   |  4 +--
 Gemfile.lock                              | 28 ++++++++++-----
 app/services/mapping_migration_service.rb | 43 ++++++-----------------
 3 files changed, 33 insertions(+), 42 deletions(-)

diff --git a/Gemfile b/Gemfile
index 135bb55cd..489548cdc 100644
--- a/Gemfile
+++ b/Gemfile
@@ -243,7 +243,7 @@ gem 'dradis-csv', '~> 4.11.0'
 # ---------------------------------------------------------------------- Upload
 gem 'dradis-acunetix', '~> 4.11.0'
 gem 'dradis-brakeman', '~> 4.11.0'
-gem 'dradis-burp', '~> 4.11.0'
+gem 'dradis-burp', github: 'dradis/dradis-burp', branch: 'mappings-manager/legacy-mapping-reference'
 gem 'dradis-coreimpact', '~> 4.11.0'
 gem 'dradis-metasploit', '~> 4.11.0'
 gem 'dradis-nessus', '~> 4.11.0'
@@ -254,7 +254,7 @@ gem 'dradis-nipper', '~> 4.11.0'
 gem 'dradis-nmap', '~> 4.11.0'
 gem 'dradis-ntospider', '~> 4.11.0'
 gem 'dradis-openvas', '~> 4.11.0'
-gem 'dradis-qualys', '~> 4.11.0'
+gem 'dradis-qualys', github: 'dradis/dradis-qualys', branch: 'mappings-manager/legacy-mapping-reference'
 gem 'dradis-saint', '~> 4.11.0'
 gem 'dradis-veracode', '~> 4.11.0'
 gem 'dradis-wpscan', '~> 4.11.0'
diff --git a/Gemfile.lock b/Gemfile.lock
index fa8383c98..b07eeb918 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,3 +1,12 @@
+GIT
+  remote: https://github.com/dradis/dradis-burp.git
+  revision: 81b48a9755b0ef63c3e979028dc917dcf2527e39
+  branch: mappings-manager/legacy-mapping-reference
+  specs:
+    dradis-burp (4.11.0)
+      dradis-plugins (~> 4.0)
+      nokogiri (~> 1.3)
+
 GIT
   remote: https://github.com/dradis/dradis-plugins.git
   revision: e69d1f42c85f647662307019b8b492152a5c3c29
@@ -5,6 +14,15 @@ GIT
   specs:
     dradis-plugins (4.11.0)
 
+GIT
+  remote: https://github.com/dradis/dradis-qualys.git
+  revision: 05376a5045acbca3da144a12ccc3609e46ccca0c
+  branch: mappings-manager/legacy-mapping-reference
+  specs:
+    dradis-qualys (4.11.0)
+      dradis-plugins (~> 4.0)
+      nokogiri (~> 1.3)
+
 PATH
   remote: engines/dradis-api
   specs:
@@ -137,9 +155,6 @@ GEM
       nokogiri (~> 1.3)
     dradis-brakeman (4.11.0)
       dradis-plugins (~> 4.0)
-    dradis-burp (4.11.0)
-      dradis-plugins (~> 4.0)
-      nokogiri (~> 1.3)
     dradis-calculator_cvss (4.11.0)
       dradis-plugins (~> 4.0)
     dradis-calculator_dread (4.11.0)
@@ -181,9 +196,6 @@ GEM
     dradis-projects (4.11.0)
       dradis-plugins (>= 4.8.0)
       rubyzip
-    dradis-qualys (4.11.0)
-      dradis-plugins (~> 4.0)
-      nokogiri (~> 1.3)
     dradis-saint (4.11.0)
       combustion (~> 0.6.0)
       dradis-plugins (~> 4.0)
@@ -534,7 +546,7 @@ DEPENDENCIES
   dradis-acunetix (~> 4.11.0)
   dradis-api!
   dradis-brakeman (~> 4.11.0)
-  dradis-burp (~> 4.11.0)
+  dradis-burp!
   dradis-calculator_cvss (~> 4.11.0)
   dradis-calculator_dread (~> 4.11.0)
   dradis-coreimpact (~> 4.11.0)
@@ -552,7 +564,7 @@ DEPENDENCIES
   dradis-openvas (~> 4.11.0)
   dradis-plugins!
   dradis-projects (~> 4.11.0)
-  dradis-qualys (~> 4.11.0)
+  dradis-qualys!
   dradis-saint (~> 4.11.0)
   dradis-veracode (~> 4.11.0)
   dradis-wpscan (~> 4.11.0)
diff --git a/app/services/mapping_migration_service.rb b/app/services/mapping_migration_service.rb
index 84dd9093e..0f435c881 100644
--- a/app/services/mapping_migration_service.rb
+++ b/app/services/mapping_migration_service.rb
@@ -1,5 +1,5 @@
 class MappingMigrationService
-  LEGACY_TEMPLATE_REGEX = /%(\S*?)%/
+  LEGACY_FIELDS_REGEX = /%(\S*?)%/
   attr_reader :integration_name, :rtp_id
 
   def call
@@ -12,7 +12,7 @@ def call
       @integration_name = integration.plugin_name.to_s
 
       if integration.uploaders.count > 1
-        migrate_multiple_uploaders(integration_name)
+        migrate_multiple_uploaders(integration)
       else
         template_files.each do |template_file|
           mapping_source = File.basename(template_file, '.template')
@@ -42,40 +42,18 @@ def create_mapping_field(mapping, field_title)
     )
   end
 
-  def migrate_multiple_uploaders(integration_name)
-    source_mapping = self.send("#{integration_name}_source_mapping")
+  def migrate_multiple_uploaders(integration)
+    legacy_mapping_reference = integration.module_parent::Mapping.legacy_mapping_reference
     integration_templates_dir = File.join(@templates_dir, integration_name)
 
-    source_mapping.each do |source_field, legacy_name|
-      template_file = Dir["#{integration_templates_dir}/#{legacy_name}.template*"]
+    legacy_mapping_reference.each do |source_field, legacy_template_name|
+      template_file = Dir["#{integration_templates_dir}/#{legacy_template_name}.template*"]
       if template_file.any? { |file| File.exist?(file) }
         migrate(template_file[0], source_field)
       end
     end
   end
 
-  def burp_source_mapping
-    # create a mapping for each legacy template type:
-    {
-      'html_evidence' => 'html_evidence',
-      'html_issue' => 'issue',
-      'xml_evidence' => 'evidence',
-      'xml_issue' => 'issue'
-    }
-  end
-
-  def qualys_source_mapping
-    # create a mapping for each legacy template type:
-    {
-      'asset_evidence' => 'asset-evidence',
-      'asset_issue' => 'asset-issue',
-      'vuln_evidence' => 'evidence',
-      'vuln_element' => 'element',
-      'was_evidence' => 'was-evidence',
-      'was_issue' => 'was-issue'
-    }
-  end
-
   def migrate(template_file, mapping_source)
     rtp_ids = defined?(Dradis::Pro) ? ReportTemplateProperties.ids : [nil]
     rtp_ids.each do |rtp_id|
@@ -83,10 +61,11 @@ def migrate(template_file, mapping_source)
       # for each file, create a mapping for the uploader&plugin_name combination
       ActiveRecord::Base.transaction do
         mapping = create_mapping(template_file, mapping_source)
-        template_fields = template_fields(template_file)
+        template_fields = parse_template_fields(template_file)
+
         template_fields.each do |field_title, field_content|
           # set source_field by taking the first match to the existing %% syntax
-          source_field = field_content.match(LEGACY_TEMPLATE_REGEX)
+          source_field = field_content.match(LEGACY_FIELDS_REGEX)
           @source_field =
             if source_field && !source_field[1].empty?
               source_field[1]
@@ -103,7 +82,7 @@ def migrate(template_file, mapping_source)
     File.rename template_file, "#{template_file}.legacy"
   end
 
-  def template_fields(template_file)
+  def parse_template_fields(template_file)
     template_content = File.read(template_file)
     FieldParser.source_to_fields(template_content)
   end
@@ -117,7 +96,7 @@ def template_files
   def update_syntax(field_content)
     # turn the %% syntax into the new
     # '{{ <integration>[was-issue.title] }}' format
-    field_content.gsub(LEGACY_TEMPLATE_REGEX) do |content|
+    field_content.gsub(LEGACY_FIELDS_REGEX) do |content|
       "{{ #{integration_name}[#{content[1..-2]}] }}"
     end
   end

From 0e1f34329dc9a6f59cf831986a651fa9d5f8c239 Mon Sep 17 00:00:00 2001
From: Caitlin <caitlinhenry.ss@gmail.com>
Date: Tue, 5 Mar 2024 11:26:07 -0500
Subject: [PATCH 15/31] move mapping_field creation logic into
 create_mapping_fields method

---
 Gemfile.lock                              |  2 +-
 app/services/mapping_migration_service.rb | 77 ++++++++++++-----------
 2 files changed, 40 insertions(+), 39 deletions(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index b07eeb918..f539fa0fd 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,6 +1,6 @@
 GIT
   remote: https://github.com/dradis/dradis-burp.git
-  revision: 81b48a9755b0ef63c3e979028dc917dcf2527e39
+  revision: b6a17cb77c294bb1af6398f6389d7bbf74d717f6
   branch: mappings-manager/legacy-mapping-reference
   specs:
     dradis-burp (4.11.0)
diff --git a/app/services/mapping_migration_service.rb b/app/services/mapping_migration_service.rb
index 0f435c881..5a505c899 100644
--- a/app/services/mapping_migration_service.rb
+++ b/app/services/mapping_migration_service.rb
@@ -12,10 +12,11 @@ def call
       @integration_name = integration.plugin_name.to_s
 
       if integration.uploaders.count > 1
-        migrate_multiple_uploaders(integration)
+        migrate_multiple_upload_integration(integration)
       else
-        template_files.each do |template_file|
+        integration_template_files.each do |template_file|
           mapping_source = File.basename(template_file, '.template')
+          # for each file, create a mapping & mapping_fields for each field defined in the .template
           migrate(template_file, mapping_source)
         end
       end
@@ -24,7 +25,7 @@ def call
 
   private
 
-  def create_mapping(template_file, mapping_source)
+  def create_mapping(mapping_source)
     destination = rtp_id ? "rtp_#{rtp_id}" : nil
 
     Mapping.find_or_create_by!(
@@ -34,23 +35,26 @@ def create_mapping(template_file, mapping_source)
     )
   end
 
-  def create_mapping_field(mapping, field_title)
-    mapping.mapping_fields.find_or_create_by!(
-      source_field: @source_field,
-      destination_field: field_title,
-      content: @updated_content
-    )
-  end
+  def create_mapping_fields(mapping, template_file)
+    template_fields = parse_template_fields(template_file)
 
-  def migrate_multiple_uploaders(integration)
-    legacy_mapping_reference = integration.module_parent::Mapping.legacy_mapping_reference
-    integration_templates_dir = File.join(@templates_dir, integration_name)
+    # create a mapping_field for each field in the .template file
+    template_fields.each do |field_title, field_content|
+      # set source_field by taking the first match to the existing %% syntax
+      source_field = field_content.match(LEGACY_FIELDS_REGEX)
+      source_field =
+        if source_field && !source_field[1].empty?
+          source_field[1]
+        else
+          'custom text'
+        end
+      updated_content = update_syntax(field_content)
 
-    legacy_mapping_reference.each do |source_field, legacy_template_name|
-      template_file = Dir["#{integration_templates_dir}/#{legacy_template_name}.template*"]
-      if template_file.any? { |file| File.exist?(file) }
-        migrate(template_file[0], source_field)
-      end
+      mapping.mapping_fields.find_or_create_by!(
+        source_field: source_field,
+        destination_field: field_title,
+        content: updated_content
+      )
     end
   end
 
@@ -58,28 +62,25 @@ def migrate(template_file, mapping_source)
     rtp_ids = defined?(Dradis::Pro) ? ReportTemplateProperties.ids : [nil]
     rtp_ids.each do |rtp_id|
       @rtp_id = rtp_id
-      # for each file, create a mapping for the uploader&plugin_name combination
+
       ActiveRecord::Base.transaction do
-        mapping = create_mapping(template_file, mapping_source)
-        template_fields = parse_template_fields(template_file)
-
-        template_fields.each do |field_title, field_content|
-          # set source_field by taking the first match to the existing %% syntax
-          source_field = field_content.match(LEGACY_FIELDS_REGEX)
-          @source_field =
-            if source_field && !source_field[1].empty?
-              source_field[1]
-            else
-              'custom text'
-            end
-          @updated_content = update_syntax(field_content)
-
-          # create a mapping field for each field in the .template file
-          create_mapping_field(mapping, field_title)
-        end
+        mapping = create_mapping(mapping_source)
+        create_mapping_fields(mapping, template_file)
+        File.rename template_file, "#{template_file}.legacy"
+      end
+    end
+  end
+
+  def migrate_multiple_upload_integration(integration)
+    legacy_mapping_reference = integration.module_parent::Mapping.legacy_mapping_reference
+    integration_templates_dir = File.join(@templates_dir, integration_name)
+
+    legacy_mapping_reference.each do |source_field, legacy_template_name|
+      template_file = Dir["#{integration_templates_dir}/#{legacy_template_name}.template*"]
+      if template_file.any? { |file| File.exist?(file) }
+        migrate(template_file[0], source_field)
       end
     end
-    File.rename template_file, "#{template_file}.legacy"
   end
 
   def parse_template_fields(template_file)
@@ -87,7 +88,7 @@ def parse_template_fields(template_file)
     FieldParser.source_to_fields(template_content)
   end
 
-  def template_files
+  def integration_template_files
     @templates_dir = Configuration.paths_templates_plugins
     plugin_templates_dir = File.join(@templates_dir, integration_name)
     Dir["#{plugin_templates_dir}/*.template"]

From 112663b6a3d8cbb5eece5d574d789bf209878f70 Mon Sep 17 00:00:00 2001
From: Caitlin <caitlinhenry.ss@gmail.com>
Date: Tue, 5 Mar 2024 11:41:57 -0500
Subject: [PATCH 16/31] don't match empty %% syntax with legacy fields regex

---
 app/services/mapping_migration_service.rb       | 12 ++++--------
 spec/services/mapping_migration_service_spec.rb |  5 +++--
 2 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/app/services/mapping_migration_service.rb b/app/services/mapping_migration_service.rb
index 5a505c899..da5aaf25e 100644
--- a/app/services/mapping_migration_service.rb
+++ b/app/services/mapping_migration_service.rb
@@ -1,5 +1,5 @@
 class MappingMigrationService
-  LEGACY_FIELDS_REGEX = /%(\S*?)%/
+  LEGACY_FIELDS_REGEX = /%(\S+?)%/
   attr_reader :integration_name, :rtp_id
 
   def call
@@ -16,7 +16,7 @@ def call
       else
         integration_template_files.each do |template_file|
           mapping_source = File.basename(template_file, '.template')
-          # for each file, create a mapping & mapping_fields for each field defined in the .template
+          # create a mapping & mapping_fields for each field in the file
           migrate(template_file, mapping_source)
         end
       end
@@ -42,12 +42,8 @@ def create_mapping_fields(mapping, template_file)
     template_fields.each do |field_title, field_content|
       # set source_field by taking the first match to the existing %% syntax
       source_field = field_content.match(LEGACY_FIELDS_REGEX)
-      source_field =
-        if source_field && !source_field[1].empty?
-          source_field[1]
-        else
-          'custom text'
-        end
+      source_field = source_field ? source_field[1] : 'custom text'
+
       updated_content = update_syntax(field_content)
 
       mapping.mapping_fields.find_or_create_by!(
diff --git a/spec/services/mapping_migration_service_spec.rb b/spec/services/mapping_migration_service_spec.rb
index 127d242c3..9f083e4bc 100644
--- a/spec/services/mapping_migration_service_spec.rb
+++ b/spec/services/mapping_migration_service_spec.rb
@@ -18,6 +18,7 @@
     after do
       FileUtils.rm_r(Rails.root.join('spec/fixtures/files/templates/plugins/qualys'))
     end
+
     it 'creates mappings and associated mapping fields' do
       migrate_templates
 
@@ -25,7 +26,7 @@
       else
         expect(Mapping.last.destination).to eq(nil)
       end
-      expect(Mapping.last.source).to eq('evidence')
+      expect(Mapping.last.source).to eq('vuln_evidence')
       expect(Mapping.last.mapping_fields.last.source_field).to eq('custom text')
       expect(Mapping.last.mapping_fields.last.destination_field).to eq('Custom')
       expect(Mapping.last.mapping_fields.first.destination_field).to eq('TestField')
@@ -35,7 +36,7 @@
     it 'renames .template files after migrating them to mappings' do
       expect(File.exist?(@templates_dir.join('qualys/evidence.template'))).to be true
       migrate_templates
-      expect(File.exist?(@templates_dir.join('qualys/evidence.template.old'))).to be false
+      expect(File.exist?(@templates_dir.join('qualys/evidence.template.legacy'))).to be true
     end
   end
 end

From 9d5dda2ca92ad3553c3bceebc09cebba233edbfa Mon Sep 17 00:00:00 2001
From: Caitlin <caitlinhenry.ss@gmail.com>
Date: Wed, 6 Mar 2024 17:21:05 -0500
Subject: [PATCH 17/31] add comment for clarity and ability to reverse the
 migration

---
 app/services/mapping_migration_service.rb     | 26 +++++++++++++++++++
 ...226193757_migrate_templates_to_mappings.rb |  1 +
 2 files changed, 27 insertions(+)

diff --git a/app/services/mapping_migration_service.rb b/app/services/mapping_migration_service.rb
index da5aaf25e..faec54eef 100644
--- a/app/services/mapping_migration_service.rb
+++ b/app/services/mapping_migration_service.rb
@@ -23,6 +23,27 @@ def call
     end
   end
 
+  def reverse_migration
+    upload_integrations = Dradis::Plugins::with_feature(:upload)
+    upload_integrations = upload_integrations - [
+      Dradis::Plugins::Projects::Engine, Dradis::Plugins::CSV::Engine
+    ]
+    templates_dir = Configuration.paths_templates_plugins
+
+    upload_integrations.each do |integration|
+      integration_name = integration.plugin_name.to_s
+
+      Mapping.where(component: integration_name).destroy_all
+
+      plugin_templates_dir = File.join(templates_dir, integration_name)
+      legacy_files = Dir["#{plugin_templates_dir}/*.template.legacy"]
+
+      legacy_files.each do |file|
+        File.rename file, file.split('.legacy').first
+      end
+    end
+  end
+
   private
 
   def create_mapping(mapping_source)
@@ -67,6 +88,11 @@ def migrate(template_file, mapping_source)
     end
   end
 
+  # previously our integrations with multiple uploaders (Burp, Qualys) had inconsistent
+  # template names (some included the uploader, some didn't)
+  # they have been renamed to follow a consistent 'uploader_entity' structure, but
+  # in order to migrate the old templates to the db with the new names as the source
+  # we need to reference an object in the integration that maps the new name to the old one
   def migrate_multiple_upload_integration(integration)
     legacy_mapping_reference = integration.module_parent::Mapping.legacy_mapping_reference
     integration_templates_dir = File.join(@templates_dir, integration_name)
diff --git a/db/migrate/20240226193757_migrate_templates_to_mappings.rb b/db/migrate/20240226193757_migrate_templates_to_mappings.rb
index 35641ead7..82816ca63 100644
--- a/db/migrate/20240226193757_migrate_templates_to_mappings.rb
+++ b/db/migrate/20240226193757_migrate_templates_to_mappings.rb
@@ -4,5 +4,6 @@ def up
   end
 
   def down
+    MappingMigrationService.new.reverse_migration
   end
 end

From 6251206d8988ee48185a699e4c60f98c7351c797 Mon Sep 17 00:00:00 2001
From: Caitlin <caitlinhenry.ss@gmail.com>
Date: Thu, 7 Mar 2024 10:03:46 -0500
Subject: [PATCH 18/31] move shared code to separate method

---
 app/services/mapping_migration_service.rb | 37 +++++++++++------------
 1 file changed, 17 insertions(+), 20 deletions(-)

diff --git a/app/services/mapping_migration_service.rb b/app/services/mapping_migration_service.rb
index faec54eef..df36ca2b4 100644
--- a/app/services/mapping_migration_service.rb
+++ b/app/services/mapping_migration_service.rb
@@ -3,17 +3,14 @@ class MappingMigrationService
   attr_reader :integration_name, :rtp_id
 
   def call
-    upload_integrations = Dradis::Plugins::with_feature(:upload)
-    upload_integrations = upload_integrations - [
-      Dradis::Plugins::Projects::Engine, Dradis::Plugins::CSV::Engine
-    ]
-
     upload_integrations.each do |integration|
       @integration_name = integration.plugin_name.to_s
+      set_integration_template_dirs
 
       if integration.uploaders.count > 1
         migrate_multiple_upload_integration(integration)
       else
+        integration_template_files = Dir["#{@integration_templates_dir}/*.template"]
         integration_template_files.each do |template_file|
           mapping_source = File.basename(template_file, '.template')
           # create a mapping & mapping_fields for each field in the file
@@ -24,19 +21,13 @@ def call
   end
 
   def reverse_migration
-    upload_integrations = Dradis::Plugins::with_feature(:upload)
-    upload_integrations = upload_integrations - [
-      Dradis::Plugins::Projects::Engine, Dradis::Plugins::CSV::Engine
-    ]
-    templates_dir = Configuration.paths_templates_plugins
-
     upload_integrations.each do |integration|
-      integration_name = integration.plugin_name.to_s
+      @integration_name = integration.plugin_name.to_s
+      set_integration_template_dirs
 
       Mapping.where(component: integration_name).destroy_all
 
-      plugin_templates_dir = File.join(templates_dir, integration_name)
-      legacy_files = Dir["#{plugin_templates_dir}/*.template.legacy"]
+      legacy_files = Dir["#{@integration_templates_dir}/*.template.legacy"]
 
       legacy_files.each do |file|
         File.rename file, file.split('.legacy').first
@@ -89,16 +80,16 @@ def migrate(template_file, mapping_source)
   end
 
   # previously our integrations with multiple uploaders (Burp, Qualys) had inconsistent
-  # template names (some included the uploader, some didn't)
+  # template names (some included the uploader, some didn't ex. burp issue vs html_evidence)
   # they have been renamed to follow a consistent 'uploader_entity' structure, but
   # in order to migrate the old templates to the db with the new names as the source
   # we need to reference an object in the integration that maps the new name to the old one
   def migrate_multiple_upload_integration(integration)
+    # pull legacy mapping reference from Burp & Qualys
     legacy_mapping_reference = integration.module_parent::Mapping.legacy_mapping_reference
-    integration_templates_dir = File.join(@templates_dir, integration_name)
 
     legacy_mapping_reference.each do |source_field, legacy_template_name|
-      template_file = Dir["#{integration_templates_dir}/#{legacy_template_name}.template*"]
+      template_file = Dir["#{@integration_templates_dir}/#{legacy_template_name}.template*"]
       if template_file.any? { |file| File.exist?(file) }
         migrate(template_file[0], source_field)
       end
@@ -110,10 +101,9 @@ def parse_template_fields(template_file)
     FieldParser.source_to_fields(template_content)
   end
 
-  def integration_template_files
+  def set_integration_template_dirs
     @templates_dir = Configuration.paths_templates_plugins
-    plugin_templates_dir = File.join(@templates_dir, integration_name)
-    Dir["#{plugin_templates_dir}/*.template"]
+    @integration_templates_dir = File.join(@templates_dir, integration_name)
   end
 
   def update_syntax(field_content)
@@ -123,4 +113,11 @@ def update_syntax(field_content)
       "{{ #{integration_name}[#{content[1..-2]}] }}"
     end
   end
+
+  def upload_integrations
+    upload_integrations = Dradis::Plugins::with_feature(:upload)
+    upload_integrations - [
+      Dradis::Plugins::Projects::Engine, Dradis::Plugins::CSV::Engine
+    ]
+  end
 end

From fa31db4cefa2f08079ee1cccc3066f83a70251ce Mon Sep 17 00:00:00 2001
From: Caitlin <caitlinhenry.ss@gmail.com>
Date: Thu, 7 Mar 2024 10:17:32 -0500
Subject: [PATCH 19/31] rename file after mappings for all rtps are created

---
 app/services/mapping_migration_service.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/services/mapping_migration_service.rb b/app/services/mapping_migration_service.rb
index df36ca2b4..45519de6d 100644
--- a/app/services/mapping_migration_service.rb
+++ b/app/services/mapping_migration_service.rb
@@ -74,9 +74,9 @@ def migrate(template_file, mapping_source)
       ActiveRecord::Base.transaction do
         mapping = create_mapping(mapping_source)
         create_mapping_fields(mapping, template_file)
-        File.rename template_file, "#{template_file}.legacy"
       end
     end
+    File.rename template_file, "#{template_file}.legacy"
   end
 
   # previously our integrations with multiple uploaders (Burp, Qualys) had inconsistent

From c583762cbf264ac18273a5c096980c708693a031 Mon Sep 17 00:00:00 2001
From: Matt Budz <matt.budz@protonmail.com>
Date: Thu, 14 Mar 2024 17:55:34 +0100
Subject: [PATCH 20/31] use mappings manager branch for each integration

---
 Gemfile      |  36 ++++----
 Gemfile.lock | 246 ++++++++++++++++++++++++++++++++++++---------------
 2 files changed, 195 insertions(+), 87 deletions(-)

diff --git a/Gemfile b/Gemfile
index 135bb55cd..5b78ae0b0 100644
--- a/Gemfile
+++ b/Gemfile
@@ -241,21 +241,21 @@ gem 'dradis-html_export', '~> 4.11.0'
 gem 'dradis-csv', '~> 4.11.0'
 
 # ---------------------------------------------------------------------- Upload
-gem 'dradis-acunetix', '~> 4.11.0'
-gem 'dradis-brakeman', '~> 4.11.0'
-gem 'dradis-burp', '~> 4.11.0'
-gem 'dradis-coreimpact', '~> 4.11.0'
-gem 'dradis-metasploit', '~> 4.11.0'
-gem 'dradis-nessus', '~> 4.11.0'
-gem 'dradis-netsparker', '~> 4.11.0'
-gem 'dradis-nexpose', '~> 4.11.0'
-gem 'dradis-nikto', '~> 4.11.0'
-gem 'dradis-nipper', '~> 4.11.0'
-gem 'dradis-nmap', '~> 4.11.0'
-gem 'dradis-ntospider', '~> 4.11.0'
-gem 'dradis-openvas', '~> 4.11.0'
-gem 'dradis-qualys', '~> 4.11.0'
-gem 'dradis-saint', '~> 4.11.0'
-gem 'dradis-veracode', '~> 4.11.0'
-gem 'dradis-wpscan', '~> 4.11.0'
-gem 'dradis-zap', '~> 4.11.0'
+gem 'dradis-acunetix', github: 'dradis/dradis-acunetix', branch: 'mappings-manager/add-uploaders'
+gem 'dradis-brakeman', github: 'dradis/dradis-brakeman', branch: 'mappings-manager/add-uploaders'
+gem 'dradis-burp', github: 'dradis/dradis-burp', branch: 'mappings-manager/add-uploaders'
+gem 'dradis-coreimpact', github: 'dradis/dradis-coreimpact', branch: 'mappings-manager/add-uploaders'
+gem 'dradis-metasploit', github: 'dradis/dradis-metasploit', branch: 'mappings-manager/add-uploaders'
+gem 'dradis-nessus', github: 'dradis/dradis-nessus', branch: 'mappings-manager/add-uploaders'
+gem 'dradis-netsparker', github: 'dradis/dradis-netsparker', branch: 'mappings-manager/add-uploaders'
+gem 'dradis-nexpose', github: 'dradis/dradis-nexpose', branch: 'mappings-manager/add-uploaders'
+gem 'dradis-nikto', github: 'dradis/dradis-nikto', branch: 'mappings-manager/add-uploaders'
+gem 'dradis-nipper', github: 'dradis/dradis-nipper', branch: 'mappings-manager/add-uploaders'
+gem 'dradis-nmap', github: 'dradis/dradis-nmap', branch: 'mappings-manager/add-uploaders'
+gem 'dradis-ntospider', github: 'dradis/dradis-ntospider', branch: 'mappings-manager/add-uploaders'
+gem 'dradis-openvas', github: 'dradis/dradis-openvas', branch: 'mappings-manager/add-uploaders'
+gem 'dradis-qualys', github: 'dradis/dradis-qualys', branch: 'mappings-manager/add-uploaders'
+gem 'dradis-saint', github: 'dradis/dradis-saint', branch: 'mappings-manager/add-uploaders'
+gem 'dradis-veracode', github: 'dradis/dradis-veracode', branch: 'mappings-manager/add-uploaders'
+gem 'dradis-wpscan', github: 'dradis/dradis-wpscan', branch: 'mappings-manager/add-uploaders'
+gem 'dradis-zap', github: 'dradis/dradis-zap', branch: 'mappings-manager/add-uploaders'
diff --git a/Gemfile.lock b/Gemfile.lock
index fa8383c98..dbb5f196d 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,3 +1,115 @@
+GIT
+  remote: https://github.com/dradis/dradis-acunetix.git
+  revision: d5f42eddd52798ca95943debd8723731ee935d2b
+  branch: mappings-manager/add-uploaders
+  specs:
+    dradis-acunetix (4.11.0)
+      dradis-plugins (~> 4.0)
+      nokogiri (~> 1.3)
+
+GIT
+  remote: https://github.com/dradis/dradis-brakeman.git
+  revision: 40c76b41e6fdf6253cb186a9bb1a18cde574b6a0
+  branch: mappings-manager/add-uploaders
+  specs:
+    dradis-brakeman (4.11.0)
+      dradis-plugins (~> 4.0)
+
+GIT
+  remote: https://github.com/dradis/dradis-burp.git
+  revision: 58a15441f796ae1611c855cb739486177c06994b
+  branch: mappings-manager/add-uploaders
+  specs:
+    dradis-burp (4.11.0)
+      dradis-plugins (~> 4.0)
+      nokogiri (~> 1.3)
+
+GIT
+  remote: https://github.com/dradis/dradis-coreimpact.git
+  revision: 5731e1da3901eacd92f9bfe4f4cc43ae1604ee98
+  branch: mappings-manager/add-uploaders
+  specs:
+    dradis-coreimpact (4.11.0)
+      dradis-plugins (~> 4.0)
+
+GIT
+  remote: https://github.com/dradis/dradis-metasploit.git
+  revision: 4eab6a3fef664705518ba435bf21857c09c47541
+  branch: mappings-manager/add-uploaders
+  specs:
+    dradis-metasploit (4.11.0)
+      dradis-plugins (~> 4.0)
+      nokogiri (~> 1.3)
+
+GIT
+  remote: https://github.com/dradis/dradis-nessus.git
+  revision: 06740a2a240ae2be5fc14da5aced25534dcca262
+  branch: mappings-manager/add-uploaders
+  specs:
+    dradis-nessus (4.11.0)
+      dradis-plugins (~> 4.0)
+      nokogiri
+
+GIT
+  remote: https://github.com/dradis/dradis-netsparker.git
+  revision: 2f6d4d40034b928a0f41b8d489697e742bdaa38e
+  branch: mappings-manager/add-uploaders
+  specs:
+    dradis-netsparker (4.11.0)
+      dradis-plugins (~> 4.0)
+      nokogiri (>= 1.12.5)
+
+GIT
+  remote: https://github.com/dradis/dradis-nexpose.git
+  revision: 439cb5dbe2825ce5461c58813348a69961c6e4a4
+  branch: mappings-manager/add-uploaders
+  specs:
+    dradis-nexpose (4.11.0)
+      dradis-plugins (~> 4.0)
+      nokogiri (~> 1.3)
+
+GIT
+  remote: https://github.com/dradis/dradis-nikto.git
+  revision: fbbca4b43f30fc241df142c55238b2feb7c7f3fe
+  branch: mappings-manager/add-uploaders
+  specs:
+    dradis-nikto (4.11.0)
+      dradis-plugins (~> 4.0)
+      nokogiri (~> 1.3)
+
+GIT
+  remote: https://github.com/dradis/dradis-nipper.git
+  revision: 169535775f078913ca3d38eb6bd7dbcc80a333dd
+  branch: mappings-manager/add-uploaders
+  specs:
+    dradis-nipper (4.11.0)
+      dradis-plugins (~> 4.0)
+
+GIT
+  remote: https://github.com/dradis/dradis-nmap.git
+  revision: a653ad66543cdef3361a1f6dd1fbb07d9e4eada7
+  branch: mappings-manager/add-uploaders
+  specs:
+    dradis-nmap (4.11.0)
+      dradis-plugins (~> 4.0)
+      ruby-nmap (~> 0.7)
+
+GIT
+  remote: https://github.com/dradis/dradis-ntospider.git
+  revision: 0c2ae49734090ce60982fe8c01658832cf96180a
+  branch: mappings-manager/add-uploaders
+  specs:
+    dradis-ntospider (4.11.0)
+      dradis-plugins (~> 4.0)
+
+GIT
+  remote: https://github.com/dradis/dradis-openvas.git
+  revision: 1678903dd6e35fd8c4cfdbf55333c6f3f18c352b
+  branch: mappings-manager/add-uploaders
+  specs:
+    dradis-openvas (4.11.0)
+      dradis-plugins (~> 4.0)
+
 GIT
   remote: https://github.com/dradis/dradis-plugins.git
   revision: e69d1f42c85f647662307019b8b492152a5c3c29
@@ -5,6 +117,53 @@ GIT
   specs:
     dradis-plugins (4.11.0)
 
+GIT
+  remote: https://github.com/dradis/dradis-qualys.git
+  revision: 82712e710418dd7e116223c9e1a29b4ba73ebf79
+  branch: mappings-manager/add-uploaders
+  specs:
+    dradis-qualys (4.11.0)
+      dradis-plugins (~> 4.0)
+      nokogiri (~> 1.3)
+
+GIT
+  remote: https://github.com/dradis/dradis-saint.git
+  revision: 7d5e45da4f61edb50798ed61c86ce6b7f4c1698b
+  branch: mappings-manager/add-uploaders
+  specs:
+    dradis-saint (4.11.0)
+      combustion (~> 0.6.0)
+      dradis-plugins (~> 4.0)
+      nokogiri
+      rake (~> 13.0)
+      rspec-rails
+
+GIT
+  remote: https://github.com/dradis/dradis-veracode.git
+  revision: 0442dee4651101bf5530dadc2e68f73faf69a381
+  branch: mappings-manager/add-uploaders
+  specs:
+    dradis-veracode (4.11.0)
+      dradis-plugins (~> 4.0)
+
+GIT
+  remote: https://github.com/dradis/dradis-wpscan.git
+  revision: 2e5ff1b8e01413dbfd3901f179f9bd62290513d2
+  branch: mappings-manager/add-uploaders
+  specs:
+    dradis-wpscan (4.11.0)
+      dradis-plugins (~> 4.0)
+      multi_json
+
+GIT
+  remote: https://github.com/dradis/dradis-zap.git
+  revision: 9deb31fe0dcfb05533414b90a64a371422340336
+  branch: mappings-manager/add-uploaders
+  specs:
+    dradis-zap (4.11.0)
+      dradis-plugins (~> 4.0)
+      nokogiri (~> 1.3)
+
 PATH
   remote: engines/dradis-api
   specs:
@@ -132,20 +291,10 @@ GEM
     date (3.3.4)
     diff-lcs (1.5.0)
     differ (0.1.2)
-    dradis-acunetix (4.11.0)
-      dradis-plugins (~> 4.0)
-      nokogiri (~> 1.3)
-    dradis-brakeman (4.11.0)
-      dradis-plugins (~> 4.0)
-    dradis-burp (4.11.0)
-      dradis-plugins (~> 4.0)
-      nokogiri (~> 1.3)
     dradis-calculator_cvss (4.11.0)
       dradis-plugins (~> 4.0)
     dradis-calculator_dread (4.11.0)
       dradis-plugins (~> 4.0)
-    dradis-coreimpact (4.11.0)
-      dradis-plugins (~> 4.0)
     dradis-csv (4.11.0)
       dradis-plugins (~> 4.0)
     dradis-csv_export (4.11.0)
@@ -154,50 +303,9 @@ GEM
       RedCloth (~> 4.3.2)
       dradis-plugins (>= 4.8.0)
       rails_autolink (~> 1.1)
-    dradis-metasploit (4.11.0)
-      dradis-plugins (~> 4.0)
-      nokogiri (~> 1.3)
-    dradis-nessus (4.11.0)
-      dradis-plugins (~> 4.0)
-      nokogiri
-    dradis-netsparker (4.11.0)
-      dradis-plugins (~> 4.0)
-      nokogiri (>= 1.12.5)
-    dradis-nexpose (4.11.0)
-      dradis-plugins (~> 4.0)
-      nokogiri (~> 1.3)
-    dradis-nikto (4.11.0)
-      dradis-plugins (~> 4.0)
-      nokogiri (~> 1.3)
-    dradis-nipper (4.11.0)
-      dradis-plugins (~> 4.0)
-    dradis-nmap (4.11.0)
-      dradis-plugins (~> 4.0)
-      ruby-nmap (~> 0.7)
-    dradis-ntospider (4.11.0)
-      dradis-plugins (~> 4.0)
-    dradis-openvas (4.11.0)
-      dradis-plugins (~> 4.0)
     dradis-projects (4.11.0)
       dradis-plugins (>= 4.8.0)
       rubyzip
-    dradis-qualys (4.11.0)
-      dradis-plugins (~> 4.0)
-      nokogiri (~> 1.3)
-    dradis-saint (4.11.0)
-      combustion (~> 0.6.0)
-      dradis-plugins (~> 4.0)
-      nokogiri
-      rake (~> 13.0)
-      rspec-rails
-    dradis-veracode (4.11.0)
-      dradis-plugins (~> 4.0)
-    dradis-wpscan (4.11.0)
-      dradis-plugins (~> 4.0)
-      multi_json
-    dradis-zap (4.11.0)
-      dradis-plugins (~> 4.0)
-      nokogiri (~> 1.3)
     erubi (1.12.0)
     execjs (2.7.0)
     factory_bot (6.2.1)
@@ -531,32 +639,32 @@ DEPENDENCIES
   coffee-rails (~> 5.0)
   database_cleaner
   differ (~> 0.1.2)
-  dradis-acunetix (~> 4.11.0)
+  dradis-acunetix!
   dradis-api!
-  dradis-brakeman (~> 4.11.0)
-  dradis-burp (~> 4.11.0)
+  dradis-brakeman!
+  dradis-burp!
   dradis-calculator_cvss (~> 4.11.0)
   dradis-calculator_dread (~> 4.11.0)
-  dradis-coreimpact (~> 4.11.0)
+  dradis-coreimpact!
   dradis-csv (~> 4.11.0)
   dradis-csv_export (~> 4.11.0)
   dradis-html_export (~> 4.11.0)
-  dradis-metasploit (~> 4.11.0)
-  dradis-nessus (~> 4.11.0)
-  dradis-netsparker (~> 4.11.0)
-  dradis-nexpose (~> 4.11.0)
-  dradis-nikto (~> 4.11.0)
-  dradis-nipper (~> 4.11.0)
-  dradis-nmap (~> 4.11.0)
-  dradis-ntospider (~> 4.11.0)
-  dradis-openvas (~> 4.11.0)
+  dradis-metasploit!
+  dradis-nessus!
+  dradis-netsparker!
+  dradis-nexpose!
+  dradis-nikto!
+  dradis-nipper!
+  dradis-nmap!
+  dradis-ntospider!
+  dradis-openvas!
   dradis-plugins!
   dradis-projects (~> 4.11.0)
-  dradis-qualys (~> 4.11.0)
-  dradis-saint (~> 4.11.0)
-  dradis-veracode (~> 4.11.0)
-  dradis-wpscan (~> 4.11.0)
-  dradis-zap (~> 4.11.0)
+  dradis-qualys!
+  dradis-saint!
+  dradis-veracode!
+  dradis-wpscan!
+  dradis-zap!
   factory_bot_rails
   font-awesome-sass (~> 6.4.0)
   foreman

From 9bd2376236d0944cd64740b64564b61f4a87b478 Mon Sep 17 00:00:00 2001
From: Caitlin <caitlinhenry.ss@gmail.com>
Date: Tue, 19 Mar 2024 09:37:10 -0400
Subject: [PATCH 21/31] remove pro only mapping in spec

---
 spec/factories/mappings.rb | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/spec/factories/mappings.rb b/spec/factories/mappings.rb
index ba21287f8..792febf3a 100644
--- a/spec/factories/mappings.rb
+++ b/spec/factories/mappings.rb
@@ -1,13 +1,7 @@
 FactoryBot.define do
   factory :mapping do
     component { 'qualys' }
-    sequence(:source) { |n| "source_#{n}" }
+    sequence(:source) { 'asset_evidence' }
     sequence(:destination) { |n| "rtp_#{n}" }
-
-    trait :export_integration do
-      component { 'jira' }
-      sequence(:source) { |n| "rtp_#{n}" }
-      sequence(:destination) { |n| "project_1_issuetype_#{n}" }
-    end
   end
 end

From 6a5ded3248320e96de7cfaebd716e8fc6c8f2ab3 Mon Sep 17 00:00:00 2001
From: Caitlin <caitlinhenry.ss@gmail.com>
Date: Thu, 4 Apr 2024 17:16:35 -0400
Subject: [PATCH 22/31] move template migration to initializer in
 dradis-plugins

---
 Gemfile                                       |   2 +-
 Gemfile.lock                                  |   4 +-
 app/services/mapping_migration_service.rb     | 123 ------------------
 config/initializers/z_11_plugin_templates.rb  |   5 +-
 ...226193757_migrate_templates_to_mappings.rb |   9 --
 5 files changed, 6 insertions(+), 137 deletions(-)
 delete mode 100644 app/services/mapping_migration_service.rb
 delete mode 100644 db/migrate/20240226193757_migrate_templates_to_mappings.rb

diff --git a/Gemfile b/Gemfile
index 489548cdc..5890103ec 100644
--- a/Gemfile
+++ b/Gemfile
@@ -213,7 +213,7 @@ end
 #
 
 # Base framework classes required by other plugins
-gem 'dradis-plugins', github: 'dradis/dradis-plugins', branch: 'fix/template-caching'
+gem 'dradis-plugins', github: 'dradis/dradis-plugins', branch: 'mappings-manager/template-migration'
 
 gem 'dradis-api', path: 'engines/dradis-api'
 
diff --git a/Gemfile.lock b/Gemfile.lock
index f539fa0fd..d19052aa7 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -9,8 +9,8 @@ GIT
 
 GIT
   remote: https://github.com/dradis/dradis-plugins.git
-  revision: e69d1f42c85f647662307019b8b492152a5c3c29
-  branch: fix/template-caching
+  revision: 160b6c1f0a3ce23e5e08ce0495baa080096ef5dd
+  branch: mappings-manager/template-migration
   specs:
     dradis-plugins (4.11.0)
 
diff --git a/app/services/mapping_migration_service.rb b/app/services/mapping_migration_service.rb
deleted file mode 100644
index 45519de6d..000000000
--- a/app/services/mapping_migration_service.rb
+++ /dev/null
@@ -1,123 +0,0 @@
-class MappingMigrationService
-  LEGACY_FIELDS_REGEX = /%(\S+?)%/
-  attr_reader :integration_name, :rtp_id
-
-  def call
-    upload_integrations.each do |integration|
-      @integration_name = integration.plugin_name.to_s
-      set_integration_template_dirs
-
-      if integration.uploaders.count > 1
-        migrate_multiple_upload_integration(integration)
-      else
-        integration_template_files = Dir["#{@integration_templates_dir}/*.template"]
-        integration_template_files.each do |template_file|
-          mapping_source = File.basename(template_file, '.template')
-          # create a mapping & mapping_fields for each field in the file
-          migrate(template_file, mapping_source)
-        end
-      end
-    end
-  end
-
-  def reverse_migration
-    upload_integrations.each do |integration|
-      @integration_name = integration.plugin_name.to_s
-      set_integration_template_dirs
-
-      Mapping.where(component: integration_name).destroy_all
-
-      legacy_files = Dir["#{@integration_templates_dir}/*.template.legacy"]
-
-      legacy_files.each do |file|
-        File.rename file, file.split('.legacy').first
-      end
-    end
-  end
-
-  private
-
-  def create_mapping(mapping_source)
-    destination = rtp_id ? "rtp_#{rtp_id}" : nil
-
-    Mapping.find_or_create_by!(
-      component: integration_name,
-      source: mapping_source,
-      destination: destination
-    )
-  end
-
-  def create_mapping_fields(mapping, template_file)
-    template_fields = parse_template_fields(template_file)
-
-    # create a mapping_field for each field in the .template file
-    template_fields.each do |field_title, field_content|
-      # set source_field by taking the first match to the existing %% syntax
-      source_field = field_content.match(LEGACY_FIELDS_REGEX)
-      source_field = source_field ? source_field[1] : 'custom text'
-
-      updated_content = update_syntax(field_content)
-
-      mapping.mapping_fields.find_or_create_by!(
-        source_field: source_field,
-        destination_field: field_title,
-        content: updated_content
-      )
-    end
-  end
-
-  def migrate(template_file, mapping_source)
-    rtp_ids = defined?(Dradis::Pro) ? ReportTemplateProperties.ids : [nil]
-    rtp_ids.each do |rtp_id|
-      @rtp_id = rtp_id
-
-      ActiveRecord::Base.transaction do
-        mapping = create_mapping(mapping_source)
-        create_mapping_fields(mapping, template_file)
-      end
-    end
-    File.rename template_file, "#{template_file}.legacy"
-  end
-
-  # previously our integrations with multiple uploaders (Burp, Qualys) had inconsistent
-  # template names (some included the uploader, some didn't ex. burp issue vs html_evidence)
-  # they have been renamed to follow a consistent 'uploader_entity' structure, but
-  # in order to migrate the old templates to the db with the new names as the source
-  # we need to reference an object in the integration that maps the new name to the old one
-  def migrate_multiple_upload_integration(integration)
-    # pull legacy mapping reference from Burp & Qualys
-    legacy_mapping_reference = integration.module_parent::Mapping.legacy_mapping_reference
-
-    legacy_mapping_reference.each do |source_field, legacy_template_name|
-      template_file = Dir["#{@integration_templates_dir}/#{legacy_template_name}.template*"]
-      if template_file.any? { |file| File.exist?(file) }
-        migrate(template_file[0], source_field)
-      end
-    end
-  end
-
-  def parse_template_fields(template_file)
-    template_content = File.read(template_file)
-    FieldParser.source_to_fields(template_content)
-  end
-
-  def set_integration_template_dirs
-    @templates_dir = Configuration.paths_templates_plugins
-    @integration_templates_dir = File.join(@templates_dir, integration_name)
-  end
-
-  def update_syntax(field_content)
-    # turn the %% syntax into the new
-    # '{{ <integration>[was-issue.title] }}' format
-    field_content.gsub(LEGACY_FIELDS_REGEX) do |content|
-      "{{ #{integration_name}[#{content[1..-2]}] }}"
-    end
-  end
-
-  def upload_integrations
-    upload_integrations = Dradis::Plugins::with_feature(:upload)
-    upload_integrations - [
-      Dradis::Plugins::Projects::Engine, Dradis::Plugins::CSV::Engine
-    ]
-  end
-end
diff --git a/config/initializers/z_11_plugin_templates.rb b/config/initializers/z_11_plugin_templates.rb
index 2c0174734..4b26ce746 100644
--- a/config/initializers/z_11_plugin_templates.rb
+++ b/config/initializers/z_11_plugin_templates.rb
@@ -11,8 +11,9 @@
     # ---------------------------------------------------------------- 3.1 Upload
     template_dir = Configuration.paths_templates_plugins
 
-    Dradis::Plugins::with_feature(:upload).each do |plugin|
-      plugin.copy_templates(to: template_dir)
+    Dradis::Plugins::with_feature(:upload).each do |integration|
+      integration.copy_templates(to: template_dir)
+      integration.migrate_templates_to_mappings(from: template_dir)
     end
 
     # ---------------------------------------------------------------- 3.2 Export
diff --git a/db/migrate/20240226193757_migrate_templates_to_mappings.rb b/db/migrate/20240226193757_migrate_templates_to_mappings.rb
deleted file mode 100644
index 82816ca63..000000000
--- a/db/migrate/20240226193757_migrate_templates_to_mappings.rb
+++ /dev/null
@@ -1,9 +0,0 @@
-class MigrateTemplatesToMappings < ActiveRecord::Migration[7.0]
-  def up
-    MappingMigrationService.new.call
-  end
-
-  def down
-    MappingMigrationService.new.reverse_migration
-  end
-end

From f079727201cd0e28abd7a3754ceb056b5e1e4626 Mon Sep 17 00:00:00 2001
From: Caitlin <caitlinhenry.ss@gmail.com>
Date: Thu, 4 Apr 2024 17:21:52 -0400
Subject: [PATCH 23/31] remove spec

---
 config/initializers/mapping_creator.rb | 121 +++++++++++++++++++++++++
 1 file changed, 121 insertions(+)
 create mode 100644 config/initializers/mapping_creator.rb

diff --git a/config/initializers/mapping_creator.rb b/config/initializers/mapping_creator.rb
new file mode 100644
index 000000000..ec2aa623f
--- /dev/null
+++ b/config/initializers/mapping_creator.rb
@@ -0,0 +1,121 @@
+  # class MappingCreator
+  # LEGACY_FIELDS_REGEX = /%(\S+?)%/
+  # LEGACY_MAPPING_REFERENCE = {
+  #   'burp' => {
+  #     'html_evidence' => 'html_evidence',
+  #     'html_issue' => 'issue',
+  #     'xml_evidence' => 'evidence',
+  #     'xml_issue' => 'issue'
+  #   },
+  #   'qualys' => {
+  #     'asset_evidence' => 'asset-evidence',
+  #     'asset_issue' => 'asset-issue',
+  #     'vuln_evidence' => 'evidence',
+  #     'vuln_element' => 'element',
+  #     'was_evidence' => 'was-evidence',
+  #     'was_issue' => 'was-issue'
+  #   }
+  # }
+
+  # Rails.application.reloader.to_prepare do
+  #   if (ActiveRecord::Base.connection rescue false) && Configuration.table_exists? && Configuration.paths_templates.exist?
+  #     upload_integrations.each do |integration|
+  #       @integration_name = integration.plugin_name.to_s
+  #       set_integration_template_dirs
+
+  #       if integration.uploaders.count > 1
+  #         migrate_multiple_upload_integration(@integration_name)
+  #       else
+  #         integration_template_files = Dir["#{@integration_templates_dir}/*.template"]
+  #         integration_template_files.each do |template_file|
+  #           mapping_source = File.basename(template_file, '.template')
+  #           # create a mapping & mapping_fields for each field in the file
+  #           migrate(template_file, mapping_source)
+  #         end
+  #       end
+  #     end
+  #   end
+  # end
+
+  # def create_mapping(mapping_source)
+  #   destination = @rtp_id ? "rtp_#{@rtp_id}" : nil
+
+  #   Mapping.find_or_create_by!(
+  #     component: @integration_name,
+  #     source: mapping_source,
+  #     destination: destination
+  #   )
+  # end
+
+  # def create_mapping_fields(mapping, template_file)
+  #   template_fields = parse_template_fields(template_file)
+
+  #   # create a mapping_field for each field in the .template file
+  #   template_fields.each do |field_title, field_content|
+  #     # set source_field by taking the first match to the existing %% syntax
+  #     source_field = field_content.match(LEGACY_FIELDS_REGEX)
+  #     source_field = source_field ? source_field[1] : 'custom text'
+
+  #     updated_content = update_syntax(field_content)
+
+  #     mapping.mapping_fields.find_or_create_by!(
+  #       source_field: source_field,
+  #       destination_field: field_title,
+  #       content: updated_content
+  #     )
+  #   end
+  # end
+
+  # def migrate(template_file, mapping_source)
+  #   rtp_ids = defined?(Dradis::Pro) ? ReportTemplateProperties.ids : [nil]
+  #   rtp_ids.each do |rtp_id|
+  #     @rtp_id = rtp_id
+
+  #     ActiveRecord::Base.transaction do
+  #       mapping = create_mapping(mapping_source)
+  #       create_mapping_fields(mapping, template_file)
+  #     end
+  #   end
+  #   File.rename template_file, "#{template_file}.legacy"
+  # end
+
+  # previously our integrations with multiple uploaders (Burp, Qualys) had inconsistent
+  # template names (some included the uploader, some didn't ex. burp issue vs html_evidence)
+  # they have been renamed to follow a consistent 'uploader_entity' structure, but
+  # in order to migrate the old templates to the db with the new names as the source
+  # we need to reference an object in the integration that maps the new name to the old one
+  # def migrate_multiple_upload_integration(integration)
+  #   byebug
+  #   LEGACY_MAPPING_REFERENCE[integration].each do |source_field, legacy_template_name|
+  #     template_file = Dir["#{@integration_templates_dir}/#{legacy_template_name}.template*"]
+  #     if template_file.any? { |file| File.exist?(file) }
+  #       migrate(template_file[0], source_field)
+  #     end
+  #   end
+  # end
+
+  # def parse_template_fields(template_file)
+  #   template_content = File.read(template_file)
+  #   FieldParser.source_to_fields(template_content)
+  # end
+
+  # def set_integration_template_dirs
+  #   @templates_dir = Configuration.paths_templates_plugins
+  #   @integration_templates_dir = File.join(@templates_dir, @integration_name)
+  # end
+
+  def update_syntax(field_content)
+    # turn the %% syntax into the new
+    # '{{ <integration>[was-issue.title] }}' format
+    field_content.gsub(LEGACY_FIELDS_REGEX) do |content|
+      "{{ #{@integration_name}[#{content[1..-2]}] }}"
+    end
+  end
+
+  def upload_integrations
+    upload_integrations = Dradis::Plugins::with_feature(:upload)
+    upload_integrations - [
+      Dradis::Plugins::Projects::Engine, Dradis::Plugins::CSV::Engine
+    ]
+  end
+# end

From d0e3dc1a725b12ee929207f59a3ab4cdf42c52b6 Mon Sep 17 00:00:00 2001
From: Caitlin <caitlinhenry.ss@gmail.com>
Date: Thu, 4 Apr 2024 17:22:51 -0400
Subject: [PATCH 24/31] remove spec

---
 config/initializers/mapping_creator.rb        | 121 ------------------
 .../mapping_migration_service_spec.rb         |  42 ------
 2 files changed, 163 deletions(-)
 delete mode 100644 config/initializers/mapping_creator.rb
 delete mode 100644 spec/services/mapping_migration_service_spec.rb

diff --git a/config/initializers/mapping_creator.rb b/config/initializers/mapping_creator.rb
deleted file mode 100644
index ec2aa623f..000000000
--- a/config/initializers/mapping_creator.rb
+++ /dev/null
@@ -1,121 +0,0 @@
-  # class MappingCreator
-  # LEGACY_FIELDS_REGEX = /%(\S+?)%/
-  # LEGACY_MAPPING_REFERENCE = {
-  #   'burp' => {
-  #     'html_evidence' => 'html_evidence',
-  #     'html_issue' => 'issue',
-  #     'xml_evidence' => 'evidence',
-  #     'xml_issue' => 'issue'
-  #   },
-  #   'qualys' => {
-  #     'asset_evidence' => 'asset-evidence',
-  #     'asset_issue' => 'asset-issue',
-  #     'vuln_evidence' => 'evidence',
-  #     'vuln_element' => 'element',
-  #     'was_evidence' => 'was-evidence',
-  #     'was_issue' => 'was-issue'
-  #   }
-  # }
-
-  # Rails.application.reloader.to_prepare do
-  #   if (ActiveRecord::Base.connection rescue false) && Configuration.table_exists? && Configuration.paths_templates.exist?
-  #     upload_integrations.each do |integration|
-  #       @integration_name = integration.plugin_name.to_s
-  #       set_integration_template_dirs
-
-  #       if integration.uploaders.count > 1
-  #         migrate_multiple_upload_integration(@integration_name)
-  #       else
-  #         integration_template_files = Dir["#{@integration_templates_dir}/*.template"]
-  #         integration_template_files.each do |template_file|
-  #           mapping_source = File.basename(template_file, '.template')
-  #           # create a mapping & mapping_fields for each field in the file
-  #           migrate(template_file, mapping_source)
-  #         end
-  #       end
-  #     end
-  #   end
-  # end
-
-  # def create_mapping(mapping_source)
-  #   destination = @rtp_id ? "rtp_#{@rtp_id}" : nil
-
-  #   Mapping.find_or_create_by!(
-  #     component: @integration_name,
-  #     source: mapping_source,
-  #     destination: destination
-  #   )
-  # end
-
-  # def create_mapping_fields(mapping, template_file)
-  #   template_fields = parse_template_fields(template_file)
-
-  #   # create a mapping_field for each field in the .template file
-  #   template_fields.each do |field_title, field_content|
-  #     # set source_field by taking the first match to the existing %% syntax
-  #     source_field = field_content.match(LEGACY_FIELDS_REGEX)
-  #     source_field = source_field ? source_field[1] : 'custom text'
-
-  #     updated_content = update_syntax(field_content)
-
-  #     mapping.mapping_fields.find_or_create_by!(
-  #       source_field: source_field,
-  #       destination_field: field_title,
-  #       content: updated_content
-  #     )
-  #   end
-  # end
-
-  # def migrate(template_file, mapping_source)
-  #   rtp_ids = defined?(Dradis::Pro) ? ReportTemplateProperties.ids : [nil]
-  #   rtp_ids.each do |rtp_id|
-  #     @rtp_id = rtp_id
-
-  #     ActiveRecord::Base.transaction do
-  #       mapping = create_mapping(mapping_source)
-  #       create_mapping_fields(mapping, template_file)
-  #     end
-  #   end
-  #   File.rename template_file, "#{template_file}.legacy"
-  # end
-
-  # previously our integrations with multiple uploaders (Burp, Qualys) had inconsistent
-  # template names (some included the uploader, some didn't ex. burp issue vs html_evidence)
-  # they have been renamed to follow a consistent 'uploader_entity' structure, but
-  # in order to migrate the old templates to the db with the new names as the source
-  # we need to reference an object in the integration that maps the new name to the old one
-  # def migrate_multiple_upload_integration(integration)
-  #   byebug
-  #   LEGACY_MAPPING_REFERENCE[integration].each do |source_field, legacy_template_name|
-  #     template_file = Dir["#{@integration_templates_dir}/#{legacy_template_name}.template*"]
-  #     if template_file.any? { |file| File.exist?(file) }
-  #       migrate(template_file[0], source_field)
-  #     end
-  #   end
-  # end
-
-  # def parse_template_fields(template_file)
-  #   template_content = File.read(template_file)
-  #   FieldParser.source_to_fields(template_content)
-  # end
-
-  # def set_integration_template_dirs
-  #   @templates_dir = Configuration.paths_templates_plugins
-  #   @integration_templates_dir = File.join(@templates_dir, @integration_name)
-  # end
-
-  def update_syntax(field_content)
-    # turn the %% syntax into the new
-    # '{{ <integration>[was-issue.title] }}' format
-    field_content.gsub(LEGACY_FIELDS_REGEX) do |content|
-      "{{ #{@integration_name}[#{content[1..-2]}] }}"
-    end
-  end
-
-  def upload_integrations
-    upload_integrations = Dradis::Plugins::with_feature(:upload)
-    upload_integrations - [
-      Dradis::Plugins::Projects::Engine, Dradis::Plugins::CSV::Engine
-    ]
-  end
-# end
diff --git a/spec/services/mapping_migration_service_spec.rb b/spec/services/mapping_migration_service_spec.rb
deleted file mode 100644
index 9f083e4bc..000000000
--- a/spec/services/mapping_migration_service_spec.rb
+++ /dev/null
@@ -1,42 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-RSpec.describe MappingMigrationService do
-  describe '.call' do
-    subject(:migrate_templates) { described_class.new.call }
-
-    before do
-      @templates_dir = Rails.root.join('spec/fixtures/files/templates/plugins/')
-      templates_path = Pathname.new(@templates_dir)
-      FileUtils.mkdir_p(templates_path)
-      allow(Configuration).to receive(:paths_templates_plugins).and_return(templates_path)
-      FileUtils.mkdir_p(templates_path.join('qualys'))
-      FileUtils.cp(templates_path.join('evidence.template') , templates_path.join('qualys/evidence.template'))
-    end
-
-    after do
-      FileUtils.rm_r(Rails.root.join('spec/fixtures/files/templates/plugins/qualys'))
-    end
-
-    it 'creates mappings and associated mapping fields' do
-      migrate_templates
-
-      if defined?(Dradis::Pro)
-      else
-        expect(Mapping.last.destination).to eq(nil)
-      end
-      expect(Mapping.last.source).to eq('vuln_evidence')
-      expect(Mapping.last.mapping_fields.last.source_field).to eq('custom text')
-      expect(Mapping.last.mapping_fields.last.destination_field).to eq('Custom')
-      expect(Mapping.last.mapping_fields.first.destination_field).to eq('TestField')
-      expect(Mapping.last.mapping_fields.first.content).to eq('{{ qualys[evidence.test_field] }}')
-    end
-
-    it 'renames .template files after migrating them to mappings' do
-      expect(File.exist?(@templates_dir.join('qualys/evidence.template'))).to be true
-      migrate_templates
-      expect(File.exist?(@templates_dir.join('qualys/evidence.template.legacy'))).to be true
-    end
-  end
-end

From 2622de8f01ccf736cc2b947ce2fcda6dda8f0a32 Mon Sep 17 00:00:00 2001
From: Caitlin <caitlinhenry.ss@gmail.com>
Date: Thu, 4 Apr 2024 17:24:39 -0400
Subject: [PATCH 25/31] revert burp and qualys gemfile change

---
 Gemfile                                                 | 4 ++--
 spec/fixtures/files/templates/plugins/evidence.template | 5 -----
 2 files changed, 2 insertions(+), 7 deletions(-)
 delete mode 100644 spec/fixtures/files/templates/plugins/evidence.template

diff --git a/Gemfile b/Gemfile
index 5890103ec..bacc1dbe8 100644
--- a/Gemfile
+++ b/Gemfile
@@ -243,7 +243,7 @@ gem 'dradis-csv', '~> 4.11.0'
 # ---------------------------------------------------------------------- Upload
 gem 'dradis-acunetix', '~> 4.11.0'
 gem 'dradis-brakeman', '~> 4.11.0'
-gem 'dradis-burp', github: 'dradis/dradis-burp', branch: 'mappings-manager/legacy-mapping-reference'
+gem 'dradis-burp', '~> 4.11.0'
 gem 'dradis-coreimpact', '~> 4.11.0'
 gem 'dradis-metasploit', '~> 4.11.0'
 gem 'dradis-nessus', '~> 4.11.0'
@@ -254,7 +254,7 @@ gem 'dradis-nipper', '~> 4.11.0'
 gem 'dradis-nmap', '~> 4.11.0'
 gem 'dradis-ntospider', '~> 4.11.0'
 gem 'dradis-openvas', '~> 4.11.0'
-gem 'dradis-qualys', github: 'dradis/dradis-qualys', branch: 'mappings-manager/legacy-mapping-reference'
+gem 'dradis-qualys', '~> 4.11.0'
 gem 'dradis-saint', '~> 4.11.0'
 gem 'dradis-veracode', '~> 4.11.0'
 gem 'dradis-wpscan', '~> 4.11.0'
diff --git a/spec/fixtures/files/templates/plugins/evidence.template b/spec/fixtures/files/templates/plugins/evidence.template
deleted file mode 100644
index 3e09d23bd..000000000
--- a/spec/fixtures/files/templates/plugins/evidence.template
+++ /dev/null
@@ -1,5 +0,0 @@
-#[TestField]#
-%evidence.test_field%
-
-#[Custom]#
-Custom text

From 524fa0880d46094c3f6d34eb8651f0bbaee71c40 Mon Sep 17 00:00:00 2001
From: Caitlin <caitlinhenry.ss@gmail.com>
Date: Thu, 4 Apr 2024 17:25:19 -0400
Subject: [PATCH 26/31] lockfile

---
 Gemfile.lock | 28 ++++++++--------------------
 1 file changed, 8 insertions(+), 20 deletions(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index d19052aa7..e3d44d6d0 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,12 +1,3 @@
-GIT
-  remote: https://github.com/dradis/dradis-burp.git
-  revision: b6a17cb77c294bb1af6398f6389d7bbf74d717f6
-  branch: mappings-manager/legacy-mapping-reference
-  specs:
-    dradis-burp (4.11.0)
-      dradis-plugins (~> 4.0)
-      nokogiri (~> 1.3)
-
 GIT
   remote: https://github.com/dradis/dradis-plugins.git
   revision: 160b6c1f0a3ce23e5e08ce0495baa080096ef5dd
@@ -14,15 +5,6 @@ GIT
   specs:
     dradis-plugins (4.11.0)
 
-GIT
-  remote: https://github.com/dradis/dradis-qualys.git
-  revision: 05376a5045acbca3da144a12ccc3609e46ccca0c
-  branch: mappings-manager/legacy-mapping-reference
-  specs:
-    dradis-qualys (4.11.0)
-      dradis-plugins (~> 4.0)
-      nokogiri (~> 1.3)
-
 PATH
   remote: engines/dradis-api
   specs:
@@ -155,6 +137,9 @@ GEM
       nokogiri (~> 1.3)
     dradis-brakeman (4.11.0)
       dradis-plugins (~> 4.0)
+    dradis-burp (4.11.0)
+      dradis-plugins (~> 4.0)
+      nokogiri (~> 1.3)
     dradis-calculator_cvss (4.11.0)
       dradis-plugins (~> 4.0)
     dradis-calculator_dread (4.11.0)
@@ -196,6 +181,9 @@ GEM
     dradis-projects (4.11.0)
       dradis-plugins (>= 4.8.0)
       rubyzip
+    dradis-qualys (4.11.0)
+      dradis-plugins (~> 4.0)
+      nokogiri (~> 1.3)
     dradis-saint (4.11.0)
       combustion (~> 0.6.0)
       dradis-plugins (~> 4.0)
@@ -546,7 +534,7 @@ DEPENDENCIES
   dradis-acunetix (~> 4.11.0)
   dradis-api!
   dradis-brakeman (~> 4.11.0)
-  dradis-burp!
+  dradis-burp (~> 4.11.0)
   dradis-calculator_cvss (~> 4.11.0)
   dradis-calculator_dread (~> 4.11.0)
   dradis-coreimpact (~> 4.11.0)
@@ -564,7 +552,7 @@ DEPENDENCIES
   dradis-openvas (~> 4.11.0)
   dradis-plugins!
   dradis-projects (~> 4.11.0)
-  dradis-qualys!
+  dradis-qualys (~> 4.11.0)
   dradis-saint (~> 4.11.0)
   dradis-veracode (~> 4.11.0)
   dradis-wpscan (~> 4.11.0)

From a4d73b3be519ee06fa520b14213c24be2a1e813e Mon Sep 17 00:00:00 2001
From: Caitlin <caitlinhenry.ss@gmail.com>
Date: Fri, 5 Apr 2024 13:53:55 -0400
Subject: [PATCH 27/31] revert unneeded change to schema version

---
 db/schema.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/db/schema.rb b/db/schema.rb
index 2710c5d35..8fdfe0362 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2024_02_26_193757) do
+ActiveRecord::Schema[7.0].define(version: 2024_02_14_201737) do
   create_table "active_storage_attachments", force: :cascade do |t|
     t.string "name", null: false
     t.string "record_type", null: false

From 5ed1a54c01d211b82cace49587bee6db5e6b21c9 Mon Sep 17 00:00:00 2001
From: Caitlin <caitlinhenry.ss@gmail.com>
Date: Tue, 9 Apr 2024 15:47:11 -0400
Subject: [PATCH 28/31] rename copy_templates to copy_samples

---
 config/initializers/z_11_plugin_templates.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/config/initializers/z_11_plugin_templates.rb b/config/initializers/z_11_plugin_templates.rb
index 4b26ce746..13033bd46 100644
--- a/config/initializers/z_11_plugin_templates.rb
+++ b/config/initializers/z_11_plugin_templates.rb
@@ -12,14 +12,14 @@
     template_dir = Configuration.paths_templates_plugins
 
     Dradis::Plugins::with_feature(:upload).each do |integration|
-      integration.copy_templates(to: template_dir)
+      integration.copy_samples(to: template_dir)
       integration.migrate_templates_to_mappings(from: template_dir)
     end
 
     # ---------------------------------------------------------------- 3.2 Export
     template_dir = Configuration.paths_templates_reports
     Dradis::Plugins::with_feature(:export).each do |plugin|
-      plugin.copy_templates(to: template_dir)
+      plugin.copy_samples(to: template_dir)
     end
   end
 end

From 21153125a7386320aa498e3a13ffd95f7d7ea3b6 Mon Sep 17 00:00:00 2001
From: Caitlin <caitlinhenry.ss@gmail.com>
Date: Tue, 9 Apr 2024 17:15:50 -0400
Subject: [PATCH 29/31] bump plugins

---
 Gemfile.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index e3d44d6d0..5249e95b9 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,6 +1,6 @@
 GIT
   remote: https://github.com/dradis/dradis-plugins.git
-  revision: 160b6c1f0a3ce23e5e08ce0495baa080096ef5dd
+  revision: c7c48f2c945f5f8d6e7bf6886dead068a7a37675
   branch: mappings-manager/template-migration
   specs:
     dradis-plugins (4.11.0)

From 73132c9ec4ab010adeab1cec8c3241b47adc05bf Mon Sep 17 00:00:00 2001
From: Caitlin <caitlinhenry.ss@gmail.com>
Date: Wed, 24 Apr 2024 16:21:37 -0400
Subject: [PATCH 30/31] point integrations to main branches

---
 Gemfile      | 38 +++++++++++-----------
 Gemfile.lock | 91 +++++++++++++++++++++-------------------------------
 2 files changed, 55 insertions(+), 74 deletions(-)

diff --git a/Gemfile b/Gemfile
index 6c83b9141..711974385 100644
--- a/Gemfile
+++ b/Gemfile
@@ -213,7 +213,7 @@ end
 #
 
 # Base framework classes required by other plugins
-gem 'dradis-plugins', github: 'dradis/dradis-plugins', branch: 'mappings-manager/template-migration'
+gem 'dradis-plugins', github: 'dradis/dradis-plugins'
 
 gem 'dradis-api', path: 'engines/dradis-api'
 
@@ -241,21 +241,21 @@ gem 'dradis-html_export', '~> 4.11.0'
 gem 'dradis-csv', '~> 4.11.0'
 
 # ---------------------------------------------------------------------- Upload
-gem 'dradis-acunetix', github: 'dradis/dradis-acunetix', branch: 'mappings-manager/add-uploaders'
-gem 'dradis-brakeman', github: 'dradis/dradis-brakeman', branch: 'mappings-manager/add-uploaders'
-gem 'dradis-burp', github: 'dradis/dradis-burp', branch: 'mappings-manager/add-uploaders'
-gem 'dradis-coreimpact', github: 'dradis/dradis-coreimpact', branch: 'mappings-manager/add-uploaders'
-gem 'dradis-metasploit', github: 'dradis/dradis-metasploit', branch: 'mappings-manager/add-uploaders'
-gem 'dradis-nessus', github: 'dradis/dradis-nessus', branch: 'mappings-manager/add-uploaders'
-gem 'dradis-netsparker', github: 'dradis/dradis-netsparker', branch: 'mappings-manager/add-uploaders'
-gem 'dradis-nexpose', github: 'dradis/dradis-nexpose', branch: 'mappings-manager/add-uploaders'
-gem 'dradis-nikto', github: 'dradis/dradis-nikto', branch: 'mappings-manager/add-uploaders'
-gem 'dradis-nipper', github: 'dradis/dradis-nipper', branch: 'mappings-manager/add-uploaders'
-gem 'dradis-nmap', github: 'dradis/dradis-nmap', branch: 'mappings-manager/add-uploaders'
-gem 'dradis-ntospider', github: 'dradis/dradis-ntospider', branch: 'mappings-manager/add-uploaders'
-gem 'dradis-openvas', github: 'dradis/dradis-openvas', branch: 'mappings-manager/add-uploaders'
-gem 'dradis-qualys', github: 'dradis/dradis-qualys', branch: 'mappings-manager/add-uploaders'
-gem 'dradis-saint', github: 'dradis/dradis-saint', branch: 'mappings-manager/add-uploaders'
-gem 'dradis-veracode', github: 'dradis/dradis-veracode', branch: 'mappings-manager/add-uploaders'
-gem 'dradis-wpscan', github: 'dradis/dradis-wpscan', branch: 'mappings-manager/add-uploaders'
-gem 'dradis-zap', github: 'dradis/dradis-zap', branch: 'mappings-manager/add-uploaders'
+gem 'dradis-acunetix', github: 'dradis/dradis-acunetix'
+gem 'dradis-brakeman', github: 'dradis/dradis-brakeman'
+gem 'dradis-burp', github: 'dradis/dradis-burp'
+gem 'dradis-coreimpact', github: 'dradis/dradis-coreimpact'
+gem 'dradis-metasploit', github: 'dradis/dradis-metasploit'
+gem 'dradis-nessus', github: 'dradis/dradis-nessus'
+gem 'dradis-netsparker', github: 'dradis/dradis-netsparker'
+gem 'dradis-nexpose', github: 'dradis/dradis-nexpose'
+gem 'dradis-nikto', github: 'dradis/dradis-nikto'
+gem 'dradis-nipper', github: 'dradis/dradis-nipper'
+gem 'dradis-nmap', github: 'dradis/dradis-nmap'
+gem 'dradis-ntospider', github: 'dradis/dradis-ntospider'
+gem 'dradis-openvas', github: 'dradis/dradis-openvas'
+gem 'dradis-qualys', github: 'dradis/dradis-qualys'
+gem 'dradis-saint', github: 'dradis/dradis-saint'
+gem 'dradis-veracode', github: 'dradis/dradis-veracode'
+gem 'dradis-wpscan', github: 'dradis/dradis-wpscan'
+gem 'dradis-zap', github: 'dradis/dradis-zap'
diff --git a/Gemfile.lock b/Gemfile.lock
index 3e3a2e379..e3f79d250 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,137 +1,121 @@
 GIT
   remote: https://github.com/dradis/dradis-acunetix.git
-  revision: d5f42eddd52798ca95943debd8723731ee935d2b
-  branch: mappings-manager/add-uploaders
+  revision: f3670e475bf3e6d171d43fb061d3c9e7185a18a5
   specs:
-    dradis-acunetix (4.11.0)
+    dradis-acunetix (4.12.0)
       dradis-plugins (~> 4.0)
       nokogiri (~> 1.3)
 
 GIT
   remote: https://github.com/dradis/dradis-brakeman.git
-  revision: 40c76b41e6fdf6253cb186a9bb1a18cde574b6a0
-  branch: mappings-manager/add-uploaders
+  revision: a59ef3403d3ad596941382990d53956b535b6442
   specs:
-    dradis-brakeman (4.11.0)
+    dradis-brakeman (4.12.0)
       dradis-plugins (~> 4.0)
 
 GIT
   remote: https://github.com/dradis/dradis-burp.git
-  revision: 58a15441f796ae1611c855cb739486177c06994b
-  branch: mappings-manager/add-uploaders
+  revision: 21eb812ea839d1f556938709dbfc25f78559ded4
   specs:
-    dradis-burp (4.11.0)
+    dradis-burp (4.12.0)
       dradis-plugins (~> 4.0)
       nokogiri (~> 1.3)
 
 GIT
   remote: https://github.com/dradis/dradis-coreimpact.git
-  revision: 5731e1da3901eacd92f9bfe4f4cc43ae1604ee98
-  branch: mappings-manager/add-uploaders
+  revision: c50f9270dd49859e5d3ee147af87f9b75fe4082d
   specs:
-    dradis-coreimpact (4.11.0)
+    dradis-coreimpact (4.12.0)
       dradis-plugins (~> 4.0)
 
 GIT
   remote: https://github.com/dradis/dradis-metasploit.git
-  revision: 4eab6a3fef664705518ba435bf21857c09c47541
-  branch: mappings-manager/add-uploaders
+  revision: 95556559e765c89155c8cf2185cd8a2946f096c5
   specs:
-    dradis-metasploit (4.11.0)
+    dradis-metasploit (4.12.0)
       dradis-plugins (~> 4.0)
       nokogiri (~> 1.3)
 
 GIT
   remote: https://github.com/dradis/dradis-nessus.git
-  revision: 06740a2a240ae2be5fc14da5aced25534dcca262
-  branch: mappings-manager/add-uploaders
+  revision: e11829da4241922bfd26136b0011b2909aaf3144
   specs:
-    dradis-nessus (4.11.0)
+    dradis-nessus (4.12.0)
       dradis-plugins (~> 4.0)
       nokogiri
 
 GIT
   remote: https://github.com/dradis/dradis-netsparker.git
-  revision: 2f6d4d40034b928a0f41b8d489697e742bdaa38e
-  branch: mappings-manager/add-uploaders
+  revision: 8e1ee7b0790bb935263dd8711de08300d6d91d5e
   specs:
-    dradis-netsparker (4.11.0)
+    dradis-netsparker (4.12.0)
       dradis-plugins (~> 4.0)
       nokogiri (>= 1.12.5)
 
 GIT
   remote: https://github.com/dradis/dradis-nexpose.git
-  revision: 439cb5dbe2825ce5461c58813348a69961c6e4a4
-  branch: mappings-manager/add-uploaders
+  revision: b44222b335af3d608f43deb8d4806fd77e7d08ee
   specs:
-    dradis-nexpose (4.11.0)
+    dradis-nexpose (4.12.0)
       dradis-plugins (~> 4.0)
       nokogiri (~> 1.3)
 
 GIT
   remote: https://github.com/dradis/dradis-nikto.git
-  revision: fbbca4b43f30fc241df142c55238b2feb7c7f3fe
-  branch: mappings-manager/add-uploaders
+  revision: bca8bc1954e74c8702f009c7a6b2cd8ef6845c8b
   specs:
-    dradis-nikto (4.11.0)
+    dradis-nikto (4.12.0)
       dradis-plugins (~> 4.0)
       nokogiri (~> 1.3)
 
 GIT
   remote: https://github.com/dradis/dradis-nipper.git
-  revision: 169535775f078913ca3d38eb6bd7dbcc80a333dd
-  branch: mappings-manager/add-uploaders
+  revision: 92c98a8a200ec056b10fbed4dba9fbfaa8dc8932
   specs:
-    dradis-nipper (4.11.0)
+    dradis-nipper (4.12.0)
       dradis-plugins (~> 4.0)
 
 GIT
   remote: https://github.com/dradis/dradis-nmap.git
-  revision: a653ad66543cdef3361a1f6dd1fbb07d9e4eada7
-  branch: mappings-manager/add-uploaders
+  revision: 10c1fa773e4c46ba34c5e75d5bb806512671b49a
   specs:
-    dradis-nmap (4.11.0)
+    dradis-nmap (4.12.0)
       dradis-plugins (~> 4.0)
       ruby-nmap (~> 0.7)
 
 GIT
   remote: https://github.com/dradis/dradis-ntospider.git
-  revision: 0c2ae49734090ce60982fe8c01658832cf96180a
-  branch: mappings-manager/add-uploaders
+  revision: 0884791fdb57d89927c383e84d4d81c4d115e7f4
   specs:
-    dradis-ntospider (4.11.0)
+    dradis-ntospider (4.12.0)
       dradis-plugins (~> 4.0)
 
 GIT
   remote: https://github.com/dradis/dradis-openvas.git
-  revision: 1678903dd6e35fd8c4cfdbf55333c6f3f18c352b
-  branch: mappings-manager/add-uploaders
+  revision: ceaef202911f8073fcf99dc0fa9717fb729078d9
   specs:
     dradis-openvas (4.11.0)
       dradis-plugins (~> 4.0)
 
 GIT
   remote: https://github.com/dradis/dradis-plugins.git
-  revision: c7c48f2c945f5f8d6e7bf6886dead068a7a37675
-  branch: mappings-manager/template-migration
+  revision: 35efe7b19670d7c75a0594a464080a8c46652f24
   specs:
     dradis-plugins (4.11.0)
 
 GIT
   remote: https://github.com/dradis/dradis-qualys.git
-  revision: 82712e710418dd7e116223c9e1a29b4ba73ebf79
-  branch: mappings-manager/add-uploaders
+  revision: 1c1f6ce6ba3348dc31f5f9567d20cee77a6004d8
   specs:
-    dradis-qualys (4.11.0)
+    dradis-qualys (4.12.0)
       dradis-plugins (~> 4.0)
       nokogiri (~> 1.3)
 
 GIT
   remote: https://github.com/dradis/dradis-saint.git
-  revision: 7d5e45da4f61edb50798ed61c86ce6b7f4c1698b
-  branch: mappings-manager/add-uploaders
+  revision: d36a55f129674ed90ab4e47b68f91bdbddd14fb3
   specs:
-    dradis-saint (4.11.0)
+    dradis-saint (4.12.0)
       combustion (~> 0.6.0)
       dradis-plugins (~> 4.0)
       nokogiri
@@ -140,27 +124,24 @@ GIT
 
 GIT
   remote: https://github.com/dradis/dradis-veracode.git
-  revision: 0442dee4651101bf5530dadc2e68f73faf69a381
-  branch: mappings-manager/add-uploaders
+  revision: 5719028296d55632d5baaac4e9f369fab684ded8
   specs:
-    dradis-veracode (4.11.0)
+    dradis-veracode (4.12.0)
       dradis-plugins (~> 4.0)
 
 GIT
   remote: https://github.com/dradis/dradis-wpscan.git
-  revision: 2e5ff1b8e01413dbfd3901f179f9bd62290513d2
-  branch: mappings-manager/add-uploaders
+  revision: 8b036f1b1c6386381fd5b2e8301d4932e8ac23f4
   specs:
-    dradis-wpscan (4.11.0)
+    dradis-wpscan (4.12.0)
       dradis-plugins (~> 4.0)
       multi_json
 
 GIT
   remote: https://github.com/dradis/dradis-zap.git
-  revision: 9deb31fe0dcfb05533414b90a64a371422340336
-  branch: mappings-manager/add-uploaders
+  revision: 05fb435b4ce26b879953a0459c9c988ad9f2739c
   specs:
-    dradis-zap (4.11.0)
+    dradis-zap (4.12.0)
       dradis-plugins (~> 4.0)
       nokogiri (~> 1.3)
 

From 7a0fa52c24ec9d184a388c2ece3562ab404aeb30 Mon Sep 17 00:00:00 2001
From: Caitlin <caitlinhenry.ss@gmail.com>
Date: Wed, 24 Apr 2024 16:30:07 -0400
Subject: [PATCH 31/31] add changelog entry

---
 CHANGELOG | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index c7d8b6245..1f5c4075a 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,5 @@
 [v#.#.#] ([month] [YYYY])
-  - [entity]:
-    - [future tense verb] [feature]
+  - Mappings: Map fields from scanner integrations to Dradis fields
   - Upgraded gems:
     - nokogiri, rails
   - Bugs fixes: