From 54b42c4bd83d4ffa2a98d86d42e15d68c88a7a8b Mon Sep 17 00:00:00 2001 From: Natnael Getahun Date: Tue, 29 Oct 2024 08:14:20 +0100 Subject: [PATCH 01/14] POC: Yabeda rmt integration --- .env.example | 1 + Gemfile | 7 +++ Gemfile.lock | 37 +++++++++++++++ config/application.rb | 1 + config/initializers/yabeda.rb | 82 ++++++++++++++++++++++++++++++++++ config/puma.rb | 5 +++ config/puma/prometheus.rb | 7 +++ config/rmt.yml | 8 ++++ config/routes.rb | 6 +++ lib/yabeda/prometheus_patch.rb | 25 +++++++++++ package/obs/rmt.conf | 7 +++ 11 files changed, 186 insertions(+) create mode 100644 config/initializers/yabeda.rb create mode 100644 config/puma/prometheus.rb create mode 100644 lib/yabeda/prometheus_patch.rb diff --git a/.env.example b/.env.example index 2229a9035..519a6834f 100644 --- a/.env.example +++ b/.env.example @@ -2,3 +2,4 @@ SECRET_KEY_BASE=8ea53ad3bc6c03923e376c8bdd85059c1885524947a7efe53d5e9c9d4e398611 EXTERNAL_PORT=8080 SCC_USERNAME= SCC_PASSWORD= +COLLECT_PROMETHEUS_METRICS= \ No newline at end of file diff --git a/Gemfile b/Gemfile index cc4b4508f..295c24165 100644 --- a/Gemfile +++ b/Gemfile @@ -32,6 +32,13 @@ gem 'repomd_parser', '~> 1.1.0' # Use Rack CORS for handling Cross-Origin Resource Sharing (CORS), making cross-origin AJAX possible # gem 'rack-cors' +# Prometheus Exporter: +gem 'yabeda' +gem 'yabeda-rails' +gem 'yabeda-puma-plugin' +gem 'yabeda-sidekiq' +gem 'yabeda-prometheus' + gem 'strong_migrations' group :development, :test do diff --git a/Gemfile.lock b/Gemfile.lock index d48697404..3445bfe5d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -32,6 +32,8 @@ GEM zeitwerk (~> 2.3) addressable (2.8.6) public_suffix (>= 2.0.2, < 6.0) + anyway_config (2.6.4) + ruby-next-core (~> 1.0) ast (2.4.2) awesome_print (1.9.2) base32 (0.3.4) @@ -48,6 +50,7 @@ GEM config (3.0.0) deep_merge (~> 1.2, >= 1.2.1) dry-validation (~> 1.0, >= 1.0.0) + connection_pool (2.4.1) coveralls (0.8.23) json (>= 1.8, < 3) simplecov (~> 0.16.1) @@ -173,6 +176,8 @@ GEM prime (0.1.2) forwardable singleton + prometheus-client (4.2.3) + base64 pry (0.14.0) coderay (~> 1.1) method_source (~> 1.0) @@ -201,6 +206,7 @@ GEM rb-inotify (0.10.1) ffi (~> 1.0) rdiscount (2.2.0.2) + redis (4.8.1) regexp_parser (2.6.0) repomd_parser (1.1.0) bzip2-ffi (~> 1.1.1) @@ -267,6 +273,7 @@ GEM rubocop (~> 1.19) rubocop-thread_safety (0.4.4) rubocop (>= 0.53.0) + ruby-next-core (1.0.3) ruby-progressbar (1.11.0) ruby_parser (3.19.2) sexp_processor (~> 4.16) @@ -281,6 +288,10 @@ GEM shellany (0.0.1) shoulda-matchers (4.5.1) activesupport (>= 4.2.0) + sidekiq (6.5.12) + connection_pool (>= 2.2.5, < 3) + rack (~> 2.0) + redis (>= 4.5.0, < 5) simplecov (0.16.1) docile (~> 1.1) json (>= 1.8, < 3) @@ -321,6 +332,27 @@ GEM addressable (>= 2.8.0) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) + yabeda (0.13.1) + anyway_config (>= 1.0, < 3) + concurrent-ruby + dry-initializer + yabeda-prometheus (0.9.1) + prometheus-client (>= 3.0, < 5.0) + rack + yabeda (~> 0.10) + yabeda-puma-plugin (0.7.1) + json + puma + yabeda (~> 0.5) + yabeda-rails (0.9.0) + activesupport + anyway_config (>= 1.3, < 3) + railties + yabeda (~> 0.8) + yabeda-sidekiq (0.12.0) + anyway_config (>= 1.3, < 3) + sidekiq + yabeda (~> 0.6) yard (0.9.35) zeitwerk (2.6.15) zstd-ruby (1.5.6.1) @@ -382,6 +414,11 @@ DEPENDENCIES vcr (~> 6.0) versionist webmock + yabeda + yabeda-prometheus + yabeda-puma-plugin + yabeda-rails + yabeda-sidekiq BUNDLED WITH 1.17.3 diff --git a/config/application.rb b/config/application.rb index 581f9eac3..ffbae9be9 100644 --- a/config/application.rb +++ b/config/application.rb @@ -15,6 +15,7 @@ # require "action_cable/engine" # require "sprockets/railtie" # require "rails/test_unit/railtie" +require_relative '../lib/yabeda/prometheus_patch' # Require the gems listed in Gemfile, including any gems # you've limited to :test, :development, or :production. diff --git a/config/initializers/yabeda.rb b/config/initializers/yabeda.rb new file mode 100644 index 000000000..e4cf51684 --- /dev/null +++ b/config/initializers/yabeda.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +return if Settings.scc.try(:metrics) && (Settings.scc.metrics.try(:enabled) && (!!Settings.scc.metrics.enabled == false)) + +# require 'yabeda' +# require 'yabeda/rails' +# require 'yabeda/sidekiq' +# require 'yabeda/prometheus' + +# Configure prometheus client +Prometheus::Client.config.data_store = Prometheus::Client::DataStores::DirectFileStore.new( + dir: './tmp/prometheus/' +) + +# Configure yabeda +Yabeda.configure do + assign_labels = lambda { + default_tag :environment, Rails.env + default_tag :application, 'rmt' + } + + group :sidekiq, &assign_labels + group :rails, &assign_labels + group :rake, &assign_labels + group :scc, &assign_labels + + group :rails do + counter :started_requests_total, + comment: 'A counter of the total number of HTTP requests rails has started to process.', + tags: %i[controller action format method] + end + + group :rake do + gauge :task_started_at, + comment: 'Time when the task started: unix time with decimals', + tags: %i[task_name] + + gauge :task_exit_status, + comment: 'Task exit status. 1 means OK, 0 means failed, not present means no data', + tags: %i[task_name] + + gauge :task_duration_ms, + comment: 'Time taken running the task', + tags: %i[task_name] + + gauge :task_finished_at, + comment: 'Time when the task finished: unix time with decimals', + tags: %i[task_name] + end + + group :pct do + + end +end + +# Instrument the request from the start +ActiveSupport::Notifications.subscribe 'start_processing.action_controller' do |*args| + # Match the same event as Yabeda + event = Yabeda::Rails::Event.new(*args) + + Yabeda.rails.started_requests_total.tap do |metric| + labels = event.labels.slice(*metric.tags) + + metric.increment(labels, by: 1) + end +end + +ActiveSupport::Notifications.subscribe 'task_run.rake' do |*args| + event = ActiveSupport::Notifications::Event.new(*args) + task_name = event.payload[:task_name] + + success = 1 + if !!(event.payload[:exception] || event.payload[:exception_class]) + success = 0 + end + + Yabeda.rake.task_duration_ms.set({ task_name: task_name }, event.duration) + Yabeda.rake.task_started_at.set({ task_name: task_name }, event.time / 1000.0) + Yabeda.rake.task_finished_at.set({ task_name: task_name }, event.end / 1000.0) + + Yabeda.rake.task_exit_status.set({ task_name: task_name }, success) +end \ No newline at end of file diff --git a/config/puma.rb b/config/puma.rb index c1fec1414..2e92007f8 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -1,4 +1,5 @@ require 'rmt/config' +require_relative 'puma/prometheus' # Puma can serve each request in a thread from an internal thread pool. # The `threads` method setting takes two numbers: a minimum and maximum. @@ -36,3 +37,7 @@ # Allow puma to be restarted by `rails restart` command. plugin :tmp_restart + +if Settings.scc.try(:metrics) && (Settings.scc.metrics.try(:enabled) && (!!Settings.scc.metrics.enabled == false)) + configure_prometheus!(self) +end \ No newline at end of file diff --git a/config/puma/prometheus.rb b/config/puma/prometheus.rb new file mode 100644 index 000000000..3e7f7b48f --- /dev/null +++ b/config/puma/prometheus.rb @@ -0,0 +1,7 @@ +def configure_prometheus!(puma) + ENV['STARTED_FROM_PUMA'] = '1' + + puma.activate_control_app + puma.plugin :yabeda + puma.plugin :yabeda_prometheus +end \ No newline at end of file diff --git a/config/rmt.yml b/config/rmt.yml index 3b1dabb2f..a3cd93a85 100644 --- a/config/rmt.yml +++ b/config/rmt.yml @@ -24,6 +24,14 @@ scc: username: <%= ENV['SCC_USERNAME'] %> password: <%= ENV['SCC_PASSWORD'] %> sync_systems: true + metrics: + enabled: true + interval: 60 + job_name: yabeda + url: <%= ENV.fetch('PROMETHEUS_ENDPOINT') { 'http://localhost:9091' } %> + username: <%= ENV.fetch('PROMETHEUS_USERNAME') { 'admin' }%> + password: <%= ENV.fetch('PROMETHEUS_PASSWORD') { 'admin' }%> + mirroring: mirror_src: false diff --git a/config/routes.rb b/config/routes.rb index 9cea050f2..7c43d4b87 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -69,6 +69,12 @@ get '/v2/_catalog', to: 'registry/registry#catalog' end + # For bin/rails s users: expose the metrics endpoint via rails instead of puma + if Rails.env.development? && ENV['STARTED_FROM_PUMA'].blank? + Rails.logger.info('Mounting Yabeda in Rails.routes') + mount Yabeda::Prometheus::Exporter, at: '/metrics' + end + if defined?(SccSumaApi::Engine) mount SccSumaApi::Engine, at: '/api/scc' diff --git a/lib/yabeda/prometheus_patch.rb b/lib/yabeda/prometheus_patch.rb new file mode 100644 index 000000000..ba4f7e392 --- /dev/null +++ b/lib/yabeda/prometheus_patch.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +require 'yabeda' +require 'yabeda/prometheus' +require 'prometheus/client/push' + +module Yabeda + module PrometheusPatch + def patched_push_gateway(grouping_key: {}) + @push_gateway ||= # rubocop:disable Naming/MemoizedInstanceVariableName + ::Prometheus::Client::Push.new( + job: Settings.scc.metrics.job_name, + gateway:Settings.scc.metrics.url, + grouping_key: grouping_key, + open_timeout: 5, read_timeout: 30 + ) + end + + def self.apply!(mod) + mod.extend(self) + end + end +end + +Yabeda::PrometheusPatch.apply!(Yabeda::Prometheus) \ No newline at end of file diff --git a/package/obs/rmt.conf b/package/obs/rmt.conf index c2f410e24..b583cfdf1 100644 --- a/package/obs/rmt.conf +++ b/package/obs/rmt.conf @@ -12,6 +12,13 @@ scc: username: password: sync_systems: true + metrics: + enabled: false + interval: 60 + job_name: yabeda + url: https://foo.com/metrics + username: + password: mirroring: mirror_src: false From 8e00dd5204edbc4f5a3c4cae7206104f03c4ef14 Mon Sep 17 00:00:00 2001 From: Natnael Getahun Date: Tue, 29 Oct 2024 14:57:33 +0100 Subject: [PATCH 02/14] Remove yabeda-sidekiq gem and update yabeda.rb initializer to use Settings.dig for metrics.enabled --- Gemfile | 1 - config/initializers/yabeda.rb | 15 ++------------- config/puma.rb | 4 ++-- config/puma/prometheus.rb | 2 +- config/rmt.yml | 5 ++--- lib/yabeda/prometheus_patch.rb | 25 ------------------------- package/obs/rmt.conf | 5 ++--- 7 files changed, 9 insertions(+), 48 deletions(-) delete mode 100644 lib/yabeda/prometheus_patch.rb diff --git a/Gemfile b/Gemfile index 295c24165..d57185c60 100644 --- a/Gemfile +++ b/Gemfile @@ -36,7 +36,6 @@ gem 'repomd_parser', '~> 1.1.0' gem 'yabeda' gem 'yabeda-rails' gem 'yabeda-puma-plugin' -gem 'yabeda-sidekiq' gem 'yabeda-prometheus' gem 'strong_migrations' diff --git a/config/initializers/yabeda.rb b/config/initializers/yabeda.rb index e4cf51684..331161ee8 100644 --- a/config/initializers/yabeda.rb +++ b/config/initializers/yabeda.rb @@ -1,11 +1,6 @@ # frozen_string_literal: true -return if Settings.scc.try(:metrics) && (Settings.scc.metrics.try(:enabled) && (!!Settings.scc.metrics.enabled == false)) - -# require 'yabeda' -# require 'yabeda/rails' -# require 'yabeda/sidekiq' -# require 'yabeda/prometheus' +return unless Settings.dig(:scc, :metrics, :enabled) # Configure prometheus client Prometheus::Client.config.data_store = Prometheus::Client::DataStores::DirectFileStore.new( @@ -19,10 +14,8 @@ default_tag :application, 'rmt' } - group :sidekiq, &assign_labels group :rails, &assign_labels group :rake, &assign_labels - group :scc, &assign_labels group :rails do counter :started_requests_total, @@ -47,10 +40,6 @@ comment: 'Time when the task finished: unix time with decimals', tags: %i[task_name] end - - group :pct do - - end end # Instrument the request from the start @@ -79,4 +68,4 @@ Yabeda.rake.task_finished_at.set({ task_name: task_name }, event.end / 1000.0) Yabeda.rake.task_exit_status.set({ task_name: task_name }, success) -end \ No newline at end of file +end diff --git a/config/puma.rb b/config/puma.rb index 2e92007f8..39ace4d2b 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -38,6 +38,6 @@ # Allow puma to be restarted by `rails restart` command. plugin :tmp_restart -if Settings.scc.try(:metrics) && (Settings.scc.metrics.try(:enabled) && (!!Settings.scc.metrics.enabled == false)) +if Settings.dig(:scc, :metrics, :enabled) configure_prometheus!(self) -end \ No newline at end of file +end diff --git a/config/puma/prometheus.rb b/config/puma/prometheus.rb index 3e7f7b48f..df5dcd9b9 100644 --- a/config/puma/prometheus.rb +++ b/config/puma/prometheus.rb @@ -4,4 +4,4 @@ def configure_prometheus!(puma) puma.activate_control_app puma.plugin :yabeda puma.plugin :yabeda_prometheus -end \ No newline at end of file +end diff --git a/config/rmt.yml b/config/rmt.yml index a3cd93a85..e64266999 100644 --- a/config/rmt.yml +++ b/config/rmt.yml @@ -25,9 +25,8 @@ scc: password: <%= ENV['SCC_PASSWORD'] %> sync_systems: true metrics: - enabled: true - interval: 60 - job_name: yabeda + enabled: <%= ENV.fetch('RMT_METRICS_ENABLED') { false } %> + job_name: <%= ENV.fetch('PROMETHEUS_JOB_NAME') { 'rmt-webserver' } %> url: <%= ENV.fetch('PROMETHEUS_ENDPOINT') { 'http://localhost:9091' } %> username: <%= ENV.fetch('PROMETHEUS_USERNAME') { 'admin' }%> password: <%= ENV.fetch('PROMETHEUS_PASSWORD') { 'admin' }%> diff --git a/lib/yabeda/prometheus_patch.rb b/lib/yabeda/prometheus_patch.rb deleted file mode 100644 index ba4f7e392..000000000 --- a/lib/yabeda/prometheus_patch.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -require 'yabeda' -require 'yabeda/prometheus' -require 'prometheus/client/push' - -module Yabeda - module PrometheusPatch - def patched_push_gateway(grouping_key: {}) - @push_gateway ||= # rubocop:disable Naming/MemoizedInstanceVariableName - ::Prometheus::Client::Push.new( - job: Settings.scc.metrics.job_name, - gateway:Settings.scc.metrics.url, - grouping_key: grouping_key, - open_timeout: 5, read_timeout: 30 - ) - end - - def self.apply!(mod) - mod.extend(self) - end - end -end - -Yabeda::PrometheusPatch.apply!(Yabeda::Prometheus) \ No newline at end of file diff --git a/package/obs/rmt.conf b/package/obs/rmt.conf index b583cfdf1..ab1bb257e 100644 --- a/package/obs/rmt.conf +++ b/package/obs/rmt.conf @@ -14,9 +14,8 @@ scc: sync_systems: true metrics: enabled: false - interval: 60 - job_name: yabeda - url: https://foo.com/metrics + job_name: rmt + url: http://localhost:9394/metrics username: password: From 94b219f7e627485f05eddf8b9402597d16bad117 Mon Sep 17 00:00:00 2001 From: Natnael Getahun Date: Tue, 29 Oct 2024 17:13:36 +0100 Subject: [PATCH 03/14] Update metrics configuration in .env.example and config/rmt.yml --- .env.example | 3 ++- Gemfile.lock | 11 ----------- config/application.rb | 1 - config/rmt.yml | 4 ---- package/obs/rmt.conf | 5 +---- 5 files changed, 3 insertions(+), 21 deletions(-) diff --git a/.env.example b/.env.example index 519a6834f..68abdc7b0 100644 --- a/.env.example +++ b/.env.example @@ -2,4 +2,5 @@ SECRET_KEY_BASE=8ea53ad3bc6c03923e376c8bdd85059c1885524947a7efe53d5e9c9d4e398611 EXTERNAL_PORT=8080 SCC_USERNAME= SCC_PASSWORD= -COLLECT_PROMETHEUS_METRICS= \ No newline at end of file +RMT_METRICS_ENABLED= +PROMETHEUS_JOB_NAME= diff --git a/Gemfile.lock b/Gemfile.lock index 3445bfe5d..0f663bea7 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -50,7 +50,6 @@ GEM config (3.0.0) deep_merge (~> 1.2, >= 1.2.1) dry-validation (~> 1.0, >= 1.0.0) - connection_pool (2.4.1) coveralls (0.8.23) json (>= 1.8, < 3) simplecov (~> 0.16.1) @@ -206,7 +205,6 @@ GEM rb-inotify (0.10.1) ffi (~> 1.0) rdiscount (2.2.0.2) - redis (4.8.1) regexp_parser (2.6.0) repomd_parser (1.1.0) bzip2-ffi (~> 1.1.1) @@ -288,10 +286,6 @@ GEM shellany (0.0.1) shoulda-matchers (4.5.1) activesupport (>= 4.2.0) - sidekiq (6.5.12) - connection_pool (>= 2.2.5, < 3) - rack (~> 2.0) - redis (>= 4.5.0, < 5) simplecov (0.16.1) docile (~> 1.1) json (>= 1.8, < 3) @@ -349,10 +343,6 @@ GEM anyway_config (>= 1.3, < 3) railties yabeda (~> 0.8) - yabeda-sidekiq (0.12.0) - anyway_config (>= 1.3, < 3) - sidekiq - yabeda (~> 0.6) yard (0.9.35) zeitwerk (2.6.15) zstd-ruby (1.5.6.1) @@ -418,7 +408,6 @@ DEPENDENCIES yabeda-prometheus yabeda-puma-plugin yabeda-rails - yabeda-sidekiq BUNDLED WITH 1.17.3 diff --git a/config/application.rb b/config/application.rb index ffbae9be9..581f9eac3 100644 --- a/config/application.rb +++ b/config/application.rb @@ -15,7 +15,6 @@ # require "action_cable/engine" # require "sprockets/railtie" # require "rails/test_unit/railtie" -require_relative '../lib/yabeda/prometheus_patch' # Require the gems listed in Gemfile, including any gems # you've limited to :test, :development, or :production. diff --git a/config/rmt.yml b/config/rmt.yml index e64266999..811e44ff2 100644 --- a/config/rmt.yml +++ b/config/rmt.yml @@ -27,10 +27,6 @@ scc: metrics: enabled: <%= ENV.fetch('RMT_METRICS_ENABLED') { false } %> job_name: <%= ENV.fetch('PROMETHEUS_JOB_NAME') { 'rmt-webserver' } %> - url: <%= ENV.fetch('PROMETHEUS_ENDPOINT') { 'http://localhost:9091' } %> - username: <%= ENV.fetch('PROMETHEUS_USERNAME') { 'admin' }%> - password: <%= ENV.fetch('PROMETHEUS_PASSWORD') { 'admin' }%> - mirroring: mirror_src: false diff --git a/package/obs/rmt.conf b/package/obs/rmt.conf index ab1bb257e..a1d6eca60 100644 --- a/package/obs/rmt.conf +++ b/package/obs/rmt.conf @@ -15,10 +15,7 @@ scc: metrics: enabled: false job_name: rmt - url: http://localhost:9394/metrics - username: - password: - + mirroring: mirror_src: false dedup_method: hardlink From 0dc9e0614f4da3223d748309d7273a53b6d0bb22 Mon Sep 17 00:00:00 2001 From: Natnael Getahun Date: Wed, 30 Oct 2024 09:10:13 +0100 Subject: [PATCH 04/14] Add :nocov: comments to RMT module and RMT::Config module --- lib/rmt.rb | 2 ++ lib/rmt/config.rb | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/rmt.rb b/lib/rmt.rb index 08a1fa97a..7d62b9bce 100644 --- a/lib/rmt.rb +++ b/lib/rmt.rb @@ -1,3 +1,4 @@ +# :nocov: module RMT VERSION ||= '2.19'.freeze @@ -7,3 +8,4 @@ module RMT DEFAULT_MIRROR_URL_PREFIX = '/repo/'.freeze CREDENTIALS_FILE_LOCATION = '/etc/zypp/credentials.d/SCCcredentials'.freeze end +# :nocov: diff --git a/lib/rmt/config.rb b/lib/rmt/config.rb index 650a316ad..a967e8f53 100644 --- a/lib/rmt/config.rb +++ b/lib/rmt/config.rb @@ -1,3 +1,4 @@ +# :nocov: require 'config' require_relative '../rmt' @@ -11,7 +12,6 @@ File.join(__dir__, '../../config/rmt.local.yml') ) - module RMT::Config class << self def db_config(key = 'database') @@ -77,3 +77,4 @@ def validate_int(value) end end end +# :nocov: From 40bcfc19eda4e497f3bb64140bc9a7a623b17500 Mon Sep 17 00:00:00 2001 From: Natnael Getahun Date: Mon, 4 Nov 2024 11:10:43 +0100 Subject: [PATCH 05/14] Add filters to .simplecov --- .simplecov | 5 +++++ lib/rmt.rb | 2 -- lib/rmt/config.rb | 3 +-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.simplecov b/.simplecov index e08fb8ee6..3757d9b5c 100644 --- a/.simplecov +++ b/.simplecov @@ -12,6 +12,11 @@ unless ENV['NO_COVERAGE'] track_files('app/**/*.rb') track_files('lib/**/*.rb') + + add_filter('lib/rmt/config.rb') + add_filter('lib/rmt/db.rb') + add_filter('lib/rmt.rb') + add_filter('config/**/*(.rb|.yml)') end end diff --git a/lib/rmt.rb b/lib/rmt.rb index 7d62b9bce..08a1fa97a 100644 --- a/lib/rmt.rb +++ b/lib/rmt.rb @@ -1,4 +1,3 @@ -# :nocov: module RMT VERSION ||= '2.19'.freeze @@ -8,4 +7,3 @@ module RMT DEFAULT_MIRROR_URL_PREFIX = '/repo/'.freeze CREDENTIALS_FILE_LOCATION = '/etc/zypp/credentials.d/SCCcredentials'.freeze end -# :nocov: diff --git a/lib/rmt/config.rb b/lib/rmt/config.rb index a967e8f53..650a316ad 100644 --- a/lib/rmt/config.rb +++ b/lib/rmt/config.rb @@ -1,4 +1,3 @@ -# :nocov: require 'config' require_relative '../rmt' @@ -12,6 +11,7 @@ File.join(__dir__, '../../config/rmt.local.yml') ) + module RMT::Config class << self def db_config(key = 'database') @@ -77,4 +77,3 @@ def validate_int(value) end end end -# :nocov: From 4aec66202235842c48855d2c0cb27cb124418fe7 Mon Sep 17 00:00:00 2001 From: Natnael Getahun Date: Mon, 4 Nov 2024 11:59:31 +0100 Subject: [PATCH 06/14] Temporary: Add coverage report export and version checks --- .github/workflows/lint-unit.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/lint-unit.yml b/.github/workflows/lint-unit.yml index fc5842d18..0f15078e9 100644 --- a/.github/workflows/lint-unit.yml +++ b/.github/workflows/lint-unit.yml @@ -60,6 +60,12 @@ jobs: with: ruby-version: ${{ matrix.ruby-version }} bundler-cache: true + + - name: Export coverage report + uses: actions/upload-artifact@v2 + with: + name: coverage + path: ${{ github.workspace }}/coverage - name: Setup database config file run: | @@ -82,6 +88,16 @@ jobs: - name: Run core tests run: | bundle exec rspec --format documentation + + - name: Run PubCloud engines tests + run: | + bundle exec rake test:engines + + - name: Run version checks + run: | + echo "::group::Version verification checks" + ruby ci/check-version-matches.rb + echo "::endgroup::" - name: Run PubCloud engines tests run: | From 63119c37f936b809385b114f4f681fd7ff7824dd Mon Sep 17 00:00:00 2001 From: Natnael Getahun Date: Mon, 4 Nov 2024 12:10:20 +0100 Subject: [PATCH 07/14] Update Ruby version and use latest artifact upload action --- .github/workflows/lint-unit.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/lint-unit.yml b/.github/workflows/lint-unit.yml index 0f15078e9..43ae648f2 100644 --- a/.github/workflows/lint-unit.yml +++ b/.github/workflows/lint-unit.yml @@ -30,7 +30,7 @@ jobs: matrix: # we need to test the Ruby versions of the products we release RMT for, # those are: SLE15 (2.5.8, 2.5.9) - ruby-version: ['2.5.8', '2.5.9'] + ruby-version: [ '2.5.9'] # skipping Ruby3 tests for now, because RMT isn't Ruby3 compatible yet, and the failing # tests confuse the team. # ruby-version: ['2.5.8', '2.5.9', '3.0.3'] @@ -60,12 +60,6 @@ jobs: with: ruby-version: ${{ matrix.ruby-version }} bundler-cache: true - - - name: Export coverage report - uses: actions/upload-artifact@v2 - with: - name: coverage - path: ${{ github.workspace }}/coverage - name: Setup database config file run: | @@ -108,3 +102,10 @@ jobs: echo "::group::Version verification checks" ruby ci/check-version-matches.rb echo "::endgroup::" + + - name: Export coverage report + uses: actions/upload-artifact@v4 + if: always() + with: + name: coverage + path: ${{ github.workspace }}/coverage \ No newline at end of file From b09244d0f32cd6c66a37aceb16601d70634225e4 Mon Sep 17 00:00:00 2001 From: Natnael Getahun Date: Mon, 4 Nov 2024 14:14:38 +0100 Subject: [PATCH 08/14] Refactor test commands and coverage configuration --- .github/workflows/lint-unit.yml | 12 +----------- .simplecov | 9 ++++----- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/.github/workflows/lint-unit.yml b/.github/workflows/lint-unit.yml index 43ae648f2..e4e8ea349 100644 --- a/.github/workflows/lint-unit.yml +++ b/.github/workflows/lint-unit.yml @@ -81,7 +81,7 @@ jobs: - name: Run core tests run: | - bundle exec rspec --format documentation + bundle exec rake test:core - name: Run PubCloud engines tests run: | @@ -93,16 +93,6 @@ jobs: ruby ci/check-version-matches.rb echo "::endgroup::" - - name: Run PubCloud engines tests - run: | - bundle exec rake test:engines - - - name: Run version checks - run: | - echo "::group::Version verification checks" - ruby ci/check-version-matches.rb - echo "::endgroup::" - - name: Export coverage report uses: actions/upload-artifact@v4 if: always() diff --git a/.simplecov b/.simplecov index 3757d9b5c..052bcb2d8 100644 --- a/.simplecov +++ b/.simplecov @@ -10,13 +10,12 @@ unless ENV['NO_COVERAGE'] # omit registration sharing (removing systems using rmt-cli) add_filter('engines/registration_sharing/lib/registration_sharing.rb') - track_files('app/**/*.rb') - track_files('lib/**/*.rb') - - add_filter('lib/rmt/config.rb') add_filter('lib/rmt/db.rb') add_filter('lib/rmt.rb') - add_filter('config/**/*(.rb|.yml)') + add_filter('config') + + track_files('app/**/*.rb') + track_files('lib/**/*.rb') end end From e51bea58302d29815e57a916885e53e3e9419be5 Mon Sep 17 00:00:00 2001 From: Natnael Getahun Date: Mon, 4 Nov 2024 17:42:10 +0100 Subject: [PATCH 09/14] Revert "Update Ruby version and use latest artifact upload action" This reverts commit 63119c37f936b809385b114f4f681fd7ff7824dd. --- .github/workflows/lint-unit.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/lint-unit.yml b/.github/workflows/lint-unit.yml index e4e8ea349..b4e7ab855 100644 --- a/.github/workflows/lint-unit.yml +++ b/.github/workflows/lint-unit.yml @@ -30,7 +30,7 @@ jobs: matrix: # we need to test the Ruby versions of the products we release RMT for, # those are: SLE15 (2.5.8, 2.5.9) - ruby-version: [ '2.5.9'] + ruby-version: ['2.5.8', '2.5.9'] # skipping Ruby3 tests for now, because RMT isn't Ruby3 compatible yet, and the failing # tests confuse the team. # ruby-version: ['2.5.8', '2.5.9', '3.0.3'] @@ -92,10 +92,3 @@ jobs: echo "::group::Version verification checks" ruby ci/check-version-matches.rb echo "::endgroup::" - - - name: Export coverage report - uses: actions/upload-artifact@v4 - if: always() - with: - name: coverage - path: ${{ github.workspace }}/coverage \ No newline at end of file From 7d84d7aefdda2fc4e64f0babe8da4fc93aaa04ae Mon Sep 17 00:00:00 2001 From: Jesus Bermudez Velazquez Date: Mon, 4 Nov 2024 16:28:16 +0000 Subject: [PATCH 10/14] Add stub request Error gets built automagically by rspec, no need to { error: }.to_json --- .../v3/systems/products_controller_spec.rb | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/engines/instance_verification/spec/requests/api/connect/v3/systems/products_controller_spec.rb b/engines/instance_verification/spec/requests/api/connect/v3/systems/products_controller_spec.rb index 6d6e15d97..243436c73 100644 --- a/engines/instance_verification/spec/requests/api/connect/v3/systems/products_controller_spec.rb +++ b/engines/instance_verification/spec/requests/api/connect/v3/systems/products_controller_spec.rb @@ -72,9 +72,15 @@ context 'when verification provider returns false' do before do - expect(InstanceVerification::Providers::Example).to receive(:new) - .with(be_a(ActiveSupport::Logger), be_a(ActionDispatch::Request), payload, instance_data).and_return(plugin_double) - expect(plugin_double).to receive(:instance_valid?).and_return(false) + stub_request(:post, scc_activate_url) + .to_return( + status: 200, + body: { error: 'Unexpected instance verification error has occurred' }.to_json, + headers: {} + ) + # expect(InstanceVerification::Providers::Example).to receive(:new) + # .with(be_a(ActiveSupport::Logger), be_a(ActionDispatch::Request), payload, instance_data).and_return(plugin_double) + # expect(plugin_double).to receive(:instance_valid?).and_return(false) post url, params: payload, headers: headers end @@ -86,9 +92,16 @@ context 'when verification provider raises an unhandled exception' do before do - expect(InstanceVerification::Providers::Example).to receive(:new) - .with(be_a(ActiveSupport::Logger), be_a(ActionDispatch::Request), payload, instance_data).and_return(plugin_double) - expect(plugin_double).to receive(:instance_valid?).and_raise('Custom plugin error') + stub_request(:post, scc_activate_url) + .to_return( + status: 422, + body: { error: 'Unexpected instance verification error has occurred' }.to_json, + headers: {} + ) + + # expect(InstanceVerification::Providers::Example).to receive(:new) + # .with(be_a(ActiveSupport::Logger), be_a(ActionDispatch::Request), payload, instance_data).and_return(plugin_double) + # expect(plugin_double).to receive(:instance_valid?).and_raise('Custom plugin error') post url, params: payload, headers: headers end @@ -623,7 +636,7 @@ .with({ headers: scc_headers, body: payload.merge({ byos_mode: 'byos' }) }) .and_return( status: 401, - body: { error: 'error_message' }.to_json, + body: 'Migration target not allowed on this instance type', headers: {} ) request From c9b6ed7227a2783b3b63754e1c04eff7eca289fb Mon Sep 17 00:00:00 2001 From: Jesus Bermudez Velazquez Date: Tue, 5 Nov 2024 14:45:59 +0000 Subject: [PATCH 11/14] Remove commented expects Expect was too granular and specific --- .../api/connect/v3/systems/products_controller_spec.rb | 6 ------ 1 file changed, 6 deletions(-) diff --git a/engines/instance_verification/spec/requests/api/connect/v3/systems/products_controller_spec.rb b/engines/instance_verification/spec/requests/api/connect/v3/systems/products_controller_spec.rb index 243436c73..9ccbcb1c2 100644 --- a/engines/instance_verification/spec/requests/api/connect/v3/systems/products_controller_spec.rb +++ b/engines/instance_verification/spec/requests/api/connect/v3/systems/products_controller_spec.rb @@ -78,9 +78,6 @@ body: { error: 'Unexpected instance verification error has occurred' }.to_json, headers: {} ) - # expect(InstanceVerification::Providers::Example).to receive(:new) - # .with(be_a(ActiveSupport::Logger), be_a(ActionDispatch::Request), payload, instance_data).and_return(plugin_double) - # expect(plugin_double).to receive(:instance_valid?).and_return(false) post url, params: payload, headers: headers end @@ -99,9 +96,6 @@ headers: {} ) - # expect(InstanceVerification::Providers::Example).to receive(:new) - # .with(be_a(ActiveSupport::Logger), be_a(ActionDispatch::Request), payload, instance_data).and_return(plugin_double) - # expect(plugin_double).to receive(:instance_valid?).and_raise('Custom plugin error') post url, params: payload, headers: headers end From 8a7d37fd06c1aa21874fb39cb835d663fb0d5a84 Mon Sep 17 00:00:00 2001 From: Natnael Getahun Date: Tue, 5 Nov 2024 17:07:51 +0100 Subject: [PATCH 12/14] Remove unused rake metrics --- config/initializers/yabeda.rb | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/config/initializers/yabeda.rb b/config/initializers/yabeda.rb index 331161ee8..f2631fbf3 100644 --- a/config/initializers/yabeda.rb +++ b/config/initializers/yabeda.rb @@ -15,31 +15,12 @@ } group :rails, &assign_labels - group :rake, &assign_labels group :rails do counter :started_requests_total, comment: 'A counter of the total number of HTTP requests rails has started to process.', tags: %i[controller action format method] end - - group :rake do - gauge :task_started_at, - comment: 'Time when the task started: unix time with decimals', - tags: %i[task_name] - - gauge :task_exit_status, - comment: 'Task exit status. 1 means OK, 0 means failed, not present means no data', - tags: %i[task_name] - - gauge :task_duration_ms, - comment: 'Time taken running the task', - tags: %i[task_name] - - gauge :task_finished_at, - comment: 'Time when the task finished: unix time with decimals', - tags: %i[task_name] - end end # Instrument the request from the start @@ -53,19 +34,3 @@ metric.increment(labels, by: 1) end end - -ActiveSupport::Notifications.subscribe 'task_run.rake' do |*args| - event = ActiveSupport::Notifications::Event.new(*args) - task_name = event.payload[:task_name] - - success = 1 - if !!(event.payload[:exception] || event.payload[:exception_class]) - success = 0 - end - - Yabeda.rake.task_duration_ms.set({ task_name: task_name }, event.duration) - Yabeda.rake.task_started_at.set({ task_name: task_name }, event.time / 1000.0) - Yabeda.rake.task_finished_at.set({ task_name: task_name }, event.end / 1000.0) - - Yabeda.rake.task_exit_status.set({ task_name: task_name }, success) -end From d364f1e35d9b616b6db97345c31c9ccc601cfcf1 Mon Sep 17 00:00:00 2001 From: Natnael Getahun Date: Thu, 7 Nov 2024 11:15:13 +0100 Subject: [PATCH 13/14] Add export artifacts step to lint-unit workflow --- .github/workflows/lint-unit.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/lint-unit.yml b/.github/workflows/lint-unit.yml index b4e7ab855..0315a9ea2 100644 --- a/.github/workflows/lint-unit.yml +++ b/.github/workflows/lint-unit.yml @@ -92,3 +92,10 @@ jobs: echo "::group::Version verification checks" ruby ci/check-version-matches.rb echo "::endgroup::" + + - name: Export artifacts + uses: actions/upload-artifact@v2 + if: always() + with: + name: test-results + path: ${{ github.workspace }}/coverage \ No newline at end of file From ddbb38f4e07a72d5bf8d13d6927a97d3754bb110 Mon Sep 17 00:00:00 2001 From: Natnael Getahun Date: Thu, 7 Nov 2024 11:35:32 +0100 Subject: [PATCH 14/14] Add :nocov: comments to ignore code coverage for specific sections --- .github/workflows/lint-unit.yml | 9 +-------- config/initializers/yabeda.rb | 2 ++ lib/rmt/config.rb | 2 ++ 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/lint-unit.yml b/.github/workflows/lint-unit.yml index f96a1ec40..f9d1eaebc 100644 --- a/.github/workflows/lint-unit.yml +++ b/.github/workflows/lint-unit.yml @@ -91,11 +91,4 @@ jobs: run: | echo "::group::Version verification checks" ruby ci/check-version-matches.rb - echo "::endgroup::" - - - name: Export artifacts - uses: actions/upload-artifact@v4 - if: always() - with: - name: test-results - path: ${{ github.workspace }}/coverage \ No newline at end of file + echo "::endgroup::" \ No newline at end of file diff --git a/config/initializers/yabeda.rb b/config/initializers/yabeda.rb index f2631fbf3..0396c27e8 100644 --- a/config/initializers/yabeda.rb +++ b/config/initializers/yabeda.rb @@ -1,3 +1,4 @@ +# :nocov: # frozen_string_literal: true return unless Settings.dig(:scc, :metrics, :enabled) @@ -34,3 +35,4 @@ metric.increment(labels, by: 1) end end +# :nocov: diff --git a/lib/rmt/config.rb b/lib/rmt/config.rb index 650a316ad..4c7ebe27c 100644 --- a/lib/rmt/config.rb +++ b/lib/rmt/config.rb @@ -1,3 +1,4 @@ +# :nocov: require 'config' require_relative '../rmt' @@ -77,3 +78,4 @@ def validate_int(value) end end end +# :nocov: