From 9bb25563a1688b15bfd764a5d37effc910dfb6de Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Tue, 4 Jun 2019 11:02:59 +0200 Subject: [PATCH 001/156] [oc] add a dockerfile with bundler 2 --- .dockerignore | 6 ++++++ .gitignore | 2 ++ Dockerfile | 20 ++++++++++++++++++++ openshift/03-buildconfig-template.yml | 4 ++-- 4 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..e5a14eb4 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +Dockerfile +.dockerignore +.bundle +log +openshift.local.clusterup +tmp diff --git a/.gitignore b/.gitignore index c559fad9..150d6277 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,5 @@ /coverage .byebug_history + +openshift.local.clusterup diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..d60df6f1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +FROM centos/ruby-24-centos7 +RUN source ${APP_ROOT}/etc/scl_enable \ + && gem install bundler --version=2.0.1 --no-document + +COPY Gemfile* ./ +RUN source ${APP_ROOT}/etc/scl_enable \ + && bundle install --deployment --path vendor/bundle --jobs $(grep -c processor /proc/cpuinfo) --retry 3 +COPY . . +ENV RAILS_LOG_TO_STDOUT=1 +USER root +RUN mkdir -p tmp log; chmod -vfR g+w tmp log +USER default +RUN source ${APP_ROOT}/etc/scl_enable \ +&& bundle exec bin/rails server -e production -d; \ +rm -rf tmp/pids +USER root +RUN chmod -fR g+w tmp/cache +USER default + +CMD [".s2i/bin/run"] diff --git a/openshift/03-buildconfig-template.yml b/openshift/03-buildconfig-template.yml index 6eb44760..63ad87d0 100644 --- a/openshift/03-buildconfig-template.yml +++ b/openshift/03-buildconfig-template.yml @@ -25,11 +25,11 @@ objects: uri: https://github.com/3scale/zync.git type: Git strategy: - sourceStrategy: + dockerStrategy: from: kind: ImageStreamTag name: ruby-24-centos7:latest - type: Source + type: Docker triggers: - github: secret: "${GITHUB_SECRET}" From 1a08c6c94c9e0cbaf72599df302aae7b1f42ac8b Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" Date: Wed, 5 Jun 2019 17:40:28 +0000 Subject: [PATCH 002/156] Bump webmock from 3.5.1 to 3.6.0 Bumps [webmock](https://github.com/bblimke/webmock) from 3.5.1 to 3.6.0. - [Release notes](https://github.com/bblimke/webmock/releases) - [Changelog](https://github.com/bblimke/webmock/blob/master/CHANGELOG.md) - [Commits](https://github.com/bblimke/webmock/compare/v3.5.1...v3.6.0) Signed-off-by: dependabot-preview[bot] --- Gemfile | 2 +- Gemfile.lock | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Gemfile b/Gemfile index d924eb93..ad14652f 100644 --- a/Gemfile +++ b/Gemfile @@ -77,6 +77,6 @@ gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby] group :test do gem 'minitest-reporters' - gem 'webmock', '~>3.5' + gem 'webmock', '~>3.6' gem 'codecov', require: false end diff --git a/Gemfile.lock b/Gemfile.lock index 2d93fdab..83ab7b59 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -89,7 +89,7 @@ GEM multipart-post (>= 1.2, < 3) globalid (0.4.2) activesupport (>= 4.2.0) - hashdiff (0.3.8) + hashdiff (0.4.0) httplog (1.3.0) rack (>= 1.0) rainbow (>= 2.0.0) @@ -165,7 +165,7 @@ GEM pry-stack_explorer (0.4.9.3) binding_of_caller (>= 0.7) pry (>= 0.9.11) - public_suffix (3.0.3) + public_suffix (3.1.0) puma (3.12.1) quantile (0.2.1) que (1.0.0.beta3) @@ -211,7 +211,7 @@ GEM railties (>= 4.2.0, < 6.0) ruby-progressbar (1.10.0) rubyzip (1.2.2) - safe_yaml (1.0.4) + safe_yaml (1.0.5) schema_monkey (2.1.5) activerecord (>= 4.2) modware (~> 0.1) @@ -253,10 +253,10 @@ GEM validate_url (1.0.6) activemodel (>= 3.0.0) public_suffix - webmock (3.5.1) + webmock (3.6.0) addressable (>= 2.3.6) crack (>= 0.3.2) - hashdiff + hashdiff (>= 0.4.0, < 2.0.0) websocket-driver (0.7.0) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.3) @@ -307,7 +307,7 @@ DEPENDENCIES spring tzinfo-data validate_url - webmock (~> 3.5) + webmock (~> 3.6) yabeda-prometheus yabeda-puma-plugin yabeda-rails From 0a295353299a9300a4360f29675da71a34d03c25 Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Mon, 10 Jun 2019 17:30:47 +0200 Subject: [PATCH 003/156] [gemfile] remove spring --- Gemfile | 5 ----- Gemfile.lock | 3 --- config/spring.rb | 28 ---------------------------- 3 files changed, 36 deletions(-) delete mode 100644 config/spring.rb diff --git a/Gemfile b/Gemfile index ad14652f..b7d3a4be 100644 --- a/Gemfile +++ b/Gemfile @@ -67,11 +67,6 @@ group :development, :test do gem 'httplog' end -group :development do - # Spring speeds up development by keeping your application running in the background. Read more: https://github.com/rails/spring - gem 'spring' -end - # Windows does not include zoneinfo files, so bundle the tzinfo-data gem gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby] diff --git a/Gemfile.lock b/Gemfile.lock index 83ab7b59..ed27821b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -233,8 +233,6 @@ GEM rack (~> 2.0) rack-protection (= 2.0.5) tilt (~> 2.0) - spring (2.0.2) - activesupport (>= 4.2) sprockets (3.7.2) concurrent-ruby (~> 1.0) rack (> 1, < 3) @@ -304,7 +302,6 @@ DEPENDENCIES rails (~> 5.2.3) responders (~> 2.4.1) schema_plus_enums - spring tzinfo-data validate_url webmock (~> 3.6) diff --git a/config/spring.rb b/config/spring.rb deleted file mode 100644 index ce5bb8a1..00000000 --- a/config/spring.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true -%w( - .ruby-version - .rbenv-vars - tmp/restart.txt - tmp/caching-dev.txt -).each { |path| Spring.watch(path) } - -require 'spring/application' - -module QueWorkers - def disconnect_database - ::Que.worker_count = 0 - ::Que.mode = :off - - ::MessageBus.off - ::MessageBus.destroy - - super - end -end - -Spring::Application.prepend(QueWorkers) - -Spring.after_fork do - ::MessageBus.on - ::MessageBus.after_fork -end From 2467a0f81527e6b199a27831c184ce5694d9d13e Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Mon, 10 Jun 2019 17:37:08 +0200 Subject: [PATCH 004/156] rename Generic integeration to REST --- .../{generic_adapter.rb => rest_adapter.rb} | 2 +- app/jobs/process_entry_job.rb | 4 +- app/models/integration/keycloak.rb | 2 +- .../integration/{generic.rb => rest.rb} | 4 +- app/services/discover_integration_service.rb | 2 +- app/services/integration/generic_service.rb | 4 +- test/adapters/generic_adapter_test.rb | 46 +++++++++---------- test/fixtures/integrations.yml | 2 +- .../integration/generic_service_test.rb | 4 +- 9 files changed, 35 insertions(+), 35 deletions(-) rename app/adapters/{generic_adapter.rb => rest_adapter.rb} (98%) rename app/models/integration/{generic.rb => rest.rb} (65%) diff --git a/app/adapters/generic_adapter.rb b/app/adapters/rest_adapter.rb similarity index 98% rename from app/adapters/generic_adapter.rb rename to app/adapters/rest_adapter.rb index 3f984d10..22ac20e2 100644 --- a/app/adapters/generic_adapter.rb +++ b/app/adapters/rest_adapter.rb @@ -3,7 +3,7 @@ require 'uri' # KeycloakAdapter adapter to create/update/delete Clients on using the KeycloakAdapter Client Registration API. -class GenericAdapter < AbstractAdapter +class RESTAdapter < AbstractAdapter def self.build_client(*attrs) Client.new(*attrs) end diff --git a/app/jobs/process_entry_job.rb b/app/jobs/process_entry_job.rb index af2dea79..33e796a2 100644 --- a/app/jobs/process_entry_job.rb +++ b/app/jobs/process_entry_job.rb @@ -74,8 +74,8 @@ def cleanup def model case type - when 'generic' - ::Integration::Generic + when 'rest' + ::Integration::REST when 'keycloak', nil ::Integration::Keycloak else raise UnknownOIDCIssuerTypeError, type diff --git a/app/models/integration/keycloak.rb b/app/models/integration/keycloak.rb index 30631aef..f412a06e 100644 --- a/app/models/integration/keycloak.rb +++ b/app/models/integration/keycloak.rb @@ -1,4 +1,4 @@ # frozen_string_literal: true -class Integration::Keycloak < Integration::Generic +class Integration::Keycloak < Integration::REST end diff --git a/app/models/integration/generic.rb b/app/models/integration/rest.rb similarity index 65% rename from app/models/integration/generic.rb rename to app/models/integration/rest.rb index 601ec98a..9801b55b 100644 --- a/app/models/integration/generic.rb +++ b/app/models/integration/rest.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true -# Generic HTTP adapter for implementing custom integrations. -class Integration::Generic < Integration +# REST HTTP adapter for implementing custom integrations. +class Integration::REST < Integration store_accessor :configuration, %i[ endpoint ] validates :endpoint, url: { allow_nil: true, no_local: true } diff --git a/app/services/discover_integration_service.rb b/app/services/discover_integration_service.rb index 936b520e..6e400f39 100644 --- a/app/services/discover_integration_service.rb +++ b/app/services/discover_integration_service.rb @@ -18,7 +18,7 @@ def call(integration) klass = case integration when Integration::Keycloak Integration::KeycloakService - when Integration::Generic + when Integration::REST Integration::GenericService when integration Integration::EchoService diff --git a/app/services/integration/generic_service.rb b/app/services/integration/generic_service.rb index 4bc9aeeb..7e9e9f3c 100644 --- a/app/services/integration/generic_service.rb +++ b/app/services/integration/generic_service.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true -# Handles persisting/removing clients using the Generic HTTP adapter. +# Handles persisting/removing clients using the REST HTTP adapter. class Integration::GenericService < Integration::AbstractService - self.adapter_class = ::GenericAdapter + self.adapter_class = ::RESTAdapter def remove(client) payload = { client: client, adapter: adapter } diff --git a/test/adapters/generic_adapter_test.rb b/test/adapters/generic_adapter_test.rb index 9a7627ac..f5ebde0e 100644 --- a/test/adapters/generic_adapter_test.rb +++ b/test/adapters/generic_adapter_test.rb @@ -3,17 +3,17 @@ class GenericAdapterTest < ActiveSupport::TestCase test 'new' do - assert GenericAdapter.new('http://id:secret@lvh.me:3000/auth/realm/name') + assert RESTAdapter.new('http://id:secret@lvh.me:3000/auth/realm/name') end test 'endpoint' do - adapter = GenericAdapter.new('http://id:secret@lvh.me:3000/auth/realm/name') + adapter = RESTAdapter.new('http://id:secret@lvh.me:3000/auth/realm/name') assert_kind_of URI, adapter.endpoint end test 'setting access token' do - subject = GenericAdapter.new('http://lvh.me:3000') + subject = RESTAdapter.new('http://lvh.me:3000') subject.authentication = 'sometoken' @@ -24,10 +24,10 @@ class GenericAdapterTest < ActiveSupport::TestCase uri = URI('http://lvh.me:3000/auth/realm/name/') assert_equal uri, - GenericAdapter.new('http://id:secret@lvh.me:3000/auth/realm/name').endpoint + RESTAdapter.new('http://id:secret@lvh.me:3000/auth/realm/name').endpoint assert_equal uri, - GenericAdapter.new('http://id:secret@lvh.me:3000/auth/realm/name/').endpoint + RESTAdapter.new('http://id:secret@lvh.me:3000/auth/realm/name/').endpoint end test 'timeout error' do @@ -37,19 +37,19 @@ class GenericAdapterTest < ActiveSupport::TestCase stub_request(:post, 'http://lvh.me:3000/auth/realm/name/protocol/openid-connect/token').to_timeout - adapter = GenericAdapter.new('http://id:secret@lvh.me:3000/auth/realm/name') + adapter = RESTAdapter.new('http://id:secret@lvh.me:3000/auth/realm/name') begin adapter.test - rescue GenericAdapter::OIDC::AuthenticationError => error + rescue RESTAdapter::OIDC::AuthenticationError => error assert_kind_of Faraday::TimeoutError, error.cause assert error.bugsnag_meta_data.presence end end test 'create client' do - adapter = GenericAdapter.new('http://example.com/adapter', authentication: 'token') - client = GenericAdapter::Client.new(name: 'Foo', id: 'foo', secret: 'bar') + adapter = RESTAdapter.new('http://example.com/adapter', authentication: 'token') + client = RESTAdapter::Client.new(name: 'Foo', id: 'foo', secret: 'bar') create = stub_request(:put, "http://example.com/adapter/clients/foo"). with( @@ -62,8 +62,8 @@ class GenericAdapterTest < ActiveSupport::TestCase end test 'update client' do - adapter = GenericAdapter.new('http://example.com/adapter', authentication: 'token') - client = GenericAdapter::Client.new(name: 'Foo', id: 'foo', secret: 'bar') + adapter = RESTAdapter.new('http://example.com/adapter', authentication: 'token') + client = RESTAdapter::Client.new(name: 'Foo', id: 'foo', secret: 'bar') update = stub_request(:put, "http://example.com/adapter/clients/foo"). with( @@ -76,8 +76,8 @@ class GenericAdapterTest < ActiveSupport::TestCase end test 'delete client' do - adapter = GenericAdapter.new('http://example.com/adapter', authentication: 'token') - client = GenericAdapter::Client.new(id: 'foo') + adapter = RESTAdapter.new('http://example.com/adapter', authentication: 'token') + client = RESTAdapter::Client.new(id: 'foo') delete = stub_request(:delete, "http://example.com/adapter/clients/foo").to_return(status: 200) @@ -87,8 +87,8 @@ class GenericAdapterTest < ActiveSupport::TestCase end test 'read client' do - adapter = GenericAdapter.new('http://example.com/adapter', authentication: 'token') - client = GenericAdapter::Client.new(id: 'foo') + adapter = RESTAdapter.new('http://example.com/adapter', authentication: 'token') + client = RESTAdapter::Client.new(id: 'foo') body = { client_id: 'foo', client_name: 'Foo'} read = stub_request(:get, "http://example.com/adapter/clients/foo") @@ -97,7 +97,7 @@ class GenericAdapterTest < ActiveSupport::TestCase client = adapter.read_client(client) - assert_kind_of GenericAdapter::Client, client + assert_kind_of RESTAdapter::Client, client assert_equal 'Foo', client.name assert_equal 'foo', client.id @@ -105,7 +105,7 @@ class GenericAdapterTest < ActiveSupport::TestCase end test 'test' do - adapter = GenericAdapter.new('http://id:secret@example.com/auth/realm/name') + adapter = RESTAdapter.new('http://id:secret@example.com/auth/realm/name') form_urlencoded = { 'Content-Type'=>'application/x-www-form-urlencoded' } token = stub_request(:post, 'http://example.com/auth/realm/name/get-token'). @@ -126,9 +126,9 @@ class GenericAdapterTest < ActiveSupport::TestCase stub_request(:get, 'http://lvh.me:3000/auth/realm/name/.well-known/openid-configuration'). to_return(status: 200, body: 'somebody', headers: {'Content-Type' => 'text/plain'} ) - adapter = GenericAdapter.new('http://id:secret@lvh.me:3000/auth/realm/name', authentication: 'something') + adapter = RESTAdapter.new('http://id:secret@lvh.me:3000/auth/realm/name', authentication: 'something') - assert_raises GenericAdapter::InvalidResponseError do + assert_raises RESTAdapter::InvalidResponseError do adapter.test end end @@ -141,26 +141,26 @@ class GenericAdapterTest < ActiveSupport::TestCase }.deep_stringify_keys Rails.application.config.x.stub(:generic, config) do - client = GenericAdapter::Client.new(name: 'foo') + client = RESTAdapter::Client.new(name: 'foo') assert_includes client.to_h.fetch(:grant_types), :client_credentials end end test 'client hash' do - client = GenericAdapter::Client.new(name: 'name') + client = RESTAdapter::Client.new(name: 'name') assert_includes client.to_h, :client_name end test 'client serialization' do - client = GenericAdapter::Client.new(name: 'name') + client = RESTAdapter::Client.new(name: 'name') assert_equal client.to_h.to_json, client.to_json end test 'oauth flows' do - client = GenericAdapter::Client.new({ + client = RESTAdapter::Client.new({ id: 'client_id', oidc_configuration: { implicit_flow_enabled: true, diff --git a/test/fixtures/integrations.yml b/test/fixtures/integrations.yml index 0f2d4a93..31e43296 100644 --- a/test/fixtures/integrations.yml +++ b/test/fixtures/integrations.yml @@ -15,6 +15,6 @@ keycloak: generic: tenant: one model: first_service - type: Integration::Generic + type: Integration::REST configuration: endpoint: 'http://id:pass@example.com/generic/api' diff --git a/test/services/integration/generic_service_test.rb b/test/services/integration/generic_service_test.rb index 0c48c987..5c39eb64 100644 --- a/test/services/integration/generic_service_test.rb +++ b/test/services/integration/generic_service_test.rb @@ -40,7 +40,7 @@ def test_new entry = entries(:client) adapter = MiniTest::Mock.new - adapter.expect(:update_client, true, [ GenericAdapter::Client ]) + adapter.expect(:update_client, true, [RESTAdapter::Client ]) subject.stub(:adapter, adapter) do |service| service.call(entry) @@ -54,7 +54,7 @@ def test_new entry.data = entry.data.except(:enabled) adapter = MiniTest::Mock.new - adapter.expect(:delete_client, true, [ GenericAdapter::Client ]) + adapter.expect(:delete_client, true, [RESTAdapter::Client ]) subject.stub(:adapter, adapter) do |service| service.call(entry) From fb091e76cb7f44664063b5eb4ac78d71c9d7c51c Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Tue, 11 Jun 2019 16:16:03 +0200 Subject: [PATCH 005/156] skip reporting lock error to bugsnag --- app/models/model.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app/models/model.rb b/app/models/model.rb index c62b467d..a8affee3 100644 --- a/app/models/model.rb +++ b/app/models/model.rb @@ -14,7 +14,13 @@ def weak_record end # Error raised when weak lock can't be acquired. - class LockTimeoutError < StandardError; end + class LockTimeoutError < StandardError + + # No need to report this error. + def skip_bugsnag + true + end + end def self.create_record!(tenant) retry_record_not_unique do From bf83df8cae4f87d12b37d9fcd1efbd029535b5da Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Tue, 11 Jun 2019 16:53:53 +0200 Subject: [PATCH 006/156] [rails] do not print that job was retried plenty of jobs are retried, becasue they are lock errors --- app/jobs/application_job.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb index cc900cb6..8dfc036d 100644 --- a/app/jobs/application_job.rb +++ b/app/jobs/application_job.rb @@ -1,4 +1,20 @@ # frozen_string_literal: true # Base class for all Jobs class ApplicationJob < ActiveJob::Base + # Copied from ActiveJob::Exceptions, but uses debug log level. + def self.retry_on(exception, wait: 3.seconds, attempts: 5, queue: nil, priority: nil) + rescue_from exception do |error| + if executions < attempts + logger.debug "Retrying #{self.class} in #{wait} seconds, due to a #{exception}. The original exception was #{error.cause.inspect}." + retry_job wait: determine_delay(wait), queue: queue, priority: priority + else + if block_given? + yield self, error + else + logger.debug "Stopped retrying #{self.class} due to a #{exception}, which reoccurred on #{executions} attempts. The original exception was #{error.cause.inspect}." + raise error + end + end + end + end end From f35c7de87bd46fed1e53228f6ce66eee5fcf7a4f Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Tue, 11 Jun 2019 19:31:44 +0200 Subject: [PATCH 007/156] [update-job] de-duplicate old update jobs when a new one is added there is no need for previous update jobs with the same parameters --- Gemfile | 1 + Gemfile.lock | 6 ++++++ app/jobs/update_job.rb | 22 ++++++++++++++++++++++ test/jobs/update_job_test.rb | 30 ++++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+) diff --git a/Gemfile b/Gemfile index b7d3a4be..97b3b770 100644 --- a/Gemfile +++ b/Gemfile @@ -32,6 +32,7 @@ gem 'bootsnap' gem 'que', '>= 1.0.0.beta3' gem 'que-web' +gem 'baby_squeel' gem 'bugsnag' # bugsnag-capistrano 2.x does not have a rake task to report deploys diff --git a/Gemfile.lock b/Gemfile.lock index ed27821b..0e459a38 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -62,6 +62,9 @@ GEM public_suffix (>= 2.0.2, < 4.0) ansi (1.5.0) arel (9.0.0) + baby_squeel (1.2.0) + activerecord (>= 4.2.0) + polyamorous (~> 1.3) binding_of_caller (0.8.0) debug_inspector (>= 0.0.1) bootsnap (1.4.4) @@ -149,6 +152,8 @@ GEM rack (>= 1.2, < 3) parslet (1.8.2) pg (1.1.4) + polyamorous (1.3.3) + activerecord (>= 3.0) prometheus-client (0.9.0) quantile (~> 0.2.1) pry (0.12.2) @@ -278,6 +283,7 @@ PLATFORMS DEPENDENCIES 3scale-api (~> 0.1.9) + baby_squeel bootsnap bugsnag bugsnag-capistrano (< 2) diff --git a/app/jobs/update_job.rb b/app/jobs/update_job.rb index 6195b8c3..cc095050 100644 --- a/app/jobs/update_job.rb +++ b/app/jobs/update_job.rb @@ -2,6 +2,8 @@ # Uses FetchService to get Entity and persist it in database. # Maintains UpdateState and can be only one running at a time by using a lock on model. +require 'que/active_record/model' + class UpdateJob < ApplicationJob include JobWithTimestamp queue_as :default @@ -16,6 +18,14 @@ def initialize(*) attr_reader :fetch + def relation + record = self.class.model + arguments = serialize.slice('arguments') + record.where.has { + args.op('@>', quoted([arguments].to_json)) + } + end + def perform(model) UpdateState.acquire_lock(model) do |state| # this is not going to be visible outside the transaction, does it matter? @@ -27,4 +37,16 @@ def perform(model) state.update_attributes(success: entry.save, finished_at: timestamp) end end + + def self.model + Que::ActiveRecord::Model.by_job_class(to_s) + end + + def self.perform_later(*args) + model.transaction do + job = job_or_instantiate(*args) + job.relation.delete_all + job.enqueue + end + end end diff --git a/test/jobs/update_job_test.rb b/test/jobs/update_job_test.rb index a685ec4f..22741e9d 100644 --- a/test/jobs/update_job_test.rb +++ b/test/jobs/update_job_test.rb @@ -13,6 +13,36 @@ class UpdateJobTest < ActiveJob::TestCase end end + test 'relation' do + application = UpdateJob.new(models(:application)) + client = UpdateJob.new(models(:client)) + + refute_equal application.relation.to_sql, client.relation.to_sql + + adapter = ActiveJob::QueueAdapters::QueAdapter.new + + assert_difference application.relation.method(:count), 2 do + adapter.enqueue(application) + adapter.enqueue(application) + + assert_difference client.relation.method(:count), 2 do + adapter.enqueue(client) + adapter.enqueue(client) + end + end + end + + test 'perform later' do + adapter = ActiveJob::QueueAdapters::QueAdapter.new + job = UpdateJob.new(models(:application)) + + adapter.enqueue(job) + + assert_difference job.relation.method(:count), -1 do + UpdateJob.perform_later(job) # this is not using the same adapter, so it actually just removes previous one + end + end + test 'creates entry' do model = Model.create!(tenant: tenants(:two), record: applications(:two)) From d375adae986950c9a2ffff05f47c8ed2a00adad2 Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Tue, 11 Jun 2019 21:17:49 +0200 Subject: [PATCH 008/156] [que] dedupliate update and process integration jobs they always take the latest data, there is no reason to have more than one in the flight --- app/jobs/application_job.rb | 26 ++++++++++++++++ app/jobs/process_integration_entry_job.rb | 2 ++ app/jobs/update_job.rb | 23 ++------------ .../process_integration_entry_job_test.rb | 30 +++++++++++++++++++ test/jobs/update_job_test.rb | 2 +- 5 files changed, 61 insertions(+), 22 deletions(-) diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb index 8dfc036d..dbe58170 100644 --- a/app/jobs/application_job.rb +++ b/app/jobs/application_job.rb @@ -1,4 +1,7 @@ # frozen_string_literal: true + +require 'que/active_record/model' + # Base class for all Jobs class ApplicationJob < ActiveJob::Base # Copied from ActiveJob::Exceptions, but uses debug log level. @@ -17,4 +20,27 @@ def self.retry_on(exception, wait: 3.seconds, attempts: 5, queue: nil, priority: end end end + + class_attribute :deduplicate + + before_enqueue :delete_duplicates, if: :deduplicate? + around_enqueue if: :deduplicate? do |job, block| + job.class.model.transaction(&block) + end + + def relation + record = self.class.model + arguments = serialize.slice('arguments') + record.where.has { + args.op('@>', quoted([arguments].to_json)) + } + end + + def delete_duplicates + relation.delete_all + end + + def self.model + Que::ActiveRecord::Model.by_job_class(to_s) + end end diff --git a/app/jobs/process_integration_entry_job.rb b/app/jobs/process_integration_entry_job.rb index 0bfdf6cf..85e90af8 100644 --- a/app/jobs/process_integration_entry_job.rb +++ b/app/jobs/process_integration_entry_job.rb @@ -5,6 +5,8 @@ class ProcessIntegrationEntryJob < ApplicationJob queue_as :default + self.deduplicate = true + delegate :instrument, to: 'ActiveSupport::Notifications' def perform(integration, model, service: DiscoverIntegrationService.call(integration)) diff --git a/app/jobs/update_job.rb b/app/jobs/update_job.rb index cc095050..73a53c0d 100644 --- a/app/jobs/update_job.rb +++ b/app/jobs/update_job.rb @@ -2,7 +2,6 @@ # Uses FetchService to get Entity and persist it in database. # Maintains UpdateState and can be only one running at a time by using a lock on model. -require 'que/active_record/model' class UpdateJob < ApplicationJob include JobWithTimestamp @@ -11,6 +10,8 @@ class UpdateJob < ApplicationJob retry_on Errno::ECONNREFUSED, wait: :exponentially_longer, attempts: 10 retry_on Model::LockTimeoutError, wait: :exponentially_longer, attempts: 10 + self.deduplicate = true + def initialize(*) super @fetch = FetchService @@ -18,14 +19,6 @@ def initialize(*) attr_reader :fetch - def relation - record = self.class.model - arguments = serialize.slice('arguments') - record.where.has { - args.op('@>', quoted([arguments].to_json)) - } - end - def perform(model) UpdateState.acquire_lock(model) do |state| # this is not going to be visible outside the transaction, does it matter? @@ -37,16 +30,4 @@ def perform(model) state.update_attributes(success: entry.save, finished_at: timestamp) end end - - def self.model - Que::ActiveRecord::Model.by_job_class(to_s) - end - - def self.perform_later(*args) - model.transaction do - job = job_or_instantiate(*args) - job.relation.delete_all - job.enqueue - end - end end diff --git a/test/jobs/process_integration_entry_job_test.rb b/test/jobs/process_integration_entry_job_test.rb index a7008885..c4ff3a35 100644 --- a/test/jobs/process_integration_entry_job_test.rb +++ b/test/jobs/process_integration_entry_job_test.rb @@ -63,4 +63,34 @@ class ProcessIntegrationEntryJobTest < ActiveJob::TestCase assert_mock service end + + test 'relation' do + service = ProcessIntegrationEntryJob.new(integrations(:keycloak), models(:service)) + application = ProcessIntegrationEntryJob.new(integrations(:keycloak), models(:application)) + + refute_equal application.relation.to_sql, service.relation.to_sql + + adapter = ActiveJob::QueueAdapters::QueAdapter.new + + assert_difference application.relation.method(:count), 2 do + adapter.enqueue(application) + adapter.enqueue(application) + + assert_difference service.relation.method(:count), 2 do + adapter.enqueue(service) + adapter.enqueue(service) + end + end + end + + test 'perform later' do + adapter = ActiveJob::QueueAdapters::QueAdapter.new + job = ProcessIntegrationEntryJob.new(integrations(:keycloak), models(:application)) + + adapter.enqueue(job) + + assert_difference job.relation.method(:count), -1 do + ApplicationJob.perform_later(job) # this is not using the same adapter, so it actually just removes previous one + end + end end diff --git a/test/jobs/update_job_test.rb b/test/jobs/update_job_test.rb index 22741e9d..989620e7 100644 --- a/test/jobs/update_job_test.rb +++ b/test/jobs/update_job_test.rb @@ -39,7 +39,7 @@ class UpdateJobTest < ActiveJob::TestCase adapter.enqueue(job) assert_difference job.relation.method(:count), -1 do - UpdateJob.perform_later(job) # this is not using the same adapter, so it actually just removes previous one + ApplicationJob.perform_later(job) # this is not using the same adapter, so it actually just removes previous one end end From 5f7839cf6c573660096a7703eda06fd89e0ca964 Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Fri, 14 Jun 2019 11:44:39 +0200 Subject: [PATCH 009/156] [gemfile] try bugsnag hotfix should fix https://github.com/bugsnag/bugsnag-ruby/issues/545 --- Gemfile | 2 +- Gemfile.lock | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Gemfile b/Gemfile index b7d3a4be..abde9b0d 100644 --- a/Gemfile +++ b/Gemfile @@ -33,7 +33,7 @@ gem 'bootsnap' gem 'que', '>= 1.0.0.beta3' gem 'que-web' -gem 'bugsnag' +gem 'bugsnag', github: 'bugsnag/bugsnag-ruby', branch: 'tobyhs/check_nil_job_in_que' # bugsnag-capistrano 2.x does not have a rake task to report deploys # https://github.com/bugsnag/bugsnag-capistrano/blob/8bcfb27cf6eaff312eef086cce729d553a431460/UPGRADING.md gem 'bugsnag-capistrano', '< 2', require: false diff --git a/Gemfile.lock b/Gemfile.lock index ed27821b..7342b9c8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -6,6 +6,14 @@ GIT license_finder_xml_reporter (0.1.0) license_finder +GIT + remote: https://github.com/bugsnag/bugsnag-ruby.git + revision: 00dac89430281610519a122dc2acbb41a1d2bed3 + branch: tobyhs/check_nil_job_in_que + specs: + bugsnag (6.11.1) + concurrent-ruby (~> 1.0) + GIT remote: https://github.com/mikz/httpclient.git revision: fec23fb32fb899b87a8b2c94e2d2069b6b4c633c @@ -66,8 +74,6 @@ GEM debug_inspector (>= 0.0.1) bootsnap (1.4.4) msgpack (~> 1.0) - bugsnag (6.11.1) - concurrent-ruby (~> 1.0) bugsnag-capistrano (1.1.2) builder (3.2.3) byebug (11.0.0) @@ -279,7 +285,7 @@ PLATFORMS DEPENDENCIES 3scale-api (~> 0.1.9) bootsnap - bugsnag + bugsnag! bugsnag-capistrano (< 2) codecov httpclient! From 7da4c289c5b8653fd34b8845284ce0384e98bc29 Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Wed, 12 Jun 2019 16:03:48 +0200 Subject: [PATCH 010/156] [que] rake tasks --- lib/tasks/que.rake | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/lib/tasks/que.rake b/lib/tasks/que.rake index b3b703cc..1972d9ef 100644 --- a/lib/tasks/que.rake +++ b/lib/tasks/que.rake @@ -1,6 +1,22 @@ # frozen_string_literal: true -desc 'Start que worker' -task que: :environment do |_, args| - exec("que ./config/environment.rb que/prometheus #{args.extras.join}") +task que: 'que:exec' + +namespace :que do + desc 'Start que worker' + task exec: :environment do |_, args| + exec("que ./config/environment.rb que/prometheus #{args.extras.join}") + end + + desc 'Reschedule all jobs to be executed now' + task reschedule: :environment do + require 'que/active_record/model' + + Que::ActiveRecord::Model.update_all(run_at: Time.now) + end + + desc 'Force updating all models' + task force_update: :environment do + Model.find_each(&UpdateJob.method(:perform_later)) + end end From d2fb71558080dc7f2c976d28ac20c3881e4167c4 Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Tue, 4 Jun 2019 10:29:12 +0200 Subject: [PATCH 011/156] [openshift] create routes on demand Create OpenShift Route objects when Provider/Proxy are created/updated/deleted. --- Gemfile | 5 +- Gemfile.lock | 57 ++++- app/jobs/process_entry_job.rb | 105 +++++++-- app/jobs/process_integration_entry_job.rb | 21 +- app/models/integration/kubernetes.rb | 17 ++ app/models/notification.rb | 2 +- app/models/provider.rb | 5 + app/services/discover_integration_service.rb | 3 + app/services/fetch_service.rb | 51 ++--- app/services/incoming_notification_service.rb | 1 + app/services/integration/echo_service.rb | 1 + app/services/integration/keycloak_service.rb | 1 - .../integration/kubernetes_service.rb | 210 ++++++++++++++++++ app/services/integration/service_base.rb | 4 + config/application.rb | 17 ++ config/openshift.yml | 18 ++ db/migrate/20190605094424_create_providers.rb | 9 + db/structure.sql | 64 +++++- test/fixtures/entries.yml | 8 + test/fixtures/models.yml | 4 + test/fixtures/providers.yml | 7 + test/jobs/process_entry_job_test.rb | 10 +- test/models/provider_test.rb | 7 + test/services/fetch_service_test.rb | 21 ++ test/services/kubernetes_service_test.rb | 68 ++++++ 25 files changed, 649 insertions(+), 67 deletions(-) create mode 100644 app/models/integration/kubernetes.rb create mode 100644 app/models/provider.rb create mode 100644 app/services/integration/kubernetes_service.rb create mode 100644 config/openshift.yml create mode 100644 db/migrate/20190605094424_create_providers.rb create mode 100644 test/fixtures/providers.yml create mode 100644 test/models/provider_test.rb create mode 100644 test/services/kubernetes_service_test.rb diff --git a/Gemfile b/Gemfile index f6857a69..448bff75 100644 --- a/Gemfile +++ b/Gemfile @@ -26,7 +26,7 @@ gem 'puma', '~> 3.12' # gem 'rack-cors' gem 'responders', '~> 2.4.1' -gem '3scale-api', '~> 0.1.9' +gem '3scale-api' gem 'bootsnap' @@ -43,6 +43,7 @@ gem 'bugsnag-capistrano', '< 2', require: false # https://github.com/nahi/httpclient/issues/369 gem 'httpclient', github: 'mikz/httpclient', branch: 'ssl-env-cert' gem 'oauth2' +gem 'k8s-client', '>= 0.10' gem 'lograge' @@ -73,6 +74,6 @@ gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby] group :test do gem 'minitest-reporters' - gem 'webmock', '~>3.6' + gem 'webmock' gem 'codecov', require: false end diff --git a/Gemfile.lock b/Gemfile.lock index febc3284..815f8932 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -24,7 +24,7 @@ GIT GEM remote: https://rubygems.org/ specs: - 3scale-api (0.1.9) + 3scale-api (0.5.0) actioncable (5.2.3) actionpack (= 5.2.3) nio4r (~> 2.0) @@ -91,23 +91,63 @@ GEM crass (1.0.4) debug_inspector (0.0.3) docile (1.3.1) + dry-configurable (0.8.3) + concurrent-ruby (~> 1.0) + dry-core (~> 0.4, >= 0.4.7) + dry-container (0.7.1) + concurrent-ruby (~> 1.0) + dry-configurable (~> 0.1, >= 0.1.3) + dry-core (0.4.7) + concurrent-ruby (~> 1.0) + dry-equalizer (0.2.2) + dry-inflector (0.1.2) dry-initializer (2.5.0) + dry-logic (0.6.1) + concurrent-ruby (~> 1.0) + dry-core (~> 0.2) + dry-equalizer (~> 0.2) + dry-struct (0.5.1) + dry-core (~> 0.4, >= 0.4.3) + dry-equalizer (~> 0.2) + dry-types (~> 0.13) + ice_nine (~> 0.11) + dry-types (0.13.4) + concurrent-ruby (~> 1.0) + dry-container (~> 0.3) + dry-core (~> 0.4, >= 0.4.4) + dry-equalizer (~> 0.2) + dry-inflector (~> 0.1, >= 0.1.2) + dry-logic (~> 0.4, >= 0.4.2) erubi (1.8.0) erubis (2.7.0) + excon (0.62.0) faraday (0.15.3) multipart-post (>= 1.2, < 3) globalid (0.4.2) activesupport (>= 4.2.0) - hashdiff (0.4.0) + hashdiff (0.3.9) httplog (1.3.0) rack (>= 1.0) rainbow (>= 2.0.0) i18n (1.6.0) concurrent-ruby (~> 1.0) + ice_nine (0.11.2) interception (0.5) its-it (1.3.0) json (2.1.0) + jsonpath (0.9.9) + multi_json + to_regexp (~> 0.2.1) jwt (2.1.0) + k8s-client (0.10.1) + dry-struct (~> 0.5.0) + dry-types (~> 0.13.0) + excon (~> 0.62.0) + hashdiff (~> 0.3.7) + jsonpath (~> 0.9.5) + recursive-open-struct (~> 1.1.0) + yajl-ruby (~> 1.4.0) + yaml-safe_load_stream (~> 0.1) key_struct (0.4.2) license_finder (5.8.0) bundler @@ -215,6 +255,7 @@ GEM thor (>= 0.19.0, < 2.0) rainbow (3.0.0) rake (12.3.2) + recursive-open-struct (1.1.0) request_store (1.4.1) rack (>= 1.4) responders (2.4.1) @@ -254,6 +295,7 @@ GEM thor (0.20.3) thread_safe (0.3.6) tilt (2.0.9) + to_regexp (0.2.1) toml (0.2.0) parslet (~> 1.8.0) tzinfo (1.2.5) @@ -262,10 +304,10 @@ GEM validate_url (1.0.6) activemodel (>= 3.0.0) public_suffix - webmock (3.6.0) + webmock (3.5.1) addressable (>= 2.3.6) crack (>= 0.3.2) - hashdiff (>= 0.4.0, < 2.0.0) + hashdiff websocket-driver (0.7.0) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.3) @@ -283,12 +325,14 @@ GEM yabeda-rails (0.1.3) rails yabeda + yajl-ruby (1.4.1) + yaml-safe_load_stream (0.1.1) PLATFORMS ruby DEPENDENCIES - 3scale-api (~> 0.1.9) + 3scale-api baby_squeel bootsnap bugsnag! @@ -296,6 +340,7 @@ DEPENDENCIES codecov httpclient! httplog + k8s-client (>= 0.10) license_finder (~> 5.8) license_finder_xml_reporter! lograge @@ -316,7 +361,7 @@ DEPENDENCIES schema_plus_enums tzinfo-data validate_url - webmock (~> 3.6) + webmock yabeda-prometheus yabeda-puma-plugin yabeda-rails diff --git a/app/jobs/process_entry_job.rb b/app/jobs/process_entry_job.rb index 33e796a2..c62cef48 100644 --- a/app/jobs/process_entry_job.rb +++ b/app/jobs/process_entry_job.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + # Process each Entry after it is created. # So schedule Integration jobs to perform the integration work. @@ -11,31 +12,98 @@ def perform(entry) end end - def model_integrations_for(entry) + def self.model_integrations_for(entry) model = entry.model integrations = Integration.retry_record_not_unique do case model.record - when Proxy then CreateProxyIntegration.new(entry).call + when Proxy then PROXY_INTEGRATIONS.map { |i| i.new(entry) }.each(&:call) + when Provider then CreateK8SIntegration.new(entry).call end Integration.for_model(model) end - integrations.each.with_object(model) end - # Wrapper for creating KeycloakAdapter when Proxy is created - CreateProxyIntegration = Struct.new(:entry) do - attr_reader :service, :data + protected + + delegate :model_integrations_for, to: :class + + class ModelIntegration + attr_reader :model, :data, :entry + + def initialize(entry) + @entry = entry + @model = Model.find_by!(record: entry.model.record) + @data = entry.data || {}.freeze + end + + def integrations + ::Integration.where(tenant: tenant, model: model) + end + + def call + raise NoMethodError, __method__ + end + + protected + + delegate :transaction, to: :model + delegate :tenant, to: :entry + end + + class ProxyIntegration + attr_reader :service, :data, :entry - def initialize(*) - super + def initialize(entry) + @entry = entry @service = Model.find_by!(record: proxy.record.service) @data = entry.data || {}.freeze end + def call + raise NoMethodError, __method__ + end + + def model + raise NoMethodError, __method__ + end + + def integrations + ::Integration.where(tenant: tenant, model: service) + end + + protected + + delegate :transaction, to: :model + delegate :tenant, to: :entry + + def proxy + entry.model + end + end + + class CreateK8SIntegration < ModelIntegration + class_attribute :integration_type, default: Integration::Kubernetes + + class_attribute :enabled, default: Rails.application.config.x.openshift.enabled + + def call + return unless enabled? + + transaction do + integration = integrations.find_or_create_by!(type: integration_type.to_s) + integration.update(state: Integration.states.fetch(:active)) + + ProcessIntegrationEntryJob.perform_later(integration, model) + end + end + end + + # Wrapper for creating Keycloak/Generic Adapter when Proxy is created + class CreateOIDCProxyIntegration < ProxyIntegration def endpoint data[:oidc_issuer_endpoint] end @@ -65,9 +133,6 @@ def call end end - delegate :transaction, to: :model - delegate :tenant, to: :entry - def cleanup integrations.update_all(state: Integration.states.fetch(:disabled)) end @@ -82,21 +147,19 @@ def model end end - def integrations - ::Integration.where(tenant: tenant, model: service) - end - # Unknown oidc_issuer_type in the entry. class UnknownOIDCIssuerTypeError < StandardError; end def find_integration model - .create_with(endpoint: endpoint) - .find_or_create_by!(integrations.where_values_hash) - end - - def proxy - entry.model + .create_with(endpoint: endpoint) + .find_or_create_by!(integrations.where_values_hash) end end + + PROXY_INTEGRATIONS = [ + CreateOIDCProxyIntegration, + CreateK8SIntegration + ].freeze + private_constant :PROXY_INTEGRATIONS end diff --git a/app/jobs/process_integration_entry_job.rb b/app/jobs/process_integration_entry_job.rb index 85e90af8..44820102 100644 --- a/app/jobs/process_integration_entry_job.rb +++ b/app/jobs/process_integration_entry_job.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + # Update Integration with latest changes to the model. # Load latest Entry and push it through the Integration. @@ -12,6 +13,11 @@ class ProcessIntegrationEntryJob < ApplicationJob def perform(integration, model, service: DiscoverIntegrationService.call(integration)) return unless service + if integration.try(:disabled?) + logger.info "#{integration.to_gid} is disabled, skipping" + return + end + result = invoke(model, integration, service) do |invocation| payload = build_payload(model, integration, invocation) @@ -31,12 +37,12 @@ def call(_payload) service.call(entry) finish(success: true) - rescue + rescue StandardError finish(success: false) raise end - def finish(success: ) + def finish(success:) state.update_attributes(success: success, finished_at: timestamp) end @@ -78,14 +84,14 @@ def call(payload, &block) value = instrument('perform.process_integration_entry', payload, &block) Result.new(true, value) - rescue => exception - Result.new(false, value, exception) + rescue StandardError => e + Result.new(false, value, e) end def build_payload(model, integration, invocation) { - entry_data: invocation.entry_data, integration: integration, model: model, - service: invocation.service_name, record: model.record + entry_data: invocation.entry_data, integration: integration, model: model, + service: invocation.service_name, record: model.record } end @@ -132,10 +138,9 @@ def transform_payload(payload) def extract_tenant(payload) tenant = payload.fetch(:model).tenant - [ tenant, { user_ids: [ tenant.to_gid_param ] } ] + [tenant, { user_ids: [tenant.to_gid_param] }] end - def build_message_bus(tenant) MessageBus::Instance.new.tap do |message_bus| message_bus.config.merge!(MessageBus.config) diff --git a/app/models/integration/kubernetes.rb b/app/models/integration/kubernetes.rb new file mode 100644 index 00000000..e5c46890 --- /dev/null +++ b/app/models/integration/kubernetes.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +# Generic HTTP adapter for implementing custom integrations. +class Integration::Kubernetes < Integration + store_accessor :configuration, %i[ server ] + + validates :server, url: { allow_nil: true, no_local: true } + + def enabled? + K8s::Client === K8s::Client.autoconfig + rescue K8s::Error::Configuration + false + rescue => error + Bugsnag.notify(error) + false + end +end diff --git a/app/models/notification.rb b/app/models/notification.rb index 696beeda..d9a28479 100644 --- a/app/models/notification.rb +++ b/app/models/notification.rb @@ -11,7 +11,7 @@ def initialize(data) @data = ActiveSupport::HashWithIndifferentAccess.new(data) end - ALLOWED_MODELS = Set.new(%w(Application Proxy Service)).freeze + ALLOWED_MODELS = Set.new(%w(Application Proxy Service Provider)).freeze NULL_TYPE = Object.new.tap do |object| def object.find_or_create_by!(*); end def object.attribute_names; end diff --git a/app/models/provider.rb b/app/models/provider.rb new file mode 100644 index 00000000..bbdb3832 --- /dev/null +++ b/app/models/provider.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +class Provider < ApplicationRecord + belongs_to :tenant +end diff --git a/app/services/discover_integration_service.rb b/app/services/discover_integration_service.rb index 6e400f39..39bd7519 100644 --- a/app/services/discover_integration_service.rb +++ b/app/services/discover_integration_service.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + # Returns a Service for each Integration. # Each Integration can be using different Service. # This class creates a mapping between Integration and Service. @@ -20,6 +21,8 @@ def call(integration) Integration::KeycloakService when Integration::REST Integration::GenericService + when Integration::Kubernetes + Integration::KubernetesService when integration Integration::EchoService else # the only one for now diff --git a/app/services/fetch_service.rb b/app/services/fetch_service.rb index a8da9315..18c13b8d 100644 --- a/app/services/fetch_service.rb +++ b/app/services/fetch_service.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + # Fetches Model information from upstream and returns the Entity. class FetchService @@ -13,7 +14,7 @@ class << self # Returned when unknown model is passed in. class UnsupportedModel < StandardError; end - # @return [ThreeScale::API] + # @return [ThreeScale::API::Client] def build_client(tenant) http_client = ThreeScale::API::InstrumentedHttpClient.new(endpoint: tenant.endpoint, provider_key: tenant.access_token) @@ -30,52 +31,52 @@ def call(model) fetch_client(model) when Proxy fetch_proxy(model) + when Provider + fetch_provider(model) else raise UnsupportedModel, "unsupported model #{record.class}" end end + def fetch_provider(model) + fetch_entry(model) do |client| + client.show_provider + end + end + def fetch_service(model) build_entry(model) end def fetch_proxy(model) - client = build_client(model.tenant) - - begin - proxy = client.show_proxy(model.record.service_id) - # right now the client raises runtime error, but rather should return a result - rescue RuntimeError - proxy = nil # 404'd + fetch_entry(model) do |client| + client.show_proxy(model.record.service_id) end - - build_entry(model, data: proxy) end def fetch_application(model) - client = build_client(model.tenant) - - begin - application = client.show_application(model.record_id) - # right now the client raises runtime error, but rather should return a result - rescue RuntimeError - application = nil # 404'd + fetch_entry(model) do |client| + client.show_application(model.record_id) end - - build_entry(model, data: application) end def fetch_client(model) + fetch_entry(model) do |client| + client.find_application(application_id: model.record.client_id) + end + end + + def fetch_entry(model) client = build_client(model.tenant) - begin - application = client.find_application(application_id: model.record.client_id) - # right now the client raises runtime error, but rather should return a result - rescue RuntimeError - application = nil # 404'd + data = begin + yield client + rescue RuntimeError => e + Rails.logger.error(e) + nil end - build_entry(model, data: application) + build_entry(model, data: data) end def build_entry(model, **attributes) diff --git a/app/services/incoming_notification_service.rb b/app/services/incoming_notification_service.rb index b5eebd72..d5e5447e 100644 --- a/app/services/incoming_notification_service.rb +++ b/app/services/incoming_notification_service.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + # This is action that is performed on incoming notificaiton. # Its purpose is to wrap the persistence and triggering update logic. diff --git a/app/services/integration/echo_service.rb b/app/services/integration/echo_service.rb index 936e5b7f..51198a90 100644 --- a/app/services/integration/echo_service.rb +++ b/app/services/integration/echo_service.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + # Example Integration that just prints what it is doing the log. class Integration::EchoService < Integration::ServiceBase diff --git a/app/services/integration/keycloak_service.rb b/app/services/integration/keycloak_service.rb index 05c7c614..19ab1b99 100644 --- a/app/services/integration/keycloak_service.rb +++ b/app/services/integration/keycloak_service.rb @@ -39,5 +39,4 @@ def update_client(client) def persist?(client) client.secret end - end diff --git a/app/services/integration/kubernetes_service.rb b/app/services/integration/kubernetes_service.rb new file mode 100644 index 00000000..b002ece3 --- /dev/null +++ b/app/services/integration/kubernetes_service.rb @@ -0,0 +1,210 @@ +# frozen_string_literal: true + +class Integration::KubernetesService < Integration::ServiceBase + attr_reader :namespace + + def initialize(integration, namespace: self.class.namespace) + super(integration) + @namespace = namespace + @client = K8s::Client.autoconfig(namespace: namespace) + end + + def self.namespace + ENV.fetch('KUBERNETES_NAMESPACE') { File.read(File.join((ENV['TELEPRESENCE_ROOT'] || '/'), 'var/run/secrets/kubernetes.io/serviceaccount/namespace')) } + end + + def call(entry) + case entry.record + when Proxy then handle_proxy(entry) + when Provider then handle_provider(entry) + end + end + + def handle_proxy(entry) + persist_proxy?(entry) ? persist_proxy(entry) : delete_proxy(entry) + end + + def handle_provider(entry) + persist?(entry) ? persist_provider(entry) : delete_provider(entry) + end + + attr_reader :client + + def persist_proxy?(entry) + entry.data&.dig('deployment_option') == 'hosted' + end + + def persist?(entry) + entry.data + end + + def owner_reference_controller(resource) + owner_references = resource.metadata.ownerReferences or return + controller = owner_references.find(&:controller) + controller.metadata = { namespace: namespace, name: controller.name } + + client.get_resource(controller) + end + + def owner_reference_root(resource) + while (owner = owner_reference_controller(resource)) + resource = owner + end + + resource + end + + def get_owner + pod_name = ENV['KUBERNETES_POD_NAME'] || ENV['POD_NAME'] || ENV['HOSTNAME'] + + pod = client.api('v1').resource('pods', namespace: namespace).get(pod_name) + owner_reference_root(pod) + end + + def as_reference(owner) + K8s::API::MetaV1::OwnerReference.new( + kind: owner.kind, + apiVersion: owner.apiVersion, + name: owner.metadata.name, + uid: owner.metadata.uid + ) + end + + def annotations_for(entry) + { + '3scale.net/gid': entry.to_gid.to_s + } + end + + def labels_for(entry) + { + '3scale.created-by': 'zync', + '3scale.tenant_id': String(entry.tenant_id) + } + end + + def labels_for_proxy(entry) + service_id = entry.last_known_data.fetch('service_id') { return } + + labels_for(entry).merge( + '3scale.ingress': 'proxy', + '3scale.service_id': String(service_id) + ) + end + + def labels_for_provider(entry) + provider_id = entry.last_known_data.fetch('id') + + labels_for(entry).merge( + '3scale.ingress': 'provider', + '3scale.provider_id': String(provider_id) + ) + end + + class Route < K8s::Resource + def initialize(attributes, **options) + super attributes.with_indifferent_access + .merge(apiVersion: 'route.openshift.io/v1', kind: 'Route') + .reverse_merge(metadata: {}), **options + end + end + + class RouteSpec < K8s::Resource + def initialize(url, service, port) + uri = URI(url) + super({ + host: uri.host || uri.path, + port: { targetPort: port }, + tls: { + insecureEdgeTerminationPolicy: 'Redirect', + termination: 'edge' + }, + to: { + kind: 'Service', + name: service + } + }) + end + end + + def build_proxy_routes(entry) + build_routes('zync-3scale-api-', [ + RouteSpec.new(entry.data.fetch('endpoint'), 'apicast-production', 'gateway'), + RouteSpec.new(entry.data.fetch('sandbox_endpoint'), 'apicast-staging', 'gateway') + ], labels: labels_for_proxy(entry), annotations: annotations_for(entry)) + end + + def build_routes(name, specs = [], owner: get_owner, **metadata) + specs.map do |spec| + Route.new( + metadata: { + generateName: name, + namespace: namespace, + labels: owner.metadata.labels, + ownerReferences: [as_reference(owner)] + }.deep_merge(metadata), + spec: spec + ) + end + end + + def build_provider_routes(entry) + data = entry.data + domain, admin_domain = data.values_at('domain', 'admin_domain') + metadata = { labels: labels_for_provider(entry), annotations: annotations_for(entry) } + + if admin_domain == domain # master account + build_routes('zync-3scale-master-', [ + RouteSpec.new(data.fetch('domain'), 'system-master', 'http') + ], **metadata) + else + build_routes('zync-3scale-provider-', [ + RouteSpec.new(data.fetch('domain'), 'system-developer', 'http'), + RouteSpec.new(data.fetch('admin_domain'), 'system-provider', 'http') + ], **metadata) + end + end + + def cleanup_but(list, label_selector) + client + .client_for_resource(list.first, namespace: namespace) + .list(labelSelector: label_selector) + .each do |resource| + client.delete_resource(resource) unless list.include?(resource) + end + end + + protected def create_resources(list) + list.map(&client.method(:create_resource)) + end + + def persist_proxy(entry) + routes = build_proxy_routes(entry) + routes = create_resources(routes) + + label_selector = labels_for_proxy(entry) + + cleanup_but(routes, label_selector) + end + + def delete_proxy(entry) + label_selector = labels_for_proxy(entry) + + cleanup_but([Route.new({})], label_selector) + end + + def persist_provider(entry) + routes = build_provider_routes(entry) + routes = create_resources(routes) + + label_selector = labels_for_provider(entry) + + cleanup_but(routes, label_selector) + end + + def delete_provider(entry) + label_selector = labels_for_provider(entry) + + cleanup_but([Route.new({})], label_selector) + end +end diff --git a/app/services/integration/service_base.rb b/app/services/integration/service_base.rb index 365c710d..f34bb611 100644 --- a/app/services/integration/service_base.rb +++ b/app/services/integration/service_base.rb @@ -7,4 +7,8 @@ class Integration::ServiceBase def initialize(integration) @integration = integration end + + def call(_entry) + raise NoMethodError, __method__ + end end diff --git a/config/application.rb b/config/application.rb index b048a03e..3b2bb089 100644 --- a/config/application.rb +++ b/config/application.rb @@ -62,6 +62,23 @@ class Application < Rails::Application config.middleware.delete(ActionDispatch::Flash) # remove it after message bus loaded end + initializer 'k8s-client.logger' do + case config.log_level + when :debug + K8s::Logging.debug! + K8s::Transport.debug! + when :info + K8s::Logging.verbose! + K8s::Transport.verbose! + when :error + K8s::Logging.quiet! + K8s::Transport.quiet! + else + K8s::Logging.log_level = K8s::Transport.log_level = Rails.logger.level + end + end + config.x.keycloak = config_for(:keycloak) || Hash.new + config.x.openshift = ActiveSupport::InheritableOptions.new(config_for(:openshift)&.deep_symbolize_keys) end end diff --git a/config/openshift.yml b/config/openshift.yml new file mode 100644 index 00000000..b8138c37 --- /dev/null +++ b/config/openshift.yml @@ -0,0 +1,18 @@ +default: &default + enabled: <%= + ENV.values_at( + 'APICAST_STAGING_SERVICE_PORT_GATEWAY', + 'APICAST_STAGING_SERVICE_PORT_GATEWAY', + 'SYSTEM_DEVELOPER_SERVICE_PORT_HTTP', + 'SYSTEM_PROVIDER_SERVICE_PORT_HTTP', + 'SYSTEM_MASTER_SERVICE_PORT_HTTP', + ).all? + %> + +development: + <<: *default +production: + <<: *default + +test: + enabled: false diff --git a/db/migrate/20190605094424_create_providers.rb b/db/migrate/20190605094424_create_providers.rb new file mode 100644 index 00000000..40e95e85 --- /dev/null +++ b/db/migrate/20190605094424_create_providers.rb @@ -0,0 +1,9 @@ +class CreateProviders < ActiveRecord::Migration[5.2] + def change + create_table :providers do |t| + t.references :tenant, foreign_key: true + + t.timestamps + end + end +end diff --git a/db/structure.sql b/db/structure.sql index 50515214..0c9e4127 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -536,6 +536,37 @@ CREATE SEQUENCE public.notifications_id_seq ALTER SEQUENCE public.notifications_id_seq OWNED BY public.notifications.id; +-- +-- Name: providers; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.providers ( + id bigint NOT NULL, + tenant_id bigint, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +-- +-- Name: providers_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.providers_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: providers_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.providers_id_seq OWNED BY public.providers.id; + + -- -- Name: proxies; Type: TABLE; Schema: public; Owner: - -- @@ -818,6 +849,13 @@ ALTER TABLE ONLY public.models ALTER COLUMN id SET DEFAULT nextval('public.model ALTER TABLE ONLY public.notifications ALTER COLUMN id SET DEFAULT nextval('public.notifications_id_seq'::regclass); +-- +-- Name: providers id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.providers ALTER COLUMN id SET DEFAULT nextval('public.providers_id_seq'::regclass); + + -- -- Name: proxies id; Type: DEFAULT; Schema: public; Owner: - -- @@ -940,6 +978,14 @@ ALTER TABLE ONLY public.notifications ADD CONSTRAINT notifications_pkey PRIMARY KEY (id); +-- +-- Name: providers providers_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.providers + ADD CONSTRAINT providers_pkey PRIMARY KEY (id); + + -- -- Name: proxies proxies_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -1159,6 +1205,13 @@ CREATE INDEX index_notifications_on_model_id ON public.notifications USING btree CREATE INDEX index_notifications_on_tenant_id ON public.notifications USING btree (tenant_id); +-- +-- Name: index_providers_on_tenant_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_providers_on_tenant_id ON public.providers USING btree (tenant_id); + + -- -- Name: index_proxies_on_service_id; Type: INDEX; Schema: public; Owner: - -- @@ -1370,6 +1423,14 @@ ALTER TABLE ONLY public.entries ADD CONSTRAINT fk_rails_acc13c3cee FOREIGN KEY (tenant_id) REFERENCES public.tenants(id); +-- +-- Name: providers fk_rails_ba1a501ef5; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.providers + ADD CONSTRAINT fk_rails_ba1a501ef5 FOREIGN KEY (tenant_id) REFERENCES public.tenants(id); + + -- -- Name: applications fk_rails_c363b8b058; Type: FK CONSTRAINT; Schema: public; Owner: - -- @@ -1449,6 +1510,7 @@ INSERT INTO "schema_migrations" (version) VALUES ('20181019101631'), ('20190410112007'), ('20190530080459'), -('20190603140450'); +('20190603140450'), +('20190605094424'); diff --git a/test/fixtures/entries.yml b/test/fixtures/entries.yml index a741a2ed..b11f4229 100644 --- a/test/fixtures/entries.yml +++ b/test/fixtures/entries.yml @@ -34,5 +34,13 @@ proxy: data: oidc_issuer_endpoint: http://example.com/auth/realm/master oidc_issuer_type: keycloak + service_id: 2 tenant: two model: proxy + + +provider: + data: + service_id: 2 + tenant: two + model: provider diff --git a/test/fixtures/models.yml b/test/fixtures/models.yml index d049467c..ef4d2a23 100644 --- a/test/fixtures/models.yml +++ b/test/fixtures/models.yml @@ -19,3 +19,7 @@ service: proxy: tenant: two record: two (Proxy) + +provider: + tenant: two + record: two (Provider) diff --git a/test/fixtures/providers.yml b/test/fixtures/providers.yml new file mode 100644 index 00000000..1500b155 --- /dev/null +++ b/test/fixtures/providers.yml @@ -0,0 +1,7 @@ +# Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html + +one: + tenant: one + +two: + tenant: two diff --git a/test/jobs/process_entry_job_test.rb b/test/jobs/process_entry_job_test.rb index f7158047..18a50bf2 100644 --- a/test/jobs/process_entry_job_test.rb +++ b/test/jobs/process_entry_job_test.rb @@ -15,9 +15,15 @@ class ProcessEntryJobTest < ActiveJob::TestCase job = ProcessEntryJob.new proxy = entries(:proxy) - integrations = job.model_integrations_for(proxy) + ProcessEntryJob::CreateK8SIntegration.stub(:enabled, true) do + integrations = job.model_integrations_for(proxy) + + integrations.each do |integration| + assert_kind_of Integration::Kubernetes, integration + end - assert_equal 0, integrations.size + assert_equal 1, integrations.size + end end test 'model integrations for client' do diff --git a/test/models/provider_test.rb b/test/models/provider_test.rb new file mode 100644 index 00000000..252fcb9b --- /dev/null +++ b/test/models/provider_test.rb @@ -0,0 +1,7 @@ +require 'test_helper' + +class ProviderTest < ActiveSupport::TestCase + # test "the truth" do + # assert true + # end +end diff --git a/test/services/fetch_service_test.rb b/test/services/fetch_service_test.rb index 4e3f0510..f9ab90ec 100644 --- a/test/services/fetch_service_test.rb +++ b/test/services/fetch_service_test.rb @@ -17,6 +17,27 @@ def setup assert_kind_of Entry, @service.call(models(:application)) end + test 'call with Provider' do + stub_request(:get, "#{tenants(:two).endpoint}/admin/api/provider.json"). + to_return(status: 200, body: '{}', headers: {}) + + assert_kind_of Entry, @service.call(models(:provider)) + end + + test 'call with Client' do + stub_request(:get, "#{tenants(:two).endpoint}/admin/api/applications/find.json?app_id=two"). + to_return(status: 200, body: '{}', headers: {}) + + assert_kind_of Entry, @service.call(models(:client)) + end + + test 'call with Proxy' do + stub_request(:get, "https://two.example.com/admin/api/services/298486374/proxy.json"). + to_return(status: 200, body: '{}', headers: {}) + + assert_kind_of Entry, @service.call(models(:proxy)) + end + test 'call returns entry that can be saved' do @service.call(models(:service)).save! end diff --git a/test/services/kubernetes_service_test.rb b/test/services/kubernetes_service_test.rb new file mode 100644 index 00000000..f073b5f7 --- /dev/null +++ b/test/services/kubernetes_service_test.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +require 'test_helper' +require 'base64' + +class Integration::KubernetesServiceTest < ActiveSupport::TestCase + include Base64 + + def before_setup + @_env = ENV.to_hash + super + end + + def after_teardown + ENV.replace(@_env) + end + + test 'create ingress ' do + ENV['KUBERNETES_NAMESPACE'] = 'zync' + ENV['KUBE_TOKEN'] = strict_encode64('token') + ENV['KUBE_SERVER'] = 'http://localhost' + ENV['KUBE_CA'] = encode64 <<~CERTIFICATE + -----BEGIN CERTIFICATE----- + MIIBZjCCAQ2gAwIBAgIQBHMSmrmlj2QTqgFRa+HP3DAKBggqhkjOPQQDAjASMRAw + DgYDVQQDEwdyb290LWNhMB4XDTE5MDQwNDExMzI1OVoXDTI5MDQwMTExMzI1OVow + EjEQMA4GA1UEAxMHcm9vdC1jYTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABGG2 + NDgiBuXNVWVVxrDNVjPsKm14wg76w4830Zn3K24u03LJthzsB3RPJN9l+kM7ryjg + dCenDYANVabMMQEy2iGjRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG + AQH/AgEBMB0GA1UdDgQWBBRfJt1t0sAlUMBwfeTWVv2v4XNcNjAKBggqhkjOPQQD + AgNHADBEAiB+MlaTocrG33AiOE8TrH4N2gVrDBo2fAyJ1qDmjxhWvAIgPOoAoWQ9 + qwUVj52L6/Ptj0Tn4Mt6u+bdVr6jEXkZ8f0= + -----END CERTIFICATE----- + CERTIFICATE + + service = Integration::KubernetesService.new(nil) + + proxy = entries(:proxy) + + stub_request(:get, 'http://localhost/apis/route.openshift.io/v1'). + with( + headers: { + 'Accept'=>'application/json', + 'Authorization'=>'Bearer token', + }). + to_return(status: 200, body: { + kind: "APIResourceList", + apiVersion: "v1", + groupVersion: "route.openshift.io/v1", + resources: [ + { name: "routes", singularName: "", namespaced: true, kind: "Route", verbs: %w(create delete deletecollection get list patch update watch), categories: ["all"] }, + ] + }.to_json, headers: { 'Content-Type' => 'application/json' }) + + stub_request(:get, 'http://localhost/apis/route.openshift.io/v1/namespaces/zync/routes?labelSelector=3scale.created-by=zync,3scale.tenant_id=298486374,3scale.ingress=proxy,3scale.service_id=2'). + with( + headers: { + 'Accept'=>'application/json', + 'Authorization'=>'Bearer token', + }). + to_return(status: 200, body: { + kind: 'RouteList', + apiVersion: 'route.openshift.io/v1', + metadata: { selfLink: '/apis/route.openshift.io/v1/namespaces/zync/routes', resourceVersion: '651341' }, + items: [] }.to_json, headers: { 'Content-Type' => 'application/json' }) + + service.call(proxy) + end +end From 62f054e712b8961eaccb7eb0b02fc7bdad0a08a8 Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Fri, 14 Jun 2019 10:53:41 +0200 Subject: [PATCH 012/156] [gemfile] disable httplog it tries to mutate frozen strings --- Gemfile | 2 +- Gemfile.lock | 7 +------ config/environments/development.rb | 8 +++++--- config/environments/test.rb | 8 +++++--- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/Gemfile b/Gemfile index 448bff75..e4031071 100644 --- a/Gemfile +++ b/Gemfile @@ -66,7 +66,7 @@ group :development, :test do gem 'license_finder', '~> 5.8' gem 'license_finder_xml_reporter', git: 'https://github.com/3scale/license_finder_xml_reporter.git', tag: '1.0.0' - gem 'httplog' + # gem 'httplog' end # Windows does not include zoneinfo files, so bundle the tzinfo-data gem diff --git a/Gemfile.lock b/Gemfile.lock index 815f8932..d1fbec05 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -126,9 +126,6 @@ GEM globalid (0.4.2) activesupport (>= 4.2.0) hashdiff (0.3.9) - httplog (1.3.0) - rack (>= 1.0) - rainbow (>= 2.0.0) i18n (1.6.0) concurrent-ruby (~> 1.0) ice_nine (0.11.2) @@ -224,7 +221,7 @@ GEM erubis que (~> 1.0.0.beta3) sinatra - rack (2.0.6) + rack (2.0.7) rack-protection (2.0.5) rack rack-test (1.1.0) @@ -253,7 +250,6 @@ GEM method_source rake (>= 0.8.7) thor (>= 0.19.0, < 2.0) - rainbow (3.0.0) rake (12.3.2) recursive-open-struct (1.1.0) request_store (1.4.1) @@ -339,7 +335,6 @@ DEPENDENCIES bugsnag-capistrano (< 2) codecov httpclient! - httplog k8s-client (>= 0.10) license_finder (~> 5.8) license_finder_xml_reporter! diff --git a/config/environments/development.rb b/config/environments/development.rb index 3331c43b..37e7742d 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -50,7 +50,9 @@ # config.file_watcher = ActiveSupport::EventedFileUpdateChecker end -HttpLog.configure do |config| - config.enabled = true - config.color = true +if defined?(HttpLog) + HttpLog.configure do |config| + config.enabled = true + config.color = true + end end diff --git a/config/environments/test.rb b/config/environments/test.rb index b5e645d4..5bcad8fa 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -36,7 +36,9 @@ # config.action_view.raise_on_missing_translations = true end -HttpLog.configure do |config| - config.enabled = true - config.color = true +if defined?(HttpLog) + HttpLog.configure do |config| + config.enabled = true + config.color = true + end end From 2ea1b98ec0c17769b068f3ce699f4368f4d3c211 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" Date: Fri, 14 Jun 2019 10:10:15 +0000 Subject: [PATCH 013/156] Bump license_finder from 5.8.0 to 5.9.1 Bumps [license_finder](https://github.com/pivotal/LicenseFinder) from 5.8.0 to 5.9.1. - [Release notes](https://github.com/pivotal/LicenseFinder/releases) - [Changelog](https://github.com/pivotal/LicenseFinder/blob/master/CHANGELOG.md) - [Commits](https://github.com/pivotal/LicenseFinder/compare/v5.8.0...v5.9.1) Signed-off-by: dependabot-preview[bot] --- Gemfile | 2 +- Gemfile.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Gemfile b/Gemfile index e4031071..7bb81308 100644 --- a/Gemfile +++ b/Gemfile @@ -63,7 +63,7 @@ group :development, :test do gem 'pry-rescue' gem 'pry-stack_explorer' - gem 'license_finder', '~> 5.8' + gem 'license_finder', '~> 5.9' gem 'license_finder_xml_reporter', git: 'https://github.com/3scale/license_finder_xml_reporter.git', tag: '1.0.0' # gem 'httplog' diff --git a/Gemfile.lock b/Gemfile.lock index d1fbec05..740e1ebb 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -146,7 +146,7 @@ GEM yajl-ruby (~> 1.4.0) yaml-safe_load_stream (~> 0.1) key_struct (0.4.2) - license_finder (5.8.0) + license_finder (5.9.1) bundler rubyzip thor @@ -258,7 +258,7 @@ GEM actionpack (>= 4.2.0, < 6.0) railties (>= 4.2.0, < 6.0) ruby-progressbar (1.10.0) - rubyzip (1.2.2) + rubyzip (1.2.3) safe_yaml (1.0.5) schema_monkey (2.1.5) activerecord (>= 4.2) @@ -336,7 +336,7 @@ DEPENDENCIES codecov httpclient! k8s-client (>= 0.10) - license_finder (~> 5.8) + license_finder (~> 5.9) license_finder_xml_reporter! lograge message_bus From 9c4599cfac35d2ddbc38ae798366b7f55db098f5 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" Date: Fri, 14 Jun 2019 18:16:18 +0000 Subject: [PATCH 014/156] Bump baby_squeel from 1.2.0 to 1.3.1 Bumps [baby_squeel](https://github.com/rzane/baby_squeel) from 1.2.0 to 1.3.1. - [Release notes](https://github.com/rzane/baby_squeel/releases) - [Changelog](https://github.com/rzane/baby_squeel/blob/master/CHANGELOG.md) - [Commits](https://github.com/rzane/baby_squeel/compare/v1.2.0...v1.3.1) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 740e1ebb..c353df36 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -70,8 +70,9 @@ GEM public_suffix (>= 2.0.2, < 4.0) ansi (1.5.0) arel (9.0.0) - baby_squeel (1.2.0) + baby_squeel (1.3.1) activerecord (>= 4.2.0) + join_dependency (~> 0.1.2) polyamorous (~> 1.3) binding_of_caller (0.8.0) debug_inspector (>= 0.0.1) @@ -131,6 +132,8 @@ GEM ice_nine (0.11.2) interception (0.5) its-it (1.3.0) + join_dependency (0.1.4) + activerecord (>= 4.2.0) json (2.1.0) jsonpath (0.9.9) multi_json From 03d8646926456deccfd2c2761e55fd05000d639b Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Thu, 20 Jun 2019 15:07:19 +0200 Subject: [PATCH 015/156] [k8s] fix accidentally deleting just created routes --- app/services/integration/kubernetes_service.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/services/integration/kubernetes_service.rb b/app/services/integration/kubernetes_service.rb index b002ece3..845cb1fb 100644 --- a/app/services/integration/kubernetes_service.rb +++ b/app/services/integration/kubernetes_service.rb @@ -170,7 +170,8 @@ def cleanup_but(list, label_selector) .client_for_resource(list.first, namespace: namespace) .list(labelSelector: label_selector) .each do |resource| - client.delete_resource(resource) unless list.include?(resource) + equal = list.any? { |object| object.metadata.uid === resource.metadata.uid && resource.metadata.selfLink == object.metadata.selfLink } + client.delete_resource(resource) unless equal end end From a877723b5823133a75baaecb392530ecb0df08aa Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Thu, 20 Jun 2019 15:11:33 +0200 Subject: [PATCH 016/156] [gemfile] use latest bugsnag --- Gemfile | 2 +- Gemfile.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index 7bb81308..14f17a3f 100644 --- a/Gemfile +++ b/Gemfile @@ -34,7 +34,7 @@ gem 'que', '>= 1.0.0.beta3' gem 'que-web' gem 'baby_squeel' -gem 'bugsnag', github: 'bugsnag/bugsnag-ruby', branch: 'tobyhs/check_nil_job_in_que' +gem 'bugsnag', github: 'bugsnag/bugsnag-ruby', branch: 'next' # bugsnag-capistrano 2.x does not have a rake task to report deploys # https://github.com/bugsnag/bugsnag-capistrano/blob/8bcfb27cf6eaff312eef086cce729d553a431460/UPGRADING.md gem 'bugsnag-capistrano', '< 2', require: false diff --git a/Gemfile.lock b/Gemfile.lock index c353df36..81d12633 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -8,8 +8,8 @@ GIT GIT remote: https://github.com/bugsnag/bugsnag-ruby.git - revision: 00dac89430281610519a122dc2acbb41a1d2bed3 - branch: tobyhs/check_nil_job_in_que + revision: 90df9965cdb9fa95a45448e8464982f457a23d34 + branch: next specs: bugsnag (6.11.1) concurrent-ruby (~> 1.0) From 0027293796fb73703eb8b20e80fe9be7a06c308d Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Sat, 22 Jun 2019 12:53:40 +0200 Subject: [PATCH 017/156] [k8s] do not recreate route, just patch existing one --- .../integration/kubernetes_service.rb | 74 ++++++++++++++----- test/services/kubernetes_service_test.rb | 2 +- 2 files changed, 55 insertions(+), 21 deletions(-) diff --git a/app/services/integration/kubernetes_service.rb b/app/services/integration/kubernetes_service.rb index 845cb1fb..c8c7d5e2 100644 --- a/app/services/integration/kubernetes_service.rb +++ b/app/services/integration/kubernetes_service.rb @@ -6,7 +6,16 @@ class Integration::KubernetesService < Integration::ServiceBase def initialize(integration, namespace: self.class.namespace) super(integration) @namespace = namespace - @client = K8s::Client.autoconfig(namespace: namespace) + @client = K8s::Client.autoconfig(namespace: namespace).extend(MergePatch) + end + + module MergePatch + # @param resource [K8s::Resource] + # @param attrs [Hash] + # @return [K8s::Client] + def merge_resource(resource, attrs) + client_for_resource(resource).merge_patch(resource.metadata.name, attrs) + end end def self.namespace @@ -72,14 +81,22 @@ def as_reference(owner) def annotations_for(entry) { - '3scale.net/gid': entry.to_gid.to_s + '3scale.net/gid': entry.to_gid.to_s, + 'zync.3scale.net/gid': entry.model.record.to_gid.to_s, } end + def label_selector_from(resource) + resource.metadata.labels.to_h.with_indifferent_access.slice( + '3scale.net/created-by', '3scale.net/tenant_id', 'zync.3scale.net/record', 'zync.3scale.net/route-to' + ) + end + def labels_for(entry) { - '3scale.created-by': 'zync', - '3scale.tenant_id': String(entry.tenant_id) + '3scale.net/created-by': 'zync', + '3scale.net/tenant_id': String(entry.tenant_id), + 'zync.3scale.net/record': entry.model.record.to_gid_param, } end @@ -87,8 +104,8 @@ def labels_for_proxy(entry) service_id = entry.last_known_data.fetch('service_id') { return } labels_for(entry).merge( - '3scale.ingress': 'proxy', - '3scale.service_id': String(service_id) + 'zync.3scale.net/ingress': 'proxy', + '3scale.net/service_id': String(service_id) ) end @@ -96,8 +113,8 @@ def labels_for_provider(entry) provider_id = entry.last_known_data.fetch('id') labels_for(entry).merge( - '3scale.ingress': 'provider', - '3scale.provider_id': String(provider_id) + 'zync.3scale.net/ingress': 'provider', + '3scale.net/provider_id': String(provider_id) ) end @@ -142,7 +159,7 @@ def build_routes(name, specs = [], owner: get_owner, **metadata) namespace: namespace, labels: owner.metadata.labels, ownerReferences: [as_reference(owner)] - }.deep_merge(metadata), + }.deep_merge(metadata.deep_merge(labels: { 'zync.3scale.net/route-to': spec.to_h.dig(:to, :name) })), spec: spec ) end @@ -170,22 +187,42 @@ def cleanup_but(list, label_selector) .client_for_resource(list.first, namespace: namespace) .list(labelSelector: label_selector) .each do |resource| - equal = list.any? { |object| object.metadata.uid === resource.metadata.uid && resource.metadata.selfLink == object.metadata.selfLink } + equal = list.any? { |object| object.metadata.uid === resource.metadata.uid && resource.metadata.selfLink == object.metadata.selfLink } + Rails.logger.warn "Deleting #{resource.metadata} from k8s because it is not on #{list}" + client.delete_resource(resource) unless equal end end - protected def create_resources(list) - list.map(&client.method(:create_resource)) + def extract_route_patch(resource) + { + metadata: resource.metadata.to_h, + spec: { host: resource.spec.host }, + } + end + + protected def persist_resources(list) + list.map do |resource| + existing = client + .client_for_resource(resource, namespace: namespace) + .list(labelSelector: label_selector_from(resource)) + + case existing.size + when 0 + client.create_resource(resource) + when 1 + client.merge_resource(existing.first, extract_route_patch(resource)) + else + existing.each(&client.method(:delete_resource)) + client.create_resource(resource) + end + end end def persist_proxy(entry) routes = build_proxy_routes(entry) - routes = create_resources(routes) - label_selector = labels_for_proxy(entry) - - cleanup_but(routes, label_selector) + persist_resources(routes) end def delete_proxy(entry) @@ -196,11 +233,8 @@ def delete_proxy(entry) def persist_provider(entry) routes = build_provider_routes(entry) - routes = create_resources(routes) - - label_selector = labels_for_provider(entry) - cleanup_but(routes, label_selector) + persist_resources(routes) end def delete_provider(entry) diff --git a/test/services/kubernetes_service_test.rb b/test/services/kubernetes_service_test.rb index f073b5f7..7cc30d04 100644 --- a/test/services/kubernetes_service_test.rb +++ b/test/services/kubernetes_service_test.rb @@ -51,7 +51,7 @@ def after_teardown ] }.to_json, headers: { 'Content-Type' => 'application/json' }) - stub_request(:get, 'http://localhost/apis/route.openshift.io/v1/namespaces/zync/routes?labelSelector=3scale.created-by=zync,3scale.tenant_id=298486374,3scale.ingress=proxy,3scale.service_id=2'). + stub_request(:get, 'http://localhost/apis/route.openshift.io/v1/namespaces/zync/routes?labelSelector=3scale.net/created-by=zync,3scale.net/tenant_id=298486374,zync.3scale.net/record=Z2lkOi8venluYy9Qcm94eS8yOTg0ODYzNzQ,zync.3scale.net/ingress=proxy,3scale.net/service_id=2'). with( headers: { 'Accept'=>'application/json', From f7414f67a88b83cd02354b0d60825deb16e88197 Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Wed, 26 Jun 2019 16:12:17 +0200 Subject: [PATCH 018/156] fix uri parse error looks like httpclient behaves differently when 'addressable' is available and webmock depends on it, but productized image does not have test only dependencies like that --- Gemfile | 1 + Gemfile.lock | 2 ++ app/adapters/abstract_adapter.rb | 2 +- test/adapters/abstract_adapter_test.rb | 6 ++++++ 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 14f17a3f..a1917783 100644 --- a/Gemfile +++ b/Gemfile @@ -74,6 +74,7 @@ gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby] group :test do gem 'minitest-reporters' + gem 'minitest-stub-const' gem 'webmock' gem 'codecov', require: false end diff --git a/Gemfile.lock b/Gemfile.lock index 81d12633..59a9fbea 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -180,6 +180,7 @@ GEM builder minitest (>= 5.0) ruby-progressbar + minitest-stub-const (0.5) modware (0.1.3) key_struct (~> 0.4) msgpack (1.2.10) @@ -344,6 +345,7 @@ DEPENDENCIES lograge message_bus minitest-reporters + minitest-stub-const oauth2 pg (>= 0.20) prometheus-client diff --git a/app/adapters/abstract_adapter.rb b/app/adapters/abstract_adapter.rb index eb49091b..b851f6c8 100644 --- a/app/adapters/abstract_adapter.rb +++ b/app/adapters/abstract_adapter.rb @@ -67,7 +67,7 @@ def build_http_client(endpoint) HTTPClient.new do self.debug_dev = $stderr if ENV.fetch('DEBUG', '0') == '1' - self.set_auth endpoint, *endpoint.auth + self.set_auth endpoint.uri.dup, *endpoint.auth Rails.application.config.x.http_client.deep_symbolize_keys .slice(:connect_timeout, :send_timeout, :receive_timeout).each do |key, value| diff --git a/test/adapters/abstract_adapter_test.rb b/test/adapters/abstract_adapter_test.rb index a56f2f9b..66b8e7db 100644 --- a/test/adapters/abstract_adapter_test.rb +++ b/test/adapters/abstract_adapter_test.rb @@ -23,4 +23,10 @@ class AbstractAdapterTest < ActiveSupport::TestCase assert_equal uri, subject.new('http://id:secret@lvh.me:3000/auth/realm/name/').endpoint end + + test 'http_client' do + HTTPClient::Util.stub_const(:AddressableEnabled, false) do + assert_kind_of subject, subject.new('http://id:secret@example.com') + end + end end From 42d41a51d71d498b3a96422b4d4f6fec2189c1ac Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Tue, 2 Jul 2019 12:58:24 +0200 Subject: [PATCH 019/156] [test] fix occasionally deadlocked test --- .../incoming_notification_service_test.rb | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/test/services/incoming_notification_service_test.rb b/test/services/incoming_notification_service_test.rb index 16a4f106..29dac322 100644 --- a/test/services/incoming_notification_service_test.rb +++ b/test/services/incoming_notification_service_test.rb @@ -37,41 +37,37 @@ class LockingTest < ActiveSupport::TestCase teardown do ::Que.clear! + ActiveRecord::Base.connection_pool.disconnect! end def test_process_locked_model notification = notifications(:two) - queue = SizedQueue.new(1) + fiber = Fiber.new do + first = Model.connection_pool.checkout + first.transaction(requires_new: true) do + first.execute('SET SESSION statement_timeout TO 100;') - Model.connection.execute('SET SESSION statement_timeout TO 100;') - - runner = Thread.new do - Model.connection_pool.with_connection do |connection| - connection.transaction do - connection.execute('SET SESSION statement_timeout TO 100;') + second = Model.connection_pool.checkout + second.transaction(requires_new: true) do + second.execute('SET SESSION statement_timeout TO 100;') model = Model.find(notification.model_id) UpdateState.acquire_lock(model) do |state| - queue.push state - queue.push model.touch - queue.push true + model.touch + Fiber.yield state end - end end end - runner.abort_on_exception = true - assert_kind_of UpdateState, queue.pop + assert_kind_of UpdateState, fiber.resume UpdateJob.stub(:perform_later, nil) do assert IncomingNotificationService.call(notification.dup) end - ensure - runner.kill - runner.join + assert_nil fiber.resume end end end From ec37abebb59c483c6f604f75aa29043c0e01becf Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Mon, 17 Jun 2019 10:09:38 +0200 Subject: [PATCH 020/156] [rest-api] allow invalid OIDC configuration --- app/adapters/abstract_adapter.rb | 2 +- app/adapters/rest_adapter.rb | 6 ++++++ test/adapters/abstract_adapter_test.rb | 9 +++++++++ test/adapters/rest_adapter_test.rb | 13 +++++++++++++ 4 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 test/adapters/rest_adapter_test.rb diff --git a/app/adapters/abstract_adapter.rb b/app/adapters/abstract_adapter.rb index b851f6c8..628f7e8e 100644 --- a/app/adapters/abstract_adapter.rb +++ b/app/adapters/abstract_adapter.rb @@ -203,7 +203,7 @@ def oauth_client def fetch_oidc_discovery response = http_client.get(well_known_url) - config = AbstractAdapter.parse_response(response) + config = response.ok? && AbstractAdapter.parse_response(response) case config when ->(obj) { obj.respond_to?(:[]) } then config diff --git a/app/adapters/rest_adapter.rb b/app/adapters/rest_adapter.rb index 22ac20e2..eacfbdfb 100644 --- a/app/adapters/rest_adapter.rb +++ b/app/adapters/rest_adapter.rb @@ -31,6 +31,12 @@ def test parse http_client.get(oidc.well_known_url, header: headers) end + def authentication + super + rescue OIDC::AuthenticationError + nil + end + # The Client entity. Mapping the OpenID Connect Client Metadata representation. # https://tools.ietf.org/html/rfc7591#section-2 class Client diff --git a/test/adapters/abstract_adapter_test.rb b/test/adapters/abstract_adapter_test.rb index 66b8e7db..e00332a0 100644 --- a/test/adapters/abstract_adapter_test.rb +++ b/test/adapters/abstract_adapter_test.rb @@ -29,4 +29,13 @@ class AbstractAdapterTest < ActiveSupport::TestCase assert_kind_of subject, subject.new('http://id:secret@example.com') end end + + test 'oidc discovery' do + stub_request(:get, "https://example.com/.well-known/openid-configuration"). + to_return(status: 404, body: '', headers: {}) + + assert_raises AbstractAdapter::OIDC::AuthenticationError do + assert_nil subject.new('https://example.com').authentication + end + end end diff --git a/test/adapters/rest_adapter_test.rb b/test/adapters/rest_adapter_test.rb new file mode 100644 index 00000000..be130bd8 --- /dev/null +++ b/test/adapters/rest_adapter_test.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true +require 'test_helper' + +class RESTAdapterTest < ActiveSupport::TestCase + class_attribute :subject, default: RESTAdapter + + test 'oidc discovery' do + stub_request(:get, "https://example.com/.well-known/openid-configuration"). + to_return(status: 404, body: '', headers: {}) + + assert_nil subject.new('https://example.com').authentication + end +end From b8f8e6effa220c98a1f7d66e9a4956ba9918be0b Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Mon, 17 Jun 2019 10:44:25 +0200 Subject: [PATCH 021/156] [rest-api] allow basic auth --- test/adapters/rest_adapter_test.rb | 34 ++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/test/adapters/rest_adapter_test.rb b/test/adapters/rest_adapter_test.rb index be130bd8..fa91acfc 100644 --- a/test/adapters/rest_adapter_test.rb +++ b/test/adapters/rest_adapter_test.rb @@ -10,4 +10,38 @@ class RESTAdapterTest < ActiveSupport::TestCase assert_nil subject.new('https://example.com').authentication end + + test 'create client without auth' do + client = RESTAdapter::Client.new(id: 'foo') + + stub_request(:get, "https://example.com/.well-known/openid-configuration"). + to_return(status: 404, body: '', headers: {}) + + stub_request(:put, "https://example.com/clients/foo"). + with( + body: client.to_json, + headers: { 'Content-Type'=>'application/json' }). + to_return(status: 200, body: { status: 'ok' }.to_json, headers: { 'Content-Type' => 'application/json' }) + + assert subject.new('https://example.com').create_client(client) + end + + test 'create client with basic auth' do + client = RESTAdapter::Client.new(id: 'foo') + adapter = subject.new('https://user:pass@example.com') + # WebMock does not support request retries on 401 status + adapter.send(:http_client).force_basic_auth = true + + stub_request(:get, "https://example.com/.well-known/openid-configuration"). + to_return(status: 404, body: '', headers: {}) + + stub_request(:put, 'https://example.com/clients/foo'). + with( + basic_auth: %w[user pass], + body: client.to_json, + headers: { 'Content-Type'=>'application/json' }). + to_return(status: 200, body: { status: 'ok' }.to_json, headers: { 'Content-Type' => 'application/json' }) + + assert adapter.create_client(client) + end end From cfdf349dfba26bac0da8e7cc688e796303a53c89 Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Mon, 17 Jun 2019 11:36:10 +0200 Subject: [PATCH 022/156] [rest-api] example of REST API integration --- examples/rest-api/.gitignore | 1 + examples/rest-api/Gemfile | 6 ++ examples/rest-api/Gemfile.lock | 32 ++++++++ examples/rest-api/README.md | 58 +++++++++++++ examples/rest-api/app.rb | 54 ++++++++++++ examples/rest-api/config.ru | 5 ++ examples/rest-api/openapi.yml | 146 +++++++++++++++++++++++++++++++++ 7 files changed, 302 insertions(+) create mode 100644 examples/rest-api/.gitignore create mode 100644 examples/rest-api/Gemfile create mode 100644 examples/rest-api/Gemfile.lock create mode 100644 examples/rest-api/README.md create mode 100644 examples/rest-api/app.rb create mode 100644 examples/rest-api/config.ru create mode 100644 examples/rest-api/openapi.yml diff --git a/examples/rest-api/.gitignore b/examples/rest-api/.gitignore new file mode 100644 index 00000000..79f33806 --- /dev/null +++ b/examples/rest-api/.gitignore @@ -0,0 +1 @@ +clients.yml diff --git a/examples/rest-api/Gemfile b/examples/rest-api/Gemfile new file mode 100644 index 00000000..c7a4cfbb --- /dev/null +++ b/examples/rest-api/Gemfile @@ -0,0 +1,6 @@ +# frozen_string_literal: true +source 'https://rubygems.org' + +gem 'sinatra' + +gem 'pry-byebug' diff --git a/examples/rest-api/Gemfile.lock b/examples/rest-api/Gemfile.lock new file mode 100644 index 00000000..62eff627 --- /dev/null +++ b/examples/rest-api/Gemfile.lock @@ -0,0 +1,32 @@ +GEM + remote: https://rubygems.org/ + specs: + byebug (11.0.1) + coderay (1.1.2) + method_source (0.9.2) + mustermann (1.0.3) + pry (0.12.2) + coderay (~> 1.1.0) + method_source (~> 0.9.0) + pry-byebug (3.7.0) + byebug (~> 11.0) + pry (~> 0.10) + rack (2.0.7) + rack-protection (2.0.5) + rack + sinatra (2.0.5) + mustermann (~> 1.0) + rack (~> 2.0) + rack-protection (= 2.0.5) + tilt (~> 2.0) + tilt (2.0.9) + +PLATFORMS + ruby + +DEPENDENCIES + pry-byebug + sinatra + +BUNDLED WITH + 2.0.1 diff --git a/examples/rest-api/README.md b/examples/rest-api/README.md new file mode 100644 index 00000000..a85f2fe9 --- /dev/null +++ b/examples/rest-api/README.md @@ -0,0 +1,58 @@ +# Zync REST API example + +This example project implements Zync's REST API protocol to synchronize OAuth2 clients. + +## Prerequisites + +Given 3scale API is configured to use: + * OpenID Connect as the Authentication, + * "REST API" as a OpenID Connect Issuer Type and + * "http://id:secret@example.com/api" as OpenID Connect Issuer. + +When a 3scale application is created/updated/deleted zync will try to replay that change to "http://example.com/api". + +## Creating, updating and deleting Clients + +Zync will make following requests to create/update/delete clients: + +* `PUT /clients/:client_id` (create, update) +* `DELETE /clients/:client_id` (delete) + +All endpoints must reply 2xx status code. Otherwise the request will be retried. + +### Payload + +The request payload in case of create and update is `application/json`: + +```json +{ + "client_id": "ee305610", + "client_secret": "ac0e42db426b4377096c6590e2b06aed", + "client_name": "oidc-app", + "redirect_uris": ["http://example.com"], + "grant_types": ["client_credentials", "password"] +} +``` + +The request to delete a client has no payload. + +## Using OAuth2 authentication + +Zync will make GET request to `/.well-known/openid-configuration` endpoint and expect an `application/json` response. +The response payload should contain following: + +```json +{ + "token_endpoint": "http://idp.example.com/auth/realm/token" +} +``` + +Zync will use that `token_endpoint` URL to exchange the client_id and client_secret provided in the OpenID Connect Issuer URL +for an access token using the OAuth2 protocol. + +If the API responds with not successful response, Zync will fallback to HTTP Basic/Digest authentication using provided credentials. + +## References + +* OpenAPI Specification document [openapi.yml](openapi.yml) +* Sinatra application [app.rb](app.rb) diff --git a/examples/rest-api/app.rb b/examples/rest-api/app.rb new file mode 100644 index 00000000..8299d005 --- /dev/null +++ b/examples/rest-api/app.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require 'sinatra' +require 'json' +require 'yaml/store' + +$store = YAML::Store.new('clients.yml') +$basic_auth = Rack::Auth::Basic.new(->(_) { [] }, 'REST API') do |username, password| + username.length > 0 && password.length > 0 +end + +def json(object) + headers 'Content-Type' => 'application/json' + body JSON(object) +end + +get '/.well-known/openid-configuration' do + # point zync where to exchange the OAuth2 access token + json({ token_endpoint: 'https://example.com/auth/realms/master/protocol/openid-connect/token' }) +end + +put '/clients/:client_id' do |client_id| + # {"client_id"=>"ee305610", + # "client_secret"=>"ac0e42db426b4377096c6590e2b06aed", + # "client_name"=>"oidc-app", + # "redirect_uris"=>["http://example.com"], + # "grant_types"=>["client_credentials", "password"]} + client = JSON.parse(request.body.read) + + # store the client + $store.transaction do + $store[client_id] = client + end + + json(client) +end + +delete '/clients/:client_id' do |client_id| + # Request HTTP Basic authentication + if (status, headers, body = $basic_auth.call(env)) + self.headers headers + error status, body + end + + client = nil + + # remove the client + $store.transaction do + client = $store[client_id] + $store.delete(client_id) + end + + json(client) +end diff --git a/examples/rest-api/config.ru b/examples/rest-api/config.ru new file mode 100644 index 00000000..221725af --- /dev/null +++ b/examples/rest-api/config.ru @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +require_relative 'app' + +run Sinatra::Application diff --git a/examples/rest-api/openapi.yml b/examples/rest-api/openapi.yml new file mode 100644 index 00000000..40f1c3b1 --- /dev/null +++ b/examples/rest-api/openapi.yml @@ -0,0 +1,146 @@ +--- +openapi: 3.0.2 +info: + title: Zync REST API + version: 1.0.0 +paths: + /clients/{clientId}: + get: + summary: Get a Client + operationId: readClient + parameters: + - name: clientId + in: path + description: client_id + required: true + schema: + type: string + responses: + 200: + description: Client resource was found. + content: + application/json: + schema: + $ref: '#/components/schemas/Client' + security: + - OIDC: [] + Basic: [] + Digest: [] + put: + summary: Create or update the Client + operationId: saveClient + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/Client' + examples: + Client: + value: + client_id: foo-bar + client_secret: some-secret + required: true + responses: + 200: + description: When the Client was updated. + content: + application/json: + schema: + $ref: '#/components/schemas/Client' + 201: + description: When the Client was created on the IDP. + content: + application/json: + schema: + $ref: '#/components/schemas/Client' + delete: + summary: Delete the Client + operationId: deleteClient + responses: + 200: + description: When the client was deleted. + content: + application/json: + schema: + $ref: '#/components/schemas/Client' + 204: + description: When the Client was already gone from the IDP. + content: + application/json: {} + parameters: + - name: clientId + in: path + required: true + /.well-known/openid-configuration: + get: + responses: + 200: + description: Enable OAuth2.0 authentication by responding with a token endpoint + of an IDP. + content: + application/json: + schema: + $ref: '#/components/schemas/OIDC' + examples: + Enable OAuth2.0 authentication: + value: + token_endpoint: https://idp.example.com/auth/realms/myrealm + security: + - {} +components: + schemas: + Client: + title: Root Type for Client + description: A Client representation. + required: [] + type: object + properties: + client_id: + type: string + client_secret: + type: string + client_name: + type: string + redirect_uris: + description: A list of allowed redirect uris. + type: array + items: + type: string + grant_types: + description: A list of allowed grant types. + type: array + items: + type: string + example: |- + { + "client_id": "foo-bar", + "client_secret": "some-secret" + } + OIDC: + title: Root Type for OIDC + description: OpenID Connect Configuration to define where to get access token. + type: object + properties: + token_endpoint: + type: string + example: |- + { + "token_endpoint": "https://idp.example.com/auth/realms/myrealm" + } + securitySchemes: + OIDC: + type: openIdConnect + description: |- + Use OpenID Connect for authentication. + Zync will try to access `/.well-known/openid-configuration` and use "token_endpoint" property from the JSON response. + Then it will exchange its' credentials for an access token and will use that access token to access this API. + Basic: + type: http + description: Zync will try to send provided credentials as HTTP Basic authentication + in case it gets a 401 response with proper WWW-Authenticate header. + scheme: basic + Digest: + type: http + description: Zync will try to send provided credentials as HTTP Basic authentication + in case it gets a 401 response with proper WWW-Authenticate header. + scheme: digest From f335d0030350f7b19868e33dabf6ef44608f3fb1 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 2 Jul 2019 11:08:09 +0000 Subject: [PATCH 023/156] Bump responders from 2.4.1 to 3.0.0 Bumps responders from 2.4.1 to 3.0.0. Signed-off-by: dependabot-preview[bot] --- Gemfile | 2 +- Gemfile.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Gemfile b/Gemfile index a1917783..bd13ec48 100644 --- a/Gemfile +++ b/Gemfile @@ -25,7 +25,7 @@ gem 'puma', '~> 3.12' # Use Rack CORS for handling Cross-Origin Resource Sharing (CORS), making cross-origin AJAX possible # gem 'rack-cors' -gem 'responders', '~> 2.4.1' +gem 'responders', '~> 3.0.0' gem '3scale-api' gem 'bootsnap' diff --git a/Gemfile.lock b/Gemfile.lock index 59a9fbea..b883a448 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -258,9 +258,9 @@ GEM recursive-open-struct (1.1.0) request_store (1.4.1) rack (>= 1.4) - responders (2.4.1) - actionpack (>= 4.2.0, < 6.0) - railties (>= 4.2.0, < 6.0) + responders (3.0.0) + actionpack (>= 5.0) + railties (>= 5.0) ruby-progressbar (1.10.0) rubyzip (1.2.3) safe_yaml (1.0.5) @@ -357,7 +357,7 @@ DEPENDENCIES que (>= 1.0.0.beta3) que-web rails (~> 5.2.3) - responders (~> 2.4.1) + responders (~> 3.0.0) schema_plus_enums tzinfo-data validate_url From aa69b62fc9b72054cc57c8faf0d51a53792b073b Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 2 Jul 2019 11:10:27 +0000 Subject: [PATCH 024/156] Bump puma from 3.12.1 to 4.0.0 Bumps [puma](https://github.com/puma/puma) from 3.12.1 to 4.0.0. - [Release notes](https://github.com/puma/puma/releases) - [Changelog](https://github.com/puma/puma/blob/master/History.md) - [Commits](https://github.com/puma/puma/compare/v3.12.1...v4.0.0) Signed-off-by: dependabot-preview[bot] --- Gemfile | 2 +- Gemfile.lock | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index bd13ec48..a45278db 100644 --- a/Gemfile +++ b/Gemfile @@ -13,7 +13,7 @@ gem 'pg', '>= 0.20' gem 'schema_plus_enums' # Use Puma as the app server -gem 'puma', '~> 3.12' +gem 'puma', '~> 4.0' # Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder # gem 'jbuilder', '~> 2.5' # Use ActiveModel has_secure_password diff --git a/Gemfile.lock b/Gemfile.lock index b883a448..cdc9d35a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -218,7 +218,8 @@ GEM binding_of_caller (>= 0.7) pry (>= 0.9.11) public_suffix (3.1.0) - puma (3.12.1) + puma (4.0.0) + nio4r (~> 2.0) quantile (0.2.1) que (1.0.0.beta3) que-web (0.8.0) @@ -353,7 +354,7 @@ DEPENDENCIES pry-rails pry-rescue pry-stack_explorer - puma (~> 3.12) + puma (~> 4.0) que (>= 1.0.0.beta3) que-web rails (~> 5.2.3) From e9a969e49e8f5908a2f9ac78396224e5397e762a Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 2 Jul 2019 11:12:43 +0000 Subject: [PATCH 025/156] Bump 3scale-api from 0.5.0 to 0.6.0 Bumps [3scale-api](https://github.com/3scale/3scale-api-ruby) from 0.5.0 to 0.6.0. - [Release notes](https://github.com/3scale/3scale-api-ruby/releases) - [Changelog](https://github.com/3scale/3scale-api-ruby/blob/master/CHANGELOG.md) - [Commits](https://github.com/3scale/3scale-api-ruby/compare/v0.5.0...v0.6.0) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index cdc9d35a..2ea4d480 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -24,7 +24,7 @@ GIT GEM remote: https://rubygems.org/ specs: - 3scale-api (0.5.0) + 3scale-api (0.6.0) actioncable (5.2.3) actionpack (= 5.2.3) nio4r (~> 2.0) From e159fe3dfd450a75a794dada4c38bea58973c883 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 2 Jul 2019 11:15:06 +0000 Subject: [PATCH 026/156] Bump minitest-stub-const from 0.5 to 0.6 Bumps [minitest-stub-const](https://github.com/adammck/minitest-stub-const) from 0.5 to 0.6. - [Release notes](https://github.com/adammck/minitest-stub-const/releases) - [Commits](https://github.com/adammck/minitest-stub-const/compare/v0.5...v0.6) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 2ea4d480..67f71fba 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -180,7 +180,7 @@ GEM builder minitest (>= 5.0) ruby-progressbar - minitest-stub-const (0.5) + minitest-stub-const (0.6) modware (0.1.3) key_struct (~> 0.4) msgpack (1.2.10) From 9a01a91644f303f6a90d47939f5e0871a12767b3 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 2 Jul 2019 11:17:25 +0000 Subject: [PATCH 027/156] Bump k8s-client from 0.10.1 to 0.10.2 Bumps [k8s-client](https://github.com/kontena/k8s-client) from 0.10.1 to 0.10.2. - [Release notes](https://github.com/kontena/k8s-client/releases) - [Commits](https://github.com/kontena/k8s-client/compare/v0.10.1...v0.10.2) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 67f71fba..b2ef8b57 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -98,7 +98,7 @@ GEM dry-container (0.7.1) concurrent-ruby (~> 1.0) dry-configurable (~> 0.1, >= 0.1.3) - dry-core (0.4.7) + dry-core (0.4.8) concurrent-ruby (~> 1.0) dry-equalizer (0.2.2) dry-inflector (0.1.2) @@ -121,7 +121,7 @@ GEM dry-logic (~> 0.4, >= 0.4.2) erubi (1.8.0) erubis (2.7.0) - excon (0.62.0) + excon (0.64.0) faraday (0.15.3) multipart-post (>= 1.2, < 3) globalid (0.4.2) @@ -139,10 +139,10 @@ GEM multi_json to_regexp (~> 0.2.1) jwt (2.1.0) - k8s-client (0.10.1) + k8s-client (0.10.2) dry-struct (~> 0.5.0) dry-types (~> 0.13.0) - excon (~> 0.62.0) + excon (~> 0.64.0) hashdiff (~> 0.3.7) jsonpath (~> 0.9.5) recursive-open-struct (~> 1.1.0) From 3eccbf532b2626baa41d69f8f8a3e2af80879fdb Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 2 Jul 2019 20:41:42 +0000 Subject: [PATCH 028/156] Bump license_finder from 5.9.1 to 5.9.2 Bumps [license_finder](https://github.com/pivotal/LicenseFinder) from 5.9.1 to 5.9.2. - [Release notes](https://github.com/pivotal/LicenseFinder/releases) - [Changelog](https://github.com/pivotal/LicenseFinder/blob/master/CHANGELOG.md) - [Commits](https://github.com/pivotal/LicenseFinder/compare/v5.9.1...v5.9.2) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index b2ef8b57..2eebbfdc 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -149,7 +149,7 @@ GEM yajl-ruby (~> 1.4.0) yaml-safe_load_stream (~> 0.1) key_struct (0.4.2) - license_finder (5.9.1) + license_finder (5.9.2) bundler rubyzip thor From 54c6882d00e1096cc3da70ce79396a2468461200 Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Wed, 10 Jul 2019 13:30:12 +0200 Subject: [PATCH 029/156] [k8s] ignore owner references that zync has no permission to --- app/services/integration/kubernetes_service.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/services/integration/kubernetes_service.rb b/app/services/integration/kubernetes_service.rb index c8c7d5e2..3a7bf3d5 100644 --- a/app/services/integration/kubernetes_service.rb +++ b/app/services/integration/kubernetes_service.rb @@ -60,6 +60,9 @@ def owner_reference_root(resource) resource = owner end + resource + rescue K8s::Error::Forbidden + # likely some resource like the operator resource end From ba9a491aad4894eaffc4b2ccf593deff64093c80 Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Wed, 17 Jul 2019 17:59:54 +0200 Subject: [PATCH 030/156] [k8s] keep existing Route objects and recreate if needed --- .../integration/kubernetes_service.rb | 54 ++++++++++++++++--- 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/app/services/integration/kubernetes_service.rb b/app/services/integration/kubernetes_service.rb index 3a7bf3d5..36ed8b30 100644 --- a/app/services/integration/kubernetes_service.rb +++ b/app/services/integration/kubernetes_service.rb @@ -160,9 +160,16 @@ def build_routes(name, specs = [], owner: get_owner, **metadata) metadata: { generateName: name, namespace: namespace, - labels: owner.metadata.labels, + labels: owner.metadata.label, ownerReferences: [as_reference(owner)] - }.deep_merge(metadata.deep_merge(labels: { 'zync.3scale.net/route-to': spec.to_h.dig(:to, :name) })), + }.deep_merge(metadata.deep_merge( + labels: { + 'zync.3scale.net/route-to': spec.to_h.dig(:to, :name), + }, + annotations: { + 'zync.3scale.net/host': spec.host, + } + )), spec: spec ) end @@ -210,11 +217,11 @@ def extract_route_patch(resource) .client_for_resource(resource, namespace: namespace) .list(labelSelector: label_selector_from(resource)) - case existing.size + client.get_resource case existing.size when 0 client.create_resource(resource) when 1 - client.merge_resource(existing.first, extract_route_patch(resource)) + update_resource(existing.first, resource) else existing.each(&client.method(:delete_resource)) client.create_resource(resource) @@ -222,10 +229,45 @@ def extract_route_patch(resource) end end + def cleanup_routes(routes) + routes.each do |route| + begin + verify_route_status(route) + rescue InvalidStatus => error + # they need to be re-created anyway, OpenShift won't re-admit them + client.delete_resource(route) if error.reason == 'HostAlreadyClaimed' && error.type == 'Admitted' + raise + end + end + end + + class InvalidStatus < StandardError + attr_reader :type, :reason + + def initialize(condition) + @type, @reason = condition.type, condition.reason + super(condition.message) + end + end + + def verify_route_status(route) + ingress = route.status.ingress.find { |ingress| ingress.host == route.spec.host } + condition = ingress.conditions.find { |condition| condition.type = 'Admitted' } + + raise InvalidStatus, condition unless condition.status == 'True' + end + + def update_resource(existing, resource) + client.merge_resource(existing, resource) + rescue K8s::Error::Invalid + client.delete_resource(existing) + client.create_resource(resource) + end + def persist_proxy(entry) routes = build_proxy_routes(entry) - persist_resources(routes) + cleanup_routes persist_resources(routes) end def delete_proxy(entry) @@ -237,7 +279,7 @@ def delete_proxy(entry) def persist_provider(entry) routes = build_provider_routes(entry) - persist_resources(routes) + cleanup_routes persist_resources(routes) end def delete_provider(entry) From 3a811c448f2a5b6bfa50dc5046986d6e78c23015 Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Wed, 17 Jul 2019 18:00:22 +0200 Subject: [PATCH 031/156] [k8s] option to keep TLS setting between Route object changes --- app/services/integration/kubernetes_service.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app/services/integration/kubernetes_service.rb b/app/services/integration/kubernetes_service.rb index 36ed8b30..cb25d3f2 100644 --- a/app/services/integration/kubernetes_service.rb +++ b/app/services/integration/kubernetes_service.rb @@ -3,6 +3,9 @@ class Integration::KubernetesService < Integration::ServiceBase attr_reader :namespace + class_attribute :maintain_tls_spec, + default: ActiveModel::Type::Boolean.new.cast(ENV['KUBERNETES_ROUTE_TLS']) + def initialize(integration, namespace: self.class.namespace) super(integration) @namespace = namespace @@ -160,7 +163,7 @@ def build_routes(name, specs = [], owner: get_owner, **metadata) metadata: { generateName: name, namespace: namespace, - labels: owner.metadata.label, + labels: owner.metadata.labels, ownerReferences: [as_reference(owner)] }.deep_merge(metadata.deep_merge( labels: { @@ -258,8 +261,11 @@ def verify_route_status(route) end def update_resource(existing, resource) + resource.spec.delete_field(:tls) if maintain_tls_spec? + client.merge_resource(existing, resource) rescue K8s::Error::Invalid + resource.spec.tls = existing.spec.tls if maintain_tls_spec? client.delete_resource(existing) client.create_resource(resource) end From 81370640bd74472f6384b98886f7b93a536ddea9 Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Mon, 5 Aug 2019 13:49:30 +0200 Subject: [PATCH 032/156] [rest] test OAuth authentication --- test/adapters/rest_adapter_test.rb | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/adapters/rest_adapter_test.rb b/test/adapters/rest_adapter_test.rb index fa91acfc..55b178a4 100644 --- a/test/adapters/rest_adapter_test.rb +++ b/test/adapters/rest_adapter_test.rb @@ -11,6 +11,26 @@ class RESTAdapterTest < ActiveSupport::TestCase assert_nil subject.new('https://example.com').authentication end + test 'create client with OAuth auth' do + stub_request(:get, "https://example.com/.well-known/openid-configuration"). + to_return(status: 200, body: { token_endpoint: 'http://auth.example.com/oauth/token' }.to_json, headers: { 'Content-Type' => 'application/json' }) + + stub_request(:post, "http://auth.example.com/oauth/token"). + with( + body: { client_id: "id", client_secret: 'secret', grant_type: "client_credentials" }, + headers: { 'Content-Type'=>'application/x-www-form-urlencoded' }). + to_return(status: 200, body: "access_token=token-value", headers: { 'Content-Type'=>'application/x-www-form-urlencoded' }) + + client = RESTAdapter::Client.new(id: 'foo') + stub_request(:put, "https://example.com/clients/foo"). + with( + body: client.to_json, + headers: { 'Content-Type'=>'application/json', 'Authorization' => 'Bearer token-value' }). + to_return(status: 200, body: { status: 'ok' }.to_json, headers: { 'Content-Type' => 'application/json' }) + + assert subject.new('https://id:secret@example.com').create_client(client) + end + test 'create client without auth' do client = RESTAdapter::Client.new(id: 'foo') From 1173ed16061736831141427f9ae04191492d7a2f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sun, 11 Aug 2019 21:24:07 +0000 Subject: [PATCH 033/156] [Security] Bump nokogiri from 1.10.3 to 1.10.4 Bumps [nokogiri](https://github.com/sparklemotion/nokogiri) from 1.10.3 to 1.10.4. **This update includes a security fix.** - [Release notes](https://github.com/sparklemotion/nokogiri/releases) - [Changelog](https://github.com/sparklemotion/nokogiri/blob/master/CHANGELOG.md) - [Commits](https://github.com/sparklemotion/nokogiri/compare/v1.10.3...v1.10.4) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 2eebbfdc..caf23abb 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -189,7 +189,7 @@ GEM multipart-post (2.0.0) mustermann (1.0.3) nio4r (2.3.1) - nokogiri (1.10.3) + nokogiri (1.10.4) mini_portile2 (~> 2.4.0) oauth2 (1.4.1) faraday (>= 0.8, < 0.16.0) From 93b65c4b7b7cfe55b5dfa2522cb126d7c94af162 Mon Sep 17 00:00:00 2001 From: Carlos Ribeiro Date: Mon, 30 Sep 2019 11:56:24 -0300 Subject: [PATCH 034/156] Use UBI7 Ruby image THREESCALE-3367 Support UBI7 (Universal Base Image) image for Zync --- .circleci/config.yml | 14 ++++++++++++++ Dockerfile | 2 +- docker-compose.yml | 30 ++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 docker-compose.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 53dd0e16..4cee5231 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,5 +1,19 @@ version: 2 jobs: + docker-build: + resource_class: small + working_directory: /home/circleci/zync + docker: + - image: circleci/postgres:10.9-alpine + environment: + POSTGRES_USER: postgres + POSTGRES_DB: zync + environment: + DATABASE_URL: postgresql://postgres:@127.0.0.1:5432/zync + steps: + - setup_remote_docker + - run: sudo docker build . --tag zync:build-{{ .Branch }} + - run: sudo docker run --environment DATABASE_URL=${DATABASE_URL} -- zync:build-{{ .Branch }} ./bin/setup build: working_directory: /home/circleci/zync diff --git a/Dockerfile b/Dockerfile index d60df6f1..e0e35133 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM centos/ruby-24-centos7 +FROM registry.access.redhat.com/ubi7/ruby-25 RUN source ${APP_ROOT}/etc/scl_enable \ && gem install bundler --version=2.0.1 --no-document diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..d8788bba --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,30 @@ +version: '3.4' + +services: + zync: + build: . + command: bash -c "rm -f tmp/pids/server.pid && UNICORN_WORKERS=2 bin/rails server -b 0.0.0.0" + depends_on: + - postgres + links: + - postgres + ports: + - "9393:9393" + environment: + - DATABASE_URL=postgresql://postgres:@postgres:5432/ + + postgres: + image: postgres:10.5 + ports: + - "5432:5432" + environment: + - PGUSER=postgres + healthcheck: + test: ["CMD", "pg_isready", "-U", "postgres"] + timeout: 20s + retries: 10 + volumes: + - postgres-data:/var/lib/postgresql/data + +volumes: + postgres-data: From 97665925f2805fc300b8ad805d59f32e3274579a Mon Sep 17 00:00:00 2001 From: Carlos Ribeiro Date: Tue, 1 Oct 2019 10:55:55 -0300 Subject: [PATCH 035/156] Extract docker build to a separate job --- .circleci/config.yml | 24 ++++++++++++++++-------- test/lib/prometheus/que_stats_test.rb | 11 ++++++----- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4cee5231..eab37e8c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,16 +4,18 @@ jobs: resource_class: small working_directory: /home/circleci/zync docker: - - image: circleci/postgres:10.9-alpine - environment: - POSTGRES_USER: postgres - POSTGRES_DB: zync - environment: - DATABASE_URL: postgresql://postgres:@127.0.0.1:5432/zync + - image: circleci/ruby:2.4.6 steps: + - run: + name: Install Docker Compose + command: | + curl -L https://github.com/docker/compose/releases/download/1.19.0/docker-compose-`uname -s`-`uname -m` > ~/docker-compose + chmod +x ~/docker-compose + sudo mv ~/docker-compose /usr/local/bin/docker-compose - setup_remote_docker - - run: sudo docker build . --tag zync:build-{{ .Branch }} - - run: sudo docker run --environment DATABASE_URL=${DATABASE_URL} -- zync:build-{{ .Branch }} ./bin/setup + - checkout + - run: docker-compose run -e RAILS_ENV=test zync bin/setup + - run: docker-compose run -e RAILS_ENV=test zync bin/rails test build: working_directory: /home/circleci/zync @@ -70,3 +72,9 @@ jobs: key: zync-branch-{{ arch }}-{{ .Branch }} paths: - vendor/bundle +workflows: + version: 2 + build_and_test_docker: + jobs: + - build + - docker-build diff --git a/test/lib/prometheus/que_stats_test.rb b/test/lib/prometheus/que_stats_test.rb index 910ecedf..c32d0720 100644 --- a/test/lib/prometheus/que_stats_test.rb +++ b/test/lib/prometheus/que_stats_test.rb @@ -21,11 +21,12 @@ class Prometheus::QueStatsTest < ActiveSupport::TestCase assert Prometheus::QueStats.job_stats('1 > 0') end - uses_transaction def test_readonly_transaction - Prometheus::QueStats.stub(:read_only_transaction, true) do - Prometheus::QueStats.worker_stats - end - end + # just testing + # uses_transaction def test_readonly_transaction + # Prometheus::QueStats.stub(:read_only_transaction, true) do + # Prometheus::QueStats.worker_stats + # end + # end test 'serialize metrics' do Que.stop! From 52c197ff40513f32894717b86abc352e39dbda30 Mon Sep 17 00:00:00 2001 From: Hallelujah Date: Wed, 2 Oct 2019 11:06:06 +0300 Subject: [PATCH 036/156] Don't use docker-compose, just use simple docker containers --- .circleci/config.yml | 24 +++++++++++++++--------- test/lib/prometheus/que_stats_test.rb | 12 ++++++------ 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index eab37e8c..f67c0155 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,18 +4,23 @@ jobs: resource_class: small working_directory: /home/circleci/zync docker: - - image: circleci/ruby:2.4.6 + - image: circleci/buildpack-deps:latest + environment: + POSGRES_CONTAINER_NAME: db + DATABASE_URL: postgresql://postgres:@db:5432/zync + POSTGRES_USER: postgres + POSTGRES_DB: zync + RAILS_ENV: production steps: + - checkout + - setup_remote_docker + - run: docker build --tag zync:build --file ./Dockerfile . + - run: docker network create net0 + - run: docker run --net net0 --name ${POSGRES_CONTAINER_NAME} -d -p 5432:5432 -e POSTGRES_USER=${POSTGRES_USER} -e POSTGRES_DB=${POSTGRES_DB} postgres:10-alpine - run: - name: Install Docker Compose command: | - curl -L https://github.com/docker/compose/releases/download/1.19.0/docker-compose-`uname -s`-`uname -m` > ~/docker-compose - chmod +x ~/docker-compose - sudo mv ~/docker-compose /usr/local/bin/docker-compose - - setup_remote_docker - - checkout - - run: docker-compose run -e RAILS_ENV=test zync bin/setup - - run: docker-compose run -e RAILS_ENV=test zync bin/rails test + docker run --net net0 -e RAILS_ENV=${RAILS_ENV} -e DATABASE_URL=${DATABASE_URL} \ + zync:build rails db:setup build: working_directory: /home/circleci/zync @@ -72,6 +77,7 @@ jobs: key: zync-branch-{{ arch }}-{{ .Branch }} paths: - vendor/bundle + workflows: version: 2 build_and_test_docker: diff --git a/test/lib/prometheus/que_stats_test.rb b/test/lib/prometheus/que_stats_test.rb index c32d0720..87aa01cc 100644 --- a/test/lib/prometheus/que_stats_test.rb +++ b/test/lib/prometheus/que_stats_test.rb @@ -21,12 +21,12 @@ class Prometheus::QueStatsTest < ActiveSupport::TestCase assert Prometheus::QueStats.job_stats('1 > 0') end - # just testing - # uses_transaction def test_readonly_transaction - # Prometheus::QueStats.stub(:read_only_transaction, true) do - # Prometheus::QueStats.worker_stats - # end - # end + uses_transaction :test_readonly_transaction + def test_readonly_transaction + Prometheus::QueStats.stub(:read_only_transaction, true) do + Prometheus::QueStats.worker_stats + end + end test 'serialize metrics' do Que.stop! From efaa9a65ebcebb564725bdb7ed2315b9ecfe2925 Mon Sep 17 00:00:00 2001 From: Guilherme Cassolato Date: Fri, 18 Oct 2019 18:27:51 +0200 Subject: [PATCH 037/156] Bump rubyzip version --- Gemfile | 2 ++ Gemfile.lock | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index a45278db..0d256f03 100644 --- a/Gemfile +++ b/Gemfile @@ -65,6 +65,8 @@ group :development, :test do gem 'license_finder', '~> 5.9' gem 'license_finder_xml_reporter', git: 'https://github.com/3scale/license_finder_xml_reporter.git', tag: '1.0.0' + # rubyzip is a transitive depencency from license_finder with vulnerability on < 1.3.0 + gem 'rubyzip', '>= 1.3.0' # gem 'httplog' end diff --git a/Gemfile.lock b/Gemfile.lock index caf23abb..50231834 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -263,7 +263,7 @@ GEM actionpack (>= 5.0) railties (>= 5.0) ruby-progressbar (1.10.0) - rubyzip (1.2.3) + rubyzip (2.0.0) safe_yaml (1.0.5) schema_monkey (2.1.5) activerecord (>= 4.2) @@ -359,6 +359,7 @@ DEPENDENCIES que-web rails (~> 5.2.3) responders (~> 3.0.0) + rubyzip (>= 1.3.0) schema_plus_enums tzinfo-data validate_url @@ -368,4 +369,4 @@ DEPENDENCIES yabeda-rails BUNDLED WITH - 2.0.1 + 2.0.2 From d143b1796a650e602a0a9efc5d0beb2bdcb33deb Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 24 Oct 2019 03:04:29 +0000 Subject: [PATCH 038/156] [Security] Bump loofah from 2.2.3 to 2.3.1 Bumps [loofah](https://github.com/flavorjones/loofah) from 2.2.3 to 2.3.1. **This update includes a security fix.** - [Release notes](https://github.com/flavorjones/loofah/releases) - [Changelog](https://github.com/flavorjones/loofah/blob/master/CHANGELOG.md) - [Commits](https://github.com/flavorjones/loofah/compare/v2.2.3...v2.3.1) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 50231834..05dcaaae 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -89,7 +89,7 @@ GEM concurrent-ruby (1.1.5) crack (0.4.3) safe_yaml (~> 1.0.0) - crass (1.0.4) + crass (1.0.5) debug_inspector (0.0.3) docile (1.3.1) dry-configurable (0.8.3) @@ -161,7 +161,7 @@ GEM activesupport (>= 4) railties (>= 4) request_store (~> 1.0) - loofah (2.2.3) + loofah (2.3.1) crass (~> 1.0.2) nokogiri (>= 1.5.9) mail (2.7.1) From ef260ecca4b8d54be6b513e0a181f4400663aa6d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 5 Dec 2019 07:38:59 +0000 Subject: [PATCH 039/156] Bump puma from 4.0.0 to 4.3.1 Bumps [puma](https://github.com/puma/puma) from 4.0.0 to 4.3.1. - [Release notes](https://github.com/puma/puma/releases) - [Changelog](https://github.com/puma/puma/blob/master/History.md) - [Commits](https://github.com/puma/puma/compare/v4.0.0...v4.3.1) Signed-off-by: dependabot-preview[bot] --- Gemfile | 2 +- Gemfile.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Gemfile b/Gemfile index 0d256f03..62662b8f 100644 --- a/Gemfile +++ b/Gemfile @@ -13,7 +13,7 @@ gem 'pg', '>= 0.20' gem 'schema_plus_enums' # Use Puma as the app server -gem 'puma', '~> 4.0' +gem 'puma', '~> 4.3' # Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder # gem 'jbuilder', '~> 2.5' # Use ActiveModel has_secure_password diff --git a/Gemfile.lock b/Gemfile.lock index 05dcaaae..41e8a684 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -188,7 +188,7 @@ GEM multi_xml (0.6.0) multipart-post (2.0.0) mustermann (1.0.3) - nio4r (2.3.1) + nio4r (2.5.2) nokogiri (1.10.4) mini_portile2 (~> 2.4.0) oauth2 (1.4.1) @@ -218,7 +218,7 @@ GEM binding_of_caller (>= 0.7) pry (>= 0.9.11) public_suffix (3.1.0) - puma (4.0.0) + puma (4.3.1) nio4r (~> 2.0) quantile (0.2.1) que (1.0.0.beta3) @@ -354,7 +354,7 @@ DEPENDENCIES pry-rails pry-rescue pry-stack_explorer - puma (~> 4.0) + puma (~> 4.3) que (>= 1.0.0.beta3) que-web rails (~> 5.2.3) From 3c2c098b4c0597c50c984c6cc93583371ec0f9ae Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 5 Dec 2019 09:03:22 +0000 Subject: [PATCH 040/156] [Security] Bump nokogiri from 1.10.4 to 1.10.7 Bumps [nokogiri](https://github.com/sparklemotion/nokogiri) from 1.10.4 to 1.10.7. **This update includes a security fix.** - [Release notes](https://github.com/sparklemotion/nokogiri/releases) - [Changelog](https://github.com/sparklemotion/nokogiri/blob/master/CHANGELOG.md) - [Commits](https://github.com/sparklemotion/nokogiri/compare/v1.10.4...v1.10.7) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 41e8a684..3719197e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -189,7 +189,7 @@ GEM multipart-post (2.0.0) mustermann (1.0.3) nio4r (2.5.2) - nokogiri (1.10.4) + nokogiri (1.10.7) mini_portile2 (~> 2.4.0) oauth2 (1.4.1) faraday (>= 0.8, < 0.16.0) From 3a9881d8f0380ffa33def69b06dfe2880b0a6f53 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 18 Dec 2019 19:33:13 +0000 Subject: [PATCH 041/156] [Security] Bump rack from 2.0.7 to 2.0.8 Bumps [rack](https://github.com/rack/rack) from 2.0.7 to 2.0.8. **This update includes a security fix.** - [Release notes](https://github.com/rack/rack/releases) - [Changelog](https://github.com/rack/rack/blob/master/CHANGELOG.md) - [Commits](https://github.com/rack/rack/compare/2.0.7...2.0.8) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 3719197e..527a7a83 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -226,7 +226,7 @@ GEM erubis que (~> 1.0.0.beta3) sinatra - rack (2.0.7) + rack (2.0.8) rack-protection (2.0.5) rack rack-test (1.1.0) From 98ba782785e8b839b40d3b04eda5e536ae7e148f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 18 Dec 2019 19:37:09 +0000 Subject: [PATCH 042/156] Bump codecov from 0.1.14 to 0.1.16 Bumps [codecov](https://github.com/codecov/codecov-ruby) from 0.1.14 to 0.1.16. - [Release notes](https://github.com/codecov/codecov-ruby/releases) - [Changelog](https://github.com/codecov/codecov-ruby/blob/master/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-ruby/compare/v0.1.14...v0.1.16) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 527a7a83..3b7dcdd1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -81,7 +81,7 @@ GEM bugsnag-capistrano (1.1.2) builder (3.2.3) byebug (11.0.0) - codecov (0.1.14) + codecov (0.1.16) json simplecov url @@ -91,7 +91,7 @@ GEM safe_yaml (~> 1.0.0) crass (1.0.5) debug_inspector (0.0.3) - docile (1.3.1) + docile (1.3.2) dry-configurable (0.8.3) concurrent-ruby (~> 1.0) dry-core (~> 0.4, >= 0.4.7) @@ -134,7 +134,7 @@ GEM its-it (1.3.0) join_dependency (0.1.4) activerecord (>= 4.2.0) - json (2.1.0) + json (2.3.0) jsonpath (0.9.9) multi_json to_regexp (~> 0.2.1) @@ -276,7 +276,7 @@ GEM activerecord (>= 4.2, < 5.3) its-it (~> 1.2) schema_plus_core - simplecov (0.16.1) + simplecov (0.17.1) docile (~> 1.1) json (>= 1.8, < 3) simplecov-html (~> 0.10.0) From faa3d04bb1b656e4f4ee507648224f160768edc3 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 31 Dec 2019 13:13:41 +0000 Subject: [PATCH 043/156] Bump minitest-reporters from 1.3.6 to 1.4.2 Bumps [minitest-reporters](https://github.com/CapnKernul/minitest-reporters) from 1.3.6 to 1.4.2. - [Release notes](https://github.com/CapnKernul/minitest-reporters/releases) - [Changelog](https://github.com/kern/minitest-reporters/blob/master/CHANGELOG.md) - [Commits](https://github.com/CapnKernul/minitest-reporters/compare/v1.3.6...v1.4.2) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 3b7dcdd1..1848dfcc 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -79,7 +79,7 @@ GEM bootsnap (1.4.4) msgpack (~> 1.0) bugsnag-capistrano (1.1.2) - builder (3.2.3) + builder (3.2.4) byebug (11.0.0) codecov (0.1.16) json @@ -174,8 +174,8 @@ GEM mimemagic (0.3.3) mini_mime (1.0.1) mini_portile2 (2.4.0) - minitest (5.11.3) - minitest-reporters (1.3.6) + minitest (5.13.0) + minitest-reporters (1.4.2) ansi builder minitest (>= 5.0) @@ -262,7 +262,7 @@ GEM responders (3.0.0) actionpack (>= 5.0) railties (>= 5.0) - ruby-progressbar (1.10.0) + ruby-progressbar (1.10.1) rubyzip (2.0.0) safe_yaml (1.0.5) schema_monkey (2.1.5) From 91dea5b22a244c9d23104b0d200a32507f4cf92b Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 31 Dec 2019 13:18:21 +0000 Subject: [PATCH 044/156] Bump license_finder from 5.9.2 to 5.11.1 Bumps [license_finder](https://github.com/pivotal/LicenseFinder) from 5.9.2 to 5.11.1. - [Release notes](https://github.com/pivotal/LicenseFinder/releases) - [Changelog](https://github.com/pivotal/LicenseFinder/blob/master/CHANGELOG.md) - [Commits](https://github.com/pivotal/LicenseFinder/compare/v5.9.2...v5.11.1) Signed-off-by: dependabot-preview[bot] --- Gemfile | 2 +- Gemfile.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Gemfile b/Gemfile index 62662b8f..57e03b86 100644 --- a/Gemfile +++ b/Gemfile @@ -63,7 +63,7 @@ group :development, :test do gem 'pry-rescue' gem 'pry-stack_explorer' - gem 'license_finder', '~> 5.9' + gem 'license_finder', '~> 5.11' gem 'license_finder_xml_reporter', git: 'https://github.com/3scale/license_finder_xml_reporter.git', tag: '1.0.0' # rubyzip is a transitive depencency from license_finder with vulnerability on < 1.3.0 gem 'rubyzip', '>= 1.3.0' diff --git a/Gemfile.lock b/Gemfile.lock index 1848dfcc..70d50168 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -149,9 +149,9 @@ GEM yajl-ruby (~> 1.4.0) yaml-safe_load_stream (~> 0.1) key_struct (0.4.2) - license_finder (5.9.2) + license_finder (5.11.1) bundler - rubyzip + rubyzip (>= 1, < 3) thor toml (= 0.2.0) with_env (= 1.1.0) @@ -293,7 +293,7 @@ GEM actionpack (>= 4.0) activesupport (>= 4.0) sprockets (>= 3.0.0) - thor (0.20.3) + thor (1.0.1) thread_safe (0.3.6) tilt (2.0.9) to_regexp (0.2.1) @@ -341,7 +341,7 @@ DEPENDENCIES codecov httpclient! k8s-client (>= 0.10) - license_finder (~> 5.9) + license_finder (~> 5.11) license_finder_xml_reporter! lograge message_bus From 55cb03646b929662589b46c849318f313a62d81c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 2 Jan 2020 11:21:41 +0000 Subject: [PATCH 045/156] Bump lograge from 0.10.0 to 0.11.2 Bumps [lograge](https://github.com/roidrage/lograge) from 0.10.0 to 0.11.2. - [Release notes](https://github.com/roidrage/lograge/releases) - [Changelog](https://github.com/roidrage/lograge/blob/master/CHANGELOG.md) - [Commits](https://github.com/roidrage/lograge/compare/v0.10.0...v0.11.2) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 70d50168..b4514200 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -119,7 +119,7 @@ GEM dry-equalizer (~> 0.2) dry-inflector (~> 0.1, >= 0.1.2) dry-logic (~> 0.4, >= 0.4.2) - erubi (1.8.0) + erubi (1.9.0) erubis (2.7.0) excon (0.64.0) faraday (0.15.3) @@ -127,7 +127,7 @@ GEM globalid (0.4.2) activesupport (>= 4.2.0) hashdiff (0.3.9) - i18n (1.6.0) + i18n (1.7.0) concurrent-ruby (~> 1.0) ice_nine (0.11.2) interception (0.5) @@ -156,12 +156,12 @@ GEM toml (= 0.2.0) with_env (= 1.1.0) xml-simple - lograge (0.10.0) + lograge (0.11.2) actionpack (>= 4) activesupport (>= 4) railties (>= 4) request_store (~> 1.0) - loofah (2.3.1) + loofah (2.4.0) crass (~> 1.0.2) nokogiri (>= 1.5.9) mail (2.7.1) @@ -247,17 +247,17 @@ GEM rails-dom-testing (2.0.3) activesupport (>= 4.2.0) nokogiri (>= 1.6) - rails-html-sanitizer (1.0.4) - loofah (~> 2.2, >= 2.2.2) + rails-html-sanitizer (1.3.0) + loofah (~> 2.3) railties (5.2.3) actionpack (= 5.2.3) activesupport (= 5.2.3) method_source rake (>= 0.8.7) thor (>= 0.19.0, < 2.0) - rake (12.3.2) + rake (13.0.1) recursive-open-struct (1.1.0) - request_store (1.4.1) + request_store (1.5.0) rack (>= 1.4) responders (3.0.0) actionpack (>= 5.0) @@ -299,7 +299,7 @@ GEM to_regexp (0.2.1) toml (0.2.0) parslet (~> 1.8.0) - tzinfo (1.2.5) + tzinfo (1.2.6) thread_safe (~> 0.1) url (0.3.2) validate_url (1.0.6) From 3206b03b3e5d33f85ca69e5b153069d7442b19e3 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 2 Jan 2020 11:35:54 +0000 Subject: [PATCH 046/156] Bump k8s-client from 0.10.2 to 0.10.4 Bumps [k8s-client](https://github.com/kontena/k8s-client) from 0.10.2 to 0.10.4. - [Release notes](https://github.com/kontena/k8s-client/releases) - [Commits](https://github.com/kontena/k8s-client/compare/v0.10.2...v0.10.4) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index b4514200..b6dcb436 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -92,16 +92,16 @@ GEM crass (1.0.5) debug_inspector (0.0.3) docile (1.3.2) - dry-configurable (0.8.3) + dry-configurable (0.9.0) concurrent-ruby (~> 1.0) dry-core (~> 0.4, >= 0.4.7) - dry-container (0.7.1) + dry-container (0.7.2) concurrent-ruby (~> 1.0) dry-configurable (~> 0.1, >= 0.1.3) - dry-core (0.4.8) + dry-core (0.4.9) concurrent-ruby (~> 1.0) - dry-equalizer (0.2.2) - dry-inflector (0.1.2) + dry-equalizer (0.3.0) + dry-inflector (0.2.0) dry-initializer (2.5.0) dry-logic (0.6.1) concurrent-ruby (~> 1.0) @@ -121,12 +121,12 @@ GEM dry-logic (~> 0.4, >= 0.4.2) erubi (1.9.0) erubis (2.7.0) - excon (0.64.0) + excon (0.71.1) faraday (0.15.3) multipart-post (>= 1.2, < 3) globalid (0.4.2) activesupport (>= 4.2.0) - hashdiff (0.3.9) + hashdiff (1.0.0) i18n (1.7.0) concurrent-ruby (~> 1.0) ice_nine (0.11.2) @@ -139,11 +139,11 @@ GEM multi_json to_regexp (~> 0.2.1) jwt (2.1.0) - k8s-client (0.10.2) + k8s-client (0.10.4) dry-struct (~> 0.5.0) dry-types (~> 0.13.0) - excon (~> 0.64.0) - hashdiff (~> 0.3.7) + excon (~> 0.66) + hashdiff (~> 1.0.0) jsonpath (~> 0.9.5) recursive-open-struct (~> 1.1.0) yajl-ruby (~> 1.4.0) @@ -184,7 +184,7 @@ GEM modware (0.1.3) key_struct (~> 0.4) msgpack (1.2.10) - multi_json (1.13.1) + multi_json (1.14.1) multi_xml (0.6.0) multipart-post (2.0.0) mustermann (1.0.3) From 28c0925d36997dcb7df650cad207374595377da8 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 2 Jan 2020 11:39:42 +0000 Subject: [PATCH 047/156] Bump validate_url from 1.0.6 to 1.0.8 Bumps [validate_url](https://github.com/perfectline/validates_url) from 1.0.6 to 1.0.8. - [Release notes](https://github.com/perfectline/validates_url/releases) - [Commits](https://github.com/perfectline/validates_url/commits) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index b6dcb436..f9125398 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -217,7 +217,7 @@ GEM pry-stack_explorer (0.4.9.3) binding_of_caller (>= 0.7) pry (>= 0.9.11) - public_suffix (3.1.0) + public_suffix (3.1.1) puma (4.3.1) nio4r (~> 2.0) quantile (0.2.1) @@ -302,7 +302,7 @@ GEM tzinfo (1.2.6) thread_safe (~> 0.1) url (0.3.2) - validate_url (1.0.6) + validate_url (1.0.8) activemodel (>= 3.0.0) public_suffix webmock (3.5.1) From efdeffd6a55cafa429886dbcbaa1b19a074d451c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 2 Jan 2020 11:58:05 +0000 Subject: [PATCH 048/156] Bump bootsnap from 1.4.4 to 1.4.5 Bumps [bootsnap](https://github.com/Shopify/bootsnap) from 1.4.4 to 1.4.5. - [Release notes](https://github.com/Shopify/bootsnap/releases) - [Changelog](https://github.com/Shopify/bootsnap/blob/master/CHANGELOG.md) - [Commits](https://github.com/Shopify/bootsnap/compare/v1.4.4...v1.4.5) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index f9125398..df21e98b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -76,7 +76,7 @@ GEM polyamorous (~> 1.3) binding_of_caller (0.8.0) debug_inspector (>= 0.0.1) - bootsnap (1.4.4) + bootsnap (1.4.5) msgpack (~> 1.0) bugsnag-capistrano (1.1.2) builder (3.2.4) @@ -183,7 +183,7 @@ GEM minitest-stub-const (0.6) modware (0.1.3) key_struct (~> 0.4) - msgpack (1.2.10) + msgpack (1.3.1) multi_json (1.14.1) multi_xml (0.6.0) multipart-post (2.0.0) From 12612a5afae577c3945862af9989a1e837c2b7f1 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 2 Jan 2020 12:02:28 +0000 Subject: [PATCH 049/156] Bump message_bus from 2.1.6 to 2.2.3 Bumps [message_bus](https://github.com/SamSaffron/message_bus) from 2.1.6 to 2.2.3. - [Release notes](https://github.com/SamSaffron/message_bus/releases) - [Changelog](https://github.com/SamSaffron/message_bus/blob/master/CHANGELOG) - [Commits](https://github.com/SamSaffron/message_bus/compare/v2.1.6...v2.2.3) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index df21e98b..6b43609d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -168,7 +168,7 @@ GEM mini_mime (>= 0.1.1) marcel (0.3.3) mimemagic (~> 0.3.2) - message_bus (2.1.6) + message_bus (2.2.3) rack (>= 1.1.3) method_source (0.9.2) mimemagic (0.3.3) From b33ce9bf1302c5c3df2c6b31fc8184fe8cff6d78 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 2 Jan 2020 12:16:22 +0000 Subject: [PATCH 050/156] Bump pg from 1.1.4 to 1.2.0 Bumps [pg](https://github.com/ged/ruby-pg) from 1.1.4 to 1.2.0. - [Release notes](https://github.com/ged/ruby-pg/releases) - [Changelog](https://github.com/ged/ruby-pg/blob/master/History.rdoc) - [Commits](https://github.com/ged/ruby-pg/compare/v1.1.4...v1.2.0) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 6b43609d..61512e8e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -198,7 +198,7 @@ GEM multi_xml (~> 0.5) rack (>= 1.2, < 3) parslet (1.8.2) - pg (1.1.4) + pg (1.2.0) polyamorous (1.3.3) activerecord (>= 3.0) prometheus-client (0.9.0) From 676018d7e2e47601012a696246a6a6e459aeba31 Mon Sep 17 00:00:00 2001 From: "depfu[bot]" <23717796+depfu[bot]@users.noreply.github.com> Date: Tue, 11 Feb 2020 11:22:35 +0000 Subject: [PATCH 051/156] Update nokogiri to version 1.10.8 --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 61512e8e..40d7c5e9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -189,7 +189,7 @@ GEM multipart-post (2.0.0) mustermann (1.0.3) nio4r (2.5.2) - nokogiri (1.10.7) + nokogiri (1.10.8) mini_portile2 (~> 2.4.0) oauth2 (1.4.1) faraday (>= 0.8, < 0.16.0) From 386e7113a089e0f37e47651eff6b28d5f8edbb00 Mon Sep 17 00:00:00 2001 From: Hallelujah Date: Tue, 22 Oct 2019 20:26:39 +0300 Subject: [PATCH 052/156] Route termination options follow public url scheme If public route was created with `http:` scheme, do not enforce secure route edge redirect Fixes THREESCALE-3545 --- .../integration/kubernetes_service.rb | 11 +++---- test/services/kubernetes_service_test.rb | 29 +++++++++++++++++++ 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/app/services/integration/kubernetes_service.rb b/app/services/integration/kubernetes_service.rb index cb25d3f2..6c778db7 100644 --- a/app/services/integration/kubernetes_service.rb +++ b/app/services/integration/kubernetes_service.rb @@ -135,18 +135,19 @@ def initialize(attributes, **options) class RouteSpec < K8s::Resource def initialize(url, service, port) uri = URI(url) + tls_options = { + insecureEdgeTerminationPolicy: 'Redirect', + termination: 'edge' + } if uri.class == URI::HTTPS + super({ host: uri.host || uri.path, port: { targetPort: port }, - tls: { - insecureEdgeTerminationPolicy: 'Redirect', - termination: 'edge' - }, to: { kind: 'Service', name: service } - }) + }.merge(tls: tls_options)) end end diff --git a/test/services/kubernetes_service_test.rb b/test/services/kubernetes_service_test.rb index 7cc30d04..29b6c7a8 100644 --- a/test/services/kubernetes_service_test.rb +++ b/test/services/kubernetes_service_test.rb @@ -65,4 +65,33 @@ def after_teardown service.call(proxy) end + + class RouteSpec < ActiveSupport::TestCase + test 'secure routes' do + url = 'https://my-api.example.com' + service_name = 'My API' + port = 7443 + spec = Integration::KubernetesService::RouteSpec.new(url, service_name, port) + json = { + host: "my-api.example.com", + port: {targetPort: 7443}, + to: {kind: "Service", name: "My API"}, + tls: {insecureEdgeTerminationPolicy: "Redirect", termination: "edge"} + } + assert_equal json, spec.to_hash + + + url = 'http://my-api.example.com' + service_name = 'My API' + port = 7780 + spec = Integration::KubernetesService::RouteSpec.new(url, service_name, port) + json = { + host: "my-api.example.com", + port: {targetPort: 7780}, + to: {kind: "Service", name: "My API"}, + tls: nil + } + assert_equal json, spec.to_hash + end + end end From cfff38f229961eeacae707f886e3baff2023bbf5 Mon Sep 17 00:00:00 2001 From: Hallelujah Date: Fri, 27 Mar 2020 15:01:21 +0300 Subject: [PATCH 053/156] [circleci] DB needs a password, defaulting to postgres Fixing: Error: Database is uninitialized and superuser password is not specified. You must specify POSTGRES_PASSWORD to a non-empty value for the superuser. For example, "-e POSTGRES_PASSWORD=password" on "docker run". You may also use "POSTGRES_HOST_AUTH_METHOD=trust" to allow all connections without a password. This is *not* recommended. See PostgreSQL documentation about "trust": https://www.postgresql.org/docs/current/auth-trust.html --- .circleci/config.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f67c0155..726b2346 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,8 +7,9 @@ jobs: - image: circleci/buildpack-deps:latest environment: POSGRES_CONTAINER_NAME: db - DATABASE_URL: postgresql://postgres:@db:5432/zync + DATABASE_URL: postgresql://postgres:postgres@db:5432/zync POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres POSTGRES_DB: zync RAILS_ENV: production steps: @@ -16,7 +17,7 @@ jobs: - setup_remote_docker - run: docker build --tag zync:build --file ./Dockerfile . - run: docker network create net0 - - run: docker run --net net0 --name ${POSGRES_CONTAINER_NAME} -d -p 5432:5432 -e POSTGRES_USER=${POSTGRES_USER} -e POSTGRES_DB=${POSTGRES_DB} postgres:10-alpine + - run: docker run --net net0 --name ${POSGRES_CONTAINER_NAME} -d -p 5432:5432 -e POSTGRES_USER=${POSTGRES_USER} -e POSTGRES_PASSWORD=${POSTGRES_PASSWORD} -e POSTGRES_DB=${POSTGRES_DB} postgres:10-alpine - run: command: | docker run --net net0 -e RAILS_ENV=${RAILS_ENV} -e DATABASE_URL=${DATABASE_URL} \ From 4d22b14a9d5cacbd40c088824d021dac41580475 Mon Sep 17 00:00:00 2001 From: Hallelujah Date: Fri, 27 Mar 2020 16:48:11 +0300 Subject: [PATCH 054/156] [random-failures] Fix random failure This happens because we create a transaction, make a query then set transaction isolation level: BEGIN INSERT INTO .... SET TRANSACTION ISOLATION LEVEL To avoid that, isolate transaction aware tests into its own class 1) Error: Prometheus::QueStatsTest#test_readonly_transaction: ActiveRecord::StatementInvalid: PG::ActiveSqlTransaction: ERROR: SET TRANSACTION ISOLATION LEVEL must be called before any query : SET TRANSACTION ISOLATION LEVEL SERIALIZABLE READ ONLY DEFERRABLE /home/circleci/zync/vendor/bundle/ruby/2.4.0/gems/activerecord-5.2.3/lib/active_record/connection_adapters/postgresql/database_statements.rb:75:in `exec' /home/circleci/zync/vendor/bundle/ruby/2.4.0/gems/activerecord-5.2.3/lib/active_record/connection_adapters/postgresql/database_statements.rb:75:in `block (2 levels) in execute' /home/circleci/zync/vendor/bundle/ruby/2.4.0/gems/activesupport-5.2.3/lib/active_support/dependencies/interlock.rb:48:in `block in permit_concurrent_loads' /home/circleci/zync/vendor/bundle/ruby/2.4.0/gems/activesupport-5.2.3/lib/active_support/concurrency/share_lock.rb:187:in `yield_shares' /home/circleci/zync/vendor/bundle/ruby/2.4.0/gems/activesupport-5.2.3/lib/active_support/dependencies/interlock.rb:47:in `permit_concurrent_loads' /home/circleci/zync/vendor/bundle/ruby/2.4.0/gems/activerecord-5.2.3/lib/active_record/connection_adapters/postgresql/database_statements.rb:74:in `block in execute' /home/circleci/zync/vendor/bundle/ruby/2.4.0/gems/activerecord-5.2.3/lib/active_record/connection_adapters/abstract_adapter.rb:581:in `block (2 levels) in log' /usr/local/lib/ruby/2.4.0/monitor.rb:214:in `mon_synchronize' /home/circleci/zync/vendor/bundle/ruby/2.4.0/gems/activerecord-5.2.3/lib/active_record/connection_adapters/abstract_adapter.rb:580:in `block in log' /home/circleci/zync/vendor/bundle/ruby/2.4.0/gems/activesupport-5.2.3/lib/active_support/notifications/instrumenter.rb:23:in `instrument' /home/circleci/zync/vendor/bundle/ruby/2.4.0/gems/activerecord-5.2.3/lib/active_record/connection_adapters/abstract_adapter.rb:571:in `log' /home/circleci/zync/vendor/bundle/ruby/2.4.0/gems/activerecord-5.2.3/lib/active_record/connection_adapters/postgresql/database_statements.rb:73:in `execute' /home/circleci/zync/lib/prometheus/que_stats.rb:41:in `block in execute' /home/circleci/zync/vendor/bundle/ruby/2.4.0/gems/activerecord-5.2.3/lib/active_record/connection_adapters/abstract/database_statements.rb:267:in `block in transaction' /home/circleci/zync/vendor/bundle/ruby/2.4.0/gems/activerecord-5.2.3/lib/active_record/connection_adapters/abstract/transaction.rb:239:in `block in within_new_transaction' /usr/local/lib/ruby/2.4.0/monitor.rb:214:in `mon_synchronize' /home/circleci/zync/vendor/bundle/ruby/2.4.0/gems/activerecord-5.2.3/lib/active_record/connection_adapters/abstract/transaction.rb:236:in `within_new_transaction' /home/circleci/zync/vendor/bundle/ruby/2.4.0/gems/activerecord-5.2.3/lib/active_record/connection_adapters/abstract/database_statements.rb:267:in `transaction' /home/circleci/zync/lib/prometheus/que_stats.rb:39:in `execute' /home/circleci/zync/lib/prometheus/que_stats.rb:15:in `worker_stats' /home/circleci/zync/test/lib/prometheus/que_stats_test.rb:28:in `test_readonly_transaction' --- test/lib/prometheus/que_stats_test.rb | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/lib/prometheus/que_stats_test.rb b/test/lib/prometheus/que_stats_test.rb index 87aa01cc..649defed 100644 --- a/test/lib/prometheus/que_stats_test.rb +++ b/test/lib/prometheus/que_stats_test.rb @@ -21,10 +21,12 @@ class Prometheus::QueStatsTest < ActiveSupport::TestCase assert Prometheus::QueStats.job_stats('1 > 0') end - uses_transaction :test_readonly_transaction - def test_readonly_transaction - Prometheus::QueStats.stub(:read_only_transaction, true) do - Prometheus::QueStats.worker_stats + class WithTransaction < ActiveSupport::TestCase + uses_transaction :test_readonly_transaction + def test_readonly_transaction + Prometheus::QueStats.stub(:read_only_transaction, true) do + Prometheus::QueStats.worker_stats + end end end From 8691ca2e5a7ab9e5f374f4b62f8320cc0b6347f4 Mon Sep 17 00:00:00 2001 From: Hallelujah Date: Fri, 27 Mar 2020 21:45:02 +0300 Subject: [PATCH 055/156] [random-failure] Do not forget super in after_teardown Otherwise ROLLBACK will not be invoked Thus transactional tests will still run inside the same global transaction --- test/services/kubernetes_service_test.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/test/services/kubernetes_service_test.rb b/test/services/kubernetes_service_test.rb index 29b6c7a8..1f33a665 100644 --- a/test/services/kubernetes_service_test.rb +++ b/test/services/kubernetes_service_test.rb @@ -13,6 +13,7 @@ def before_setup def after_teardown ENV.replace(@_env) + super end test 'create ingress ' do From def8e97c4ebed90c37e4e297d5c1365f0470e62e Mon Sep 17 00:00:00 2001 From: Rael Garcia Date: Sat, 28 Mar 2020 03:22:16 +0100 Subject: [PATCH 056/156] Upgrade postgresql client to 9.6 --- Dockerfile | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Dockerfile b/Dockerfile index e0e35133..4e22c123 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,13 @@ FROM registry.access.redhat.com/ubi7/ruby-25 + +USER root +RUN rpm -Uvh http://yum.postgresql.org/9.6/redhat/rhel-7-x86_64/pgdg-centos96-9.6-3.noarch.rpm \ + && yum update -y \ + && yum remove -y postgresql \ + && yum install -y postgresql96 postgresql96-devel postgresql96-libs \ + && yum clean all \ + && rm -rf /var/cache/yum + RUN source ${APP_ROOT}/etc/scl_enable \ && gem install bundler --version=2.0.1 --no-document From 1efcacb08511aeb0f7813e7839007be4f998b05f Mon Sep 17 00:00:00 2001 From: Rael Garcia Date: Sat, 28 Mar 2020 03:22:47 +0100 Subject: [PATCH 057/156] Configure pg gem pg_config path --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4e22c123..6fd03bf7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,8 @@ RUN source ${APP_ROOT}/etc/scl_enable \ COPY Gemfile* ./ RUN source ${APP_ROOT}/etc/scl_enable \ - && bundle install --deployment --path vendor/bundle --jobs $(grep -c processor /proc/cpuinfo) --retry 3 + && bundle config build.pg --with-pg-config=/usr/pgsql-9.6/bin/pg_config \ + && bundle install --deployment --path vendor/bundle --jobs $(grep -c processor /proc/cpuinfo) --retry 3 COPY . . ENV RAILS_LOG_TO_STDOUT=1 USER root From 41b51464579567b16e3d6f42c0516a7aabe7a3f8 Mon Sep 17 00:00:00 2001 From: Rael Garcia Date: Sat, 28 Mar 2020 03:23:21 +0100 Subject: [PATCH 058/156] Simplify user and permissions setup --- Dockerfile | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6fd03bf7..8aaa44bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,30 +1,33 @@ FROM registry.access.redhat.com/ubi7/ruby-25 USER root -RUN rpm -Uvh http://yum.postgresql.org/9.6/redhat/rhel-7-x86_64/pgdg-centos96-9.6-3.noarch.rpm \ + && yum update -y \ && yum remove -y postgresql \ && yum install -y postgresql96 postgresql96-devel postgresql96-libs \ && yum clean all \ && rm -rf /var/cache/yum +USER default +WORKDIR ${APP_ROOT} + RUN source ${APP_ROOT}/etc/scl_enable \ - && gem install bundler --version=2.0.1 --no-document + && gem install bundler --version=2.0.1 --no-document -COPY Gemfile* ./ +COPY --chown=default:root Gemfile* ./ RUN source ${APP_ROOT}/etc/scl_enable \ && bundle config build.pg --with-pg-config=/usr/pgsql-9.6/bin/pg_config \ && bundle install --deployment --path vendor/bundle --jobs $(grep -c processor /proc/cpuinfo) --retry 3 -COPY . . + +COPY --chown=default:root . . + ENV RAILS_LOG_TO_STDOUT=1 -USER root -RUN mkdir -p tmp log; chmod -vfR g+w tmp log -USER default RUN source ${APP_ROOT}/etc/scl_enable \ -&& bundle exec bin/rails server -e production -d; \ -rm -rf tmp/pids -USER root -RUN chmod -fR g+w tmp/cache -USER default + && bundle exec bin/rails server -e production -d; \ + rm -rf tmp/pids + +RUN mkdir -p -m 0775 tmp/cache log \ + && chown -fR default tmp log db \ + && chmod -fR g+w tmp log db CMD [".s2i/bin/run"] From 4ced0ffae81c9fe22adb53f195f6f65925abab64 Mon Sep 17 00:00:00 2001 From: Rael Garcia Date: Mon, 30 Mar 2020 16:09:27 +0200 Subject: [PATCH 059/156] Update to PostgreSQL10 --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8aaa44bf..98935fb9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ FROM registry.access.redhat.com/ubi7/ruby-25 USER root - +RUN rpm -Uvh https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm\ && yum update -y \ && yum remove -y postgresql \ - && yum install -y postgresql96 postgresql96-devel postgresql96-libs \ + && yum install -y postgresql1pg 0 postgresql10-devel postgresql10-libs \ && yum clean all \ && rm -rf /var/cache/yum @@ -16,7 +16,7 @@ RUN source ${APP_ROOT}/etc/scl_enable \ COPY --chown=default:root Gemfile* ./ RUN source ${APP_ROOT}/etc/scl_enable \ - && bundle config build.pg --with-pg-config=/usr/pgsql-9.6/bin/pg_config \ + && bundle config build.pg --with-pg-config=/usr/pgsql-10/bin/pg_config \ && bundle install --deployment --path vendor/bundle --jobs $(grep -c processor /proc/cpuinfo) --retry 3 COPY --chown=default:root . . From 34e35588233d738b32bd4c84802b314e69570d38 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 23 Apr 2020 14:25:11 +0000 Subject: [PATCH 060/156] [Security] Bump puma from 4.3.1 to 4.3.3 Bumps [puma](https://github.com/puma/puma) from 4.3.1 to 4.3.3. **This update includes security fixes.** - [Release notes](https://github.com/puma/puma/releases) - [Changelog](https://github.com/puma/puma/blob/master/History.md) - [Commits](https://github.com/puma/puma/compare/v4.3.1...v4.3.3) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 40d7c5e9..ddd39f29 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -218,7 +218,7 @@ GEM binding_of_caller (>= 0.7) pry (>= 0.9.11) public_suffix (3.1.1) - puma (4.3.1) + puma (4.3.3) nio4r (~> 2.0) quantile (0.2.1) que (1.0.0.beta3) From f410a265cd0b56dbc753d6882cbe4e8c5f7f8bbb Mon Sep 17 00:00:00 2001 From: Guilherme Cassolato Date: Thu, 7 May 2020 12:37:56 +0200 Subject: [PATCH 061/156] Makes Kubernetes routes created by Zync to default to https when the scheme is missing This is the default case of Porta account routes, which [don't accept non-TLS routes on OpenShift](https://github.com/3scale/porta/blob/933da0da4b12d1296e4960b83ca60f3f2af07f68/openshift/system/config/settings.yml#L5), yet the API returns scheme-less provider domains (`domain` and `admin_domain`). --- app/services/integration/kubernetes_service.rb | 2 +- test/services/kubernetes_service_test.rb | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/app/services/integration/kubernetes_service.rb b/app/services/integration/kubernetes_service.rb index 6c778db7..d581f0d0 100644 --- a/app/services/integration/kubernetes_service.rb +++ b/app/services/integration/kubernetes_service.rb @@ -138,7 +138,7 @@ def initialize(url, service, port) tls_options = { insecureEdgeTerminationPolicy: 'Redirect', termination: 'edge' - } if uri.class == URI::HTTPS + } if uri.class == URI::HTTPS || uri.scheme.blank? super({ host: uri.host || uri.path, diff --git a/test/services/kubernetes_service_test.rb b/test/services/kubernetes_service_test.rb index 1f33a665..42478ed5 100644 --- a/test/services/kubernetes_service_test.rb +++ b/test/services/kubernetes_service_test.rb @@ -81,7 +81,6 @@ class RouteSpec < ActiveSupport::TestCase } assert_equal json, spec.to_hash - url = 'http://my-api.example.com' service_name = 'My API' port = 7780 @@ -94,5 +93,19 @@ class RouteSpec < ActiveSupport::TestCase } assert_equal json, spec.to_hash end + + test 'defaults to https when scheme is missing' do + url = 'my-api.example.com' + service_name = 'My API' + port = 7443 + spec = Integration::KubernetesService::RouteSpec.new(url, service_name, port) + json = { + host: "my-api.example.com", + port: {targetPort: 7443}, + to: {kind: "Service", name: "My API"}, + tls: {insecureEdgeTerminationPolicy: "Redirect", termination: "edge"} + } + assert_equal json, spec.to_hash + end end end From 8ca73d368127e098f4bf83e6e5ed3c017b4349f9 Mon Sep 17 00:00:00 2001 From: Guilherme Cassolato Date: Thu, 7 May 2020 13:12:40 +0200 Subject: [PATCH 062/156] rbenv's .ruby-version added to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 150d6277..b8c88b24 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,4 @@ .byebug_history openshift.local.clusterup +.ruby-version From f25436aeeade5cdffbd9f0745b5681a553ded676 Mon Sep 17 00:00:00 2001 From: "depfu[bot]" <23717796+depfu[bot]@users.noreply.github.com> Date: Wed, 13 May 2020 00:26:43 +0000 Subject: [PATCH 063/156] Update rack to version 2.1.3 --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index ddd39f29..64ffaf3f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -226,7 +226,7 @@ GEM erubis que (~> 1.0.0.beta3) sinatra - rack (2.0.8) + rack (2.1.3) rack-protection (2.0.5) rack rack-test (1.1.0) From 74b309e09e85896ea15627837381b585c754938d Mon Sep 17 00:00:00 2001 From: "depfu[bot]" <23717796+depfu[bot]@users.noreply.github.com> Date: Wed, 13 May 2020 09:42:04 +0000 Subject: [PATCH 064/156] Update all of rails to version 5.2.4.2 --- Gemfile.lock | 100 +++++++++++++++++++++++++-------------------------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 64ffaf3f..528b28c5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -25,43 +25,43 @@ GEM remote: https://rubygems.org/ specs: 3scale-api (0.6.0) - actioncable (5.2.3) - actionpack (= 5.2.3) + actioncable (5.2.4.2) + actionpack (= 5.2.4.2) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailer (5.2.3) - actionpack (= 5.2.3) - actionview (= 5.2.3) - activejob (= 5.2.3) + actionmailer (5.2.4.2) + actionpack (= 5.2.4.2) + actionview (= 5.2.4.2) + activejob (= 5.2.4.2) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (5.2.3) - actionview (= 5.2.3) - activesupport (= 5.2.3) - rack (~> 2.0) + actionpack (5.2.4.2) + actionview (= 5.2.4.2) + activesupport (= 5.2.4.2) + rack (~> 2.0, >= 2.0.8) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.0.2) - actionview (5.2.3) - activesupport (= 5.2.3) + actionview (5.2.4.2) + activesupport (= 5.2.4.2) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.0.3) - activejob (5.2.3) - activesupport (= 5.2.3) + activejob (5.2.4.2) + activesupport (= 5.2.4.2) globalid (>= 0.3.6) - activemodel (5.2.3) - activesupport (= 5.2.3) - activerecord (5.2.3) - activemodel (= 5.2.3) - activesupport (= 5.2.3) + activemodel (5.2.4.2) + activesupport (= 5.2.4.2) + activerecord (5.2.4.2) + activemodel (= 5.2.4.2) + activesupport (= 5.2.4.2) arel (>= 9.0) - activestorage (5.2.3) - actionpack (= 5.2.3) - activerecord (= 5.2.3) + activestorage (5.2.4.2) + actionpack (= 5.2.4.2) + activerecord (= 5.2.4.2) marcel (~> 0.3.1) - activesupport (5.2.3) + activesupport (5.2.4.2) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 0.7, < 2) minitest (~> 5.1) @@ -86,10 +86,10 @@ GEM simplecov url coderay (1.1.2) - concurrent-ruby (1.1.5) + concurrent-ruby (1.1.6) crack (0.4.3) safe_yaml (~> 1.0.0) - crass (1.0.5) + crass (1.0.6) debug_inspector (0.0.3) docile (1.3.2) dry-configurable (0.9.0) @@ -127,7 +127,7 @@ GEM globalid (0.4.2) activesupport (>= 4.2.0) hashdiff (1.0.0) - i18n (1.7.0) + i18n (1.8.2) concurrent-ruby (~> 1.0) ice_nine (0.11.2) interception (0.5) @@ -161,7 +161,7 @@ GEM activesupport (>= 4) railties (>= 4) request_store (~> 1.0) - loofah (2.4.0) + loofah (2.5.0) crass (~> 1.0.2) nokogiri (>= 1.5.9) mail (2.7.1) @@ -171,10 +171,10 @@ GEM message_bus (2.2.3) rack (>= 1.1.3) method_source (0.9.2) - mimemagic (0.3.3) - mini_mime (1.0.1) + mimemagic (0.3.5) + mini_mime (1.0.2) mini_portile2 (2.4.0) - minitest (5.13.0) + minitest (5.14.0) minitest-reporters (1.4.2) ansi builder @@ -189,7 +189,7 @@ GEM multipart-post (2.0.0) mustermann (1.0.3) nio4r (2.5.2) - nokogiri (1.10.8) + nokogiri (1.10.9) mini_portile2 (~> 2.4.0) oauth2 (1.4.1) faraday (>= 0.8, < 0.16.0) @@ -226,32 +226,32 @@ GEM erubis que (~> 1.0.0.beta3) sinatra - rack (2.1.3) + rack (2.2.2) rack-protection (2.0.5) rack rack-test (1.1.0) rack (>= 1.0, < 3) - rails (5.2.3) - actioncable (= 5.2.3) - actionmailer (= 5.2.3) - actionpack (= 5.2.3) - actionview (= 5.2.3) - activejob (= 5.2.3) - activemodel (= 5.2.3) - activerecord (= 5.2.3) - activestorage (= 5.2.3) - activesupport (= 5.2.3) + rails (5.2.4.2) + actioncable (= 5.2.4.2) + actionmailer (= 5.2.4.2) + actionpack (= 5.2.4.2) + actionview (= 5.2.4.2) + activejob (= 5.2.4.2) + activemodel (= 5.2.4.2) + activerecord (= 5.2.4.2) + activestorage (= 5.2.4.2) + activesupport (= 5.2.4.2) bundler (>= 1.3.0) - railties (= 5.2.3) + railties (= 5.2.4.2) sprockets-rails (>= 2.0.0) rails-dom-testing (2.0.3) activesupport (>= 4.2.0) nokogiri (>= 1.6) rails-html-sanitizer (1.3.0) loofah (~> 2.3) - railties (5.2.3) - actionpack (= 5.2.3) - activesupport (= 5.2.3) + railties (5.2.4.2) + actionpack (= 5.2.4.2) + activesupport (= 5.2.4.2) method_source rake (>= 0.8.7) thor (>= 0.19.0, < 2.0) @@ -286,7 +286,7 @@ GEM rack (~> 2.0) rack-protection (= 2.0.5) tilt (~> 2.0) - sprockets (3.7.2) + sprockets (4.0.0) concurrent-ruby (~> 1.0) rack (> 1, < 3) sprockets-rails (3.2.1) @@ -299,7 +299,7 @@ GEM to_regexp (0.2.1) toml (0.2.0) parslet (~> 1.8.0) - tzinfo (1.2.6) + tzinfo (1.2.7) thread_safe (~> 0.1) url (0.3.2) validate_url (1.0.8) @@ -309,9 +309,9 @@ GEM addressable (>= 2.3.6) crack (>= 0.3.2) hashdiff - websocket-driver (0.7.0) + websocket-driver (0.7.1) websocket-extensions (>= 0.1.0) - websocket-extensions (0.1.3) + websocket-extensions (0.1.4) with_env (1.1.0) xml-simple (1.1.5) yabeda (0.1.3) From c52c4bde5f944eea80d6c8dc16bc2fc9494a5125 Mon Sep 17 00:00:00 2001 From: Guilherme Cassolato Date: Wed, 13 May 2020 16:20:50 +0200 Subject: [PATCH 065/156] Bump circleci/ruby build image to v2.5.5 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 726b2346..07abfd36 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -26,7 +26,7 @@ jobs: build: working_directory: /home/circleci/zync docker: - - image: circleci/ruby:2.4.6 + - image: circleci/ruby:2.5.5 - image: circleci/postgres:10-alpine-ram environment: RAILS_ENV: test From ea20cfe06b55f085886a4c3e33221c6c31850bf4 Mon Sep 17 00:00:00 2001 From: Guilherme Cassolato Date: Thu, 14 May 2020 18:25:33 +0200 Subject: [PATCH 066/156] Use UBI base image when building in Openshift --- ...entos7-imagestream.yml => 02-ruby-25-ubi7-imagestream.yml} | 4 ++-- openshift/03-buildconfig-template.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename openshift/{02-ruby-24-centos7-imagestream.yml => 02-ruby-25-ubi7-imagestream.yml} (70%) diff --git a/openshift/02-ruby-24-centos7-imagestream.yml b/openshift/02-ruby-25-ubi7-imagestream.yml similarity index 70% rename from openshift/02-ruby-24-centos7-imagestream.yml rename to openshift/02-ruby-25-ubi7-imagestream.yml index aafa68f8..1de8dc5e 100644 --- a/openshift/02-ruby-24-centos7-imagestream.yml +++ b/openshift/02-ruby-25-ubi7-imagestream.yml @@ -3,12 +3,12 @@ kind: ImageStream metadata: labels: app: zync - name: ruby-24-centos7 + name: ruby-25-ubi7 spec: tags: - from: kind: DockerImage - name: centos/ruby-24-centos7 + name: registry.access.redhat.com/ubi7/ruby-25 name: latest referencePolicy: type: Source diff --git a/openshift/03-buildconfig-template.yml b/openshift/03-buildconfig-template.yml index 63ad87d0..64afa80a 100644 --- a/openshift/03-buildconfig-template.yml +++ b/openshift/03-buildconfig-template.yml @@ -28,7 +28,7 @@ objects: dockerStrategy: from: kind: ImageStreamTag - name: ruby-24-centos7:latest + name: ruby-25-ubi7:latest type: Docker triggers: - github: @@ -36,7 +36,7 @@ objects: type: GitHub - type: ImageChange - type: ConfigChange - + parameters: - name: GITHUB_SECRET displayName: GitHub WebHook Secret From fce7b86e9e2b067b78f25a5db8c2a49d2bb44bf5 Mon Sep 17 00:00:00 2001 From: Yorgos Saslis <2420882+gsaslis@users.noreply.github.com> Date: Fri, 15 May 2020 13:07:33 +0300 Subject: [PATCH 067/156] Switched CI image to UBI7-based ruby 2.5 image Same image as used during productization --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 07abfd36..da90c15b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,9 +24,9 @@ jobs: zync:build rails db:setup build: - working_directory: /home/circleci/zync + working_directory: /opt/app-root/zync docker: - - image: circleci/ruby:2.5.5 + - image: registry.access.redhat.com/ubi7/ruby-25 - image: circleci/postgres:10-alpine-ram environment: RAILS_ENV: test From d4a24affeb501c04b10663aec1419c7081090474 Mon Sep 17 00:00:00 2001 From: Yorgos Saslis Date: Fri, 15 May 2020 13:17:50 +0300 Subject: [PATCH 068/156] registry.access.redhat.com/ubi7/ruby-25 already contains psql --- .circleci/config.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index da90c15b..f4d9c165 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -33,8 +33,6 @@ jobs: DISABLE_SPRING: 1 # we can't really run spring as it hangs on local circleci build DATABASE_URL: postgres://postgres:@localhost/circle_test steps: - - run: sudo sh -c 'apt update && apt install -y --no-install-recommends -y postgresql-client' - - checkout # Restore bundle cache From 70f2b17f242dafe2ae096a39f931b887992fd0f4 Mon Sep 17 00:00:00 2001 From: Yorgos Saslis Date: Fri, 15 May 2020 13:25:40 +0300 Subject: [PATCH 069/156] introduced version number in cache key this allows us to be able to invalidate the cache (by incrementing the number), in cases like the failing build on the previous commit. That build tries to restore a cache on the older /home/circleci/... path, whereas with the new builder image we're using in this PR, that path no longer exists. The fix is really quite simple: don't use the cache. And this is how we can do that. --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f4d9c165..a8d06784 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -38,9 +38,9 @@ jobs: # Restore bundle cache - restore_cache: keys: - - zync-bundle-{{ arch }}-{{ checksum "Gemfile.lock" }} - - zync-bundle-{{ arch }}-{{ .Branch }} - - zync-branch-{{ arch }}-master + - zync-bundle-v2-{{ arch }}-{{ checksum "Gemfile.lock" }} + - zync-bundle-v2-{{ arch }}-{{ .Branch }} + - zync-branch-v2-{{ arch }}-master - run: name: bundle install @@ -52,7 +52,7 @@ jobs: command: BUNDLE_WITHOUT=development:test bundle exec bin/rails runner --environment=production 'puts Rails.env' - save_cache: - key: zync-bundle-{{ arch }}-{{ checksum "Gemfile.lock" }} + key: zync-bundle-v2-{{ arch }}-{{ checksum "Gemfile.lock" }} paths: - vendor/bundle @@ -73,7 +73,7 @@ jobs: path: test/reports - save_cache: - key: zync-branch-{{ arch }}-{{ .Branch }} + key: zync-branch-v2-{{ arch }}-{{ .Branch }} paths: - vendor/bundle From 07930791c0e86416b33560ee0f3880ff0e936eba Mon Sep 17 00:00:00 2001 From: "depfu[bot]" <23717796+depfu[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 18:27:22 +0000 Subject: [PATCH 070/156] Update all of rails to version 5.2.4.3 --- Gemfile.lock | 74 ++++++++++++++++++++++++++-------------------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 528b28c5..c5812bd6 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -25,43 +25,43 @@ GEM remote: https://rubygems.org/ specs: 3scale-api (0.6.0) - actioncable (5.2.4.2) - actionpack (= 5.2.4.2) + actioncable (5.2.4.3) + actionpack (= 5.2.4.3) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailer (5.2.4.2) - actionpack (= 5.2.4.2) - actionview (= 5.2.4.2) - activejob (= 5.2.4.2) + actionmailer (5.2.4.3) + actionpack (= 5.2.4.3) + actionview (= 5.2.4.3) + activejob (= 5.2.4.3) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (5.2.4.2) - actionview (= 5.2.4.2) - activesupport (= 5.2.4.2) + actionpack (5.2.4.3) + actionview (= 5.2.4.3) + activesupport (= 5.2.4.3) rack (~> 2.0, >= 2.0.8) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.0.2) - actionview (5.2.4.2) - activesupport (= 5.2.4.2) + actionview (5.2.4.3) + activesupport (= 5.2.4.3) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.0.3) - activejob (5.2.4.2) - activesupport (= 5.2.4.2) + activejob (5.2.4.3) + activesupport (= 5.2.4.3) globalid (>= 0.3.6) - activemodel (5.2.4.2) - activesupport (= 5.2.4.2) - activerecord (5.2.4.2) - activemodel (= 5.2.4.2) - activesupport (= 5.2.4.2) + activemodel (5.2.4.3) + activesupport (= 5.2.4.3) + activerecord (5.2.4.3) + activemodel (= 5.2.4.3) + activesupport (= 5.2.4.3) arel (>= 9.0) - activestorage (5.2.4.2) - actionpack (= 5.2.4.2) - activerecord (= 5.2.4.2) + activestorage (5.2.4.3) + actionpack (= 5.2.4.3) + activerecord (= 5.2.4.3) marcel (~> 0.3.1) - activesupport (5.2.4.2) + activesupport (5.2.4.3) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 0.7, < 2) minitest (~> 5.1) @@ -174,7 +174,7 @@ GEM mimemagic (0.3.5) mini_mime (1.0.2) mini_portile2 (2.4.0) - minitest (5.14.0) + minitest (5.14.1) minitest-reporters (1.4.2) ansi builder @@ -231,27 +231,27 @@ GEM rack rack-test (1.1.0) rack (>= 1.0, < 3) - rails (5.2.4.2) - actioncable (= 5.2.4.2) - actionmailer (= 5.2.4.2) - actionpack (= 5.2.4.2) - actionview (= 5.2.4.2) - activejob (= 5.2.4.2) - activemodel (= 5.2.4.2) - activerecord (= 5.2.4.2) - activestorage (= 5.2.4.2) - activesupport (= 5.2.4.2) + rails (5.2.4.3) + actioncable (= 5.2.4.3) + actionmailer (= 5.2.4.3) + actionpack (= 5.2.4.3) + actionview (= 5.2.4.3) + activejob (= 5.2.4.3) + activemodel (= 5.2.4.3) + activerecord (= 5.2.4.3) + activestorage (= 5.2.4.3) + activesupport (= 5.2.4.3) bundler (>= 1.3.0) - railties (= 5.2.4.2) + railties (= 5.2.4.3) sprockets-rails (>= 2.0.0) rails-dom-testing (2.0.3) activesupport (>= 4.2.0) nokogiri (>= 1.6) rails-html-sanitizer (1.3.0) loofah (~> 2.3) - railties (5.2.4.2) - actionpack (= 5.2.4.2) - activesupport (= 5.2.4.2) + railties (5.2.4.3) + actionpack (= 5.2.4.3) + activesupport (= 5.2.4.3) method_source rake (>= 0.8.7) thor (>= 0.19.0, < 2.0) From e5be877f87109e18d377b2490a27a9d3de03582d Mon Sep 17 00:00:00 2001 From: Guilherme Cassolato Date: Mon, 18 May 2020 18:43:37 +0200 Subject: [PATCH 071/156] [refactoring] Que stats Prometheus metrics - Renames the following measurement types of the `jobs_scheduled_total` Prometheus (gauge) metric: - `/retry` to `/retried` (Que jobs with at least 1 retry attempt) - `/future` to `/scheduled` (Que jobs scheduled to be executed in the future) - Introduces the following new measurement types of the `jobs_scheduled_total` Prometheus (gauge) metric: - `/ready`: Que jobs ready to be executed for the first time (excludes: errored, expired, finished, performed) - `/finished`: Que jobs already performed - `/failed`: Que jobs that failed at least once --- lib/prometheus/que_stats.rb | 46 +++++++++++------- test/lib/prometheus/que_stats_test.rb | 69 ++++++++++++++++++++++++++- 2 files changed, 97 insertions(+), 18 deletions(-) diff --git a/lib/prometheus/que_stats.rb b/lib/prometheus/que_stats.rb index a00850a4..55a839ee 100644 --- a/lib/prometheus/que_stats.rb +++ b/lib/prometheus/que_stats.rb @@ -17,8 +17,8 @@ def worker_stats end end - def job_stats(filter = nil) - filter = "WHERE #{filter}" if filter + def job_stats(*filters) + filter = "WHERE #{filters.join(' AND ')}" if filters.presence sql = <<~SQL SELECT args->0->>'job_class' AS job_class, COUNT(*) as count FROM que_jobs #{filter} @@ -30,6 +30,26 @@ def job_stats(filter = nil) end end + def job_stats_ready + job_stats('error_count = 0', 'expired_at IS NULL', 'finished_at IS NULL', 'run_at <= now()') + end + + def job_stats_scheduled + job_stats('run_at > now()') + end + + def job_stats_finished + job_stats('finished_at IS NOT NULL') + end + + def job_stats_retried + job_stats(%q[(args->0->>'retries')::integer > 0]) + end + + def job_stats_failed + job_stats('error_count > 0') + end + mattr_accessor :read_only_transaction, default: true, instance_accessor: false class << self @@ -54,23 +74,17 @@ def execute Yabeda.configure do group :que do scheduled_jobs = gauge :jobs_scheduled_total, comment: 'Que Jobs to be executed' - workers = gauge :workers_total, comment: 'Que Workers running' - - collect do - Prometheus::QueStats.job_stats.each do |stats| - scheduled_jobs.set({ job: stats.fetch('job_class') }, stats.fetch('count')) - end + workers = gauge :workers_total, comment: 'Que Workers running' - Prometheus::QueStats.job_stats(%q[(args->0->>'retries')::integer > 0]).each do |stats| - scheduled_jobs.set({ job: stats.fetch('job_class'), type: 'retry' }, stats.fetch('count')) - end + set_stats = ->(type, stats) { scheduled_jobs.set({ job: stats.fetch('job_class') }.merge(type), stats.fetch('count')) } - Prometheus::QueStats.job_stats('run_at < now()').each do |stats| - scheduled_jobs.set({ job: stats.fetch('job_class'), type: 'scheduled' }, stats.fetch('count')) - end + collect do + set_stats_all = set_stats.curry.call({}) + Prometheus::QueStats.job_stats.each(&set_stats_all) - Prometheus::QueStats.job_stats('run_at > now()').each do |stats| - scheduled_jobs.set({ job: stats.fetch('job_class'), type: 'future' }, stats.fetch('count')) + %w[ready scheduled finished retried failed].each do |type| + set_stats_type = set_stats.curry.call(type: type) + Prometheus::QueStats.public_send("job_stats_#{type}").each(&set_stats_type) end workers.set({ }, Prometheus::QueStats.worker_stats.fetch('workers')) diff --git a/test/lib/prometheus/que_stats_test.rb b/test/lib/prometheus/que_stats_test.rb index 649defed..90a73403 100644 --- a/test/lib/prometheus/que_stats_test.rb +++ b/test/lib/prometheus/que_stats_test.rb @@ -17,8 +17,67 @@ class Prometheus::QueStatsTest < ActiveSupport::TestCase end test 'job stats' do - assert Prometheus::QueStats.job_stats - assert Prometheus::QueStats.job_stats('1 > 0') + Que.stop! + ApplicationJob.perform_later + assert Prometheus::QueStats.job_stats.any? + assert Prometheus::QueStats.job_stats('1 > 0').any? + assert Prometheus::QueStats.job_stats('1 > 0', '2 > 1').any? + assert Prometheus::QueStats.job_stats('1 > 0', '2 < 1').empty? + end + + test 'ready jobs stats' do + Que.stop! + assert Prometheus::QueStats.job_stats_ready.empty? + jobs = Array.new(3) { ApplicationJob.perform_later } + jobs << ApplicationJob.set(wait_until: 1.day.from_now).perform_later + assert_equal 3, Prometheus::QueStats.job_stats_ready.first['count'] + update_job(jobs[0], error_count: 1) + assert_equal 2, Prometheus::QueStats.job_stats_ready.first['count'] + update_job(jobs[1], expired_at: 1.minute.ago) + assert_equal 1, Prometheus::QueStats.job_stats_ready.first['count'] + update_job(jobs[2], finished_at: 1.minute.ago) + assert Prometheus::QueStats.job_stats_ready.empty? + end + + test 'scheduled jobs stats' do + Que.stop! + assert Prometheus::QueStats.job_stats_scheduled.empty? + jobs = [ApplicationJob, ApplicationJob.set(wait_until: 1.day.from_now)].map(&:perform_later) + assert_equal 1, Prometheus::QueStats.job_stats_scheduled.first['count'] + update_job(jobs.last, run_at: 1.minute.ago) + assert Prometheus::QueStats.job_stats_scheduled.empty? + end + + test 'finished jobs stats' do + Que.stop! + assert Prometheus::QueStats.job_stats_finished.empty? + jobs = Array.new(2) { ApplicationJob.perform_later } + assert Prometheus::QueStats.job_stats_finished.empty? + update_job(jobs.first, finished_at: Time.now) + assert_equal 1, Prometheus::QueStats.job_stats_finished.first['count'] + end + + test 'retried jobs stats' do + Que.stop! + assert Prometheus::QueStats.job_stats_retried.empty? + jobs = Array.new(2) { ApplicationJob.perform_later } + assert Prometheus::QueStats.job_stats_retried.empty? + + job = jobs.first + job_model = ApplicationJob.model.where("args->0->>'job_id' = ?", job.job_id).first + job_model.args = [job_model.args.first.merge('retries' => 1)] + job_model.save! + + assert_equal 1, Prometheus::QueStats.job_stats_retried.first['count'] + end + + test 'failed jobs stats' do + Que.stop! + assert Prometheus::QueStats.job_stats_failed.empty? + jobs = Array.new(2) { ApplicationJob.perform_later } + assert Prometheus::QueStats.job_stats_failed.empty? + update_job(jobs.first, error_count: 1) + assert_equal 1, Prometheus::QueStats.job_stats_failed.first['count'] end class WithTransaction < ActiveSupport::TestCase @@ -46,4 +105,10 @@ def test_readonly_transaction assert Prometheus::Client::Formats::Text.marshal(Yabeda::Prometheus.registry) end + + protected + + def update_job(job, attributes = {}) + ApplicationJob.model.where("args->0->>'job_id' = ?", job.job_id).update_all(attributes) + end end From bdc9836ef751fde9197286d3f4be2df8ad7b9d32 Mon Sep 17 00:00:00 2001 From: Guilherme Cassolato Date: Thu, 21 May 2020 17:03:22 +0200 Subject: [PATCH 072/156] Updates the count of a job class and type back to zero --- lib/prometheus/que_stats.rb | 15 +++++++-- test/lib/prometheus/que_stats_test.rb | 48 +++++++++++++++------------ 2 files changed, 39 insertions(+), 24 deletions(-) diff --git a/lib/prometheus/que_stats.rb b/lib/prometheus/que_stats.rb index 55a839ee..cac94e1b 100644 --- a/lib/prometheus/que_stats.rb +++ b/lib/prometheus/que_stats.rb @@ -17,12 +17,21 @@ def worker_stats end end + # ApplicationJob did not have to be here, but it's just harder to test otherwise because of ApplicationJob#delete_duplicates + JOB_CLASSES = %w[ApplicationJob ProcessEntryJob ProcessIntegrationEntryJob UpdateJob].inspect.tr('"', "'").freeze + def job_stats(*filters) filter = "WHERE #{filters.join(' AND ')}" if filters.presence sql = <<~SQL - SELECT args->0->>'job_class' AS job_class, COUNT(*) as count - FROM que_jobs #{filter} - GROUP BY args->0->>'job_class' + WITH + jobs AS (SELECT unnest(array#{JOB_CLASSES}) AS job_class), + stats AS ( + SELECT args->0->>'job_class' AS job_class, COUNT(*) as count + FROM que_jobs #{filter} + GROUP BY args->0->>'job_class' + ) + SELECT jobs.job_class, COALESCE(stats.count, 0) AS count + FROM jobs LEFT JOIN stats ON jobs.job_class = stats.job_class SQL execute do |connection| diff --git a/test/lib/prometheus/que_stats_test.rb b/test/lib/prometheus/que_stats_test.rb index 90a73403..866e3788 100644 --- a/test/lib/prometheus/que_stats_test.rb +++ b/test/lib/prometheus/que_stats_test.rb @@ -19,65 +19,65 @@ class Prometheus::QueStatsTest < ActiveSupport::TestCase test 'job stats' do Que.stop! ApplicationJob.perform_later - assert Prometheus::QueStats.job_stats.any? - assert Prometheus::QueStats.job_stats('1 > 0').any? - assert Prometheus::QueStats.job_stats('1 > 0', '2 > 1').any? - assert Prometheus::QueStats.job_stats('1 > 0', '2 < 1').empty? + assert_equal 1, stats_count + assert_equal 1, stats_count(where: ['1 > 0']) + assert_equal 1, stats_count(where: ['1 > 0', '2 > 1']) + assert_equal 0, stats_count(where: ['1 > 0', '2 < 1']) end test 'ready jobs stats' do Que.stop! - assert Prometheus::QueStats.job_stats_ready.empty? + assert_equal 0, stats_count(type: :ready) jobs = Array.new(3) { ApplicationJob.perform_later } jobs << ApplicationJob.set(wait_until: 1.day.from_now).perform_later - assert_equal 3, Prometheus::QueStats.job_stats_ready.first['count'] + assert_equal 3, stats_count(type: :ready) update_job(jobs[0], error_count: 1) - assert_equal 2, Prometheus::QueStats.job_stats_ready.first['count'] + assert_equal 2, stats_count(type: :ready) update_job(jobs[1], expired_at: 1.minute.ago) - assert_equal 1, Prometheus::QueStats.job_stats_ready.first['count'] + assert_equal 1, stats_count(type: :ready) update_job(jobs[2], finished_at: 1.minute.ago) - assert Prometheus::QueStats.job_stats_ready.empty? + assert_equal 0, stats_count(type: :ready) end test 'scheduled jobs stats' do Que.stop! - assert Prometheus::QueStats.job_stats_scheduled.empty? + assert_equal 0, stats_count(type: :scheduled) jobs = [ApplicationJob, ApplicationJob.set(wait_until: 1.day.from_now)].map(&:perform_later) - assert_equal 1, Prometheus::QueStats.job_stats_scheduled.first['count'] + assert_equal 1, stats_count(type: :scheduled) update_job(jobs.last, run_at: 1.minute.ago) - assert Prometheus::QueStats.job_stats_scheduled.empty? + assert_equal 0, stats_count(type: :scheduled) end test 'finished jobs stats' do Que.stop! - assert Prometheus::QueStats.job_stats_finished.empty? + assert_equal 0, stats_count(type: :finished) jobs = Array.new(2) { ApplicationJob.perform_later } - assert Prometheus::QueStats.job_stats_finished.empty? + assert_equal 0, stats_count(type: :finished) update_job(jobs.first, finished_at: Time.now) - assert_equal 1, Prometheus::QueStats.job_stats_finished.first['count'] + assert_equal 1, stats_count(type: :finished) end test 'retried jobs stats' do Que.stop! - assert Prometheus::QueStats.job_stats_retried.empty? + assert_equal 0, stats_count(type: :retried) jobs = Array.new(2) { ApplicationJob.perform_later } - assert Prometheus::QueStats.job_stats_retried.empty? + assert_equal 0, stats_count(type: :retried) job = jobs.first job_model = ApplicationJob.model.where("args->0->>'job_id' = ?", job.job_id).first job_model.args = [job_model.args.first.merge('retries' => 1)] job_model.save! - assert_equal 1, Prometheus::QueStats.job_stats_retried.first['count'] + assert_equal 1, stats_count(type: :retried) end test 'failed jobs stats' do Que.stop! - assert Prometheus::QueStats.job_stats_failed.empty? + assert_equal 0, stats_count(type: :failed) jobs = Array.new(2) { ApplicationJob.perform_later } - assert Prometheus::QueStats.job_stats_failed.empty? + assert_equal 0, stats_count(type: :failed) update_job(jobs.first, error_count: 1) - assert_equal 1, Prometheus::QueStats.job_stats_failed.first['count'] + assert_equal 1, stats_count(type: :failed) end class WithTransaction < ActiveSupport::TestCase @@ -111,4 +111,10 @@ def test_readonly_transaction def update_job(job, attributes = {}) ApplicationJob.model.where("args->0->>'job_id' = ?", job.job_id).update_all(attributes) end + + def stats_count(job_class: ApplicationJob.name, type: nil, where: []) + stats = type ? Prometheus::QueStats.public_send("job_stats_#{type}") : Prometheus::QueStats.job_stats(*where) + record = stats.find { |record| record['job_class'] == job_class } + record['count'] + end end From 57290e879df9f8da0094f79c6d8bbef6cc4c435c Mon Sep 17 00:00:00 2001 From: Guilherme Cassolato Date: Fri, 29 May 2020 18:52:08 +0200 Subject: [PATCH 073/156] Refactoring Prometheus::QueStats Replaces raw SQL in Prometheus::QueStats#job_stats with some Arel and BabySqueel --- lib/prometheus/que_stats.rb | 39 +++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/lib/prometheus/que_stats.rb b/lib/prometheus/que_stats.rb index cac94e1b..b9ad8469 100644 --- a/lib/prometheus/que_stats.rb +++ b/lib/prometheus/que_stats.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true require 'prometheus/client/metric' +require 'que/active_record/model' module Prometheus @@ -21,30 +22,34 @@ def worker_stats JOB_CLASSES = %w[ApplicationJob ProcessEntryJob ProcessIntegrationEntryJob UpdateJob].inspect.tr('"', "'").freeze def job_stats(*filters) - filter = "WHERE #{filters.join(' AND ')}" if filters.presence - sql = <<~SQL - WITH - jobs AS (SELECT unnest(array#{JOB_CLASSES}) AS job_class), - stats AS ( - SELECT args->0->>'job_class' AS job_class, COUNT(*) as count - FROM que_jobs #{filter} - GROUP BY args->0->>'job_class' - ) - SELECT jobs.job_class, COALESCE(stats.count, 0) AS count - FROM jobs LEFT JOIN stats ON jobs.job_class = stats.job_class - SQL + job_classes_expression = Arel.sql("SELECT unnest(ARRAY#{JOB_CLASSES})").as('job_class') + job_classes_table = Arel::Table.new(:jobs) + job_class_column = job_classes_table['job_class'] + + stats_count_relation = Que::ActiveRecord::Model.selecting { [Arel.sql("args->0->>'job_class'").as('job_class'), Arel.star.count.as('count')] }.group("args->0->>'job_class'") + stats_count_relation = filters.reduce(stats_count_relation) { |relation, filter| relation.where(filter) } + stats_count_table = Arel::Table.new(:stats) + + relation = job_classes_table.join(stats_count_table, Arel::Nodes::OuterJoin). + on(job_class_column.eq(stats_count_table['job_class'])). + project([job_class_column, Arel::Nodes::NamedFunction.new('coalesce', [stats_count_table['count'], 0]).as('count')]). + with([ + Arel::Nodes::As.new(job_classes_table, Arel.sql("(#{job_classes_expression.to_sql})")), + Arel::Nodes::As.new(stats_count_table, stats_count_relation.arel) + ]) execute do |connection| - connection.select_all(sql) + connection.select_all(relation.to_sql) end end def job_stats_ready - job_stats('error_count = 0', 'expired_at IS NULL', 'finished_at IS NULL', 'run_at <= now()') + conditions = [['error_count = ?', 0], { expired_at: nil, finished_at: nil }, ['run_at <= ?', Time.zone.now]] + job_stats(*conditions) end def job_stats_scheduled - job_stats('run_at > now()') + job_stats(['run_at > ?', Time.zone.now]) end def job_stats_finished @@ -52,11 +57,11 @@ def job_stats_finished end def job_stats_retried - job_stats(%q[(args->0->>'retries')::integer > 0]) + job_stats(["(args->0->>'retries')::integer > ?", 0]) end def job_stats_failed - job_stats('error_count > 0') + job_stats(['error_count > ?', 0]) end mattr_accessor :read_only_transaction, default: true, instance_accessor: false From 559aac9e7dae50bf80760ec5fe71cdb89d3097de Mon Sep 17 00:00:00 2001 From: Guilherme Cassolato Date: Tue, 2 Jun 2020 13:52:26 +0200 Subject: [PATCH 074/156] Proper classes to calculate and collect the Prometheus metrics - `Prometheus::QueStats::WorkerStats#call` to replace `Prometheus::QueStats.worker_stats` - `Prometheus::QueStats::JobStats#call` to replace `Prometheus::QueStats.job_stats` - New `Prometheus::QueStats::StatsCollector` and `Prometheus::QueStats::GroupedStatsCollector` to replace explicit call of `gauge.set` inside the Yabeda collector block --- lib/prometheus/que_stats.rb | 205 ++++++++++++++++++-------- test/lib/prometheus/que_stats_test.rb | 9 +- 2 files changed, 145 insertions(+), 69 deletions(-) diff --git a/lib/prometheus/que_stats.rb b/lib/prometheus/que_stats.rb index b9ad8469..34e2546b 100644 --- a/lib/prometheus/que_stats.rb +++ b/lib/prometheus/que_stats.rb @@ -3,106 +3,181 @@ require 'que/active_record/model' module Prometheus - # Prometheus metric to get job stats from Que. module QueStats - module_function + mattr_accessor :read_only_transaction, default: true, instance_accessor: false - WORKER_STATS = <<~SQL - SELECT SUM(worker_count) AS workers, COUNT(*) AS nodes FROM que_lockers - SQL + class Stats + def call + raise NoMethodError, __method__ + end + + protected - def worker_stats - execute do |connection| - connection.select_one(WORKER_STATS) + DEFAULT_STATEMENT_TIMEOUT = 'SET LOCAL statement_timeout TO DEFAULT' + READ_ONLY_TRANSACTION = 'SET TRANSACTION ISOLATION LEVEL SERIALIZABLE READ ONLY DEFERRABLE' + + def execute + connection.transaction(requires_new: true, joinable: false) do + connection.execute(DEFAULT_STATEMENT_TIMEOUT) + connection.execute(READ_ONLY_TRANSACTION) if Prometheus::QueStats.read_only_transaction + yield connection + end end + + delegate :connection, to: ActiveRecord::Base.name end - # ApplicationJob did not have to be here, but it's just harder to test otherwise because of ApplicationJob#delete_duplicates - JOB_CLASSES = %w[ApplicationJob ProcessEntryJob ProcessIntegrationEntryJob UpdateJob].inspect.tr('"', "'").freeze + class WorkerStats < Stats + WORKER_STATS = <<~SQL + SELECT SUM(worker_count) AS workers_count, COUNT(*) AS nodes_count FROM que_lockers + SQL + private_constant :WORKER_STATS - def job_stats(*filters) - job_classes_expression = Arel.sql("SELECT unnest(ARRAY#{JOB_CLASSES})").as('job_class') - job_classes_table = Arel::Table.new(:jobs) - job_class_column = job_classes_table['job_class'] + def call + execute do |connection| + connection.select_one(WORKER_STATS) + end + end - stats_count_relation = Que::ActiveRecord::Model.selecting { [Arel.sql("args->0->>'job_class'").as('job_class'), Arel.star.count.as('count')] }.group("args->0->>'job_class'") - stats_count_relation = filters.reduce(stats_count_relation) { |relation, filter| relation.where(filter) } - stats_count_table = Arel::Table.new(:stats) + def workers + call.fetch('workers_count') + end - relation = job_classes_table.join(stats_count_table, Arel::Nodes::OuterJoin). - on(job_class_column.eq(stats_count_table['job_class'])). - project([job_class_column, Arel::Nodes::NamedFunction.new('coalesce', [stats_count_table['count'], 0]).as('count')]). - with([ - Arel::Nodes::As.new(job_classes_table, Arel.sql("(#{job_classes_expression.to_sql})")), - Arel::Nodes::As.new(stats_count_table, stats_count_relation.arel) - ]) + alias_method :all, :workers - execute do |connection| - connection.select_all(relation.to_sql) + def nodes + call.fetch('nodes_count') end end - def job_stats_ready - conditions = [['error_count = ?', 0], { expired_at: nil, finished_at: nil }, ['run_at <= ?', Time.zone.now]] - job_stats(*conditions) - end + class JobStats < Stats + def initialize + @job_classes_table = Arel::Table.new(:jobs) + @stats_count_table = Arel::Table.new(:stats) + end - def job_stats_scheduled - job_stats(['run_at > ?', Time.zone.now]) - end + attr_reader :job_classes_table, :stats_count_table - def job_stats_finished - job_stats('finished_at IS NOT NULL') - end + def call(*filters) + filtered_stats_count_relation = filters.reduce(build_stats_count_relation) { |relation, filter| relation.where(filter) } + common_tables = [job_classes_arel, Arel::Nodes::As.new(stats_count_table, filtered_stats_count_relation.arel)] - def job_stats_retried - job_stats(["(args->0->>'retries')::integer > ?", 0]) - end + relation = job_classes_table.join(stats_count_table, Arel::Nodes::OuterJoin). + on(job_class_column.eq(stats_count_table[JOB_CLASS_COLUMN_NAME])). + project([job_class_column, Arel::Nodes::NamedFunction.new('coalesce', [stats_count_table['count'], 0]).as('count')]). + with(common_tables) - def job_stats_failed - job_stats(['error_count > ?', 0]) - end + execute do |connection| + connection.select_all(relation.to_sql) + end + end - mattr_accessor :read_only_transaction, default: true, instance_accessor: false + alias_method :all, :call + + def ready + conditions = [['error_count = ?', 0], { expired_at: nil, finished_at: nil }, ['run_at <= ?', Time.zone.now]] + call(*conditions) + end + + def scheduled + call(['run_at > ?', Time.zone.now]) + end + + def finished + call('finished_at IS NOT NULL') + end + + def retried + call(["(args->0->>'retries')::integer > ?", 0]) + end + + def failed + call(['error_count > ?', 0]) + end - class << self protected - def execute - connection.transaction(requires_new: true, joinable: false) do - connection.execute(DEFAULT_STATEMENT_TIMEOUT) - connection.execute(READ_ONLY_TRANSACTION) if read_only_transaction - yield connection + # ApplicationJob did not have to be here, but it's just harder to test otherwise because of ApplicationJob#delete_duplicates + JOB_CLASSES = %w[ApplicationJob ProcessEntryJob ProcessIntegrationEntryJob UpdateJob].inspect.tr('"', "'").freeze + private_constant :JOB_CLASSES + + JOB_CLASS_COLUMN_NAME = 'job' + private_constant :JOB_CLASS_COLUMN_NAME + + def job_class_column + @job_classes_table[JOB_CLASS_COLUMN_NAME] + end + + def job_classes_arel + @job_classes_arel ||= begin + job_classes = Arel.sql("SELECT unnest(ARRAY#{JOB_CLASSES})").as(JOB_CLASS_COLUMN_NAME) + Arel::Nodes::As.new(job_classes_table, Arel.sql("(#{job_classes.to_sql})")) end end - delegate :connection, to: ActiveRecord::Base.name + def build_stats_count_relation + Que::ActiveRecord::Model.selecting { [Arel.sql("args->0->>'job_class'").as(JOB_CLASS_COLUMN_NAME), Arel.star.count.as('count')] }.group("args->0->>'job_class'") + end end - DEFAULT_STATEMENT_TIMEOUT = 'SET LOCAL statement_timeout TO DEFAULT' - READ_ONLY_TRANSACTION = 'SET TRANSACTION ISOLATION LEVEL SERIALIZABLE READ ONLY DEFERRABLE' + class StatsCollector + def initialize(gauge, stats) + @gauge = gauge + @stats = stats + end + + attr_reader :gauge, :stats + + def call(type = nil) + type_hash, stats_count = type_hash_and_stats_count(type) + gauge.set({ }.merge(type_hash), stats_count) + end + + protected + + def type_hash_and_stats_count(type) + type_hash = type ? { type: type } : {} + stats_count = stats.public_send(type ? type : :all) + [type_hash, stats_count] + end + end + + class GroupedStatsCollector < StatsCollector + def initialize(gauge, stats, grouped_by:) + super(gauge, stats) + @grouped_by = grouped_by + end + + attr_reader :grouped_by + + def call(type = nil) + type_hash, grouped_stats_count = type_hash_and_stats_count(type) + grouped_stats_count.each do |stats_count| + gauge.set({ grouped_by => stats_count.fetch(grouped_by) }.merge(type_hash), stats_count.fetch('count')) + end + end + end end end Yabeda.configure do group :que do - scheduled_jobs = gauge :jobs_scheduled_total, comment: 'Que Jobs to be executed' workers = gauge :workers_total, comment: 'Que Workers running' + worker_stats = Prometheus::QueStats::WorkerStats.new + collector = Prometheus::QueStats::StatsCollector.new(workers, worker_stats) + collect(&collector.method(:call)) + end +end - set_stats = ->(type, stats) { scheduled_jobs.set({ job: stats.fetch('job_class') }.merge(type), stats.fetch('count')) } - +Yabeda.configure do + group :que do + jobs = gauge :jobs_scheduled_total, comment: 'Que Jobs to be executed' + job_stats = Prometheus::QueStats::JobStats.new + collector = Prometheus::QueStats::GroupedStatsCollector.new(jobs, job_stats, grouped_by: 'job') collect do - set_stats_all = set_stats.curry.call({}) - Prometheus::QueStats.job_stats.each(&set_stats_all) - - %w[ready scheduled finished retried failed].each do |type| - set_stats_type = set_stats.curry.call(type: type) - Prometheus::QueStats.public_send("job_stats_#{type}").each(&set_stats_type) - end - - workers.set({ }, Prometheus::QueStats.worker_stats.fetch('workers')) + collector.call + %w[ready scheduled finished retried failed].each(&collector.method(:call)) end end end - diff --git a/test/lib/prometheus/que_stats_test.rb b/test/lib/prometheus/que_stats_test.rb index 866e3788..0cdcccea 100644 --- a/test/lib/prometheus/que_stats_test.rb +++ b/test/lib/prometheus/que_stats_test.rb @@ -13,7 +13,7 @@ class Prometheus::QueStatsTest < ActiveSupport::TestCase end test 'worker stats' do - assert Prometheus::QueStats.worker_stats + assert Prometheus::QueStats::WorkerStats.new.call end test 'job stats' do @@ -84,7 +84,7 @@ class WithTransaction < ActiveSupport::TestCase uses_transaction :test_readonly_transaction def test_readonly_transaction Prometheus::QueStats.stub(:read_only_transaction, true) do - Prometheus::QueStats.worker_stats + Prometheus::QueStats::WorkerStats.new.call end end end @@ -113,8 +113,9 @@ def update_job(job, attributes = {}) end def stats_count(job_class: ApplicationJob.name, type: nil, where: []) - stats = type ? Prometheus::QueStats.public_send("job_stats_#{type}") : Prometheus::QueStats.job_stats(*where) - record = stats.find { |record| record['job_class'] == job_class } + job_stats = Prometheus::QueStats::JobStats.new + stats = type ? job_stats.public_send(type) : job_stats.call(*where) + record = stats.find { |record| record['job'] == job_class } record['count'] end end From 5c784500163e24327cab5f40434be7d682b48b39 Mon Sep 17 00:00:00 2001 From: "depfu[bot]" <23717796+depfu[bot]@users.noreply.github.com> Date: Fri, 5 Jun 2020 18:37:18 +0000 Subject: [PATCH 075/156] Update websocket-extensions to version 0.1.5 --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index c5812bd6..fca92327 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -311,7 +311,7 @@ GEM hashdiff websocket-driver (0.7.1) websocket-extensions (>= 0.1.0) - websocket-extensions (0.1.4) + websocket-extensions (0.1.5) with_env (1.1.0) xml-simple (1.1.5) yabeda (0.1.3) From 6a24e0a9a07f7780398717c8fc8996d7095b7e6c Mon Sep 17 00:00:00 2001 From: Guilherme Cassolato Date: Thu, 25 Jun 2020 12:39:22 +0200 Subject: [PATCH 076/156] Update Prometheus metric que_jobs_scheduled_total Added type="expired" -> jobs that failed and ran out of retry attempts, so they won't be retried any longer --- lib/prometheus/que_stats.rb | 6 +++++- test/lib/prometheus/que_stats_test.rb | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/prometheus/que_stats.rb b/lib/prometheus/que_stats.rb index 34e2546b..b0c0dd21 100644 --- a/lib/prometheus/que_stats.rb +++ b/lib/prometheus/que_stats.rb @@ -96,6 +96,10 @@ def failed call(['error_count > ?', 0]) end + def expired + call('expired_at IS NOT NULL') + end + protected # ApplicationJob did not have to be here, but it's just harder to test otherwise because of ApplicationJob#delete_duplicates @@ -177,7 +181,7 @@ def call(type = nil) collector = Prometheus::QueStats::GroupedStatsCollector.new(jobs, job_stats, grouped_by: 'job') collect do collector.call - %w[ready scheduled finished retried failed].each(&collector.method(:call)) + %w[ready scheduled finished retried failed expired].each(&collector.method(:call)) end end end diff --git a/test/lib/prometheus/que_stats_test.rb b/test/lib/prometheus/que_stats_test.rb index 0cdcccea..b395931f 100644 --- a/test/lib/prometheus/que_stats_test.rb +++ b/test/lib/prometheus/que_stats_test.rb @@ -80,6 +80,15 @@ class Prometheus::QueStatsTest < ActiveSupport::TestCase assert_equal 1, stats_count(type: :failed) end + test 'expired jobs stats' do + Que.stop! + assert_equal 0, stats_count(type: :expired) + jobs = Array.new(2) { ApplicationJob.perform_later } + assert_equal 0, stats_count(type: :expired) + update_job(jobs.first, error_count: 16, expired_at: Time.now.utc) + assert_equal 1, stats_count(type: :expired) + end + class WithTransaction < ActiveSupport::TestCase uses_transaction :test_readonly_transaction def test_readonly_transaction From c1f089079a2f170e50ce9596bfde6640d4f97ca4 Mon Sep 17 00:00:00 2001 From: Guilherme Cassolato Date: Thu, 25 Jun 2020 12:47:48 +0200 Subject: [PATCH 077/156] Failed jobs no longer counted within 'scheduled' Prometheus metric --- lib/prometheus/que_stats.rb | 2 +- test/lib/prometheus/que_stats_test.rb | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/prometheus/que_stats.rb b/lib/prometheus/que_stats.rb index b0c0dd21..1554b6dc 100644 --- a/lib/prometheus/que_stats.rb +++ b/lib/prometheus/que_stats.rb @@ -81,7 +81,7 @@ def ready end def scheduled - call(['run_at > ?', Time.zone.now]) + call(['error_count = ?', 0], ['run_at > ?', Time.zone.now]) end def finished diff --git a/test/lib/prometheus/que_stats_test.rb b/test/lib/prometheus/que_stats_test.rb index b395931f..4fbfa48e 100644 --- a/test/lib/prometheus/que_stats_test.rb +++ b/test/lib/prometheus/que_stats_test.rb @@ -42,7 +42,9 @@ class Prometheus::QueStatsTest < ActiveSupport::TestCase test 'scheduled jobs stats' do Que.stop! assert_equal 0, stats_count(type: :scheduled) - jobs = [ApplicationJob, ApplicationJob.set(wait_until: 1.day.from_now)].map(&:perform_later) + jobs = [ApplicationJob, ApplicationJob.set(wait_until: 1.day.from_now), ApplicationJob.set(wait_until: 2.days.from_now)].map(&:perform_later) + assert_equal 2, stats_count(type: :scheduled) + update_job(jobs[1], error_count: 16, expired_at: 1.minute.ago) assert_equal 1, stats_count(type: :scheduled) update_job(jobs.last, run_at: 1.minute.ago) assert_equal 0, stats_count(type: :scheduled) From a688f936ff0571e4830c95f78e8dd0309a0e2da9 Mon Sep 17 00:00:00 2001 From: Guilherme Cassolato Date: Thu, 25 Jun 2020 12:54:39 +0200 Subject: [PATCH 078/156] Removes 'retried' type from the que prometheus metrics --- lib/prometheus/que_stats.rb | 6 +----- test/lib/prometheus/que_stats_test.rb | 14 -------------- 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/lib/prometheus/que_stats.rb b/lib/prometheus/que_stats.rb index 1554b6dc..d1f99b42 100644 --- a/lib/prometheus/que_stats.rb +++ b/lib/prometheus/que_stats.rb @@ -88,10 +88,6 @@ def finished call('finished_at IS NOT NULL') end - def retried - call(["(args->0->>'retries')::integer > ?", 0]) - end - def failed call(['error_count > ?', 0]) end @@ -181,7 +177,7 @@ def call(type = nil) collector = Prometheus::QueStats::GroupedStatsCollector.new(jobs, job_stats, grouped_by: 'job') collect do collector.call - %w[ready scheduled finished retried failed expired].each(&collector.method(:call)) + %w[ready scheduled finished failed expired].each(&collector.method(:call)) end end end diff --git a/test/lib/prometheus/que_stats_test.rb b/test/lib/prometheus/que_stats_test.rb index 4fbfa48e..c29b8ce3 100644 --- a/test/lib/prometheus/que_stats_test.rb +++ b/test/lib/prometheus/que_stats_test.rb @@ -59,20 +59,6 @@ class Prometheus::QueStatsTest < ActiveSupport::TestCase assert_equal 1, stats_count(type: :finished) end - test 'retried jobs stats' do - Que.stop! - assert_equal 0, stats_count(type: :retried) - jobs = Array.new(2) { ApplicationJob.perform_later } - assert_equal 0, stats_count(type: :retried) - - job = jobs.first - job_model = ApplicationJob.model.where("args->0->>'job_id' = ?", job.job_id).first - job_model.args = [job_model.args.first.merge('retries' => 1)] - job_model.save! - - assert_equal 1, stats_count(type: :retried) - end - test 'failed jobs stats' do Que.stop! assert_equal 0, stats_count(type: :failed) From 7e84400ab1a198479a13daabacec03ac1f272a8a Mon Sep 17 00:00:00 2001 From: Guilherme Cassolato Date: Thu, 25 Jun 2020 12:59:52 +0200 Subject: [PATCH 079/156] Removed expired jobs from the 'failed' metric --- lib/prometheus/que_stats.rb | 2 +- test/lib/prometheus/que_stats_test.rb | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/prometheus/que_stats.rb b/lib/prometheus/que_stats.rb index d1f99b42..56579756 100644 --- a/lib/prometheus/que_stats.rb +++ b/lib/prometheus/que_stats.rb @@ -89,7 +89,7 @@ def finished end def failed - call(['error_count > ?', 0]) + call(['error_count > ?', 0], { expired_at: nil }) end def expired diff --git a/test/lib/prometheus/que_stats_test.rb b/test/lib/prometheus/que_stats_test.rb index c29b8ce3..09d102cf 100644 --- a/test/lib/prometheus/que_stats_test.rb +++ b/test/lib/prometheus/que_stats_test.rb @@ -66,6 +66,10 @@ class Prometheus::QueStatsTest < ActiveSupport::TestCase assert_equal 0, stats_count(type: :failed) update_job(jobs.first, error_count: 1) assert_equal 1, stats_count(type: :failed) + update_job(jobs.first, error_count: 15) + assert_equal 1, stats_count(type: :failed) + update_job(jobs.first, error_count: 16, expired_at: Time.now.utc) + assert_equal 0, stats_count(type: :failed) end test 'expired jobs stats' do From 5bec6fdb06f7aff5193b648cafd28bb910288c40 Mon Sep 17 00:00:00 2001 From: Guilherme Cassolato Date: Thu, 25 Jun 2020 13:02:17 +0200 Subject: [PATCH 080/156] Refactoring of 'scheduled' jobs Prometheus metric type --- lib/prometheus/que_stats.rb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/prometheus/que_stats.rb b/lib/prometheus/que_stats.rb index 56579756..3af4008f 100644 --- a/lib/prometheus/que_stats.rb +++ b/lib/prometheus/que_stats.rb @@ -76,12 +76,11 @@ def call(*filters) alias_method :all, :call def ready - conditions = [['error_count = ?', 0], { expired_at: nil, finished_at: nil }, ['run_at <= ?', Time.zone.now]] - call(*conditions) + call(['error_count = ?', 0], { expired_at: nil, finished_at: nil }, ['run_at <= ?', Time.zone.now]) end def scheduled - call(['error_count = ?', 0], ['run_at > ?', Time.zone.now]) + call(['error_count = ?', 0], { expired_at: nil, finished_at: nil }, ['run_at > ?', Time.zone.now]) end def finished @@ -89,7 +88,7 @@ def finished end def failed - call(['error_count > ?', 0], { expired_at: nil }) + call(['error_count > ?', 0], { expired_at: nil, finished_at: nil }) end def expired From 46708e5a8cf5925bdaacb1116582c0acae5bf5cb Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 2 Jul 2020 08:08:12 +0000 Subject: [PATCH 081/156] [Security] Bump rack from 2.2.2 to 2.2.3 Bumps [rack](https://github.com/rack/rack) from 2.2.2 to 2.2.3. **This update includes a security fix.** - [Release notes](https://github.com/rack/rack/releases) - [Changelog](https://github.com/rack/rack/blob/master/CHANGELOG.md) - [Commits](https://github.com/rack/rack/compare/v2.2.2...2.2.3) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index fca92327..4220de21 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -226,7 +226,7 @@ GEM erubis que (~> 1.0.0.beta3) sinatra - rack (2.2.2) + rack (2.2.3) rack-protection (2.0.5) rack rack-test (1.1.0) From 12d128a83b718aea06a9efc4be9d402d437e1325 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 2 Jul 2020 08:16:04 +0000 Subject: [PATCH 082/156] [Security] Bump puma from 4.3.3 to 4.3.5 Bumps [puma](https://github.com/puma/puma) from 4.3.3 to 4.3.5. **This update includes security fixes.** - [Release notes](https://github.com/puma/puma/releases) - [Changelog](https://github.com/puma/puma/blob/master/History.md) - [Commits](https://github.com/puma/puma/commits) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 4220de21..caf8fb3b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -218,7 +218,7 @@ GEM binding_of_caller (>= 0.7) pry (>= 0.9.11) public_suffix (3.1.1) - puma (4.3.3) + puma (4.3.5) nio4r (~> 2.0) quantile (0.2.1) que (1.0.0.beta3) From bf44a845f6e7a92633133be5192df4728d88014b Mon Sep 17 00:00:00 2001 From: Guilherme Cassolato Date: Thu, 2 Jul 2020 10:45:00 +0200 Subject: [PATCH 083/156] Bump example rest-api's rack to 2.2.3 --- examples/rest-api/Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/rest-api/Gemfile.lock b/examples/rest-api/Gemfile.lock index 62eff627..ec0224ae 100644 --- a/examples/rest-api/Gemfile.lock +++ b/examples/rest-api/Gemfile.lock @@ -11,7 +11,7 @@ GEM pry-byebug (3.7.0) byebug (~> 11.0) pry (~> 0.10) - rack (2.0.7) + rack (2.2.3) rack-protection (2.0.5) rack sinatra (2.0.5) @@ -29,4 +29,4 @@ DEPENDENCIES sinatra BUNDLED WITH - 2.0.1 + 2.1.4 From 64022dea84dd47a5d791a789882e909d4540617b Mon Sep 17 00:00:00 2001 From: Guilherme Cassolato Date: Mon, 13 Jul 2020 20:30:07 +0200 Subject: [PATCH 084/156] Test for race condition while processing entry jobs that concur to create the same proxy integration --- app/jobs/process_entry_job.rb | 2 +- test/jobs/process_entry_job_test.rb | 50 +++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/app/jobs/process_entry_job.rb b/app/jobs/process_entry_job.rb index c62cef48..88dc32d6 100644 --- a/app/jobs/process_entry_job.rb +++ b/app/jobs/process_entry_job.rb @@ -17,7 +17,7 @@ def self.model_integrations_for(entry) integrations = Integration.retry_record_not_unique do case model.record - when Proxy then PROXY_INTEGRATIONS.map { |i| i.new(entry) }.each(&:call) + when Proxy then self.const_get(:PROXY_INTEGRATIONS).map { |i| i.new(entry) }.each(&:call) when Provider then CreateK8SIntegration.new(entry).call end diff --git a/test/jobs/process_entry_job_test.rb b/test/jobs/process_entry_job_test.rb index 18a50bf2..27cbfa71 100644 --- a/test/jobs/process_entry_job_test.rb +++ b/test/jobs/process_entry_job_test.rb @@ -59,6 +59,56 @@ class ProcessEntryJobTest < ActiveJob::TestCase end end + class ActiveRecordRelationWithFiber < ActiveRecord::Relation + def find_by(*attributes) + record = super + Fiber.yield + record + end + end + + class IntegrationWithFiber < ::Integration::REST + def self.relation + ActiveRecordRelationWithFiber.new(self) + end + end + + class CreateProxyIntegrationWithFiber < ProcessEntryJob::CreateOIDCProxyIntegration + def model + IntegrationWithFiber + end + end + + class ProcessEntryJobWithFiber < ProcessEntryJob + PROXY_INTEGRATIONS = [CreateProxyIntegrationWithFiber] + end + + test 'race condition between entry jobs to create same proxy integration' do + entry = entries(:proxy) + + existing_integrations = Integration.where(tenant: entry.tenant) + UpdateState.where(model: existing_integrations).delete_all + existing_integrations.delete_all + + fiber1 = Fiber.new { ProcessEntryJobWithFiber.model_integrations_for(entry) } + fiber2 = Fiber.new { ProcessEntryJobWithFiber.model_integrations_for(entry) } + + fiber1.resume + fiber2.resume # right now, both jobs believe the integration must be created + + assert_difference(Integration.where(type: 'ProcessEntryJobTest::IntegrationWithFiber').method(:count)) do + fiber2.resume # creates the integration first + end + + old_logger = ::Integration.logger + tmp_logger = Minitest::Mock.new(old_logger) + ::Integration.logger = tmp_logger + tmp_logger.expect(:warn, true) { |error| ActiveRecord::RecordNotUnique === error } + + fiber1.resume # raises ActiveRecord::RecordNotUnique + ::Integration.logger = old_logger + end + test 'skips deleted proxy' do proxy = entries(:proxy) From 347ee9a2e9afc4dda6af95675564b8f9c5a22758 Mon Sep 17 00:00:00 2001 From: Guilherme Cassolato Date: Mon, 13 Jul 2020 20:31:04 +0200 Subject: [PATCH 085/156] Fix race condition between entry jobs trying to create a same proxy integration --- app/jobs/process_entry_job.rb | 24 ++++++++++--------- config/initializers/rails_6.rb | 24 +++++++++++++++++++ test/jobs/process_entry_job_test.rb | 37 ++++++++--------------------- 3 files changed, 47 insertions(+), 38 deletions(-) create mode 100644 config/initializers/rails_6.rb diff --git a/app/jobs/process_entry_job.rb b/app/jobs/process_entry_job.rb index 88dc32d6..44451199 100644 --- a/app/jobs/process_entry_job.rb +++ b/app/jobs/process_entry_job.rb @@ -13,18 +13,20 @@ def perform(entry) end def self.model_integrations_for(entry) - model = entry.model + self.ensure_integrations_for(entry) - integrations = Integration.retry_record_not_unique do - case model.record - when Proxy then self.const_get(:PROXY_INTEGRATIONS).map { |i| i.new(entry) }.each(&:call) - when Provider then CreateK8SIntegration.new(entry).call - end + model = entry.model + integrations = Integration.for_model(model) + integrations.each.with_object(model) + end - Integration.for_model(model) + def self.ensure_integrations_for(entry) + case entry.model.record + when Proxy + self.const_get(:PROXY_INTEGRATIONS).map { |integration| integration.new(entry) }.each(&:call) + when Provider + CreateK8SIntegration.new(entry).call end - - integrations.each.with_object(model) end protected @@ -94,7 +96,7 @@ def call return unless enabled? transaction do - integration = integrations.find_or_create_by!(type: integration_type.to_s) + integration = integrations.create_or_find_by!(type: integration_type.to_s) integration.update(state: Integration.states.fetch(:active)) ProcessIntegrationEntryJob.perform_later(integration, model) @@ -153,7 +155,7 @@ class UnknownOIDCIssuerTypeError < StandardError; end def find_integration model .create_with(endpoint: endpoint) - .find_or_create_by!(integrations.where_values_hash) + .create_or_find_by!(integrations.where_values_hash) end end diff --git a/config/initializers/rails_6.rb b/config/initializers/rails_6.rb new file mode 100644 index 00000000..9e101eca --- /dev/null +++ b/config/initializers/rails_6.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +# Remove this when upgrading to Rails 6 +module Rails6 + module ActiveRecord + module Relation + # File activerecord/lib/active_record/relation.rb, line 218 + def create_or_find_by!(attributes, &block) + transaction(requires_new: true) { create!(attributes, &block) } + rescue ::ActiveRecord::RecordNotUnique + find_by!(attributes) + end + + # File activerecord/lib/active_record/relation.rb, line 209 + def create_or_find_by(attributes, &block) + transaction(requires_new: true) { create(attributes, &block) } + rescue ::ActiveRecord::RecordNotUnique + find_by!(attributes) + end + end + end +end + +ActiveRecord::Relation.prepend(Rails6::ActiveRecord::Relation) diff --git a/test/jobs/process_entry_job_test.rb b/test/jobs/process_entry_job_test.rb index 27cbfa71..993985d8 100644 --- a/test/jobs/process_entry_job_test.rb +++ b/test/jobs/process_entry_job_test.rb @@ -59,23 +59,10 @@ class ProcessEntryJobTest < ActiveJob::TestCase end end - class ActiveRecordRelationWithFiber < ActiveRecord::Relation - def find_by(*attributes) - record = super - Fiber.yield - record - end - end - - class IntegrationWithFiber < ::Integration::REST - def self.relation - ActiveRecordRelationWithFiber.new(self) - end - end - class CreateProxyIntegrationWithFiber < ProcessEntryJob::CreateOIDCProxyIntegration - def model - IntegrationWithFiber + def find_integration + Fiber.yield + super end end @@ -90,23 +77,19 @@ class ProcessEntryJobWithFiber < ProcessEntryJob UpdateState.where(model: existing_integrations).delete_all existing_integrations.delete_all - fiber1 = Fiber.new { ProcessEntryJobWithFiber.model_integrations_for(entry) } - fiber2 = Fiber.new { ProcessEntryJobWithFiber.model_integrations_for(entry) } + fiber1 = Fiber.new { ProcessEntryJobWithFiber.ensure_integrations_for(entry) } + fiber2 = Fiber.new { ProcessEntryJobWithFiber.ensure_integrations_for(entry) } fiber1.resume - fiber2.resume # right now, both jobs believe the integration must be created + fiber2.resume - assert_difference(Integration.where(type: 'ProcessEntryJobTest::IntegrationWithFiber').method(:count)) do + assert_difference(existing_integrations.method(:count)) do fiber2.resume # creates the integration first end - old_logger = ::Integration.logger - tmp_logger = Minitest::Mock.new(old_logger) - ::Integration.logger = tmp_logger - tmp_logger.expect(:warn, true) { |error| ActiveRecord::RecordNotUnique === error } - - fiber1.resume # raises ActiveRecord::RecordNotUnique - ::Integration.logger = old_logger + assert_no_difference(existing_integrations.method(:count)) do + fiber1.resume + end end test 'skips deleted proxy' do From 50bd3df932305c0d392bd0e09aea26a270c8b4a5 Mon Sep 17 00:00:00 2001 From: Guilherme Cassolato Date: Tue, 14 Jul 2020 11:12:56 +0200 Subject: [PATCH 086/156] Turn ProcessEntryJob::PROXY_INTEGRATIONS into a class attribute --- app/jobs/process_entry_job.rb | 7 ++++--- test/jobs/process_entry_job_test.rb | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/app/jobs/process_entry_job.rb b/app/jobs/process_entry_job.rb index 44451199..057395d0 100644 --- a/app/jobs/process_entry_job.rb +++ b/app/jobs/process_entry_job.rb @@ -6,6 +6,8 @@ class ProcessEntryJob < ApplicationJob queue_as :default + class_attribute :proxy_integration_services + def perform(entry) model_integrations_for(entry).each do |integration, model| ProcessIntegrationEntryJob.perform_later(integration, model) @@ -23,7 +25,7 @@ def self.model_integrations_for(entry) def self.ensure_integrations_for(entry) case entry.model.record when Proxy - self.const_get(:PROXY_INTEGRATIONS).map { |integration| integration.new(entry) }.each(&:call) + proxy_integration_services.map { |integration| integration.new(entry) }.each(&:call) when Provider CreateK8SIntegration.new(entry).call end @@ -159,9 +161,8 @@ def find_integration end end - PROXY_INTEGRATIONS = [ + self.proxy_integration_services = [ CreateOIDCProxyIntegration, CreateK8SIntegration ].freeze - private_constant :PROXY_INTEGRATIONS end diff --git a/test/jobs/process_entry_job_test.rb b/test/jobs/process_entry_job_test.rb index 993985d8..997b1f72 100644 --- a/test/jobs/process_entry_job_test.rb +++ b/test/jobs/process_entry_job_test.rb @@ -67,7 +67,7 @@ def find_integration end class ProcessEntryJobWithFiber < ProcessEntryJob - PROXY_INTEGRATIONS = [CreateProxyIntegrationWithFiber] + self.proxy_integration_services = [CreateProxyIntegrationWithFiber] end test 'race condition between entry jobs to create same proxy integration' do From a3905cff9a55f99489dfe8578aacf1027e6cab03 Mon Sep 17 00:00:00 2001 From: PoojaChandak Date: Tue, 25 Aug 2020 12:31:37 +0530 Subject: [PATCH 087/156] Update README.md Observed a few changes, suggesting the same. --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index b7da14ab..9d5e00f8 100644 --- a/README.md +++ b/README.md @@ -16,13 +16,13 @@ Before describing the design it is important to define the terms. * **3scale** - 3scale API Manager (Porta). -* **Tenant** - mapping of Provider id registered in 3scale API Manager to domain and access token. +* **Tenant** - mapping of Provider id registered in 3scale API Manager to the domain and access token. * **Model** - relevant object in **3scale** like Application, Limit, Metric, Service. * **Notification** - Message sent to **Zync** describing the **model** that changed and having all required properties to fetch it from the API later. * Example: Application 3456, tenant_id: 26 * Example: Limit 4566, metric_id: 36, application_id: 46, tenant_id: 16 * **Update** - **Zync** fetches updated **Model** from the **Notifier** via the API using the information provided by the **Notification**. -* **Lock** - mechanism that preventing concurrent data access to the same scope. +* **Lock** - the mechanism that prevents concurrent data access to the same scope. * Example: **Tenant Lock** would mean only one can be running for one **Tenant**. * Example: **Model** **Lock** - only one per uniquely identified **Model**. * **Entry** - The information from the API provided by the **Update**. @@ -35,9 +35,9 @@ Before describing the design it is important to define the terms. ## Design -**Zync** is meant to synchronize data from **3scale** to external systems (like IDPs). Some people use Web-hooks for this but without further logic they can be unreliable and arrive out of order. This tool is meant to synchronize the final state to a different systems. +**Zync** is meant to synchronize data from **3scale** to external systems (like IDPs). Some people use Web-hooks for this but without further logic, they can be unreliable and arrive out of order. This tool is meant to synchronize the final state to a different system. -The flow is defined as **3scale** -> **Zync** ( <- **3scale**) -> **Integration**. So **3scale** notifies **Zync** there was a change to a **model** but does not say more than primary key and information required to fetch it from the **3scale** API. In some cases **model** needs just its primary key (**id**) and in some it needs other metadata (usually primary keys of its parents) to compose the API call (service_id, metric_id, …). +The flow is defined as **3scale** -> **Zync** ( <- **3scale**) -> **Integration**. So **3scale** notifies **Zync** there was a change to a **model** but does not say more than the primary key and information required to fetch it from the **3scale** API. In some cases **model** needs just its primary key (**id**) and in some, it needs other metadata (usually primary keys of its parents) to compose the API call (service_id, metric_id, …). **Zync** upon receiving the notification will acquire an **update model lock** and try to perform an **update**. Any information received this way is added as an **entry** to the **log** and the **model lock** is released. That **entry** can be either new data or information that the record is no longer there (404 from the API). If new **notification** came when the **model lock** was acquired, it is going to be processed after the lock is released. @@ -45,13 +45,13 @@ After adding **entry** to the **log** an **integration** is triggered and acquir ## Properties -Given the locking on the **model** there will be some parallelization, but also updates to one object will be serialized. This needs to be done to limit the network issues and ensure the request was delivered before issuing new one. +Given the locking on the **model** there will be some parallelization, but also updates to one object will be serialized. This needs to be done to limit the network issues and ensure the request was delivered before issuing a new one. -Because **Zync** will keep a **log** of **events** it will be able to replay changes and recover last state just taking last revisions of each **model** and even remove the ones that have been created before but have been deleted. +Because **Zync** will keep a **log** of **events** it will be able to replay changes and recover the last state just taking last revisions of each **model** and even remove the ones that have been created before but have been deleted. ## Data Model -**Record** types are for enforcing correctness of data on the database level and referential integrity. There is one relationship (Model -> Record) that can't have foreign constraint but can be recreated from other data. +**Record** types are for enforcing correctness of data on the database level and referential integrity. There is one relationship (Model -> Record) that can't have foreign constraints but can be recreated from other data. ### Tenant From 421998d0de88e3bcbc037a4ad293a9998de9288f Mon Sep 17 00:00:00 2001 From: Hallelujah Date: Tue, 25 Aug 2020 10:55:45 +0200 Subject: [PATCH 088/156] [gem] Update codecov As the current version is yanked --- Gemfile.lock | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index caf8fb3b..5461365a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -81,10 +81,9 @@ GEM bugsnag-capistrano (1.1.2) builder (3.2.4) byebug (11.0.0) - codecov (0.1.16) + codecov (0.2.8) json simplecov - url coderay (1.1.2) concurrent-ruby (1.1.6) crack (0.4.3) @@ -134,7 +133,7 @@ GEM its-it (1.3.0) join_dependency (0.1.4) activerecord (>= 4.2.0) - json (2.3.0) + json (2.3.1) jsonpath (0.9.9) multi_json to_regexp (~> 0.2.1) @@ -276,11 +275,10 @@ GEM activerecord (>= 4.2, < 5.3) its-it (~> 1.2) schema_plus_core - simplecov (0.17.1) + simplecov (0.19.0) docile (~> 1.1) - json (>= 1.8, < 3) - simplecov-html (~> 0.10.0) - simplecov-html (0.10.2) + simplecov-html (~> 0.11) + simplecov-html (0.12.2) sinatra (2.0.5) mustermann (~> 1.0) rack (~> 2.0) @@ -301,7 +299,6 @@ GEM parslet (~> 1.8.0) tzinfo (1.2.7) thread_safe (~> 0.1) - url (0.3.2) validate_url (1.0.8) activemodel (>= 3.0.0) public_suffix @@ -369,4 +366,4 @@ DEPENDENCIES yabeda-rails BUNDLED WITH - 2.0.2 + 2.1.4 From 7963c151209b8227df6a693a5a1dcae100f32bf7 Mon Sep 17 00:00:00 2001 From: Hallelujah Date: Mon, 9 Dec 2019 17:28:52 +0100 Subject: [PATCH 089/156] Allow disabling integrations Disable routes creations for Kubernetes Integration by setting: DISABLE_K8S_ROUTES_CREATION=1 --- app/models/integration.rb | 2 +- app/models/integration/kubernetes.rb | 2 +- app/models/integration/rest.rb | 2 +- config/application.rb | 2 ++ config/integrations.yml | 11 ++++++ test/models/integration_test.rb | 52 ++++++++++++++++++++++++++++ 6 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 config/integrations.yml diff --git a/app/models/integration.rb b/app/models/integration.rb index d8f2bee7..e417f123 100644 --- a/app/models/integration.rb +++ b/app/models/integration.rb @@ -17,6 +17,6 @@ def self.for_model(model) end def enabled? - true + Rails.application.config.integrations.fetch(self.class.name.demodulize.downcase, true) end end diff --git a/app/models/integration/kubernetes.rb b/app/models/integration/kubernetes.rb index e5c46890..d335df05 100644 --- a/app/models/integration/kubernetes.rb +++ b/app/models/integration/kubernetes.rb @@ -7,7 +7,7 @@ class Integration::Kubernetes < Integration validates :server, url: { allow_nil: true, no_local: true } def enabled? - K8s::Client === K8s::Client.autoconfig + super && K8s::Client === K8s::Client.autoconfig rescue K8s::Error::Configuration false rescue => error diff --git a/app/models/integration/rest.rb b/app/models/integration/rest.rb index 9801b55b..19cd4bf0 100644 --- a/app/models/integration/rest.rb +++ b/app/models/integration/rest.rb @@ -7,6 +7,6 @@ class Integration::REST < Integration validates :endpoint, url: { allow_nil: true, no_local: true } def enabled? - endpoint.present? + super && endpoint.present? end end diff --git a/config/application.rb b/config/application.rb index 3b2bb089..3af4d664 100644 --- a/config/application.rb +++ b/config/application.rb @@ -37,6 +37,8 @@ class Application < Rails::Application # Initialize configuration defaults for originally generated Rails version. config.load_defaults 5.1 + config.integrations = config_for(:integrations) + # Settings in config/environments/* take precedence over those specified here. # Application configuration should go into files in config/initializers # -- all .rb files in that directory are automatically loaded. diff --git a/config/integrations.yml b/config/integrations.yml new file mode 100644 index 00000000..85bcb928 --- /dev/null +++ b/config/integrations.yml @@ -0,0 +1,11 @@ +shared: &shared + kubernetes: <%= ENV.fetch('DISABLE_K8S_ROUTES_CREATION', '0') == '0' %> + +production: + <<: *shared + +development: + <<: *shared + +test: + <<: *shared diff --git a/test/models/integration_test.rb b/test/models/integration_test.rb index 09a38ec9..3290b7ff 100644 --- a/test/models/integration_test.rb +++ b/test/models/integration_test.rb @@ -13,4 +13,56 @@ class IntegrationTest < ActiveSupport::TestCase assert_equal [ keycloak ], Integration.tenant_or_model(nil, model) assert_equal [ one ], Integration.tenant_or_model(tenant, nil) end + + test 'enabled is by integration' do + with_integration keycloak: false, rest: true, kubernetes: true do + assert Integration.new.enabled? + refute Integration::Keycloak.new.enabled? + end + end + + test 'rest enabled?' do + integration = Integration::REST.new + integration.endpoint = 'https://rest.example.com/endpoint' + assert integration.enabled? + + integration.endpoint = nil + refute integration.enabled? + + with_integration rest: false do + integration = Integration::REST.new + integration.endpoint = 'https://rest.example.com/endpoint' + refute integration.enabled? + + integration.endpoint = nil + refute integration.enabled? + end + end + + test 'kubernetes enabled?' do + client = K8s::Client.new(nil) + integration = Integration::Kubernetes.new + + K8s::Client.stub(:autoconfig, client) do + assert integration.enabled? + + with_integration kubernetes: false do + refute integration.enabled? + end + end + + K8s::Client.stub(:autoconfig, nil) do + refute integration.enabled? + end + + with_integration kubernetes: false do + refute integration.enabled? + end + end + + protected + + def with_integration(opts = {}, &block) + Rails.application.config.stub(:integrations, opts.with_indifferent_access, &block) + end end From fc6477043ed16169972c7edfa0a1f5f7ffa09a60 Mon Sep 17 00:00:00 2001 From: Ramihajamalala Hery Date: Tue, 25 Aug 2020 16:54:43 +0200 Subject: [PATCH 090/156] Update test/models/integration_test.rb Co-authored-by: Guilherme Cassolato --- test/models/integration_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/models/integration_test.rb b/test/models/integration_test.rb index 3290b7ff..4e5604d0 100644 --- a/test/models/integration_test.rb +++ b/test/models/integration_test.rb @@ -22,7 +22,7 @@ class IntegrationTest < ActiveSupport::TestCase end test 'rest enabled?' do - integration = Integration::REST.new + integration = Integration::REST.new integration.endpoint = 'https://rest.example.com/endpoint' assert integration.enabled? From 32376badf338fc89950ff6d5b78b2a1c3dac332c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 14 Sep 2020 07:37:33 +0000 Subject: [PATCH 091/156] Bump webmock from 3.5.1 to 3.9.1 Bumps [webmock](https://github.com/bblimke/webmock) from 3.5.1 to 3.9.1. - [Release notes](https://github.com/bblimke/webmock/releases) - [Changelog](https://github.com/bblimke/webmock/blob/master/CHANGELOG.md) - [Commits](https://github.com/bblimke/webmock/compare/v3.5.1...v3.9.1) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 5461365a..916efd3a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -66,8 +66,8 @@ GEM i18n (>= 0.7, < 2) minitest (~> 5.1) tzinfo (~> 1.1) - addressable (2.6.0) - public_suffix (>= 2.0.2, < 4.0) + addressable (2.7.0) + public_suffix (>= 2.0.2, < 5.0) ansi (1.5.0) arel (9.0.0) baby_squeel (1.3.1) @@ -125,7 +125,7 @@ GEM multipart-post (>= 1.2, < 3) globalid (0.4.2) activesupport (>= 4.2.0) - hashdiff (1.0.0) + hashdiff (1.0.1) i18n (1.8.2) concurrent-ruby (~> 1.0) ice_nine (0.11.2) @@ -216,7 +216,7 @@ GEM pry-stack_explorer (0.4.9.3) binding_of_caller (>= 0.7) pry (>= 0.9.11) - public_suffix (3.1.1) + public_suffix (4.0.6) puma (4.3.5) nio4r (~> 2.0) quantile (0.2.1) @@ -302,10 +302,10 @@ GEM validate_url (1.0.8) activemodel (>= 3.0.0) public_suffix - webmock (3.5.1) + webmock (3.9.1) addressable (>= 2.3.6) crack (>= 0.3.2) - hashdiff + hashdiff (>= 0.4.0, < 2.0.0) websocket-driver (0.7.1) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) From d3debbf8b9aefdf212e2c2a6f18132772f9c32f2 Mon Sep 17 00:00:00 2001 From: PoojaChandak Date: Tue, 15 Sep 2020 10:26:01 +0530 Subject: [PATCH 092/156] Update README.md grammatical/typo changes --- examples/rest-api/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/rest-api/README.md b/examples/rest-api/README.md index a85f2fe9..86ad2e62 100644 --- a/examples/rest-api/README.md +++ b/examples/rest-api/README.md @@ -13,12 +13,12 @@ When a 3scale application is created/updated/deleted zync will try to replay tha ## Creating, updating and deleting Clients -Zync will make following requests to create/update/delete clients: +Zync will make the following requests to create/update/delete clients: * `PUT /clients/:client_id` (create, update) * `DELETE /clients/:client_id` (delete) -All endpoints must reply 2xx status code. Otherwise the request will be retried. +All endpoints must reply 2xx status code. Otherwise, the request will be retried. ### Payload @@ -39,7 +39,7 @@ The request to delete a client has no payload. ## Using OAuth2 authentication Zync will make GET request to `/.well-known/openid-configuration` endpoint and expect an `application/json` response. -The response payload should contain following: +The response payload should contain the following: ```json { @@ -50,7 +50,7 @@ The response payload should contain following: Zync will use that `token_endpoint` URL to exchange the client_id and client_secret provided in the OpenID Connect Issuer URL for an access token using the OAuth2 protocol. -If the API responds with not successful response, Zync will fallback to HTTP Basic/Digest authentication using provided credentials. +If the API responds with a not successful response, Zync will fallback to HTTP Basic/Digest authentication using provided credentials. ## References From ec2307e715a42b234df0d605fcc899720d0f4c28 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2020 12:57:12 +0000 Subject: [PATCH 093/156] Bump validate_url from 1.0.8 to 1.0.11 Bumps [validate_url](https://github.com/perfectline/validates_url) from 1.0.8 to 1.0.11. - [Release notes](https://github.com/perfectline/validates_url/releases) - [Commits](https://github.com/perfectline/validates_url/commits) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 916efd3a..93fa17f1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -85,7 +85,7 @@ GEM json simplecov coderay (1.1.2) - concurrent-ruby (1.1.6) + concurrent-ruby (1.1.7) crack (0.4.3) safe_yaml (~> 1.0.0) crass (1.0.6) @@ -126,7 +126,7 @@ GEM globalid (0.4.2) activesupport (>= 4.2.0) hashdiff (1.0.1) - i18n (1.8.2) + i18n (1.8.5) concurrent-ruby (~> 1.0) ice_nine (0.11.2) interception (0.5) @@ -173,7 +173,7 @@ GEM mimemagic (0.3.5) mini_mime (1.0.2) mini_portile2 (2.4.0) - minitest (5.14.1) + minitest (5.14.2) minitest-reporters (1.4.2) ansi builder @@ -299,7 +299,7 @@ GEM parslet (~> 1.8.0) tzinfo (1.2.7) thread_safe (~> 0.1) - validate_url (1.0.8) + validate_url (1.0.11) activemodel (>= 3.0.0) public_suffix webmock (3.9.1) From 3c3a8dc8c669bd2206fc585da6ec61e9883ced23 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2020 13:12:22 +0000 Subject: [PATCH 094/156] Bump que-web from 0.8.0 to 0.9.3 Bumps [que-web](https://github.com/statianzo/que-web) from 0.8.0 to 0.9.3. - [Release notes](https://github.com/statianzo/que-web/releases) - [Changelog](https://github.com/statianzo/que-web/blob/master/CHANGELOG.md) - [Commits](https://github.com/statianzo/que-web/compare/v0.8.0...v0.9.3) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 93fa17f1..a917d81f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -119,7 +119,6 @@ GEM dry-inflector (~> 0.1, >= 0.1.2) dry-logic (~> 0.4, >= 0.4.2) erubi (1.9.0) - erubis (2.7.0) excon (0.71.1) faraday (0.15.3) multipart-post (>= 1.2, < 3) @@ -186,7 +185,8 @@ GEM multi_json (1.14.1) multi_xml (0.6.0) multipart-post (2.0.0) - mustermann (1.0.3) + mustermann (1.1.1) + ruby2_keywords (~> 0.0.1) nio4r (2.5.2) nokogiri (1.10.9) mini_portile2 (~> 2.4.0) @@ -221,12 +221,11 @@ GEM nio4r (~> 2.0) quantile (0.2.1) que (1.0.0.beta3) - que-web (0.8.0) - erubis + que-web (0.9.3) que (~> 1.0.0.beta3) sinatra rack (2.2.3) - rack-protection (2.0.5) + rack-protection (2.1.0) rack rack-test (1.1.0) rack (>= 1.0, < 3) @@ -262,6 +261,7 @@ GEM actionpack (>= 5.0) railties (>= 5.0) ruby-progressbar (1.10.1) + ruby2_keywords (0.0.2) rubyzip (2.0.0) safe_yaml (1.0.5) schema_monkey (2.1.5) @@ -279,10 +279,10 @@ GEM docile (~> 1.1) simplecov-html (~> 0.11) simplecov-html (0.12.2) - sinatra (2.0.5) + sinatra (2.1.0) mustermann (~> 1.0) - rack (~> 2.0) - rack-protection (= 2.0.5) + rack (~> 2.2) + rack-protection (= 2.1.0) tilt (~> 2.0) sprockets (4.0.0) concurrent-ruby (~> 1.0) @@ -293,7 +293,7 @@ GEM sprockets (>= 3.0.0) thor (1.0.1) thread_safe (0.3.6) - tilt (2.0.9) + tilt (2.0.10) to_regexp (0.2.1) toml (0.2.0) parslet (~> 1.8.0) From 68cc29f0677b6e1d91ba6eabf2dd023f765bcee2 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2020 14:38:57 +0000 Subject: [PATCH 095/156] Bump pg from 1.2.0 to 1.2.3 Bumps [pg](https://github.com/ged/ruby-pg) from 1.2.0 to 1.2.3. - [Release notes](https://github.com/ged/ruby-pg/releases) - [Changelog](https://github.com/ged/ruby-pg/blob/master/History.rdoc) - [Commits](https://github.com/ged/ruby-pg/commits) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index a917d81f..05f66da4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -197,7 +197,7 @@ GEM multi_xml (~> 0.5) rack (>= 1.2, < 3) parslet (1.8.2) - pg (1.2.0) + pg (1.2.3) polyamorous (1.3.3) activerecord (>= 3.0) prometheus-client (0.9.0) From edc9a517bcfea6cd427a407b7b1d2c6f5da13381 Mon Sep 17 00:00:00 2001 From: "depfu[bot]" <23717796+depfu[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2020 14:40:59 +0000 Subject: [PATCH 096/156] Update all of rails to version 5.2.4.4 --- Gemfile.lock | 84 ++++++++++++++++++++++++++-------------------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index a917d81f..9ec6d623 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -25,43 +25,43 @@ GEM remote: https://rubygems.org/ specs: 3scale-api (0.6.0) - actioncable (5.2.4.3) - actionpack (= 5.2.4.3) + actioncable (5.2.4.4) + actionpack (= 5.2.4.4) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailer (5.2.4.3) - actionpack (= 5.2.4.3) - actionview (= 5.2.4.3) - activejob (= 5.2.4.3) + actionmailer (5.2.4.4) + actionpack (= 5.2.4.4) + actionview (= 5.2.4.4) + activejob (= 5.2.4.4) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (5.2.4.3) - actionview (= 5.2.4.3) - activesupport (= 5.2.4.3) + actionpack (5.2.4.4) + actionview (= 5.2.4.4) + activesupport (= 5.2.4.4) rack (~> 2.0, >= 2.0.8) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.0.2) - actionview (5.2.4.3) - activesupport (= 5.2.4.3) + actionview (5.2.4.4) + activesupport (= 5.2.4.4) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.0.3) - activejob (5.2.4.3) - activesupport (= 5.2.4.3) + activejob (5.2.4.4) + activesupport (= 5.2.4.4) globalid (>= 0.3.6) - activemodel (5.2.4.3) - activesupport (= 5.2.4.3) - activerecord (5.2.4.3) - activemodel (= 5.2.4.3) - activesupport (= 5.2.4.3) + activemodel (5.2.4.4) + activesupport (= 5.2.4.4) + activerecord (5.2.4.4) + activemodel (= 5.2.4.4) + activesupport (= 5.2.4.4) arel (>= 9.0) - activestorage (5.2.4.3) - actionpack (= 5.2.4.3) - activerecord (= 5.2.4.3) + activestorage (5.2.4.4) + actionpack (= 5.2.4.4) + activerecord (= 5.2.4.4) marcel (~> 0.3.1) - activesupport (5.2.4.3) + activesupport (5.2.4.4) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 0.7, < 2) minitest (~> 5.1) @@ -159,7 +159,7 @@ GEM activesupport (>= 4) railties (>= 4) request_store (~> 1.0) - loofah (2.5.0) + loofah (2.7.0) crass (~> 1.0.2) nokogiri (>= 1.5.9) mail (2.7.1) @@ -187,8 +187,8 @@ GEM multipart-post (2.0.0) mustermann (1.1.1) ruby2_keywords (~> 0.0.1) - nio4r (2.5.2) - nokogiri (1.10.9) + nio4r (2.5.3) + nokogiri (1.10.10) mini_portile2 (~> 2.4.0) oauth2 (1.4.1) faraday (>= 0.8, < 0.16.0) @@ -229,27 +229,27 @@ GEM rack rack-test (1.1.0) rack (>= 1.0, < 3) - rails (5.2.4.3) - actioncable (= 5.2.4.3) - actionmailer (= 5.2.4.3) - actionpack (= 5.2.4.3) - actionview (= 5.2.4.3) - activejob (= 5.2.4.3) - activemodel (= 5.2.4.3) - activerecord (= 5.2.4.3) - activestorage (= 5.2.4.3) - activesupport (= 5.2.4.3) + rails (5.2.4.4) + actioncable (= 5.2.4.4) + actionmailer (= 5.2.4.4) + actionpack (= 5.2.4.4) + actionview (= 5.2.4.4) + activejob (= 5.2.4.4) + activemodel (= 5.2.4.4) + activerecord (= 5.2.4.4) + activestorage (= 5.2.4.4) + activesupport (= 5.2.4.4) bundler (>= 1.3.0) - railties (= 5.2.4.3) + railties (= 5.2.4.4) sprockets-rails (>= 2.0.0) rails-dom-testing (2.0.3) activesupport (>= 4.2.0) nokogiri (>= 1.6) rails-html-sanitizer (1.3.0) loofah (~> 2.3) - railties (5.2.4.3) - actionpack (= 5.2.4.3) - activesupport (= 5.2.4.3) + railties (5.2.4.4) + actionpack (= 5.2.4.4) + activesupport (= 5.2.4.4) method_source rake (>= 0.8.7) thor (>= 0.19.0, < 2.0) @@ -284,10 +284,10 @@ GEM rack (~> 2.2) rack-protection (= 2.1.0) tilt (~> 2.0) - sprockets (4.0.0) + sprockets (4.0.2) concurrent-ruby (~> 1.0) rack (> 1, < 3) - sprockets-rails (3.2.1) + sprockets-rails (3.2.2) actionpack (>= 4.0) activesupport (>= 4.0) sprockets (>= 3.0.0) @@ -306,7 +306,7 @@ GEM addressable (>= 2.3.6) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) - websocket-driver (0.7.1) + websocket-driver (0.7.3) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) with_env (1.1.0) From 8ca0bbb399273ffc67dd9348aecd9fa5b6abc009 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2020 14:45:40 +0000 Subject: [PATCH 097/156] Bump que from 1.0.0.beta3 to 1.0.0.beta4 Bumps [que](https://github.com/chanks/que) from 1.0.0.beta3 to 1.0.0.beta4. - [Release notes](https://github.com/chanks/que/releases) - [Changelog](https://github.com/que-rb/que/blob/master/CHANGELOG.1.0.beta.md) - [Commits](https://github.com/chanks/que/compare/v1.0.0.beta3...v1.0.0.beta4) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 05f66da4..8c07e53c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -220,7 +220,7 @@ GEM puma (4.3.5) nio4r (~> 2.0) quantile (0.2.1) - que (1.0.0.beta3) + que (1.0.0.beta4) que-web (0.9.3) que (~> 1.0.0.beta3) sinatra From dc8492907eee420f69ada3c2406572a22604fb6f Mon Sep 17 00:00:00 2001 From: "depfu[bot]" <23717796+depfu[bot]@users.noreply.github.com> Date: Thu, 11 Feb 2021 00:40:59 +0000 Subject: [PATCH 098/156] Update all of rails to version 5.2.4.5 --- Gemfile.lock | 98 +++++++++++++++++++++++++++------------------------- 1 file changed, 50 insertions(+), 48 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 96daa1b3..9c8e1744 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -25,43 +25,43 @@ GEM remote: https://rubygems.org/ specs: 3scale-api (0.6.0) - actioncable (5.2.4.4) - actionpack (= 5.2.4.4) + actioncable (5.2.4.5) + actionpack (= 5.2.4.5) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailer (5.2.4.4) - actionpack (= 5.2.4.4) - actionview (= 5.2.4.4) - activejob (= 5.2.4.4) + actionmailer (5.2.4.5) + actionpack (= 5.2.4.5) + actionview (= 5.2.4.5) + activejob (= 5.2.4.5) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (5.2.4.4) - actionview (= 5.2.4.4) - activesupport (= 5.2.4.4) + actionpack (5.2.4.5) + actionview (= 5.2.4.5) + activesupport (= 5.2.4.5) rack (~> 2.0, >= 2.0.8) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.0.2) - actionview (5.2.4.4) - activesupport (= 5.2.4.4) + actionview (5.2.4.5) + activesupport (= 5.2.4.5) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.0.3) - activejob (5.2.4.4) - activesupport (= 5.2.4.4) + activejob (5.2.4.5) + activesupport (= 5.2.4.5) globalid (>= 0.3.6) - activemodel (5.2.4.4) - activesupport (= 5.2.4.4) - activerecord (5.2.4.4) - activemodel (= 5.2.4.4) - activesupport (= 5.2.4.4) + activemodel (5.2.4.5) + activesupport (= 5.2.4.5) + activerecord (5.2.4.5) + activemodel (= 5.2.4.5) + activesupport (= 5.2.4.5) arel (>= 9.0) - activestorage (5.2.4.4) - actionpack (= 5.2.4.4) - activerecord (= 5.2.4.4) + activestorage (5.2.4.5) + actionpack (= 5.2.4.5) + activerecord (= 5.2.4.5) marcel (~> 0.3.1) - activesupport (5.2.4.4) + activesupport (5.2.4.5) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 0.7, < 2) minitest (~> 5.1) @@ -85,7 +85,7 @@ GEM json simplecov coderay (1.1.2) - concurrent-ruby (1.1.7) + concurrent-ruby (1.1.8) crack (0.4.3) safe_yaml (~> 1.0.0) crass (1.0.6) @@ -118,14 +118,14 @@ GEM dry-equalizer (~> 0.2) dry-inflector (~> 0.1, >= 0.1.2) dry-logic (~> 0.4, >= 0.4.2) - erubi (1.9.0) + erubi (1.10.0) excon (0.71.1) faraday (0.15.3) multipart-post (>= 1.2, < 3) globalid (0.4.2) activesupport (>= 4.2.0) hashdiff (1.0.1) - i18n (1.8.5) + i18n (1.8.8) concurrent-ruby (~> 1.0) ice_nine (0.11.2) interception (0.5) @@ -159,7 +159,7 @@ GEM activesupport (>= 4) railties (>= 4) request_store (~> 1.0) - loofah (2.7.0) + loofah (2.9.0) crass (~> 1.0.2) nokogiri (>= 1.5.9) mail (2.7.1) @@ -171,8 +171,8 @@ GEM method_source (0.9.2) mimemagic (0.3.5) mini_mime (1.0.2) - mini_portile2 (2.4.0) - minitest (5.14.2) + mini_portile2 (2.5.0) + minitest (5.14.3) minitest-reporters (1.4.2) ansi builder @@ -187,9 +187,10 @@ GEM multipart-post (2.0.0) mustermann (1.1.1) ruby2_keywords (~> 0.0.1) - nio4r (2.5.3) - nokogiri (1.10.10) - mini_portile2 (~> 2.4.0) + nio4r (2.5.5) + nokogiri (1.11.1) + mini_portile2 (~> 2.5.0) + racc (~> 1.4) oauth2 (1.4.1) faraday (>= 0.8, < 0.16.0) jwt (>= 1.0, < 3.0) @@ -224,36 +225,37 @@ GEM que-web (0.9.3) que (~> 1.0.0.beta3) sinatra + racc (1.5.2) rack (2.2.3) rack-protection (2.1.0) rack rack-test (1.1.0) rack (>= 1.0, < 3) - rails (5.2.4.4) - actioncable (= 5.2.4.4) - actionmailer (= 5.2.4.4) - actionpack (= 5.2.4.4) - actionview (= 5.2.4.4) - activejob (= 5.2.4.4) - activemodel (= 5.2.4.4) - activerecord (= 5.2.4.4) - activestorage (= 5.2.4.4) - activesupport (= 5.2.4.4) + rails (5.2.4.5) + actioncable (= 5.2.4.5) + actionmailer (= 5.2.4.5) + actionpack (= 5.2.4.5) + actionview (= 5.2.4.5) + activejob (= 5.2.4.5) + activemodel (= 5.2.4.5) + activerecord (= 5.2.4.5) + activestorage (= 5.2.4.5) + activesupport (= 5.2.4.5) bundler (>= 1.3.0) - railties (= 5.2.4.4) + railties (= 5.2.4.5) sprockets-rails (>= 2.0.0) rails-dom-testing (2.0.3) activesupport (>= 4.2.0) nokogiri (>= 1.6) rails-html-sanitizer (1.3.0) loofah (~> 2.3) - railties (5.2.4.4) - actionpack (= 5.2.4.4) - activesupport (= 5.2.4.4) + railties (5.2.4.5) + actionpack (= 5.2.4.5) + activesupport (= 5.2.4.5) method_source rake (>= 0.8.7) thor (>= 0.19.0, < 2.0) - rake (13.0.1) + rake (13.0.3) recursive-open-struct (1.1.0) request_store (1.5.0) rack (>= 1.4) @@ -291,13 +293,13 @@ GEM actionpack (>= 4.0) activesupport (>= 4.0) sprockets (>= 3.0.0) - thor (1.0.1) + thor (1.1.0) thread_safe (0.3.6) tilt (2.0.10) to_regexp (0.2.1) toml (0.2.0) parslet (~> 1.8.0) - tzinfo (1.2.7) + tzinfo (1.2.9) thread_safe (~> 0.1) validate_url (1.0.11) activemodel (>= 3.0.0) From 6253a70ce5c81a7827ce27b87b07dcfe4468eaeb Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 16 Feb 2021 06:03:05 +0000 Subject: [PATCH 099/156] Bump codecov from 0.2.8 to 0.4.3 Bumps [codecov](https://github.com/codecov/codecov-ruby) from 0.2.8 to 0.4.3. - [Release notes](https://github.com/codecov/codecov-ruby/releases) - [Changelog](https://github.com/codecov/codecov-ruby/blob/master/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-ruby/compare/v0.2.8...v0.4.3) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 9c8e1744..1af90ed4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -81,16 +81,15 @@ GEM bugsnag-capistrano (1.1.2) builder (3.2.4) byebug (11.0.0) - codecov (0.2.8) - json - simplecov + codecov (0.4.3) + simplecov (>= 0.15, < 0.22) coderay (1.1.2) concurrent-ruby (1.1.8) crack (0.4.3) safe_yaml (~> 1.0.0) crass (1.0.6) debug_inspector (0.0.3) - docile (1.3.2) + docile (1.3.5) dry-configurable (0.9.0) concurrent-ruby (~> 1.0) dry-core (~> 0.4, >= 0.4.7) @@ -132,7 +131,7 @@ GEM its-it (1.3.0) join_dependency (0.1.4) activerecord (>= 4.2.0) - json (2.3.1) + json (2.5.1) jsonpath (0.9.9) multi_json to_regexp (~> 0.2.1) @@ -277,10 +276,12 @@ GEM activerecord (>= 4.2, < 5.3) its-it (~> 1.2) schema_plus_core - simplecov (0.19.0) + simplecov (0.21.2) docile (~> 1.1) simplecov-html (~> 0.11) - simplecov-html (0.12.2) + simplecov_json_formatter (~> 0.1) + simplecov-html (0.12.3) + simplecov_json_formatter (0.1.2) sinatra (2.1.0) mustermann (~> 1.0) rack (~> 2.2) From e905b00655f182250d1875f5443c0bd213286033 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 16 Feb 2021 07:06:16 +0000 Subject: [PATCH 100/156] Bump bugsnag from next to 6.19.0 Bumps [bugsnag](https://github.com/bugsnag/bugsnag-ruby) from next to 6.19.0. This release includes the previously tagged commit. - [Release notes](https://github.com/bugsnag/bugsnag-ruby/releases) - [Changelog](https://github.com/bugsnag/bugsnag-ruby/blob/master/CHANGELOG.md) - [Commits](https://github.com/bugsnag/bugsnag-ruby/compare/90df9965cdb9fa95a45448e8464982f457a23d34...v6.19.0) Signed-off-by: dependabot-preview[bot] --- Gemfile | 2 +- Gemfile.lock | 12 +++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/Gemfile b/Gemfile index 57e03b86..ab40e8b0 100644 --- a/Gemfile +++ b/Gemfile @@ -34,7 +34,7 @@ gem 'que', '>= 1.0.0.beta3' gem 'que-web' gem 'baby_squeel' -gem 'bugsnag', github: 'bugsnag/bugsnag-ruby', branch: 'next' +gem 'bugsnag' # bugsnag-capistrano 2.x does not have a rake task to report deploys # https://github.com/bugsnag/bugsnag-capistrano/blob/8bcfb27cf6eaff312eef086cce729d553a431460/UPGRADING.md gem 'bugsnag-capistrano', '< 2', require: false diff --git a/Gemfile.lock b/Gemfile.lock index 1af90ed4..f916ac93 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -6,14 +6,6 @@ GIT license_finder_xml_reporter (0.1.0) license_finder -GIT - remote: https://github.com/bugsnag/bugsnag-ruby.git - revision: 90df9965cdb9fa95a45448e8464982f457a23d34 - branch: next - specs: - bugsnag (6.11.1) - concurrent-ruby (~> 1.0) - GIT remote: https://github.com/mikz/httpclient.git revision: fec23fb32fb899b87a8b2c94e2d2069b6b4c633c @@ -78,6 +70,8 @@ GEM debug_inspector (>= 0.0.1) bootsnap (1.4.5) msgpack (~> 1.0) + bugsnag (6.19.0) + concurrent-ruby (~> 1.0) bugsnag-capistrano (1.1.2) builder (3.2.4) byebug (11.0.0) @@ -336,7 +330,7 @@ DEPENDENCIES 3scale-api baby_squeel bootsnap - bugsnag! + bugsnag bugsnag-capistrano (< 2) codecov httpclient! From 965142dfab9b6eb5077ac09e82d0a98c79aee95e Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 16 Feb 2021 07:10:06 +0000 Subject: [PATCH 101/156] Bump puma from 4.3.5 to 5.2.1 Bumps [puma](https://github.com/puma/puma) from 4.3.5 to 5.2.1. - [Release notes](https://github.com/puma/puma/releases) - [Changelog](https://github.com/puma/puma/blob/master/History.md) - [Commits](https://github.com/puma/puma/compare/v4.3.5...v5.2.1) Signed-off-by: dependabot-preview[bot] --- Gemfile | 2 +- Gemfile.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index ab40e8b0..499cb7fd 100644 --- a/Gemfile +++ b/Gemfile @@ -13,7 +13,7 @@ gem 'pg', '>= 0.20' gem 'schema_plus_enums' # Use Puma as the app server -gem 'puma', '~> 4.3' +gem 'puma', '~> 5.2' # Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder # gem 'jbuilder', '~> 2.5' # Use ActiveModel has_secure_password diff --git a/Gemfile.lock b/Gemfile.lock index f916ac93..6f203ae7 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -211,7 +211,7 @@ GEM binding_of_caller (>= 0.7) pry (>= 0.9.11) public_suffix (4.0.6) - puma (4.3.5) + puma (5.2.1) nio4r (~> 2.0) quantile (0.2.1) que (1.0.0.beta4) @@ -348,7 +348,7 @@ DEPENDENCIES pry-rails pry-rescue pry-stack_explorer - puma (~> 4.3) + puma (~> 5.2) que (>= 1.0.0.beta3) que-web rails (~> 5.2.3) From ec28af77dd0b173706fab000631ee1efbda9edde Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 17 Feb 2021 13:59:13 +0000 Subject: [PATCH 102/156] Bump minitest-reporters from 1.4.2 to 1.4.3 Bumps [minitest-reporters](https://github.com/CapnKernul/minitest-reporters) from 1.4.2 to 1.4.3. - [Release notes](https://github.com/CapnKernul/minitest-reporters/releases) - [Changelog](https://github.com/minitest-reporters/minitest-reporters/blob/master/CHANGELOG.md) - [Commits](https://github.com/CapnKernul/minitest-reporters/compare/v1.4.2...v1.4.3) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 6f203ae7..6f1a436a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -166,7 +166,7 @@ GEM mini_mime (1.0.2) mini_portile2 (2.5.0) minitest (5.14.3) - minitest-reporters (1.4.2) + minitest-reporters (1.4.3) ansi builder minitest (>= 5.0) @@ -255,7 +255,7 @@ GEM responders (3.0.0) actionpack (>= 5.0) railties (>= 5.0) - ruby-progressbar (1.10.1) + ruby-progressbar (1.11.0) ruby2_keywords (0.0.2) rubyzip (2.0.0) safe_yaml (1.0.5) From 7480451e318001303d3d33ccc9a402cb3461e85d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 18 Feb 2021 13:23:49 +0000 Subject: [PATCH 103/156] Bump validate_url from 1.0.11 to 1.0.13 Bumps [validate_url](https://github.com/perfectline/validates_url) from 1.0.11 to 1.0.13. - [Release notes](https://github.com/perfectline/validates_url/releases) - [Commits](https://github.com/perfectline/validates_url/commits) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 6f1a436a..6a9d7bed 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -118,7 +118,7 @@ GEM globalid (0.4.2) activesupport (>= 4.2.0) hashdiff (1.0.1) - i18n (1.8.8) + i18n (1.8.9) concurrent-ruby (~> 1.0) ice_nine (0.11.2) interception (0.5) @@ -296,7 +296,7 @@ GEM parslet (~> 1.8.0) tzinfo (1.2.9) thread_safe (~> 0.1) - validate_url (1.0.11) + validate_url (1.0.13) activemodel (>= 3.0.0) public_suffix webmock (3.9.1) From d217c7be03a1b5042950b6a196a7ba36de78c967 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 22 Feb 2021 11:47:44 +0000 Subject: [PATCH 104/156] Bump rubyzip from 2.0.0 to 2.3.0 Bumps [rubyzip](https://github.com/rubyzip/rubyzip) from 2.0.0 to 2.3.0. - [Release notes](https://github.com/rubyzip/rubyzip/releases) - [Changelog](https://github.com/rubyzip/rubyzip/blob/master/Changelog.md) - [Commits](https://github.com/rubyzip/rubyzip/compare/v2.0.0...v2.3.0) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 6a9d7bed..df5e20b8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -257,7 +257,7 @@ GEM railties (>= 5.0) ruby-progressbar (1.11.0) ruby2_keywords (0.0.2) - rubyzip (2.0.0) + rubyzip (2.3.0) safe_yaml (1.0.5) schema_monkey (2.1.5) activerecord (>= 4.2) From 75a51512b804b8fb4f64e41b920e80213daddf65 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 22 Feb 2021 11:51:55 +0000 Subject: [PATCH 105/156] Bump bootsnap from 1.4.5 to 1.7.2 Bumps [bootsnap](https://github.com/Shopify/bootsnap) from 1.4.5 to 1.7.2. - [Release notes](https://github.com/Shopify/bootsnap/releases) - [Changelog](https://github.com/Shopify/bootsnap/blob/master/CHANGELOG.md) - [Commits](https://github.com/Shopify/bootsnap/compare/v1.4.5...v1.7.2) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index df5e20b8..4f5ab59b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -68,7 +68,7 @@ GEM polyamorous (~> 1.3) binding_of_caller (0.8.0) debug_inspector (>= 0.0.1) - bootsnap (1.4.5) + bootsnap (1.7.2) msgpack (~> 1.0) bugsnag (6.19.0) concurrent-ruby (~> 1.0) @@ -174,7 +174,7 @@ GEM minitest-stub-const (0.6) modware (0.1.3) key_struct (~> 0.4) - msgpack (1.3.1) + msgpack (1.4.2) multi_json (1.14.1) multi_xml (0.6.0) multipart-post (2.0.0) From a2c3143e5d36389cc94d8c9229f211aacde87f64 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 22 Feb 2021 12:34:48 +0000 Subject: [PATCH 106/156] Bump pry-byebug from 3.7.0 to 3.9.0 Bumps [pry-byebug](https://github.com/deivid-rodriguez/pry-byebug) from 3.7.0 to 3.9.0. - [Release notes](https://github.com/deivid-rodriguez/pry-byebug/releases) - [Changelog](https://github.com/deivid-rodriguez/pry-byebug/blob/master/CHANGELOG.md) - [Commits](https://github.com/deivid-rodriguez/pry-byebug/compare/v3.7.0...v3.9.0) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 4f5ab59b..6d89340d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -74,10 +74,10 @@ GEM concurrent-ruby (~> 1.0) bugsnag-capistrano (1.1.2) builder (3.2.4) - byebug (11.0.0) + byebug (11.1.3) codecov (0.4.3) simplecov (>= 0.15, < 0.22) - coderay (1.1.2) + coderay (1.1.3) concurrent-ruby (1.1.8) crack (0.4.3) safe_yaml (~> 1.0.0) @@ -161,7 +161,7 @@ GEM mimemagic (~> 0.3.2) message_bus (2.2.3) rack (>= 1.1.3) - method_source (0.9.2) + method_source (1.0.0) mimemagic (0.3.5) mini_mime (1.0.2) mini_portile2 (2.5.0) @@ -196,12 +196,12 @@ GEM activerecord (>= 3.0) prometheus-client (0.9.0) quantile (~> 0.2.1) - pry (0.12.2) - coderay (~> 1.1.0) - method_source (~> 0.9.0) - pry-byebug (3.7.0) + pry (0.13.1) + coderay (~> 1.1) + method_source (~> 1.0) + pry-byebug (3.9.0) byebug (~> 11.0) - pry (~> 0.10) + pry (~> 0.13.0) pry-rails (0.3.9) pry (>= 0.10.4) pry-rescue (1.5.0) From 33ee70d4a2dae084af0e6fcbaa3594fc8cdbc289 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 22 Feb 2021 13:13:49 +0000 Subject: [PATCH 107/156] Bump webmock from 3.9.1 to 3.11.2 Bumps [webmock](https://github.com/bblimke/webmock) from 3.9.1 to 3.11.2. - [Release notes](https://github.com/bblimke/webmock/releases) - [Changelog](https://github.com/bblimke/webmock/blob/master/CHANGELOG.md) - [Commits](https://github.com/bblimke/webmock/compare/v3.9.1...v3.11.2) Signed-off-by: dependabot-preview[bot] --- Gemfile.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 6d89340d..2c14a6dd 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -79,8 +79,8 @@ GEM simplecov (>= 0.15, < 0.22) coderay (1.1.3) concurrent-ruby (1.1.8) - crack (0.4.3) - safe_yaml (~> 1.0.0) + crack (0.4.5) + rexml crass (1.0.6) debug_inspector (0.0.3) docile (1.3.5) @@ -255,10 +255,10 @@ GEM responders (3.0.0) actionpack (>= 5.0) railties (>= 5.0) + rexml (3.2.4) ruby-progressbar (1.11.0) ruby2_keywords (0.0.2) rubyzip (2.3.0) - safe_yaml (1.0.5) schema_monkey (2.1.5) activerecord (>= 4.2) modware (~> 0.1) @@ -299,7 +299,7 @@ GEM validate_url (1.0.13) activemodel (>= 3.0.0) public_suffix - webmock (3.9.1) + webmock (3.11.2) addressable (>= 2.3.6) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) From 7609f04eb08757cd59a2f6cca92e9fcb377b0682 Mon Sep 17 00:00:00 2001 From: HemaHG Date: Tue, 23 Feb 2021 17:23:21 +0530 Subject: [PATCH 108/156] Bump ruby build image to v2.6 Signed-off-by: HemaHG --- .circleci/config.yml | 2 +- Dockerfile | 2 +- openshift/02-ruby-25-ubi7-imagestream.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a8d06784..8307cb9d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -26,7 +26,7 @@ jobs: build: working_directory: /opt/app-root/zync docker: - - image: registry.access.redhat.com/ubi7/ruby-25 + - image: registry.access.redhat.com/ubi7/ruby-26 - image: circleci/postgres:10-alpine-ram environment: RAILS_ENV: test diff --git a/Dockerfile b/Dockerfile index 98935fb9..53a7dce8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM registry.access.redhat.com/ubi7/ruby-25 +FROM registry.access.redhat.com/ubi7/ruby-26 USER root RUN rpm -Uvh https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm\ diff --git a/openshift/02-ruby-25-ubi7-imagestream.yml b/openshift/02-ruby-25-ubi7-imagestream.yml index 1de8dc5e..c445361b 100644 --- a/openshift/02-ruby-25-ubi7-imagestream.yml +++ b/openshift/02-ruby-25-ubi7-imagestream.yml @@ -3,12 +3,12 @@ kind: ImageStream metadata: labels: app: zync - name: ruby-25-ubi7 + name: ruby-26-ubi7 spec: tags: - from: kind: DockerImage - name: registry.access.redhat.com/ubi7/ruby-25 + name: registry.access.redhat.com/ubi7/ruby-26 name: latest referencePolicy: type: Source From d1b52111fc40df952c4baa882b74f6a8af0cceeb Mon Sep 17 00:00:00 2001 From: HemaHG Date: Tue, 23 Feb 2021 18:55:30 +0530 Subject: [PATCH 109/156] Bump ruby build image to v2.7 Signed-off-by: HemaHG --- .circleci/config.yml | 2 +- Dockerfile | 2 +- openshift/02-ruby-25-ubi7-imagestream.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8307cb9d..7e6d4944 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -26,7 +26,7 @@ jobs: build: working_directory: /opt/app-root/zync docker: - - image: registry.access.redhat.com/ubi7/ruby-26 + - image: registry.access.redhat.com/ubi7/ruby-27 - image: circleci/postgres:10-alpine-ram environment: RAILS_ENV: test diff --git a/Dockerfile b/Dockerfile index 53a7dce8..6987c249 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM registry.access.redhat.com/ubi7/ruby-26 +FROM registry.access.redhat.com/ubi7/ruby-27 USER root RUN rpm -Uvh https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm\ diff --git a/openshift/02-ruby-25-ubi7-imagestream.yml b/openshift/02-ruby-25-ubi7-imagestream.yml index c445361b..93160b08 100644 --- a/openshift/02-ruby-25-ubi7-imagestream.yml +++ b/openshift/02-ruby-25-ubi7-imagestream.yml @@ -3,12 +3,12 @@ kind: ImageStream metadata: labels: app: zync - name: ruby-26-ubi7 + name: ruby-27-ubi7 spec: tags: - from: kind: DockerImage - name: registry.access.redhat.com/ubi7/ruby-26 + name: registry.access.redhat.com/ubi7/ruby-27 name: latest referencePolicy: type: Source From d0b5910743492e70ca15669b3c8c98c92a2efb1d Mon Sep 17 00:00:00 2001 From: HemaHG Date: Mon, 1 Mar 2021 17:00:58 +0530 Subject: [PATCH 110/156] Bump responders from 3.0.0 to 3.0.1 Signed-off-by: HemaHG --- Gemfile | 2 +- Gemfile.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index 499cb7fd..4d04cce7 100644 --- a/Gemfile +++ b/Gemfile @@ -25,7 +25,7 @@ gem 'puma', '~> 5.2' # Use Rack CORS for handling Cross-Origin Resource Sharing (CORS), making cross-origin AJAX possible # gem 'rack-cors' -gem 'responders', '~> 3.0.0' +gem 'responders', '~> 3.0.1' gem '3scale-api' gem 'bootsnap' diff --git a/Gemfile.lock b/Gemfile.lock index 2c14a6dd..e1560538 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -252,7 +252,7 @@ GEM recursive-open-struct (1.1.0) request_store (1.5.0) rack (>= 1.4) - responders (3.0.0) + responders (3.0.1) actionpack (>= 5.0) railties (>= 5.0) rexml (3.2.4) @@ -352,7 +352,7 @@ DEPENDENCIES que (>= 1.0.0.beta3) que-web rails (~> 5.2.3) - responders (~> 3.0.0) + responders (~> 3.0.1) rubyzip (>= 1.3.0) schema_plus_enums tzinfo-data From d171d32d49300e3ae960a86ecfd0abbcca05392b Mon Sep 17 00:00:00 2001 From: HemaHG Date: Mon, 1 Mar 2021 19:52:12 +0530 Subject: [PATCH 111/156] Bump 3scale-api from 0.6.0 to 1.4.0 Signed-off-by: HemaHG --- Gemfile | 1 + Gemfile.lock | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 499cb7fd..f76695b4 100644 --- a/Gemfile +++ b/Gemfile @@ -79,4 +79,5 @@ group :test do gem 'minitest-stub-const' gem 'webmock' gem 'codecov', require: false + gem 'simplecov', '~> 0.21.2', require: false end diff --git a/Gemfile.lock b/Gemfile.lock index 2c14a6dd..67e09ac8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -16,7 +16,7 @@ GIT GEM remote: https://rubygems.org/ specs: - 3scale-api (0.6.0) + 3scale-api (1.4.0) actioncable (5.2.4.5) actionpack (= 5.2.4.5) nio4r (~> 2.0) @@ -355,6 +355,7 @@ DEPENDENCIES responders (~> 3.0.0) rubyzip (>= 1.3.0) schema_plus_enums + simplecov (~> 0.21.2) tzinfo-data validate_url webmock From 2a2e094aed7231636c427f95793ab42590b6a142 Mon Sep 17 00:00:00 2001 From: HemaHG Date: Mon, 1 Mar 2021 20:07:12 +0530 Subject: [PATCH 112/156] Bump yabeda-puma-plugin from 0.1.0 to 0.6.0 Signed-off-by: HemaHG --- Gemfile.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 2c14a6dd..1564cd39 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -94,7 +94,7 @@ GEM concurrent-ruby (~> 1.0) dry-equalizer (0.3.0) dry-inflector (0.2.0) - dry-initializer (2.5.0) + dry-initializer (3.0.4) dry-logic (0.6.1) concurrent-ruby (~> 1.0) dry-core (~> 0.2) @@ -308,15 +308,15 @@ GEM websocket-extensions (0.1.5) with_env (1.1.0) xml-simple (1.1.5) - yabeda (0.1.3) + yabeda (0.8.0) concurrent-ruby dry-initializer yabeda-prometheus (0.1.4) yabeda - yabeda-puma-plugin (0.1.0) + yabeda-puma-plugin (0.6.0) json puma - yabeda + yabeda (~> 0.5) yabeda-rails (0.1.3) rails yabeda From 7aae374d65841d25d61261ed5f141a4b9bf9c8d6 Mon Sep 17 00:00:00 2001 From: HemaHG Date: Wed, 24 Feb 2021 13:10:44 +0530 Subject: [PATCH 113/156] Bump yabeda-prometheus from 0.1.4 to 0.1.5 Signed-off-by: HemaHG --- Gemfile.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 78367fc7..633b6697 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -94,7 +94,7 @@ GEM concurrent-ruby (~> 1.0) dry-equalizer (0.3.0) dry-inflector (0.2.0) - dry-initializer (2.5.0) + dry-initializer (3.0.4) dry-logic (0.6.1) concurrent-ruby (~> 1.0) dry-core (~> 0.2) @@ -308,10 +308,10 @@ GEM websocket-extensions (0.1.5) with_env (1.1.0) xml-simple (1.1.5) - yabeda (0.1.3) + yabeda (0.8.0) concurrent-ruby dry-initializer - yabeda-prometheus (0.1.4) + yabeda-prometheus (0.1.5) yabeda yabeda-puma-plugin (0.1.0) json From 7706422490ca1973b3c4d101a382caebb97d3809 Mon Sep 17 00:00:00 2001 From: Hallelujah Date: Tue, 9 Mar 2021 12:48:27 +0100 Subject: [PATCH 114/156] [yabeda] Fix configuration of yabeda prometheus * COALESCE a sum for que workers * Rename `job` label to `job_name` * Call `Yabeda.configure!`` after all definition * `attach_to :active_job` subscriber after Yabeda.configure! otherwise `Yabeda.que` is not defined --- config/initializers/prometheus.rb | 3 +++ lib/prometheus/active_job_subscriber.rb | 2 -- lib/prometheus/active_record.rb | 13 +++++++------ lib/prometheus/que_stats.rb | 10 +++++----- test/lib/prometheus/que_stats_test.rb | 2 +- 5 files changed, 16 insertions(+), 14 deletions(-) diff --git a/config/initializers/prometheus.rb b/config/initializers/prometheus.rb index e3d706f8..db59c1f9 100644 --- a/config/initializers/prometheus.rb +++ b/config/initializers/prometheus.rb @@ -1,2 +1,5 @@ require 'prometheus/active_job_subscriber' require 'prometheus/active_record' + +Yabeda.configure! if Yabeda.respond_to?(:configure!) +Prometheus::ActiveJobSubscriber.attach_to :active_job diff --git a/lib/prometheus/active_job_subscriber.rb b/lib/prometheus/active_job_subscriber.rb index 37527a1e..a09e482b 100644 --- a/lib/prometheus/active_job_subscriber.rb +++ b/lib/prometheus/active_job_subscriber.rb @@ -77,7 +77,5 @@ def extract_labels(payload) to: :@metrics attr_reader :job_runtime_seconds - - attach_to :active_job end end diff --git a/lib/prometheus/active_record.rb b/lib/prometheus/active_record.rb index 0848f89c..c9dd388d 100644 --- a/lib/prometheus/active_record.rb +++ b/lib/prometheus/active_record.rb @@ -2,18 +2,19 @@ Yabeda.configure do group :rails_connection_pool do - size = gauge :size, comment: 'Size of the connection pool' - connections = gauge :connections, comment: 'Number of connections in the connection pool' - waiting = gauge :waiting, comment: 'Number of waiting in the queue of the connection pool' - - no_labels = {}.freeze + # Empty label values SHOULD be treated as if the label was not present. + # @see https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#label + no_labels = { state: nil }.freeze busy = { state: :busy }.freeze dead = { state: :dead }.freeze idle = { state: :idle }.freeze + size = gauge :size, comment: 'Size of the connection pool' + connections = gauge :connections, comment: 'Number of connections in the connection pool' + waiting = gauge :waiting, comment: 'Number of waiting in the queue of the connection pool' + collect do stat = ActiveRecord::Base.connection_pool.stat - size.set(no_labels, stat.fetch(:size)) connections.set(no_labels, stat.fetch(:connections)) connections.set(busy, stat.fetch(:busy)) diff --git a/lib/prometheus/que_stats.rb b/lib/prometheus/que_stats.rb index 3af4008f..88e1feb2 100644 --- a/lib/prometheus/que_stats.rb +++ b/lib/prometheus/que_stats.rb @@ -30,7 +30,7 @@ def execute class WorkerStats < Stats WORKER_STATS = <<~SQL - SELECT SUM(worker_count) AS workers_count, COUNT(*) AS nodes_count FROM que_lockers + SELECT COALESCE(SUM("worker_count"),0) AS workers_count, COUNT(*) AS nodes_count FROM que_lockers SQL private_constant :WORKER_STATS @@ -101,7 +101,7 @@ def expired JOB_CLASSES = %w[ApplicationJob ProcessEntryJob ProcessIntegrationEntryJob UpdateJob].inspect.tr('"', "'").freeze private_constant :JOB_CLASSES - JOB_CLASS_COLUMN_NAME = 'job' + JOB_CLASS_COLUMN_NAME = 'job_name' private_constant :JOB_CLASS_COLUMN_NAME def job_class_column @@ -136,7 +136,7 @@ def call(type = nil) protected def type_hash_and_stats_count(type) - type_hash = type ? { type: type } : {} + type_hash = { type: type } stats_count = stats.public_send(type ? type : :all) [type_hash, stats_count] end @@ -153,7 +153,7 @@ def initialize(gauge, stats, grouped_by:) def call(type = nil) type_hash, grouped_stats_count = type_hash_and_stats_count(type) grouped_stats_count.each do |stats_count| - gauge.set({ grouped_by => stats_count.fetch(grouped_by) }.merge(type_hash), stats_count.fetch('count')) + gauge.set({ grouped_by.to_sym => stats_count.fetch(grouped_by) }.merge(type_hash), stats_count.fetch('count')) end end end @@ -173,7 +173,7 @@ def call(type = nil) group :que do jobs = gauge :jobs_scheduled_total, comment: 'Que Jobs to be executed' job_stats = Prometheus::QueStats::JobStats.new - collector = Prometheus::QueStats::GroupedStatsCollector.new(jobs, job_stats, grouped_by: 'job') + collector = Prometheus::QueStats::GroupedStatsCollector.new(jobs, job_stats, grouped_by: 'job_name') collect do collector.call %w[ready scheduled finished failed expired].each(&collector.method(:call)) diff --git a/test/lib/prometheus/que_stats_test.rb b/test/lib/prometheus/que_stats_test.rb index 09d102cf..0bdef172 100644 --- a/test/lib/prometheus/que_stats_test.rb +++ b/test/lib/prometheus/que_stats_test.rb @@ -116,7 +116,7 @@ def update_job(job, attributes = {}) def stats_count(job_class: ApplicationJob.name, type: nil, where: []) job_stats = Prometheus::QueStats::JobStats.new stats = type ? job_stats.public_send(type) : job_stats.call(*where) - record = stats.find { |record| record['job'] == job_class } + record = stats.find { |record| record['job_name'] == job_class } record['count'] end end From 031b641872730ed0778e91bdb473a2413e0ad8c0 Mon Sep 17 00:00:00 2001 From: HemaHG Date: Wed, 10 Mar 2021 11:37:18 +0530 Subject: [PATCH 115/156] Bump yabeda-rails from 0.1.3 to 0.7.1 Signed-off-by: HemaHG --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index ad5c1834..ed4930f4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -317,9 +317,9 @@ GEM json puma yabeda (~> 0.5) - yabeda-rails (0.1.3) + yabeda-rails (0.7.1) rails - yabeda + yabeda (~> 0.8) yajl-ruby (1.4.1) yaml-safe_load_stream (0.1.1) From ff5b68b124932fc26ec08ecb8a0a0830dc8d59be Mon Sep 17 00:00:00 2001 From: HemaHG Date: Wed, 10 Mar 2021 11:52:58 +0530 Subject: [PATCH 116/156] Bump license_finder from 5.11.1 to 6.10.1 Signed-off-by: HemaHG --- Gemfile | 2 +- Gemfile.lock | 17 +++++++------ doc/dependency_decisions.yml | 46 ++++++++++++++++++------------------ 3 files changed, 32 insertions(+), 33 deletions(-) diff --git a/Gemfile b/Gemfile index 96c29e15..1de40237 100644 --- a/Gemfile +++ b/Gemfile @@ -63,7 +63,7 @@ group :development, :test do gem 'pry-rescue' gem 'pry-stack_explorer' - gem 'license_finder', '~> 5.11' + gem 'license_finder', '~> 6.10' gem 'license_finder_xml_reporter', git: 'https://github.com/3scale/license_finder_xml_reporter.git', tag: '1.0.0' # rubyzip is a transitive depencency from license_finder with vulnerability on < 1.3.0 gem 'rubyzip', '>= 1.3.0' diff --git a/Gemfile.lock b/Gemfile.lock index ad5c1834..757601d7 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -140,13 +140,14 @@ GEM yajl-ruby (~> 1.4.0) yaml-safe_load_stream (~> 0.1) key_struct (0.4.2) - license_finder (5.11.1) + license_finder (6.10.1) bundler rubyzip (>= 1, < 3) thor - toml (= 0.2.0) + thor (~> 1.0.1) + tomlrb (>= 1.3, < 2.1) with_env (= 1.1.0) - xml-simple + xml-simple (~> 1.1.5) lograge (0.11.2) actionpack (>= 4) activesupport (>= 4) @@ -190,7 +191,6 @@ GEM multi_json (~> 1.3) multi_xml (~> 0.5) rack (>= 1.2, < 3) - parslet (1.8.2) pg (1.2.3) polyamorous (1.3.3) activerecord (>= 3.0) @@ -288,12 +288,11 @@ GEM actionpack (>= 4.0) activesupport (>= 4.0) sprockets (>= 3.0.0) - thor (1.1.0) + thor (1.0.1) thread_safe (0.3.6) tilt (2.0.10) to_regexp (0.2.1) - toml (0.2.0) - parslet (~> 1.8.0) + tomlrb (2.0.1) tzinfo (1.2.9) thread_safe (~> 0.1) validate_url (1.0.13) @@ -307,7 +306,7 @@ GEM websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) with_env (1.1.0) - xml-simple (1.1.5) + xml-simple (1.1.8) yabeda (0.8.0) concurrent-ruby dry-initializer @@ -335,7 +334,7 @@ DEPENDENCIES codecov httpclient! k8s-client (>= 0.10) - license_finder (~> 5.11) + license_finder (~> 6.10) license_finder_xml_reporter! lograge message_bus diff --git a/doc/dependency_decisions.yml b/doc/dependency_decisions.yml index cb2e1682..d9322b48 100644 --- a/doc/dependency_decisions.yml +++ b/doc/dependency_decisions.yml @@ -1,56 +1,56 @@ --- -- - :whitelist +- - :permit - MIT - - :who: - :why: + - :who: + :why: :versions: [] :when: 2017-06-23 06:12:33.037106000 Z -- - :whitelist +- - :permit - Apache 2.0 - - :who: - :why: + - :who: + :why: :versions: [] :when: 2017-06-23 06:12:54.025599000 Z -- - :whitelist +- - :permit - Simplified BSD - - :who: - :why: + - :who: + :why: :versions: [] :when: 2017-06-23 06:12:59.933965000 Z -- - :whitelist +- - :permit - New BSD - - :who: - :why: + - :who: + :why: :versions: [] :when: 2017-06-23 06:13:02.855385000 Z -- - :whitelist +- - :permit - ruby - - :who: - :why: + - :who: + :why: :versions: [] :when: 2017-06-23 06:13:06.311014000 Z - - :ignore_group - development - - :who: - :why: + - :who: + :why: :versions: [] :when: 2017-06-23 06:15:58.484416000 Z - - :ignore_group - test - - :who: - :why: + - :who: + :why: :versions: [] :when: 2017-06-23 06:16:00.472944000 Z - - :license - bundler - MIT - - :who: + - :who: :why: Bundler is distributed under MIT license https://github.com/bundler/bundler/blob/1102ec1a06d5971fc3c870b1ed50de8f826c36e4/bundler.gemspec\#L10 :versions: [] :when: 2017-06-23 06:34:01.682953000 Z -- - :whitelist +- - :permit - BSD - - :who: - :why: + - :who: + :why: :versions: [] :when: 2019-04-11 17:56:14.390898000 Z From 6a881dd3d6c7ded0c7221ea54beac14b32fedaa9 Mon Sep 17 00:00:00 2001 From: Guilherme Cassolato Date: Thu, 11 Mar 2021 12:19:16 +0100 Subject: [PATCH 117/156] Kubernetes resource status missing 'ingress' property --- .../integration/kubernetes_service.rb | 10 +- test/fixtures/entries.yml | 6 +- test/services/kubernetes_service_test.rb | 218 ++++++++++++++++-- 3 files changed, 213 insertions(+), 21 deletions(-) diff --git a/app/services/integration/kubernetes_service.rb b/app/services/integration/kubernetes_service.rb index d581f0d0..4fc436d0 100644 --- a/app/services/integration/kubernetes_service.rb +++ b/app/services/integration/kubernetes_service.rb @@ -254,8 +254,16 @@ def initialize(condition) end end + class MissingStatusIngress < InvalidStatus + MISSING_STATUS_INGRESS_CONDITION = ActiveSupport::OrderedOptions.new.merge(type: 'unknown', reason: 'unknown', message: "Kubernetes resource status missing 'ingress' property").freeze + + def initialize + super(MISSING_STATUS_INGRESS_CONDITION) + end + end + def verify_route_status(route) - ingress = route.status.ingress.find { |ingress| ingress.host == route.spec.host } + ingress = (route.status.ingress or raise MissingStatusIngress).find { |ingress| ingress.host == route.spec.host } condition = ingress.conditions.find { |condition| condition.type = 'Admitted' } raise InvalidStatus, condition unless condition.status == 'True' diff --git a/test/fixtures/entries.yml b/test/fixtures/entries.yml index b11f4229..cbcaecc6 100644 --- a/test/fixtures/entries.yml +++ b/test/fixtures/entries.yml @@ -26,7 +26,7 @@ client: created_at: <%= 1.week.ago %> service: - data: + data: tenant: two model: service @@ -41,6 +41,8 @@ proxy: provider: data: - service_id: 2 + id: 2 + domain: provider.example.com + admin_domain: provider-admin.example.com tenant: two model: provider diff --git a/test/services/kubernetes_service_test.rb b/test/services/kubernetes_service_test.rb index 42478ed5..d8a6951c 100644 --- a/test/services/kubernetes_service_test.rb +++ b/test/services/kubernetes_service_test.rb @@ -16,7 +16,7 @@ def after_teardown super end - test 'create ingress ' do + setup do ENV['KUBERNETES_NAMESPACE'] = 'zync' ENV['KUBE_TOKEN'] = strict_encode64('token') ENV['KUBE_SERVER'] = 'http://localhost' @@ -33,40 +33,208 @@ def after_teardown -----END CERTIFICATE----- CERTIFICATE - service = Integration::KubernetesService.new(nil) + @service = Integration::KubernetesService.new(nil) + end + + attr_reader :service + test 'create ingress' do proxy = entries(:proxy) stub_request(:get, 'http://localhost/apis/route.openshift.io/v1'). - with( - headers: { - 'Accept'=>'application/json', - 'Authorization'=>'Bearer token', - }). + with(headers: request_headers). to_return(status: 200, body: { - kind: "APIResourceList", - apiVersion: "v1", - groupVersion: "route.openshift.io/v1", + kind: 'APIResourceList', + apiVersion: 'v1', + groupVersion: 'route.openshift.io/v1', resources: [ - { name: "routes", singularName: "", namespaced: true, kind: "Route", verbs: %w(create delete deletecollection get list patch update watch), categories: ["all"] }, + { name: 'routes', singularName: '', namespaced: true, kind: 'Route', verbs: %w(create delete deletecollection get list patch update watch), categories: ['all'] }, ] - }.to_json, headers: { 'Content-Type' => 'application/json' }) + }.to_json, headers: response_headers) stub_request(:get, 'http://localhost/apis/route.openshift.io/v1/namespaces/zync/routes?labelSelector=3scale.net/created-by=zync,3scale.net/tenant_id=298486374,zync.3scale.net/record=Z2lkOi8venluYy9Qcm94eS8yOTg0ODYzNzQ,zync.3scale.net/ingress=proxy,3scale.net/service_id=2'). - with( - headers: { - 'Accept'=>'application/json', - 'Authorization'=>'Bearer token', - }). + with(headers: request_headers). to_return(status: 200, body: { kind: 'RouteList', apiVersion: 'route.openshift.io/v1', metadata: { selfLink: '/apis/route.openshift.io/v1/namespaces/zync/routes', resourceVersion: '651341' }, - items: [] }.to_json, headers: { 'Content-Type' => 'application/json' }) + items: [] + }.to_json, headers: response_headers) service.call(proxy) end + test 'route status missing ingress' do + # stub api resource list requests (kinds 'pods' and 'routes') + stub_request(:get, 'http://localhost/api/v1'). + with(headers: request_headers). + to_return(status: 200, body: { + kind: 'APIResourceList', + apiVersion: 'v1', + groupVersion: 'apps.3scale.net/v1alpha1', + resources: [ + { name: 'pods', singularName: '', namespaced: true, kind: 'pod', verbs: %w(create delete deletecollection get list patch update watch), categories: ['all'] }, + ] + }.to_json, headers: response_headers) + + stub_request(:get, 'http://localhost/apis/route.openshift.io/v1'). + with(headers: request_headers). + to_return(status: 200, body: { + kind: 'APIResourceList', + apiVersion: 'v1', + groupVersion: 'route.openshift.io/v1', + resources: [ + { name: 'routes', singularName: '', namespaced: true, kind: 'Route', verbs: %w(create delete deletecollection get list patch update watch), categories: ['all'] }, + ] + }.to_json, headers: response_headers) + + # stub route owner + ENV['POD_NAME'] = 'zync-que-123' + route_owner = { kind: 'Pod', apiVersion: 'v1', metadata: { name: 'zync-que-123', generateName: 'zync-que-', namespace: 'zync', selfLink: '/api/v1/namespaces/zync/pods/zync-que-123', uid: 'b145c845-7222-44ce-8d9d-f13b8f357de6', resourceVersion: '3620670' } } + + stub_request(:get, "http://localhost/api/v1/namespaces/zync/pods/#{route_owner.dig(:metadata, :name)}"). + with(headers: request_headers). + to_return(status: 200, body: route_owner.to_json, headers: response_headers) + + route_owner_reference = route_owner.slice(:kind, :apiVersion).merge(**route_owner[:metadata].slice(:name, :uid), controller: nil, blockOwnerDeletion: nil) + + # base objects for creating provider routes + entry = entries(:provider) + provider_id = entry.data.fetch('id') + provider = entry.model.record + tenant_id = entry.tenant_id + record_gid = provider.to_gid_param + + provider_route_labels = { + '3scale.net/created-by' => 'zync', + '3scale.net/tenant_id' => tenant_id.to_s, + 'zync.3scale.net/record' => record_gid, + 'zync.3scale.net/ingress' => 'provider', + '3scale.net/provider_id' => provider_id.to_s + } + + provider_route_annotations = { + '3scale.net/gid' => entry.to_gid.to_s, + 'zync.3scale.net/gid' => provider.to_gid.to_s + } + + route_list = { + kind: 'RouteList', + apiVersion: 'route.openshift.io/v1', + metadata: { selfLink: '/apis/route.openshift.io/v1/namespaces/zync/routes', resourceVersion: '651341' }, + items: [] + } + + # stub for creating provider route to system-developer + system_developer_route_labels = provider_route_labels.merge('zync.3scale.net/route-to' => 'system-developer') + system_developer_route_annotations = provider_route_annotations.merge('zync.3scale.net/host' => 'provider.example.com') + system_developer_route = { + kind: 'Route', + apiVersion: 'route.openshift.io/v1', + metadata: { + namespace: 'zync', + name: 'zync-3scale-provider-grvkd', + uid: '3882e5dc-1f8f-460e-a1cc-ee4c5f35a709', + selfLink: '/apis/route.openshift.io/v1/namespaces/zync/routes/zync-3scale-provider-grvkd', + labels: system_developer_route_labels, + annotations: system_developer_route_annotations + }, + status: {} + } + + stub_request(:get, "http://localhost/apis/route.openshift.io/v1/namespaces/zync/routes?labelSelector=3scale.net/created-by=zync,3scale.net/tenant_id=#{tenant_id},zync.3scale.net/record=#{record_gid},zync.3scale.net/route-to=system-developer"). + with(headers: request_headers). + to_return(status: 200, body: route_list.to_json, headers: response_headers) + + stub_request(:post, 'http://localhost/apis/route.openshift.io/v1/namespaces/zync/routes'). + with(headers: request_headers, body: { + metadata: { + generateName: 'zync-3scale-provider-', + namespace: 'zync', + labels: system_developer_route_labels, + ownerReferences: [route_owner_reference], + annotations: system_developer_route_annotations + }, + spec: { + host: 'provider.example.com', + port: { targetPort: 'http' }, + to: { kind: 'Service', name: 'system-developer' }, + tls: { insecureEdgeTerminationPolicy: 'Redirect', termination: 'edge' } + }, + apiVersion: 'route.openshift.io/v1', + kind: 'Route' + }.to_json). + to_return(status: 201, body: system_developer_route.to_json, headers: response_headers) + + route_list[:metadata][:resourceVersion] = '651342' + route_list[:items] = [system_developer_route] + + stub_request(:get, 'http://localhost/apis/route.openshift.io/v1/namespaces/zync/routes'). + with(headers: request_headers). + to_return(status: 200, body: route_list.to_json, headers: response_headers) + + stub_request(:get, "http://localhost/apis/route.openshift.io/v1/namespaces/zync/routes/#{system_developer_route.dig(:metadata, :name)}"). + with(headers: request_headers). + to_return(status: 200, body: system_developer_route.to_json, headers: response_headers) + + # stub for creating provider route to system-provider + system_provider_route_labels = provider_route_labels.merge('zync.3scale.net/route-to' => 'system-provider') + system_provider_route_annotations = provider_route_annotations.merge('zync.3scale.net/host' => 'provider-admin.example.com') + system_provider_route = { + kind: 'Route', + apiVersion: 'route.openshift.io/v1', + metadata: { + namespace: 'zync', + name: 'zync-3scale-provider-rbpqw', + uid: 'f741703c-7ca5-4480-8a32-074fcc759583', + selfLink: '/apis/route.openshift.io/v1/namespaces/zync/routes/zync-3scale-provider-rbpqw', + labels: system_developer_route_labels, + annotations: system_developer_route_annotations + }, + status: {} + } + + stub_request(:get, "http://localhost/apis/route.openshift.io/v1/namespaces/zync/routes?labelSelector=3scale.net/created-by=zync,3scale.net/tenant_id=#{tenant_id},zync.3scale.net/record=#{record_gid},zync.3scale.net/route-to=system-provider"). + with(headers: request_headers). + to_return(status: 200, body: route_list.merge(items: []).to_json, headers: response_headers) + + stub_request(:post, 'http://localhost/apis/route.openshift.io/v1/namespaces/zync/routes'). + with(headers: request_headers, body: { + metadata: { + generateName: 'zync-3scale-provider-', + namespace: 'zync', + labels: system_provider_route_labels, + ownerReferences: [route_owner_reference], + annotations: system_provider_route_annotations + }, + spec: { + host: 'provider-admin.example.com', + port: { targetPort: 'http' }, + to: { kind: 'Service', name: 'system-provider' }, + tls: { insecureEdgeTerminationPolicy: 'Redirect', termination: 'edge' } + }, + apiVersion: 'route.openshift.io/v1', + kind: 'Route' + }.to_json). + to_return(status: 201, body: system_provider_route.to_json, headers: response_headers) + + route_list[:metadata][:resourceVersion] = '651343' + route_list[:items] = [system_developer_route, system_provider_route] + + stub_request(:get, 'http://localhost/apis/route.openshift.io/v1/namespaces/zync/routes'). + with(headers: request_headers). + to_return(status: 200, body: route_list.to_json, headers: response_headers) + + stub_request(:get, "http://localhost/apis/route.openshift.io/v1/namespaces/zync/routes/#{system_provider_route.dig(:metadata, :name)}"). + with(headers: request_headers). + to_return(status: 200, body: system_provider_route.to_json, headers: response_headers) + + # create both routes + assert_raises(Integration::KubernetesService::MissingStatusIngress) do + service.call(entry) + end + end + class RouteSpec < ActiveSupport::TestCase test 'secure routes' do url = 'https://my-api.example.com' @@ -108,4 +276,18 @@ class RouteSpec < ActiveSupport::TestCase assert_equal json, spec.to_hash end end + + protected + + def request_headers + { + 'Accept' => 'application/json', + 'Authorization' => 'Bearer token', + 'Host' => 'localhost:80' + } + end + + def response_headers + { 'Content-Type' => 'application/json' } + end end From 488200374186ce4b4a3f7ef1dd4ea6905b7f46ec Mon Sep 17 00:00:00 2001 From: Guilherme Cassolato Date: Thu, 11 Mar 2021 14:58:29 +0100 Subject: [PATCH 118/156] Fix verification of the conditions of the ingress based on its 'type' property --- app/services/integration/kubernetes_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/integration/kubernetes_service.rb b/app/services/integration/kubernetes_service.rb index 4fc436d0..afcb1c2f 100644 --- a/app/services/integration/kubernetes_service.rb +++ b/app/services/integration/kubernetes_service.rb @@ -264,7 +264,7 @@ def initialize def verify_route_status(route) ingress = (route.status.ingress or raise MissingStatusIngress).find { |ingress| ingress.host == route.spec.host } - condition = ingress.conditions.find { |condition| condition.type = 'Admitted' } + condition = ingress.conditions.find { |condition| condition.type == 'Admitted' } raise InvalidStatus, condition unless condition.status == 'True' end From db1000d97851954fcbd2f78301f1de832378f966 Mon Sep 17 00:00:00 2001 From: HemaHG Date: Wed, 10 Mar 2021 11:46:29 +0530 Subject: [PATCH 119/156] Bump prometheus-client from 0.9.0 to 2.1.0 Signed-off-by: HemaHG --- Gemfile.lock | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 66900c9a..44ea281e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -194,8 +194,7 @@ GEM pg (1.2.3) polyamorous (1.3.3) activerecord (>= 3.0) - prometheus-client (0.9.0) - quantile (~> 0.2.1) + prometheus-client (2.1.0) pry (0.13.1) coderay (~> 1.1) method_source (~> 1.0) @@ -213,7 +212,6 @@ GEM public_suffix (4.0.6) puma (5.2.1) nio4r (~> 2.0) - quantile (0.2.1) que (1.0.0.beta4) que-web (0.9.3) que (~> 1.0.0.beta3) From fe8797286ed4a41d4d4d43bac38f3c712d3a0995 Mon Sep 17 00:00:00 2001 From: Hallelujah Date: Tue, 16 Mar 2021 14:40:56 +0100 Subject: [PATCH 120/156] [prometheus] Update to yabeda-prometheus gem 0.6.1 * Updates the yabeda-prometheus gem to 0.6.1 * Fixes API issues, declaring tags before being able to use the metrics --- Gemfile | 4 ++-- Gemfile.lock | 10 ++++++---- lib/prometheus/active_job_subscriber.rb | 20 +++++++++++++------- lib/prometheus/active_record.rb | 9 ++++++--- lib/prometheus/que_stats.rb | 12 ++++++++++-- 5 files changed, 37 insertions(+), 18 deletions(-) diff --git a/Gemfile b/Gemfile index 1de40237..8ba31d03 100644 --- a/Gemfile +++ b/Gemfile @@ -51,9 +51,9 @@ gem 'message_bus' # for publishing notifications about integration status gem 'validate_url' -gem 'prometheus-client', require: %w[prometheus/client] +gem 'prometheus-client', '~> 2.1.0', require: %w[prometheus/client] gem 'yabeda-rails' -gem 'yabeda-prometheus' +gem 'yabeda-prometheus', '~> 0.6.1' gem 'yabeda-puma-plugin' group :development, :test do diff --git a/Gemfile.lock b/Gemfile.lock index 44ea281e..dd17f2f1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -308,8 +308,10 @@ GEM yabeda (0.8.0) concurrent-ruby dry-initializer - yabeda-prometheus (0.1.5) - yabeda + yabeda-prometheus (0.6.1) + prometheus-client (>= 0.10, < 3.0) + rack + yabeda (~> 0.5) yabeda-puma-plugin (0.6.0) json puma @@ -340,7 +342,7 @@ DEPENDENCIES minitest-stub-const oauth2 pg (>= 0.20) - prometheus-client + prometheus-client (~> 2.1.0) pry-byebug pry-rails pry-rescue @@ -356,7 +358,7 @@ DEPENDENCIES tzinfo-data validate_url webmock - yabeda-prometheus + yabeda-prometheus (~> 0.6.1) yabeda-puma-plugin yabeda-rails diff --git a/lib/prometheus/active_job_subscriber.rb b/lib/prometheus/active_job_subscriber.rb index a09e482b..2901e378 100644 --- a/lib/prometheus/active_job_subscriber.rb +++ b/lib/prometheus/active_job_subscriber.rb @@ -5,13 +5,15 @@ module Prometheus ## ActiveJob Subscriber to record Prometheus metrics. ## Those metrics are per process, so they have to be aggregated by Prometheus. class ActiveJobSubscriber < ActiveSupport::Subscriber + PROMETHEUS_TAGS = %i[adapter job_name].freeze + Yabeda.configure do group :que do - counter :job_retries_total, comment: 'A number of Jobs retried by this process' - counter :job_failures_total, comment: 'A number of Jobs errored by this process' - counter :job_performed_total, comment: 'A number of Jobs performed by this process' - counter :job_enqueued_total, comment: 'A number of Jobs enqueued by this process' - histogram :job_duration_seconds do + counter :job_retries_total, comment: 'A number of Jobs retried by this process', tags: ActiveJobSubscriber::PROMETHEUS_TAGS + counter :job_failures_total, comment: 'A number of Jobs errored by this process', tags: ActiveJobSubscriber::PROMETHEUS_TAGS + counter :job_performed_total, comment: 'A number of Jobs performed by this process', tags: ActiveJobSubscriber::PROMETHEUS_TAGS + counter :job_enqueued_total, comment: 'A number of Jobs enqueued by this process', tags: ActiveJobSubscriber::PROMETHEUS_TAGS + histogram :job_duration_seconds, tags: ActiveJobSubscriber::PROMETHEUS_TAGS do comment 'A histogram of Jobs perform times by this process' buckets false end @@ -22,7 +24,11 @@ class ActiveJobSubscriber < ActiveSupport::Subscriber def initialize super @metrics = Yabeda.que - @job_runtime_seconds = Yabeda::Prometheus.registry.summary(:que_job_runtime_seconds, 'A summary of Jobs perform times') + @job_runtime_seconds = Yabeda::Prometheus.registry.summary( + :que_job_runtime_seconds, + docstring: 'A summary of Jobs perform times', + labels: ActiveJobSubscriber::PROMETHEUS_TAGS + ) end def enqueue(event) @@ -52,7 +58,7 @@ def observe_duration(event, labels) duration = event.duration / 1000.0 job_duration_seconds.measure(labels, duration) - job_runtime_seconds.observe(labels, duration) + job_runtime_seconds.observe(duration, labels: labels) end def observe_perform(payload, labels) diff --git a/lib/prometheus/active_record.rb b/lib/prometheus/active_record.rb index c9dd388d..99fa9bb0 100644 --- a/lib/prometheus/active_record.rb +++ b/lib/prometheus/active_record.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +RAILS_CONNECTION_PROMETHEUS_TAGS = %i[state].freeze + Yabeda.configure do group :rails_connection_pool do # Empty label values SHOULD be treated as if the label was not present. @@ -8,10 +10,11 @@ busy = { state: :busy }.freeze dead = { state: :dead }.freeze idle = { state: :idle }.freeze + tags = RAILS_CONNECTION_PROMETHEUS_TAGS - size = gauge :size, comment: 'Size of the connection pool' - connections = gauge :connections, comment: 'Number of connections in the connection pool' - waiting = gauge :waiting, comment: 'Number of waiting in the queue of the connection pool' + size = gauge :size, comment: 'Size of the connection pool', tags: tags + connections = gauge :connections, comment: 'Number of connections in the connection pool', tags: tags + waiting = gauge :waiting, comment: 'Number of waiting in the queue of the connection pool', tags: tags collect do stat = ActiveRecord::Base.connection_pool.stat diff --git a/lib/prometheus/que_stats.rb b/lib/prometheus/que_stats.rb index 88e1feb2..75e2bc4d 100644 --- a/lib/prometheus/que_stats.rb +++ b/lib/prometheus/que_stats.rb @@ -121,6 +121,8 @@ def build_stats_count_relation end class StatsCollector + PROMETHEUS_TAGS = %i[type].freeze + def initialize(gauge, stats) @gauge = gauge @stats = stats @@ -143,6 +145,8 @@ def type_hash_and_stats_count(type) end class GroupedStatsCollector < StatsCollector + PROMETHEUS_TAGS = %i[job_name type].freeze + def initialize(gauge, stats, grouped_by:) super(gauge, stats) @grouped_by = grouped_by @@ -162,7 +166,9 @@ def call(type = nil) Yabeda.configure do group :que do - workers = gauge :workers_total, comment: 'Que Workers running' + workers = gauge :workers_total, + comment: 'Que Workers running', + tags: Prometheus::QueStats::StatsCollector::PROMETHEUS_TAGS worker_stats = Prometheus::QueStats::WorkerStats.new collector = Prometheus::QueStats::StatsCollector.new(workers, worker_stats) collect(&collector.method(:call)) @@ -171,7 +177,9 @@ def call(type = nil) Yabeda.configure do group :que do - jobs = gauge :jobs_scheduled_total, comment: 'Que Jobs to be executed' + jobs = gauge :jobs_scheduled_total, + comment: 'Que Jobs to be executed', + tags: Prometheus::QueStats::GroupedStatsCollector::PROMETHEUS_TAGS job_stats = Prometheus::QueStats::JobStats.new collector = Prometheus::QueStats::GroupedStatsCollector.new(jobs, job_stats, grouped_by: 'job_name') collect do From a88d47d44ff94d830f30c3b8c0a8932f35342ad3 Mon Sep 17 00:00:00 2001 From: HemaHG Date: Tue, 30 Mar 2021 17:41:18 +0530 Subject: [PATCH 121/156] Updating mimemagic gem Signed-off-by: HemaHG --- Gemfile.lock | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index dd17f2f1..73932aae 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -143,7 +143,6 @@ GEM license_finder (6.10.1) bundler rubyzip (>= 1, < 3) - thor thor (~> 1.0.1) tomlrb (>= 1.3, < 2.1) with_env (= 1.1.0) @@ -163,7 +162,9 @@ GEM message_bus (2.2.3) rack (>= 1.1.3) method_source (1.0.0) - mimemagic (0.3.5) + mimemagic (0.3.10) + nokogiri (~> 1) + rake mini_mime (1.0.2) mini_portile2 (2.5.0) minitest (5.14.3) @@ -363,4 +364,4 @@ DEPENDENCIES yabeda-rails BUNDLED WITH - 2.1.4 + 2.2.15 From cb1742949b293fbc146c43d55b5f2a32452c91da Mon Sep 17 00:00:00 2001 From: HemaHG Date: Tue, 30 Mar 2021 17:47:15 +0530 Subject: [PATCH 122/156] updating mimemagic version Signed-off-by: HemaHG --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 73932aae..013164d3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -364,4 +364,4 @@ DEPENDENCIES yabeda-rails BUNDLED WITH - 2.2.15 + 2.1.4 From 5a06abf6540a9b062129fbcdbbf380177e17c71f Mon Sep 17 00:00:00 2001 From: HemaHG Date: Fri, 9 Apr 2021 18:32:17 +0530 Subject: [PATCH 123/156] Updating Rails to 6.1.2 Signed-off-by: HemaHG --- Gemfile | 8 +- Gemfile.lock | 170 +++++++++--------- app/adapters/abstract_adapter.rb | 9 +- app/adapters/keycloak_adapter.rb | 2 +- app/jobs/application_job.rb | 4 +- app/jobs/process_integration_entry_job.rb | 4 +- app/jobs/update_job.rb | 4 +- app/models/tenant.rb | 2 +- bin/rails | 5 +- bin/rake | 4 +- bin/setup | 12 +- bin/spring | 18 -- config.ru | 3 +- config/application.rb | 15 +- config/boot.rb | 4 +- config/environment.rb | 2 +- config/environments/development.rb | 27 ++- config/environments/production.rb | 48 +++-- config/environments/test.rb | 25 ++- config/initializers/backtrace_silencers.rb | 7 +- .../new_framework_defaults_5_2.rb | 38 ---- .../new_framework_defaults_6_1.rb | 67 +++++++ config/locales/en.yml | 2 +- config/puma.rb | 42 ++--- config/que.yml | 4 + db/structure.sql | 2 - lib/prometheus/que_stats.rb | 2 +- 27 files changed, 294 insertions(+), 236 deletions(-) delete mode 100755 bin/spring delete mode 100644 config/initializers/new_framework_defaults_5_2.rb create mode 100644 config/initializers/new_framework_defaults_6_1.rb diff --git a/Gemfile b/Gemfile index 8ba31d03..2833d01d 100644 --- a/Gemfile +++ b/Gemfile @@ -8,11 +8,10 @@ end # Bundle edge Rails instead: gem 'rails', github: 'rails/rails' -gem 'rails', '~> 5.2.3' +gem 'rails', '~> 6.1.2' gem 'pg', '>= 0.20' -gem 'schema_plus_enums' - # Use Puma as the app server + gem 'puma', '~> 5.2' # Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder # gem 'jbuilder', '~> 2.5' @@ -28,11 +27,10 @@ gem 'puma', '~> 5.2' gem 'responders', '~> 3.0.1' gem '3scale-api' -gem 'bootsnap' +gem 'bootsnap', '>= 1.4.4', require: false gem 'que', '>= 1.0.0.beta3' gem 'que-web' -gem 'baby_squeel' gem 'bugsnag' # bugsnag-capistrano 2.x does not have a rake task to report deploys diff --git a/Gemfile.lock b/Gemfile.lock index 013164d3..69d4279b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -17,55 +17,73 @@ GEM remote: https://rubygems.org/ specs: 3scale-api (1.4.0) +<<<<<<< HEAD actioncable (5.2.4.5) actionpack (= 5.2.4.5) +======= + actioncable (6.1.3) + actionpack (= 6.1.3) + activesupport (= 6.1.3) +>>>>>>> f4d383f... Updating Rails to 6.1.2 nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailer (5.2.4.5) - actionpack (= 5.2.4.5) - actionview (= 5.2.4.5) - activejob (= 5.2.4.5) + actionmailbox (6.1.3) + actionpack (= 6.1.3) + activejob (= 6.1.3) + activerecord (= 6.1.3) + activestorage (= 6.1.3) + activesupport (= 6.1.3) + mail (>= 2.7.1) + actionmailer (6.1.3) + actionpack (= 6.1.3) + actionview (= 6.1.3) + activejob (= 6.1.3) + activesupport (= 6.1.3) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (5.2.4.5) - actionview (= 5.2.4.5) - activesupport (= 5.2.4.5) - rack (~> 2.0, >= 2.0.8) + actionpack (6.1.3) + actionview (= 6.1.3) + activesupport (= 6.1.3) + rack (~> 2.0, >= 2.0.9) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) - rails-html-sanitizer (~> 1.0, >= 1.0.2) - actionview (5.2.4.5) - activesupport (= 5.2.4.5) + rails-html-sanitizer (~> 1.0, >= 1.2.0) + actiontext (6.1.3) + actionpack (= 6.1.3) + activerecord (= 6.1.3) + activestorage (= 6.1.3) + activesupport (= 6.1.3) + nokogiri (>= 1.8.5) + actionview (6.1.3) + activesupport (= 6.1.3) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) - rails-html-sanitizer (~> 1.0, >= 1.0.3) - activejob (5.2.4.5) - activesupport (= 5.2.4.5) + rails-html-sanitizer (~> 1.1, >= 1.2.0) + activejob (6.1.3) + activesupport (= 6.1.3) globalid (>= 0.3.6) - activemodel (5.2.4.5) - activesupport (= 5.2.4.5) - activerecord (5.2.4.5) - activemodel (= 5.2.4.5) - activesupport (= 5.2.4.5) - arel (>= 9.0) - activestorage (5.2.4.5) - actionpack (= 5.2.4.5) - activerecord (= 5.2.4.5) + activemodel (6.1.3) + activesupport (= 6.1.3) + activerecord (6.1.3) + activemodel (= 6.1.3) + activesupport (= 6.1.3) + activestorage (6.1.3) + actionpack (= 6.1.3) + activejob (= 6.1.3) + activerecord (= 6.1.3) + activesupport (= 6.1.3) marcel (~> 0.3.1) - activesupport (5.2.4.5) + mimemagic (~> 0.3.2) + activesupport (6.1.3) concurrent-ruby (~> 1.0, >= 1.0.2) - i18n (>= 0.7, < 2) - minitest (~> 5.1) - tzinfo (~> 1.1) + i18n (>= 1.6, < 2) + minitest (>= 5.1) + tzinfo (~> 2.0) + zeitwerk (~> 2.3) addressable (2.7.0) public_suffix (>= 2.0.2, < 5.0) ansi (1.5.0) - arel (9.0.0) - baby_squeel (1.3.1) - activerecord (>= 4.2.0) - join_dependency (~> 0.1.2) - polyamorous (~> 1.3) binding_of_caller (0.8.0) debug_inspector (>= 0.0.1) bootsnap (1.7.2) @@ -113,8 +131,11 @@ GEM dry-logic (~> 0.4, >= 0.4.2) erubi (1.10.0) excon (0.71.1) - faraday (0.15.3) + faraday (1.3.0) + faraday-net_http (~> 1.0) multipart-post (>= 1.2, < 3) + ruby2_keywords + faraday-net_http (1.0.1) globalid (0.4.2) activesupport (>= 4.2.0) hashdiff (1.0.1) @@ -122,14 +143,11 @@ GEM concurrent-ruby (~> 1.0) ice_nine (0.11.2) interception (0.5) - its-it (1.3.0) - join_dependency (0.1.4) - activerecord (>= 4.2.0) json (2.5.1) jsonpath (0.9.9) multi_json to_regexp (~> 0.2.1) - jwt (2.1.0) + jwt (2.2.2) k8s-client (0.10.4) dry-struct (~> 0.5.0) dry-types (~> 0.13.0) @@ -139,7 +157,6 @@ GEM recursive-open-struct (~> 1.1.0) yajl-ruby (~> 1.4.0) yaml-safe_load_stream (~> 0.1) - key_struct (0.4.2) license_finder (6.10.1) bundler rubyzip (>= 1, < 3) @@ -167,34 +184,30 @@ GEM rake mini_mime (1.0.2) mini_portile2 (2.5.0) - minitest (5.14.3) + minitest (5.14.4) minitest-reporters (1.4.3) ansi builder minitest (>= 5.0) ruby-progressbar minitest-stub-const (0.6) - modware (0.1.3) - key_struct (~> 0.4) msgpack (1.4.2) - multi_json (1.14.1) + multi_json (1.15.0) multi_xml (0.6.0) - multipart-post (2.0.0) + multipart-post (2.1.1) mustermann (1.1.1) ruby2_keywords (~> 0.0.1) - nio4r (2.5.5) - nokogiri (1.11.1) + nio4r (2.5.7) + nokogiri (1.11.2) mini_portile2 (~> 2.5.0) racc (~> 1.4) - oauth2 (1.4.1) - faraday (>= 0.8, < 0.16.0) + oauth2 (1.4.7) + faraday (>= 0.8, < 2.0) jwt (>= 1.0, < 3.0) multi_json (~> 1.3) multi_xml (~> 0.5) rack (>= 1.2, < 3) pg (1.2.3) - polyamorous (1.3.3) - activerecord (>= 3.0) prometheus-client (2.1.0) pry (0.13.1) coderay (~> 1.1) @@ -223,30 +236,32 @@ GEM rack rack-test (1.1.0) rack (>= 1.0, < 3) - rails (5.2.4.5) - actioncable (= 5.2.4.5) - actionmailer (= 5.2.4.5) - actionpack (= 5.2.4.5) - actionview (= 5.2.4.5) - activejob (= 5.2.4.5) - activemodel (= 5.2.4.5) - activerecord (= 5.2.4.5) - activestorage (= 5.2.4.5) - activesupport (= 5.2.4.5) - bundler (>= 1.3.0) - railties (= 5.2.4.5) + rails (6.1.3) + actioncable (= 6.1.3) + actionmailbox (= 6.1.3) + actionmailer (= 6.1.3) + actionpack (= 6.1.3) + actiontext (= 6.1.3) + actionview (= 6.1.3) + activejob (= 6.1.3) + activemodel (= 6.1.3) + activerecord (= 6.1.3) + activestorage (= 6.1.3) + activesupport (= 6.1.3) + bundler (>= 1.15.0) + railties (= 6.1.3) sprockets-rails (>= 2.0.0) rails-dom-testing (2.0.3) activesupport (>= 4.2.0) nokogiri (>= 1.6) rails-html-sanitizer (1.3.0) loofah (~> 2.3) - railties (5.2.4.5) - actionpack (= 5.2.4.5) - activesupport (= 5.2.4.5) + railties (6.1.3) + actionpack (= 6.1.3) + activesupport (= 6.1.3) method_source rake (>= 0.8.7) - thor (>= 0.19.0, < 2.0) + thor (~> 1.0) rake (13.0.3) recursive-open-struct (1.1.0) request_store (1.5.0) @@ -258,17 +273,6 @@ GEM ruby-progressbar (1.11.0) ruby2_keywords (0.0.2) rubyzip (2.3.0) - schema_monkey (2.1.5) - activerecord (>= 4.2) - modware (~> 0.1) - schema_plus_core (2.2.3) - activerecord (~> 5.0) - its-it (~> 1.2) - schema_monkey (~> 2.1) - schema_plus_enums (0.1.8) - activerecord (>= 4.2, < 5.3) - its-it (~> 1.2) - schema_plus_core simplecov (0.21.2) docile (~> 1.1) simplecov-html (~> 0.11) @@ -288,12 +292,11 @@ GEM activesupport (>= 4.0) sprockets (>= 3.0.0) thor (1.0.1) - thread_safe (0.3.6) tilt (2.0.10) to_regexp (0.2.1) tomlrb (2.0.1) - tzinfo (1.2.9) - thread_safe (~> 0.1) + tzinfo (2.0.4) + concurrent-ruby (~> 1.0) validate_url (1.0.13) activemodel (>= 3.0.0) public_suffix @@ -322,14 +325,14 @@ GEM yabeda (~> 0.8) yajl-ruby (1.4.1) yaml-safe_load_stream (0.1.1) + zeitwerk (2.4.2) PLATFORMS ruby DEPENDENCIES 3scale-api - baby_squeel - bootsnap + bootsnap (>= 1.4.4) bugsnag bugsnag-capistrano (< 2) codecov @@ -351,10 +354,9 @@ DEPENDENCIES puma (~> 5.2) que (>= 1.0.0.beta3) que-web - rails (~> 5.2.3) + rails (~> 6.1.2) responders (~> 3.0.1) rubyzip (>= 1.3.0) - schema_plus_enums simplecov (~> 0.21.2) tzinfo-data validate_url @@ -364,4 +366,4 @@ DEPENDENCIES yabeda-rails BUNDLED WITH - 2.1.4 + 2.2.15 diff --git a/app/adapters/abstract_adapter.rb b/app/adapters/abstract_adapter.rb index 628f7e8e..b58953c6 100644 --- a/app/adapters/abstract_adapter.rb +++ b/app/adapters/abstract_adapter.rb @@ -59,8 +59,6 @@ def headers JSON_TYPE = Mime[:json] private_constant :JSON_TYPE - NULL_TYPE = Mime::Type.lookup(nil) - attr_reader :http_client def build_http_client(endpoint) @@ -94,9 +92,10 @@ def parse_client(_) def self.parse_response(response) body = response.body - case Mime::Type.lookup(response.content_type) + content_type = response.content_type.presence or return body + + case Mime::Type.lookup(content_type) when JSON_TYPE then JSON.parse(body) - when NULL_TYPE then body else raise InvalidResponseError, { response: response, message: 'Unknown Content-Type' } end end @@ -195,7 +194,7 @@ def access_token def oauth_client OAuth2::Client.new(@endpoint.client_id, @endpoint.client_secret, site: @endpoint.uri.dup, token_url: token_endpoint) do |builder| - builder.adapter(:httpclient).last.instance_variable_set(:@client, http_client) + builder.adapter(:httpclient).instance_variable_set(:@client, http_client) end end diff --git a/app/adapters/keycloak_adapter.rb b/app/adapters/keycloak_adapter.rb index a3e9618f..1126a6ed 100644 --- a/app/adapters/keycloak_adapter.rb +++ b/app/adapters/keycloak_adapter.rb @@ -65,7 +65,7 @@ def redirectUris=(uris) end def oidc_configuration=(params) - write_attribute :oidc_configuration, OAuthConfiguration.new(params) + _write_attribute 'oidc_configuration', OAuthConfiguration.new(params) end def persisted? diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb index dbe58170..a454c503 100644 --- a/app/jobs/application_job.rb +++ b/app/jobs/application_job.rb @@ -31,9 +31,7 @@ def self.retry_on(exception, wait: 3.seconds, attempts: 5, queue: nil, priority: def relation record = self.class.model arguments = serialize.slice('arguments') - record.where.has { - args.op('@>', quoted([arguments].to_json)) - } + record.where('args @> ?', [arguments].to_json) end def delete_duplicates diff --git a/app/jobs/process_integration_entry_job.rb b/app/jobs/process_integration_entry_job.rb index 44820102..901771fb 100644 --- a/app/jobs/process_integration_entry_job.rb +++ b/app/jobs/process_integration_entry_job.rb @@ -43,11 +43,11 @@ def call(_payload) end def finish(success:) - state.update_attributes(success: success, finished_at: timestamp) + state.update(success: success, finished_at: timestamp) end def start - state.update_attributes(started_at: timestamp, entry: entry, success: nil) + state.update(started_at: timestamp, entry: entry, success: nil) end def to_proc diff --git a/app/jobs/update_job.rb b/app/jobs/update_job.rb index 73a53c0d..2a88bd72 100644 --- a/app/jobs/update_job.rb +++ b/app/jobs/update_job.rb @@ -23,11 +23,11 @@ def perform(model) UpdateState.acquire_lock(model) do |state| # this is not going to be visible outside the transaction, does it matter? # what matters is that it could be rolled back - state.update_attributes(started_at: timestamp) + state.update(started_at: timestamp) entry = fetch.call(model) - state.update_attributes(success: entry.save, finished_at: timestamp) + state.update(success: entry.save, finished_at: timestamp) end end end diff --git a/app/models/tenant.rb b/app/models/tenant.rb index d1a63e1d..81de0998 100644 --- a/app/models/tenant.rb +++ b/app/models/tenant.rb @@ -7,7 +7,7 @@ class Tenant < ApplicationRecord def self.upsert(params) retry_record_not_unique do tenant = find_or_create_by(id: params.require(:id)) - tenant.update_attributes(params) + tenant.update(params) tenant end end diff --git a/bin/rails b/bin/rails index 07396602..225ac226 100755 --- a/bin/rails +++ b/bin/rails @@ -1,4 +1,5 @@ #!/usr/bin/env ruby + APP_PATH = File.expand_path('../config/application', __dir__) -require_relative '../config/boot' -require 'rails/commands' +require_relative "../config/boot" +require "rails/commands" diff --git a/bin/rake b/bin/rake index 17240489..4fbf10b9 100755 --- a/bin/rake +++ b/bin/rake @@ -1,4 +1,4 @@ #!/usr/bin/env ruby -require_relative '../config/boot' -require 'rake' +require_relative "../config/boot" +require "rake" Rake.application.run diff --git a/bin/setup b/bin/setup index a334d86a..57923026 100755 --- a/bin/setup +++ b/bin/setup @@ -1,6 +1,5 @@ #!/usr/bin/env ruby -require 'fileutils' -include FileUtils +require "fileutils" # path to your application root. APP_ROOT = File.expand_path('..', __dir__) @@ -9,8 +8,9 @@ def system!(*args) system(*args) || abort("\n== Command #{args} failed ==") end -chdir APP_ROOT do - # This script is a starting point to setup your application. +FileUtils.chdir APP_ROOT do + # This script is a way to set up or update your development environment automatically. + # This script is idempotent, so that you can run it at any time and get an expectable outcome. # Add necessary setup steps to this file. puts '== Installing dependencies ==' @@ -19,11 +19,11 @@ chdir APP_ROOT do # puts "\n== Copying sample files ==" # unless File.exist?('config/database.yml') - # cp 'config/database.yml.sample', 'config/database.yml' + # FileUtils.cp 'config/database.yml.sample', 'config/database.yml' # end puts "\n== Preparing database ==" - system! 'bin/rails db:setup' + system! 'bin/rails db:prepare' puts "\n== Removing old logs and tempfiles ==" system! 'bin/rails log:clear tmp:clear' diff --git a/bin/spring b/bin/spring deleted file mode 100755 index df22d97b..00000000 --- a/bin/spring +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -# This file loads spring without using Bundler, in order to be fast. -# It gets overwritten when you run the `spring binstub` command. - -unless defined?(Spring) - require 'rubygems' - require 'bundler' - - lockfile = Bundler::LockfileParser.new(Bundler.default_lockfile.read) - spring = lockfile.specs.detect { |spec| spec.name == "spring" } - if spring - Gem.use_paths Gem.dir, Bundler.bundle_path.to_s, *Gem.path - gem 'spring', spring.version - require 'spring/binstub' - end -end diff --git a/config.ru b/config.ru index 7eae2644..a97566a7 100644 --- a/config.ru +++ b/config.ru @@ -1,6 +1,7 @@ # frozen_string_literal: true # This file is used by Rack-based servers to start the application. -require_relative 'config/environment' +require_relative "config/environment" run Rails.application +Rails.application.load_server diff --git a/config/application.rb b/config/application.rb index 3af4d664..a0f58b68 100644 --- a/config/application.rb +++ b/config/application.rb @@ -1,13 +1,16 @@ # frozen_string_literal: true -require_relative 'boot' +require_relative "boot" require "rails" # Pick the frameworks you want: require "active_model/railtie" require "active_job/railtie" require "active_record/railtie" +# require "active_storage/engine" require "action_controller/railtie" # require "action_mailer/railtie" +# require "action_mailbox/engine" +# require "action_text/engine" require "action_view/railtie" # require "action_cable/engine" # require "sprockets/railtie" @@ -39,9 +42,13 @@ class Application < Rails::Application config.integrations = config_for(:integrations) - # Settings in config/environments/* take precedence over those specified here. - # Application configuration should go into files in config/initializers - # -- all .rb files in that directory are automatically loaded. + # Configuration for the application, engines, and railties goes here. + # + # These settings can be overridden in specific environments using the files + # in config/environments, which are processed later. + # + # config.time_zone = "Central Time (US & Canada)" + # config.eager_load_paths << Rails.root.join("extras") # Only loads a smaller set of middleware suitable for API only apps. # Middleware like session, flash, cookies can be added back manually. diff --git a/config/boot.rb b/config/boot.rb index c04863fa..38a47b2c 100644 --- a/config/boot.rb +++ b/config/boot.rb @@ -2,5 +2,5 @@ ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__) -require 'bundler/setup' # Set up gems listed in the Gemfile. -require 'bootsnap/setup' # Speed up boot time by caching expensive operations. +require "bundler/setup" # Set up gems listed in the Gemfile. +require "bootsnap/setup" # Speed up boot time by caching expensive operations. diff --git a/config/environment.rb b/config/environment.rb index 12ea62f8..a21c0c58 100644 --- a/config/environment.rb +++ b/config/environment.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true # Load the Rails application. -require_relative 'application' +require_relative "application" # Initialize the Rails application. Rails.application.initialize! diff --git a/config/environments/development.rb b/config/environments/development.rb index 37e7742d..d8e8b3ef 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -1,13 +1,14 @@ # frozen_string_literal: true +require "active_support/core_ext/integer/time" + Rails.application.configure do config.middleware.insert_before Rack::Sendfile, ActionDispatch::DebugLocks - # Settings specified here will take precedence over those in config/application.rb. - # In the development environment your application's code is reloaded on - # every request. This slows down response time but is perfect for development + # In the development environment your application's code is reloaded any time + # it changes. This slows down response time but is perfect for development # since you don't have to restart the web server when you make code changes. config.cache_classes = false @@ -20,8 +21,6 @@ # Enable/disable caching. By default caching is disabled. # Run rails dev:cache to toggle caching. if Rails.root.join('tmp', 'caching-dev.txt').exist? - config.action_controller.perform_caching = true - config.cache_store = :memory_store config.public_file_server.headers = { 'Cache-Control' => "public, max-age=#{2.days.to_i}" @@ -35,6 +34,12 @@ # Print deprecation notices to the Rails logger. config.active_support.deprecation = :log + # Raise exceptions for disallowed deprecations. + config.active_support.disallowed_deprecation = :raise + + # Tell Active Support which deprecation messages to disallow. + config.active_support.disallowed_deprecation_warnings = [] + # Raise an error on page load if there are pending migrations. config.active_record.migration_error = :page_load @@ -42,12 +47,18 @@ config.active_record.verbose_query_logs = true - # Raises error for missing translations - # config.action_view.raise_on_missing_translations = true + # Raises error for missing translations. + # config.i18n.raise_on_missing_translations = true + + # Annotate rendered view with file names. + # config.action_view.annotate_rendered_view_with_filenames = true # Use an evented file watcher to asynchronously detect changes in source code, # routes, locales, etc. This feature depends on the listen gem. - # config.file_watcher = ActiveSupport::EventedFileUpdateChecker + #config.file_watcher = ActiveSupport::EventedFileUpdateChecker + + # Uncomment if you wish to allow Action Cable access from any origin. + # config.action_cable.disable_request_forgery_protection = true end if defined?(HttpLog) diff --git a/config/environments/production.rb b/config/environments/production.rb index 26edf5d0..9873f7c5 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -1,4 +1,6 @@ # frozen_string_literal: true +require "active_support/core_ext/integer/time" + Rails.application.configure do # Settings specified here will take precedence over those in config/application.rb. @@ -13,7 +15,6 @@ # Full error reports are disabled and caching is turned on. config.consider_all_requests_local = false - config.action_controller.perform_caching = true # Ensures that a master key has been made available in either ENV["RAILS_MASTER_KEY"] # or in config/master.key. This key is used to decrypt credentials (and other encrypted files). @@ -24,31 +25,28 @@ config.public_file_server.enabled = ENV['RAILS_SERVE_STATIC_FILES'].present? # Enable serving of images, stylesheets, and JavaScripts from an asset server. - # config.action_controller.asset_host = 'http://assets.example.com' + # config.asset_host = 'http://assets.example.com' # Specifies the header that your server uses for sending files. # config.action_dispatch.x_sendfile_header = 'X-Sendfile' # for Apache # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for NGINX - # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. # config.force_ssl = true - # Use the lowest log level to ensure availability of diagnostic information - # when problems arise. + # Include generic and useful information about system operation, but avoid logging too much + # information to avoid inadvertent exposure of personally identifiable information (PII). config.log_level = ENV.fetch('RAILS_LOG_LEVEL', 'debug').to_sym # Prepend all log lines with the following tags. config.log_tags = [ ] - config.lograge.enabled = true - # Use a different cache store in production. # config.cache_store = :mem_cache_store - # Use a real queuing backend for Active Job (and separate queues per environment) + # Use a real queuing backend for Active Job (and separate queues per environment). # config.active_job.queue_adapter = :resque - # config.active_job.queue_name_prefix = "zync_#{Rails.env}" + # config.active_job.queue_name_prefix = "zync_production" # Enable locale fallbacks for I18n (makes lookups for any locale fall back to # the I18n.default_locale when a translation cannot be found). @@ -57,8 +55,17 @@ # Send deprecation notices to registered listeners. config.active_support.deprecation = :notify + # Log disallowed deprecations. + config.active_support.disallowed_deprecation = :log + + # Tell Active Support which deprecation messages to disallow. + config.active_support.disallowed_deprecation_warnings = [] + + # Use default logging formatter so that PID and timestamp are not suppressed. + config.log_formatter = ::Logger::Formatter.new + # Use a different logger for distributed setups. - # require 'syslog/logger' + # require "syslog/logger" # config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new 'app-name') if ENV["RAILS_LOG_TO_STDOUT"].present? @@ -69,4 +76,25 @@ # Do not dump schema after migrations. config.active_record.dump_schema_after_migration = false + + # Inserts middleware to perform automatic connection switching. + # The `database_selector` hash is used to pass options to the DatabaseSelector + # middleware. The `delay` is used to determine how long to wait after a write + # to send a subsequent read to the primary. + # + # The `database_resolver` class is used by the middleware to determine which + # database is appropriate to use based on the time delay. + # + # The `database_resolver_context` class is used by the middleware to set + # timestamps for the last write to the primary. The resolver uses the context + # class timestamps to determine how long to wait before reading from the + # replica. + # + # By default Rails will store a last write timestamp in the session. The + # DatabaseSelector middleware is designed as such you can define your own + # strategy for connection switching and pass that into the middleware through + # these configuration options. + # config.active_record.database_selector = { delay: 2.seconds } + # config.active_record.database_resolver = ActiveRecord::Middleware::DatabaseSelector::Resolver + # config.active_record.database_resolver_context = ActiveRecord::Middleware::DatabaseSelector::Resolver::Session end diff --git a/config/environments/test.rb b/config/environments/test.rb index 5bcad8fa..7528b333 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -1,11 +1,14 @@ # frozen_string_literal: true +require "active_support/core_ext/integer/time" + +# The test environment is used exclusively to run your application's +# test suite. You never need to work with it otherwise. Remember that +# your test database is "scratch space" for the test suite and is wiped +# and recreated between test runs. Don't rely on the data there! + Rails.application.configure do # Settings specified here will take precedence over those in config/application.rb. - # The test environment is used exclusively to run your application's - # test suite. You never need to work with it otherwise. Remember that - # your test database is "scratch space" for the test suite and is wiped - # and recreated between test runs. Don't rely on the data there! config.cache_classes = true # Do not eager load code on boot. This avoids loading your whole application @@ -22,6 +25,7 @@ # Show full error reports and disable caching. config.consider_all_requests_local = true config.action_controller.perform_caching = false + config.cache_store = :null_store # Raise exceptions instead of rendering exception templates. config.action_dispatch.show_exceptions = false @@ -32,8 +36,17 @@ # Print deprecation notices to the stderr. config.active_support.deprecation = :stderr - # Raises error for missing translations - # config.action_view.raise_on_missing_translations = true + # Raise exceptions for disallowed deprecations. + config.active_support.disallowed_deprecation = :raise + + # Tell Active Support which deprecation messages to disallow. + config.active_support.disallowed_deprecation_warnings = [] + + # Raises error for missing translations. + # config.i18n.raise_on_missing_translations = true + + # Annotate rendered view with file names. + # config.action_view.annotate_rendered_view_with_filenames = true end if defined?(HttpLog) diff --git a/config/initializers/backtrace_silencers.rb b/config/initializers/backtrace_silencers.rb index d0f0d3b5..6bea00bc 100644 --- a/config/initializers/backtrace_silencers.rb +++ b/config/initializers/backtrace_silencers.rb @@ -2,7 +2,8 @@ # Be sure to restart your server when you modify this file. # You can add backtrace silencers for libraries that you're using but don't wish to see in your backtraces. -# Rails.backtrace_cleaner.add_silencer { |line| line =~ /my_noisy_library/ } +# Rails.backtrace_cleaner.add_silencer { |line| /my_noisy_library/.match?(line) } -# You can also remove all the silencers if you're trying to debug a problem that might stem from framework code. -# Rails.backtrace_cleaner.remove_silencers! +# You can also remove all the silencers if you're trying to debug a problem that might stem from framework code +# by setting BACKTRACE=1 before calling your invocation, like "BACKTRACE=1 ./bin/rails runner 'MyClass.perform'". +# Rails.backtrace_cleaner.remove_silencers! if ENV["BACKTRACE"] diff --git a/config/initializers/new_framework_defaults_5_2.rb b/config/initializers/new_framework_defaults_5_2.rb deleted file mode 100644 index 3262bea3..00000000 --- a/config/initializers/new_framework_defaults_5_2.rb +++ /dev/null @@ -1,38 +0,0 @@ -# Be sure to restart your server when you modify this file. -# -# This file contains migration options to ease your Rails 5.2 upgrade. -# -# Once upgraded flip defaults one by one to migrate to the new default. -# -# Read the Guide for Upgrading Ruby on Rails for more info on each option. - -# Make Active Record use stable #cache_key alongside new #cache_version method. -# This is needed for recyclable cache keys. -Rails.application.config.active_record.cache_versioning = true - -# Use AES-256-GCM authenticated encryption for encrypted cookies. -# Also, embed cookie expiry in signed or encrypted cookies for increased security. -# -# This option is not backwards compatible with earlier Rails versions. -# It's best enabled when your entire app is migrated and stable on 5.2. -# -# Existing cookies will be converted on read then written with the new scheme. -Rails.application.config.action_dispatch.use_authenticated_cookie_encryption = true - -# Use AES-256-GCM authenticated encryption as default cipher for encrypting messages -# instead of AES-256-CBC, when use_authenticated_message_encryption is set to true. -Rails.application.config.active_support.use_authenticated_message_encryption = true - -# Add default protection from forgery to ActionController::Base instead of in -# ApplicationController. -Rails.application.config.action_controller.default_protect_from_forgery = true - -# Store boolean values are in sqlite3 databases as 1 and 0 instead of 't' and -# 'f' after migrating old data. -Rails.application.config.active_record.sqlite3.represent_boolean_as_integer = true - -# Use SHA-1 instead of MD5 to generate non-sensitive digests, such as the ETag header. -Rails.application.config.active_support.use_sha1_digests = true - -# Make `form_with` generate id attributes for any generated HTML tags. -Rails.application.config.action_view.form_with_generates_ids = true diff --git a/config/initializers/new_framework_defaults_6_1.rb b/config/initializers/new_framework_defaults_6_1.rb new file mode 100644 index 00000000..9526b835 --- /dev/null +++ b/config/initializers/new_framework_defaults_6_1.rb @@ -0,0 +1,67 @@ +# Be sure to restart your server when you modify this file. +# +# This file contains migration options to ease your Rails 6.1 upgrade. +# +# Once upgraded flip defaults one by one to migrate to the new default. +# +# Read the Guide for Upgrading Ruby on Rails for more info on each option. + +# Support for inversing belongs_to -> has_many Active Record associations. +# Rails.application.config.active_record.has_many_inversing = true + +# Track Active Storage variants in the database. +# Rails.application.config.active_storage.track_variants = true + +# Apply random variation to the delay when retrying failed jobs. +# Rails.application.config.active_job.retry_jitter = 0.15 + +# Stop executing `after_enqueue`/`after_perform` callbacks if +# `before_enqueue`/`before_perform` respectively halts with `throw :abort`. +# Rails.application.config.active_job.skip_after_callbacks_if_terminated = true + +# Specify cookies SameSite protection level: either :none, :lax, or :strict. +# +# This change is not backwards compatible with earlier Rails versions. +# It's best enabled when your entire app is migrated and stable on 6.1. +# Rails.application.config.action_dispatch.cookies_same_site_protection = :lax + +# Generate CSRF tokens that are encoded in URL-safe Base64. +# +# This change is not backwards compatible with earlier Rails versions. +# It's best enabled when your entire app is migrated and stable on 6.1. +# Rails.application.config.action_controller.urlsafe_csrf_tokens = true + +# Specify whether `ActiveSupport::TimeZone.utc_to_local` returns a time with an +# UTC offset or a UTC time. +# ActiveSupport.utc_to_local_returns_utc_offset_times = true + +# Change the default HTTP status code to `308` when redirecting non-GET/HEAD +# requests to HTTPS in `ActionDispatch::SSL` middleware. +# Rails.application.config.action_dispatch.ssl_default_redirect_status = 308 + +# Use new connection handling API. For most applications this won't have any +# effect. For applications using multiple databases, this new API provides +# support for granular connection swapping. +# Rails.application.config.active_record.legacy_connection_handling = false + +# Make `form_with` generate non-remote forms by default. +# Rails.application.config.action_view.form_with_generates_remote_forms = false + +# Set the default queue name for the analysis job to the queue adapter default. +# Rails.application.config.active_storage.queues.analysis = nil + +# Set the default queue name for the purge job to the queue adapter default. +# Rails.application.config.active_storage.queues.purge = nil + +# Set the default queue name for the incineration job to the queue adapter default. +# Rails.application.config.action_mailbox.queues.incineration = nil + +# Set the default queue name for the routing job to the queue adapter default. +# Rails.application.config.action_mailbox.queues.routing = nil + +# Set the default queue name for the mail deliver job to the queue adapter default. +# Rails.application.config.action_mailer.deliver_later_queue_name = nil + +# Generate a `Link` header that gives a hint to modern browsers about +# preloading assets when using `javascript_include_tag` and `stylesheet_link_tag`. +# Rails.application.config.action_view.preload_links_header = true diff --git a/config/locales/en.yml b/config/locales/en.yml index decc5a85..cf9b342d 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -27,7 +27,7 @@ # 'true': 'foo' # # To learn more, please read the Rails Internationalization guide -# available at http://guides.rubyonrails.org/i18n.html. +# available at https://guides.rubyonrails.org/i18n.html. en: hello: "Hello world" diff --git a/config/puma.rb b/config/puma.rb index 2fb90cb4..0c0ba2d2 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -1,23 +1,31 @@ -# frozen_string_literal: true # Puma can serve each request in a thread from an internal thread pool. # The `threads` method setting takes two numbers: a minimum and maximum. # Any libraries that use thread pools should be configured to match # the maximum value specified for Puma. Default is set to 5 threads for minimum # and maximum; this matches the default thread size of Active Record. # -threads_count = ENV.fetch("RAILS_MAX_THREADS") { 5 } -threads threads_count, threads_count +max_threads_count = ENV.fetch("RAILS_MAX_THREADS") { 5 } +min_threads_count = ENV.fetch("RAILS_MIN_THREADS") { max_threads_count } +threads min_threads_count, max_threads_count + +# Specifies the `worker_timeout` threshold that Puma will use to wait before +# terminating a worker in development environments. +# +worker_timeout 3600 if ENV.fetch("RAILS_ENV", "development") == "development" # Specifies the `port` that Puma will listen on to receive requests; default is 3000. # -port ENV.fetch("PORT") { 3000 } +port ENV.fetch("PORT") { 3000 } # Specifies the `environment` that Puma will run in. # environment ENV.fetch("RAILS_ENV") { "development" } +# Specifies the `pidfile` that Puma will use. +pidfile ENV.fetch("PIDFILE") { "tmp/pids/server.pid" } + # Specifies the number of `workers` to boot in clustered mode. -# Workers are forked webserver processes. If using threads and workers together +# Workers are forked web server processes. If using threads and workers together # the concurrency of the application would be max `threads` * `workers`. # Workers do not work on JRuby or Windows (both of which do not support # processes). @@ -27,32 +35,10 @@ # Use the `preload_app!` method when specifying a `workers` number. # This directive tells Puma to first boot the application and load code # before forking the application. This takes advantage of Copy On Write -# process behavior so workers use less memory. If you use this option -# you need to make sure to reconnect any threads in the `on_worker_boot` -# block. +# process behavior so workers use less memory. # # preload_app! -# If you are preloading your application and using Active Record, it's -# recommended that you close any connections to the database before workers -# are forked to prevent connection leakage. -# -# before_fork do -# ActiveRecord::Base.connection_pool.disconnect! if defined?(ActiveRecord) -# end - -# The code in the `on_worker_boot` will be called if you are using -# clustered mode by specifying a number of `workers`. After each worker -# process is booted, this block will be run. If you are using the `preload_app!` -# option, you will want to use this block to reconnect to any threads -# or connections that may have been created at application boot, as Ruby -# cannot share connections between processes. -# -# on_worker_boot do -# ActiveRecord::Base.establish_connection if defined?(ActiveRecord) -# end -# - queue_requests false # let the higher layer figure that out # plugin 'metrics' diff --git a/config/que.yml b/config/que.yml index b77128a6..f75528b0 100644 --- a/config/que.yml +++ b/config/que.yml @@ -1,3 +1,7 @@ +production: + worker_count: 3 + wait_period: 100 + development: worker_count: 3 wait_period: 100 diff --git a/db/structure.sql b/db/structure.sql index 0c9e4127..188825a8 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -1512,5 +1512,3 @@ INSERT INTO "schema_migrations" (version) VALUES ('20190530080459'), ('20190603140450'), ('20190605094424'); - - diff --git a/lib/prometheus/que_stats.rb b/lib/prometheus/que_stats.rb index 75e2bc4d..b61f91cf 100644 --- a/lib/prometheus/que_stats.rb +++ b/lib/prometheus/que_stats.rb @@ -116,7 +116,7 @@ def job_classes_arel end def build_stats_count_relation - Que::ActiveRecord::Model.selecting { [Arel.sql("args->0->>'job_class'").as(JOB_CLASS_COLUMN_NAME), Arel.star.count.as('count')] }.group("args->0->>'job_class'") + Que::ActiveRecord::Model.select([Arel.sql("args->0->>'job_class'").as(JOB_CLASS_COLUMN_NAME), Arel.star.count.as('count')]).group("args->0->>'job_class'") end end From 87a4c88d0eec159b82513f9b974a5954ca1f1aa0 Mon Sep 17 00:00:00 2001 From: HemaHG Date: Fri, 9 Apr 2021 18:43:37 +0530 Subject: [PATCH 124/156] resolving merge conflicts Signed-off-by: HemaHG --- Gemfile | 2 +- Gemfile.lock | 5 ----- db/structure.sql | 8 +++++--- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/Gemfile b/Gemfile index 2833d01d..a6c6e6a9 100644 --- a/Gemfile +++ b/Gemfile @@ -27,7 +27,7 @@ gem 'puma', '~> 5.2' gem 'responders', '~> 3.0.1' gem '3scale-api' -gem 'bootsnap', '>= 1.4.4', require: false +gem 'bootsnap', '>= 1.4.4' gem 'que', '>= 1.0.0.beta3' gem 'que-web' diff --git a/Gemfile.lock b/Gemfile.lock index 69d4279b..9da2e517 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -17,14 +17,9 @@ GEM remote: https://rubygems.org/ specs: 3scale-api (1.4.0) -<<<<<<< HEAD - actioncable (5.2.4.5) - actionpack (= 5.2.4.5) -======= actioncable (6.1.3) actionpack (= 6.1.3) activesupport (= 6.1.3) ->>>>>>> f4d383f... Updating Rails to 6.1.2 nio4r (~> 2.0) websocket-driver (>= 0.6.1) actionmailbox (6.1.3) diff --git a/db/structure.sql b/db/structure.sql index 188825a8..31951722 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -37,7 +37,7 @@ $$; SET default_tablespace = ''; -SET default_with_oids = false; +SET default_table_access_method = heap; -- -- Name: que_jobs; Type: TABLE; Schema: public; Owner: - @@ -1293,14 +1293,14 @@ CREATE INDEX table_channel_id_index ON public.message_bus USING btree (channel, -- Name: que_jobs que_job_notify; Type: TRIGGER; Schema: public; Owner: - -- -CREATE TRIGGER que_job_notify AFTER INSERT ON public.que_jobs FOR EACH ROW EXECUTE PROCEDURE public.que_job_notify(); +CREATE TRIGGER que_job_notify AFTER INSERT ON public.que_jobs FOR EACH ROW EXECUTE FUNCTION public.que_job_notify(); -- -- Name: que_jobs que_state_notify; Type: TRIGGER; Schema: public; Owner: - -- -CREATE TRIGGER que_state_notify AFTER INSERT OR DELETE OR UPDATE ON public.que_jobs FOR EACH ROW EXECUTE PROCEDURE public.que_state_notify(); +CREATE TRIGGER que_state_notify AFTER INSERT OR DELETE OR UPDATE ON public.que_jobs FOR EACH ROW EXECUTE FUNCTION public.que_state_notify(); -- @@ -1512,3 +1512,5 @@ INSERT INTO "schema_migrations" (version) VALUES ('20190530080459'), ('20190603140450'), ('20190605094424'); + + From d9986cf802ec921bfbfb44af183179db6147e791 Mon Sep 17 00:00:00 2001 From: HemaHG Date: Fri, 9 Apr 2021 18:48:35 +0530 Subject: [PATCH 125/156] Undoing changes db/structure.sql Signed-off-by: HemaHG --- db/structure.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db/structure.sql b/db/structure.sql index 31951722..0c9e4127 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -37,7 +37,7 @@ $$; SET default_tablespace = ''; -SET default_table_access_method = heap; +SET default_with_oids = false; -- -- Name: que_jobs; Type: TABLE; Schema: public; Owner: - @@ -1293,14 +1293,14 @@ CREATE INDEX table_channel_id_index ON public.message_bus USING btree (channel, -- Name: que_jobs que_job_notify; Type: TRIGGER; Schema: public; Owner: - -- -CREATE TRIGGER que_job_notify AFTER INSERT ON public.que_jobs FOR EACH ROW EXECUTE FUNCTION public.que_job_notify(); +CREATE TRIGGER que_job_notify AFTER INSERT ON public.que_jobs FOR EACH ROW EXECUTE PROCEDURE public.que_job_notify(); -- -- Name: que_jobs que_state_notify; Type: TRIGGER; Schema: public; Owner: - -- -CREATE TRIGGER que_state_notify AFTER INSERT OR DELETE OR UPDATE ON public.que_jobs FOR EACH ROW EXECUTE FUNCTION public.que_state_notify(); +CREATE TRIGGER que_state_notify AFTER INSERT OR DELETE OR UPDATE ON public.que_jobs FOR EACH ROW EXECUTE PROCEDURE public.que_state_notify(); -- From d277bd6cae6a5030d50a9b12a4eeb3d9a93fd522 Mon Sep 17 00:00:00 2001 From: HemaHG Date: Tue, 27 Apr 2021 12:12:58 +0530 Subject: [PATCH 126/156] Adding activerecord-pg_enum gem Signed-off-by: HemaHG --- Gemfile | 2 ++ Gemfile.lock | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/Gemfile b/Gemfile index a6c6e6a9..f4868fc9 100644 --- a/Gemfile +++ b/Gemfile @@ -24,6 +24,8 @@ gem 'puma', '~> 5.2' # Use Rack CORS for handling Cross-Origin Resource Sharing (CORS), making cross-origin AJAX possible # gem 'rack-cors' +gem 'activerecord-pg_enum' + gem 'responders', '~> 3.0.1' gem '3scale-api' diff --git a/Gemfile.lock b/Gemfile.lock index 9da2e517..922ac728 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -63,6 +63,10 @@ GEM activerecord (6.1.3) activemodel (= 6.1.3) activesupport (= 6.1.3) + activerecord-pg_enum (1.2.2) + activerecord (>= 4.1.0) + activesupport + pg activestorage (6.1.3) actionpack (= 6.1.3) activejob (= 6.1.3) @@ -327,6 +331,7 @@ PLATFORMS DEPENDENCIES 3scale-api + activerecord-pg_enum bootsnap (>= 1.4.4) bugsnag bugsnag-capistrano (< 2) From e60a550872e8955a1f990d277110cc4313f33b44 Mon Sep 17 00:00:00 2001 From: HemaHG Date: Tue, 27 Apr 2021 14:20:55 +0530 Subject: [PATCH 127/156] updating migration file Signed-off-by: HemaHG --- db/migrate/20190530080459_add_integration_state.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/migrate/20190530080459_add_integration_state.rb b/db/migrate/20190530080459_add_integration_state.rb index c6b828c4..3abfb2d5 100644 --- a/db/migrate/20190530080459_add_integration_state.rb +++ b/db/migrate/20190530080459_add_integration_state.rb @@ -1,6 +1,6 @@ class AddIntegrationState < ActiveRecord::Migration[5.2] def up - create_enum :integration_state, 'active', 'disabled' + create_enum :integration_state, %w[active disabled] add_column :integrations, :state, :integration_state default = 'active' Integration.in_batches.update_all(state: default) From 44f1cf80f3bcd4fae6fc378e568f35e63353a3f6 Mon Sep 17 00:00:00 2001 From: HemaHG Date: Thu, 29 Apr 2021 13:55:20 +0530 Subject: [PATCH 128/156] updating db/structure.sql Signed-off-by: HemaHG --- db/structure.sql | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/db/structure.sql b/db/structure.sql index 0c9e4127..6e24b252 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -37,8 +37,6 @@ $$; SET default_tablespace = ''; -SET default_with_oids = false; - -- -- Name: que_jobs; Type: TABLE; Schema: public; Owner: - -- @@ -264,8 +262,8 @@ ALTER SEQUENCE public.applications_id_seq OWNED BY public.applications.id; CREATE TABLE public.ar_internal_metadata ( key character varying NOT NULL, value character varying, - created_at timestamp without time zone NOT NULL, - updated_at timestamp without time zone NOT NULL + created_at timestamp(6) without time zone NOT NULL, + updated_at timestamp(6) without time zone NOT NULL ); @@ -1178,10 +1176,10 @@ CREATE INDEX index_metrics_on_tenant_id ON public.metrics USING btree (tenant_id -- --- Name: index_models_on_record_type_and_record_id; Type: INDEX; Schema: public; Owner: - +-- Name: index_models_on_record; Type: INDEX; Schema: public; Owner: - -- -CREATE UNIQUE INDEX index_models_on_record_type_and_record_id ON public.models USING btree (record_type, record_id); +CREATE UNIQUE INDEX index_models_on_record ON public.models USING btree (record_type, record_id); -- From da3206c89cce17846912e317a362d454a18f92e5 Mon Sep 17 00:00:00 2001 From: Guilherme Cassolato Date: Tue, 4 May 2021 13:21:09 +0200 Subject: [PATCH 129/156] Change source of the httpclient gem --- Gemfile | 2 +- Gemfile.lock | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Gemfile b/Gemfile index f4868fc9..134c5b58 100644 --- a/Gemfile +++ b/Gemfile @@ -41,7 +41,7 @@ gem 'bugsnag-capistrano', '< 2', require: false # This fork allows setting SSL_CERT_FILE and SSL_CERT_DIR # https://github.com/nahi/httpclient/issues/369 -gem 'httpclient', github: 'mikz/httpclient', branch: 'ssl-env-cert' +gem 'httpclient', github: '3scale/httpclient', branch: 'ssl-env-cert' gem 'oauth2' gem 'k8s-client', '>= 0.10' diff --git a/Gemfile.lock b/Gemfile.lock index 922ac728..181b1a38 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,3 +1,10 @@ +GIT + remote: https://github.com/3scale/httpclient.git + revision: fec23fb32fb899b87a8b2c94e2d2069b6b4c633c + branch: ssl-env-cert + specs: + httpclient (2.8.3) + GIT remote: https://github.com/3scale/license_finder_xml_reporter.git revision: ab8072d11344b5e6d05565ccc84cb4745d01e6f8 @@ -6,13 +13,6 @@ GIT license_finder_xml_reporter (0.1.0) license_finder -GIT - remote: https://github.com/mikz/httpclient.git - revision: fec23fb32fb899b87a8b2c94e2d2069b6b4c633c - branch: ssl-env-cert - specs: - httpclient (2.8.3) - GEM remote: https://rubygems.org/ specs: From 0011e9f58654383c3a1fafc8fee2a46baa7b2532 Mon Sep 17 00:00:00 2001 From: "depfu[bot]" <23717796+depfu[bot]@users.noreply.github.com> Date: Tue, 4 May 2021 15:34:31 +0000 Subject: [PATCH 130/156] Update rexml to version 3.2.5 --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 181b1a38..36286569 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -268,7 +268,7 @@ GEM responders (3.0.1) actionpack (>= 5.0) railties (>= 5.0) - rexml (3.2.4) + rexml (3.2.5) ruby-progressbar (1.11.0) ruby2_keywords (0.0.2) rubyzip (2.3.0) From 3e097a72a3983cdad05c81747b4664717221e9ab Mon Sep 17 00:00:00 2001 From: hemahg Date: Wed, 5 May 2021 17:13:02 +0530 Subject: [PATCH 131/156] removing index on remove_index_models_on_record_type_and_record_id and adding new index Signed-off-by: hemahg --- ...1301_remove_index_models_on_record_type_and_record_id.rb | 5 +++++ db/migrate/20210504152609_add_index_models_on_record.rb | 5 +++++ db/structure.sql | 6 +++--- 3 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 db/migrate/20210504151301_remove_index_models_on_record_type_and_record_id.rb create mode 100644 db/migrate/20210504152609_add_index_models_on_record.rb diff --git a/db/migrate/20210504151301_remove_index_models_on_record_type_and_record_id.rb b/db/migrate/20210504151301_remove_index_models_on_record_type_and_record_id.rb new file mode 100644 index 00000000..02a3140f --- /dev/null +++ b/db/migrate/20210504151301_remove_index_models_on_record_type_and_record_id.rb @@ -0,0 +1,5 @@ +class RemoveIndexModelsOnRecordTypeAndRecordId < ActiveRecord::Migration[6.1] + def change + remove_index :models, name: "index_models_on_record_type_and_record_id" , if_exists: true + end +end diff --git a/db/migrate/20210504152609_add_index_models_on_record.rb b/db/migrate/20210504152609_add_index_models_on_record.rb new file mode 100644 index 00000000..81406219 --- /dev/null +++ b/db/migrate/20210504152609_add_index_models_on_record.rb @@ -0,0 +1,5 @@ +class AddIndexModelsOnRecord < ActiveRecord::Migration[6.1] + def change + add_index :models, [:record_id, :record_type], name: "index_models_on_record", if_not_exists: true + end +end diff --git a/db/structure.sql b/db/structure.sql index 6e24b252..2b4abda3 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -1509,6 +1509,6 @@ INSERT INTO "schema_migrations" (version) VALUES ('20190410112007'), ('20190530080459'), ('20190603140450'), -('20190605094424'); - - +('20190605094424'), +('20210504151301'), +('20210504152609'); From a6b4d5ee7665e6d544b96b68499079356d544b94 Mon Sep 17 00:00:00 2001 From: hemahg Date: Wed, 5 May 2021 20:09:51 +0530 Subject: [PATCH 132/156] Incorporating review comment Signed-off-by: hemahg --- ...51301_remove_index_models_on_record_type_and_record_id.rb | 5 ----- db/migrate/20210504152609_add_index_models_on_record.rb | 3 ++- db/structure.sql | 1 - 3 files changed, 2 insertions(+), 7 deletions(-) delete mode 100644 db/migrate/20210504151301_remove_index_models_on_record_type_and_record_id.rb diff --git a/db/migrate/20210504151301_remove_index_models_on_record_type_and_record_id.rb b/db/migrate/20210504151301_remove_index_models_on_record_type_and_record_id.rb deleted file mode 100644 index 02a3140f..00000000 --- a/db/migrate/20210504151301_remove_index_models_on_record_type_and_record_id.rb +++ /dev/null @@ -1,5 +0,0 @@ -class RemoveIndexModelsOnRecordTypeAndRecordId < ActiveRecord::Migration[6.1] - def change - remove_index :models, name: "index_models_on_record_type_and_record_id" , if_exists: true - end -end diff --git a/db/migrate/20210504152609_add_index_models_on_record.rb b/db/migrate/20210504152609_add_index_models_on_record.rb index 81406219..f692272b 100644 --- a/db/migrate/20210504152609_add_index_models_on_record.rb +++ b/db/migrate/20210504152609_add_index_models_on_record.rb @@ -1,5 +1,6 @@ class AddIndexModelsOnRecord < ActiveRecord::Migration[6.1] def change - add_index :models, [:record_id, :record_type], name: "index_models_on_record", if_not_exists: true + remove_index :models, name: "index_models_on_record_type_and_record_id" , if_exists: true + add_index :models, [:record_id, :record_type], name: "index_models_on_record", if_not_exists: true, unique: true end end diff --git a/db/structure.sql b/db/structure.sql index 2b4abda3..907f2388 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -1510,5 +1510,4 @@ INSERT INTO "schema_migrations" (version) VALUES ('20190530080459'), ('20190603140450'), ('20190605094424'), -('20210504151301'), ('20210504152609'); From 59534a54a9fe3dfc045a63c800e66e2440341e03 Mon Sep 17 00:00:00 2001 From: "depfu[bot]" <23717796+depfu[bot]@users.noreply.github.com> Date: Thu, 20 May 2021 10:54:52 +0000 Subject: [PATCH 133/156] Update nokogiri to version 1.11.5 --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 36286569..2a4780ca 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -182,7 +182,7 @@ GEM nokogiri (~> 1) rake mini_mime (1.0.2) - mini_portile2 (2.5.0) + mini_portile2 (2.5.1) minitest (5.14.4) minitest-reporters (1.4.3) ansi @@ -197,7 +197,7 @@ GEM mustermann (1.1.1) ruby2_keywords (~> 0.0.1) nio4r (2.5.7) - nokogiri (1.11.2) + nokogiri (1.11.5) mini_portile2 (~> 2.5.0) racc (~> 1.4) oauth2 (1.4.7) From 64cc8763d902b978ee66aa0b2960492e1dd80818 Mon Sep 17 00:00:00 2001 From: "Aleksandar N. Kostadinov" Date: Fri, 14 May 2021 21:09:05 +0300 Subject: [PATCH 134/156] update local setup instructions --- INSTALL.md | 58 ++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 10 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index f7ac31f6..168299cd 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,17 +1,55 @@ -# Install Zync (only explained for MacOS so far) -1. Download this repository. +# Install Zync + +## Download this repository. `git clone git@github.com:3scale/zync.git` -2. Move to the folder of the project. -`cd zync` +## Setup PostgreSQL on Mac + +There is `Brewfile` containing all the dependencies. + +``` +cd zync +brew bundle +brew services start postgresql +``` + +The command `brew services start postgresql` starts the service of PostgreSQL. If later `./bin/setup` aborts, make sure that the `PostgreSQL` service is running. Verify with `brew services list` that has a status `started` and looking green. If the status is `started` but coloured orange, fix the errors indicated in the log located in `/usr/local/var/log/postgres.log` + +## Setup PostgreSQL on Fedora 34 -3. Install the dependencies. There is `Brewfile` containing all the dependencies. -`brew bundle` +```shell +sudo dnf module install postgresql:10 +sudo dnf install libpq-devel +sudo /usr/bin/postgresql-setup --initdb +systemctl enable postgresql +systemctl start postgresql +psql -U postgres -c "create user ${USER}" +psql -U postgres -c "create database zync_development with owner ${USER}" +psql -U postgres -c "create database zync_test with owner ${USER}" +psql -U postgres -c "create database zync_production with owner ${USER}" +``` -4. Start postgres. -`brew services start postgresql` +## Setup PostgreSQL as a container with Docker or Podman -5. Setup Zync. +``` +docker run -d -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_DB=zync --name postgres10-zync docker.io/circleci/postgres:10.5-alpine +``` + +When running any Zync command, make sure to have `DATABASE_URL` environment variable set in prior. + +``` +export DATABASE_URL=postgresql://postgres:@localhost:5432/zync +``` + +**Note:** You will also have to install on the host machine `psql` client tool (needed for `db:setup`) and `libpq-devel` (needed to build `pg` gem). + +## Setup Zync `./bin/setup` -The command `brew services start postgresql` starts the service of PostgreSQL. If `./bin/setup` aborts, make sure that the `PostgreSQL` service is running. Verify with `brew services list` that has a status `started` and looking green. If the status is `started` but coloured orange, fix the errors indicated in the log located in `/usr/local/var/log/postgres.log` + +## Start Zync +``` +export ZYNC_AUTHENTICATION_TOKEN=token # must match porta config +bundle exec rails server -p 5000 +PROMETHEUS_EXPORTER_PORT=9395 bundle exec rake que +``` From f1cc6feeb00b1b54e1046f3a1a95649a4ac56185 Mon Sep 17 00:00:00 2001 From: "Aleksandar N. Kostadinov" Date: Tue, 25 May 2021 08:06:37 +0300 Subject: [PATCH 135/156] ignore .env and .tool-versions --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index b8c88b24..0d07a04c 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,8 @@ /coverage .byebug_history +.env openshift.local.clusterup .ruby-version +.tool-versions From c5a90ba0bc14933230739f81ba04d0d16fe5b76f Mon Sep 17 00:00:00 2001 From: Hallelujah Date: Thu, 3 Jun 2021 13:01:15 +0200 Subject: [PATCH 136/156] Fix CVE 2020-3627 Relates to #454 Refers to https://issues.redhat.com/browse/THREESCALE-7085 --- Dockerfile | 2 +- Gemfile.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6987c249..9269eaa4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ USER default WORKDIR ${APP_ROOT} RUN source ${APP_ROOT}/etc/scl_enable \ - && gem install bundler --version=2.0.1 --no-document + && gem install bundler --version=2.2.19 --no-document COPY --chown=default:root Gemfile* ./ RUN source ${APP_ROOT}/etc/scl_enable \ diff --git a/Gemfile.lock b/Gemfile.lock index 36286569..e7378789 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -366,4 +366,4 @@ DEPENDENCIES yabeda-rails BUNDLED WITH - 2.2.15 + 2.2.19 From 9ba7aef075d84f5a793e54eaceb0f86871860f22 Mon Sep 17 00:00:00 2001 From: "Aleksandar N. Kostadinov" Date: Mon, 7 Jun 2021 03:00:49 +0300 Subject: [PATCH 137/156] move OS specific commands to Quickstart guide --- INSTALL.md | 60 +++++++++++++++++++++-------------------------- README.md | 2 +- doc/Quickstart.md | 56 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 34 deletions(-) create mode 100644 doc/Quickstart.md diff --git a/INSTALL.md b/INSTALL.md index 168299cd..e93bde3b 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,55 +1,49 @@ # Install Zync -## Download this repository. -`git clone git@github.com:3scale/zync.git` - -## Setup PostgreSQL on Mac - -There is `Brewfile` containing all the dependencies. +**Note:** you can check the [Quickstart guide](doc/Quickstart.md) for list of commands that can quickly get you going. +## Download this repository. ``` -cd zync -brew bundle -brew services start postgresql +git clone git@github.com:3scale/zync.git ``` -The command `brew services start postgresql` starts the service of PostgreSQL. If later `./bin/setup` aborts, make sure that the `PostgreSQL` service is running. Verify with `brew services list` that has a status `started` and looking green. If the status is `started` but coloured orange, fix the errors indicated in the log located in `/usr/local/var/log/postgres.log` +## Install dependencies. -## Setup PostgreSQL on Fedora 34 +To run Zync you need access to a running [PostgreSQL](https://www.postgresql.org) server. You can install one with your operating system package +manager, as a container or run it remotely. -```shell -sudo dnf module install postgresql:10 -sudo dnf install libpq-devel -sudo /usr/bin/postgresql-setup --initdb -systemctl enable postgresql -systemctl start postgresql -psql -U postgres -c "create user ${USER}" -psql -U postgres -c "create database zync_development with owner ${USER}" -psql -U postgres -c "create database zync_test with owner ${USER}" -psql -U postgres -c "create database zync_production with owner ${USER}" -``` +The minimum requirement for the machine running Zync is to have + - Ruby 2.7.x + - `psql` client tool - needed when running for `db:setup` + - `libpq-devel` - needed to build `pg` gem during `bundle install`. -## Setup PostgreSQL as a container with Docker or Podman +You may have to adjust `config/database.yml` or `DATABASE_URL` environment variable, see below. -``` -docker run -d -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_DB=zync --name postgres10-zync docker.io/circleci/postgres:10.5-alpine -``` +## Setup Zync -When running any Zync command, make sure to have `DATABASE_URL` environment variable set in prior. +There is a `setup` script to install gem dependencies, +seed the database and run Zync server. ``` -export DATABASE_URL=postgresql://postgres:@localhost:5432/zync +./bin/setup ``` -**Note:** You will also have to install on the host machine `psql` client tool (needed for `db:setup`) and `libpq-devel` (needed to build `pg` gem). - -## Setup Zync -`./bin/setup` +Make sure to edit configuration or set needed environment variables +beforehand. Most important environment variables you can use: + - `ZYNC_AUTHENTICATION_TOKEN` - this one must match your running [Porta](https://github.com/3scale/porta) configuration + - `DATABASE_URL` - depending on your PostgreSQL and `database.yml` configuration, you may want to set this one + - `PROMETHEUS_EXPORTER_PORT` - in case you are running other 3scale components like `que` and Porta, you may need to set a different port for each of them through this variable to avoid conflict between them + - `PORT` - change port where Zync is running (e.g. `5000`) to avoid conflict with a locally running [Porta](https://github.com/3scale/porta) server or other software, you can also use the `-p 5000` command line option ## Start Zync + +When starting Zync, make sure to use a non-conflicting port on your machine ``` -export ZYNC_AUTHENTICATION_TOKEN=token # must match porta config bundle exec rails server -p 5000 +``` + +When starting que, make sure to set a non-conflicting Prometheus port +``` PROMETHEUS_EXPORTER_PORT=9395 bundle exec rake que ``` diff --git a/README.md b/README.md index 9d5e00f8..d8c9f6f4 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Zync is going to take your 3scale data and pushes it somewhere else, reliably. O This component is deployed as part of the overall 3scale API Management solution. -Please follow [these instructions](INSTALL.md) on how to set up your development environment locally and [these instructions](INTEGRATE.md) to integrate it with [Porta](https://github.com/3scale/porta) and Keycloak. +Please see [these instructions](INSTALL.md) and [Quickstart guide](doc/Quickstart.md) on how to set up your development environment locally and [these instructions](INTEGRATE.md) to integrate it with [Porta](https://github.com/3scale/porta) and Keycloak. ## Terminology diff --git a/doc/Quickstart.md b/doc/Quickstart.md new file mode 100644 index 00000000..afc03b41 --- /dev/null +++ b/doc/Quickstart.md @@ -0,0 +1,56 @@ +## Overview + +A quick way to get Zync running locally without many details. +Check [README](../README.md) and [INSTALL](../INSTALL.md) for +more details. + +## Download this repository. +``` +git clone git@github.com:3scale/zync.git +``` + +## Run PostgreSQL on Mac + +``` +cd zync +brew bundle +brew services start postgresql +``` + +**Note:** The command `brew services start postgresql` starts the service of PostgreSQL. If later `./bin/setup` aborts, make sure that the `PostgreSQL` service is running. Verify with `brew services list` that has a status `started` and looking green. If the status is `started` but coloured orange, fix the errors indicated in the log located in `/usr/local/var/log/postgres.log` + +## Run PostgreSQL on Fedora 34 + +```shell +sudo dnf module install postgresql:10 +sudo dnf install libpq-devel +sudo /usr/bin/postgresql-setup --initdb +sudo systemctl enable postgresql +sudo systemctl start postgresql +sudo -i -u postgres createuser $USER +sudo -i -u postgres createdb -O $USER zync_development +sudo -i -u postgres createdb -O $USER zync_test +sudo -i -u postgres createdb -O $USER zync_production +``` + +## Run PostgreSQL as a container with Docker or Podman + +``` +docker run -d -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_DB=zync --name postgres10-zync docker.io/circleci/postgres:10.5-alpine +``` + +**Note:** With such a setup make sure to have `DATABASE_URL` environment variable set +prior starting Zync. You will also have to install on the host machine `psql` client +tool (needed for `db:setup`) and `libpq-devel` (needed to build `pg` gem). + +``` +export DATABASE_URL=postgresql://postgres:@localhost:5432/zync +``` + +## Start Zync + +``` +export ZYNC_AUTHENTICATION_TOKEN=token # must match porta config +./bin/setup +PROMETHEUS_EXPORTER_PORT=9395 bundle exec rake que +``` From d4157e57c32041ac30e3e8e003c42ccbee3d259e Mon Sep 17 00:00:00 2001 From: Hallelujah Date: Fri, 18 Jun 2021 13:40:29 +0200 Subject: [PATCH 138/156] Allow dumping to another structure file for Postgresql 12 --- db/structure-12.sql | 1517 +++++++++++++++++++++++++++++++++++++++++++ lib/tasks/db.rake | 20 + 2 files changed, 1537 insertions(+) create mode 100644 db/structure-12.sql diff --git a/db/structure-12.sql b/db/structure-12.sql new file mode 100644 index 00000000..bfdb861f --- /dev/null +++ b/db/structure-12.sql @@ -0,0 +1,1517 @@ +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SELECT pg_catalog.set_config('search_path', '', false); +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +-- +-- Name: integration_state; Type: TYPE; Schema: public; Owner: - +-- + +CREATE TYPE public.integration_state AS ENUM ( + 'active', + 'disabled' +); + + +-- +-- Name: que_validate_tags(jsonb); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.que_validate_tags(tags_array jsonb) RETURNS boolean + LANGUAGE sql + AS $$ + SELECT bool_and( + jsonb_typeof(value) = 'string' + AND + char_length(value::text) <= 100 + ) + FROM jsonb_array_elements(tags_array) +$$; + + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: que_jobs; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.que_jobs ( + priority smallint DEFAULT 100 NOT NULL, + run_at timestamp with time zone DEFAULT now() NOT NULL, + id bigint NOT NULL, + job_class text NOT NULL, + error_count integer DEFAULT 0 NOT NULL, + last_error_message text, + queue text DEFAULT 'default'::text NOT NULL, + last_error_backtrace text, + finished_at timestamp with time zone, + expired_at timestamp with time zone, + args jsonb DEFAULT '[]'::jsonb NOT NULL, + data jsonb DEFAULT '{}'::jsonb NOT NULL, + CONSTRAINT error_length CHECK (((char_length(last_error_message) <= 500) AND (char_length(last_error_backtrace) <= 10000))), + CONSTRAINT job_class_length CHECK ((char_length( +CASE job_class + WHEN 'ActiveJob::QueueAdapters::QueAdapter::JobWrapper'::text THEN ((args -> 0) ->> 'job_class'::text) + ELSE job_class +END) <= 200)), + CONSTRAINT queue_length CHECK ((char_length(queue) <= 100)), + CONSTRAINT valid_args CHECK ((jsonb_typeof(args) = 'array'::text)), + CONSTRAINT valid_data CHECK (((jsonb_typeof(data) = 'object'::text) AND ((NOT (data ? 'tags'::text)) OR ((jsonb_typeof((data -> 'tags'::text)) = 'array'::text) AND (jsonb_array_length((data -> 'tags'::text)) <= 5) AND public.que_validate_tags((data -> 'tags'::text)))))) +) +WITH (fillfactor='90'); + + +-- +-- Name: TABLE que_jobs; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON TABLE public.que_jobs IS '4'; + + +-- +-- Name: que_determine_job_state(public.que_jobs); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.que_determine_job_state(job public.que_jobs) RETURNS text + LANGUAGE sql + AS $$ + SELECT + CASE + WHEN job.expired_at IS NOT NULL THEN 'expired' + WHEN job.finished_at IS NOT NULL THEN 'finished' + WHEN job.error_count > 0 THEN 'errored' + WHEN job.run_at > CURRENT_TIMESTAMP THEN 'scheduled' + ELSE 'ready' + END +$$; + + +-- +-- Name: que_job_notify(); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.que_job_notify() RETURNS trigger + LANGUAGE plpgsql + AS $$ + DECLARE + locker_pid integer; + sort_key json; + BEGIN + -- Don't do anything if the job is scheduled for a future time. + IF NEW.run_at IS NOT NULL AND NEW.run_at > now() THEN + RETURN null; + END IF; + + -- Pick a locker to notify of the job's insertion, weighted by their number + -- of workers. Should bounce pseudorandomly between lockers on each + -- invocation, hence the md5-ordering, but still touch each one equally, + -- hence the modulo using the job_id. + SELECT pid + INTO locker_pid + FROM ( + SELECT *, last_value(row_number) OVER () + 1 AS count + FROM ( + SELECT *, row_number() OVER () - 1 AS row_number + FROM ( + SELECT * + FROM public.que_lockers ql, generate_series(1, ql.worker_count) AS id + WHERE listening AND queues @> ARRAY[NEW.queue] + ORDER BY md5(pid::text || id::text) + ) t1 + ) t2 + ) t3 + WHERE NEW.id % count = row_number; + + IF locker_pid IS NOT NULL THEN + -- There's a size limit to what can be broadcast via LISTEN/NOTIFY, so + -- rather than throw errors when someone enqueues a big job, just + -- broadcast the most pertinent information, and let the locker query for + -- the record after it's taken the lock. The worker will have to hit the + -- DB in order to make sure the job is still visible anyway. + SELECT row_to_json(t) + INTO sort_key + FROM ( + SELECT + 'job_available' AS message_type, + NEW.queue AS queue, + NEW.priority AS priority, + NEW.id AS id, + -- Make sure we output timestamps as UTC ISO 8601 + to_char(NEW.run_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS run_at + ) t; + + PERFORM pg_notify('que_listener_' || locker_pid::text, sort_key::text); + END IF; + + RETURN null; + END +$$; + + +-- +-- Name: que_state_notify(); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.que_state_notify() RETURNS trigger + LANGUAGE plpgsql + AS $$ + DECLARE + row record; + message json; + previous_state text; + current_state text; + BEGIN + IF TG_OP = 'INSERT' THEN + previous_state := 'nonexistent'; + current_state := public.que_determine_job_state(NEW); + row := NEW; + ELSIF TG_OP = 'DELETE' THEN + previous_state := public.que_determine_job_state(OLD); + current_state := 'nonexistent'; + row := OLD; + ELSIF TG_OP = 'UPDATE' THEN + previous_state := public.que_determine_job_state(OLD); + current_state := public.que_determine_job_state(NEW); + + -- If the state didn't change, short-circuit. + IF previous_state = current_state THEN + RETURN null; + END IF; + + row := NEW; + ELSE + RAISE EXCEPTION 'Unrecognized TG_OP: %', TG_OP; + END IF; + + SELECT row_to_json(t) + INTO message + FROM ( + SELECT + 'job_change' AS message_type, + row.id AS id, + row.queue AS queue, + + coalesce(row.data->'tags', '[]'::jsonb) AS tags, + + to_char(row.run_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS run_at, + to_char(now() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS time, + + CASE row.job_class + WHEN 'ActiveJob::QueueAdapters::QueAdapter::JobWrapper' THEN + coalesce( + row.args->0->>'job_class', + 'ActiveJob::QueueAdapters::QueAdapter::JobWrapper' + ) + ELSE + row.job_class + END AS job_class, + + previous_state AS previous_state, + current_state AS current_state + ) t; + + PERFORM pg_notify('que_state', message::text); + + RETURN null; + END +$$; + + +-- +-- Name: applications; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.applications ( + id bigint NOT NULL, + tenant_id bigint NOT NULL, + service_id bigint NOT NULL, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +-- +-- Name: applications_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.applications_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: applications_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.applications_id_seq OWNED BY public.applications.id; + + +-- +-- Name: ar_internal_metadata; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.ar_internal_metadata ( + key character varying NOT NULL, + value character varying, + created_at timestamp(6) without time zone NOT NULL, + updated_at timestamp(6) without time zone NOT NULL +); + + +-- +-- Name: clients; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.clients ( + id bigint NOT NULL, + service_id bigint NOT NULL, + tenant_id bigint NOT NULL, + client_id character varying NOT NULL, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +-- +-- Name: clients_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.clients_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: clients_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.clients_id_seq OWNED BY public.clients.id; + + +-- +-- Name: entries; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.entries ( + id bigint NOT NULL, + data jsonb, + tenant_id bigint NOT NULL, + model_id bigint NOT NULL, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +-- +-- Name: entries_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.entries_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: entries_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.entries_id_seq OWNED BY public.entries.id; + + +-- +-- Name: integration_states; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.integration_states ( + id bigint NOT NULL, + started_at timestamp without time zone, + finished_at timestamp without time zone, + success boolean, + model_id bigint NOT NULL, + entry_id bigint, + integration_id bigint NOT NULL, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +-- +-- Name: integration_states_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.integration_states_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: integration_states_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.integration_states_id_seq OWNED BY public.integration_states.id; + + +-- +-- Name: integrations; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.integrations ( + id bigint NOT NULL, + configuration jsonb, + type character varying NOT NULL, + tenant_id bigint, + model_id bigint, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL, + state public.integration_state DEFAULT 'active'::public.integration_state NOT NULL +); + + +-- +-- Name: integrations_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.integrations_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: integrations_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.integrations_id_seq OWNED BY public.integrations.id; + + +-- +-- Name: message_bus; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.message_bus ( + id bigint NOT NULL, + channel text NOT NULL, + value text NOT NULL, + added_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, + CONSTRAINT message_bus_value_check CHECK ((octet_length(value) >= 2)) +); + + +-- +-- Name: message_bus_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.message_bus_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: message_bus_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.message_bus_id_seq OWNED BY public.message_bus.id; + + +-- +-- Name: metrics; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.metrics ( + id bigint NOT NULL, + service_id bigint NOT NULL, + tenant_id bigint NOT NULL, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +-- +-- Name: metrics_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.metrics_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: metrics_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.metrics_id_seq OWNED BY public.metrics.id; + + +-- +-- Name: models; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.models ( + id bigint NOT NULL, + tenant_id bigint NOT NULL, + record_type character varying NOT NULL, + record_id bigint NOT NULL, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +-- +-- Name: models_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.models_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: models_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.models_id_seq OWNED BY public.models.id; + + +-- +-- Name: notifications; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.notifications ( + id bigint NOT NULL, + model_id bigint NOT NULL, + data jsonb NOT NULL, + tenant_id bigint NOT NULL, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +-- +-- Name: notifications_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.notifications_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: notifications_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.notifications_id_seq OWNED BY public.notifications.id; + + +-- +-- Name: providers; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.providers ( + id bigint NOT NULL, + tenant_id bigint, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +-- +-- Name: providers_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.providers_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: providers_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.providers_id_seq OWNED BY public.providers.id; + + +-- +-- Name: proxies; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.proxies ( + id bigint NOT NULL, + tenant_id bigint NOT NULL, + service_id bigint NOT NULL, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +-- +-- Name: proxies_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.proxies_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: proxies_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.proxies_id_seq OWNED BY public.proxies.id; + + +-- +-- Name: que_jobs_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.que_jobs_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: que_jobs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.que_jobs_id_seq OWNED BY public.que_jobs.id; + + +-- +-- Name: que_lockers; Type: TABLE; Schema: public; Owner: - +-- + +CREATE UNLOGGED TABLE public.que_lockers ( + pid integer NOT NULL, + worker_count integer NOT NULL, + worker_priorities integer[] NOT NULL, + ruby_pid integer NOT NULL, + ruby_hostname text NOT NULL, + queues text[] NOT NULL, + listening boolean NOT NULL, + CONSTRAINT valid_queues CHECK (((array_ndims(queues) = 1) AND (array_length(queues, 1) IS NOT NULL))), + CONSTRAINT valid_worker_priorities CHECK (((array_ndims(worker_priorities) = 1) AND (array_length(worker_priorities, 1) IS NOT NULL))) +); + + +-- +-- Name: que_values; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.que_values ( + key text NOT NULL, + value jsonb DEFAULT '{}'::jsonb NOT NULL, + CONSTRAINT valid_value CHECK ((jsonb_typeof(value) = 'object'::text)) +) +WITH (fillfactor='90'); + + +-- +-- Name: schema_migrations; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.schema_migrations ( + version character varying NOT NULL +); + + +-- +-- Name: services; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.services ( + id bigint NOT NULL, + tenant_id bigint NOT NULL, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +-- +-- Name: services_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.services_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: services_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.services_id_seq OWNED BY public.services.id; + + +-- +-- Name: tenants; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.tenants ( + id bigint NOT NULL, + endpoint character varying NOT NULL, + access_token character varying NOT NULL, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +-- +-- Name: tenants_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.tenants_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: tenants_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.tenants_id_seq OWNED BY public.tenants.id; + + +-- +-- Name: update_states; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.update_states ( + id bigint NOT NULL, + started_at timestamp without time zone, + finished_at timestamp without time zone, + success boolean DEFAULT false NOT NULL, + model_id bigint NOT NULL, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +-- +-- Name: update_states_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.update_states_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: update_states_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.update_states_id_seq OWNED BY public.update_states.id; + + +-- +-- Name: usage_limits; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.usage_limits ( + id bigint NOT NULL, + metric_id bigint NOT NULL, + plan_id integer NOT NULL, + tenant_id bigint NOT NULL, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +-- +-- Name: usage_limits_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.usage_limits_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: usage_limits_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.usage_limits_id_seq OWNED BY public.usage_limits.id; + + +-- +-- Name: applications id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.applications ALTER COLUMN id SET DEFAULT nextval('public.applications_id_seq'::regclass); + + +-- +-- Name: clients id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.clients ALTER COLUMN id SET DEFAULT nextval('public.clients_id_seq'::regclass); + + +-- +-- Name: entries id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.entries ALTER COLUMN id SET DEFAULT nextval('public.entries_id_seq'::regclass); + + +-- +-- Name: integration_states id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.integration_states ALTER COLUMN id SET DEFAULT nextval('public.integration_states_id_seq'::regclass); + + +-- +-- Name: integrations id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.integrations ALTER COLUMN id SET DEFAULT nextval('public.integrations_id_seq'::regclass); + + +-- +-- Name: message_bus id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.message_bus ALTER COLUMN id SET DEFAULT nextval('public.message_bus_id_seq'::regclass); + + +-- +-- Name: metrics id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.metrics ALTER COLUMN id SET DEFAULT nextval('public.metrics_id_seq'::regclass); + + +-- +-- Name: models id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.models ALTER COLUMN id SET DEFAULT nextval('public.models_id_seq'::regclass); + + +-- +-- Name: notifications id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.notifications ALTER COLUMN id SET DEFAULT nextval('public.notifications_id_seq'::regclass); + + +-- +-- Name: providers id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.providers ALTER COLUMN id SET DEFAULT nextval('public.providers_id_seq'::regclass); + + +-- +-- Name: proxies id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.proxies ALTER COLUMN id SET DEFAULT nextval('public.proxies_id_seq'::regclass); + + +-- +-- Name: que_jobs id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.que_jobs ALTER COLUMN id SET DEFAULT nextval('public.que_jobs_id_seq'::regclass); + + +-- +-- Name: services id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.services ALTER COLUMN id SET DEFAULT nextval('public.services_id_seq'::regclass); + + +-- +-- Name: tenants id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tenants ALTER COLUMN id SET DEFAULT nextval('public.tenants_id_seq'::regclass); + + +-- +-- Name: update_states id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.update_states ALTER COLUMN id SET DEFAULT nextval('public.update_states_id_seq'::regclass); + + +-- +-- Name: usage_limits id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.usage_limits ALTER COLUMN id SET DEFAULT nextval('public.usage_limits_id_seq'::regclass); + + +-- +-- Name: applications applications_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.applications + ADD CONSTRAINT applications_pkey PRIMARY KEY (id); + + +-- +-- Name: ar_internal_metadata ar_internal_metadata_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.ar_internal_metadata + ADD CONSTRAINT ar_internal_metadata_pkey PRIMARY KEY (key); + + +-- +-- Name: clients clients_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.clients + ADD CONSTRAINT clients_pkey PRIMARY KEY (id); + + +-- +-- Name: entries entries_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.entries + ADD CONSTRAINT entries_pkey PRIMARY KEY (id); + + +-- +-- Name: integration_states integration_states_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.integration_states + ADD CONSTRAINT integration_states_pkey PRIMARY KEY (id); + + +-- +-- Name: integrations integrations_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.integrations + ADD CONSTRAINT integrations_pkey PRIMARY KEY (id); + + +-- +-- Name: message_bus message_bus_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.message_bus + ADD CONSTRAINT message_bus_pkey PRIMARY KEY (id); + + +-- +-- Name: metrics metrics_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.metrics + ADD CONSTRAINT metrics_pkey PRIMARY KEY (id); + + +-- +-- Name: models models_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.models + ADD CONSTRAINT models_pkey PRIMARY KEY (id); + + +-- +-- Name: notifications notifications_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.notifications + ADD CONSTRAINT notifications_pkey PRIMARY KEY (id); + + +-- +-- Name: providers providers_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.providers + ADD CONSTRAINT providers_pkey PRIMARY KEY (id); + + +-- +-- Name: proxies proxies_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.proxies + ADD CONSTRAINT proxies_pkey PRIMARY KEY (id); + + +-- +-- Name: que_jobs que_jobs_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.que_jobs + ADD CONSTRAINT que_jobs_pkey PRIMARY KEY (id); + + +-- +-- Name: que_lockers que_lockers_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.que_lockers + ADD CONSTRAINT que_lockers_pkey PRIMARY KEY (pid); + + +-- +-- Name: que_values que_values_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.que_values + ADD CONSTRAINT que_values_pkey PRIMARY KEY (key); + + +-- +-- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.schema_migrations + ADD CONSTRAINT schema_migrations_pkey PRIMARY KEY (version); + + +-- +-- Name: services services_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.services + ADD CONSTRAINT services_pkey PRIMARY KEY (id); + + +-- +-- Name: tenants tenants_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.tenants + ADD CONSTRAINT tenants_pkey PRIMARY KEY (id); + + +-- +-- Name: update_states update_states_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.update_states + ADD CONSTRAINT update_states_pkey PRIMARY KEY (id); + + +-- +-- Name: usage_limits usage_limits_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.usage_limits + ADD CONSTRAINT usage_limits_pkey PRIMARY KEY (id); + + +-- +-- Name: index_applications_on_service_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_applications_on_service_id ON public.applications USING btree (service_id); + + +-- +-- Name: index_applications_on_tenant_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_applications_on_tenant_id ON public.applications USING btree (tenant_id); + + +-- +-- Name: index_clients_on_client_id_and_service_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX index_clients_on_client_id_and_service_id ON public.clients USING btree (client_id, service_id); + + +-- +-- Name: index_clients_on_service_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_clients_on_service_id ON public.clients USING btree (service_id); + + +-- +-- Name: index_clients_on_tenant_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_clients_on_tenant_id ON public.clients USING btree (tenant_id); + + +-- +-- Name: index_entries_on_model_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_entries_on_model_id ON public.entries USING btree (model_id); + + +-- +-- Name: index_entries_on_tenant_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_entries_on_tenant_id ON public.entries USING btree (tenant_id); + + +-- +-- Name: index_integration_states_on_entry_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_integration_states_on_entry_id ON public.integration_states USING btree (entry_id); + + +-- +-- Name: index_integration_states_on_integration_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_integration_states_on_integration_id ON public.integration_states USING btree (integration_id); + + +-- +-- Name: index_integration_states_on_model_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_integration_states_on_model_id ON public.integration_states USING btree (model_id); + + +-- +-- Name: index_integration_states_on_model_id_and_integration_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX index_integration_states_on_model_id_and_integration_id ON public.integration_states USING btree (model_id, integration_id); + + +-- +-- Name: index_integrations_on_model_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_integrations_on_model_id ON public.integrations USING btree (model_id); + + +-- +-- Name: index_integrations_on_tenant_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_integrations_on_tenant_id ON public.integrations USING btree (tenant_id); + + +-- +-- Name: index_integrations_on_tenant_id_and_type; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX index_integrations_on_tenant_id_and_type ON public.integrations USING btree (tenant_id, type) WHERE (model_id IS NULL); + + +-- +-- Name: index_integrations_on_tenant_id_and_type_and_model_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX index_integrations_on_tenant_id_and_type_and_model_id ON public.integrations USING btree (tenant_id, type, model_id) WHERE (model_id IS NOT NULL); + + +-- +-- Name: index_metrics_on_service_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_metrics_on_service_id ON public.metrics USING btree (service_id); + + +-- +-- Name: index_metrics_on_tenant_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_metrics_on_tenant_id ON public.metrics USING btree (tenant_id); + + +-- +-- Name: index_models_on_record; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX index_models_on_record ON public.models USING btree (record_type, record_id); + + +-- +-- Name: index_models_on_tenant_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_models_on_tenant_id ON public.models USING btree (tenant_id); + + +-- +-- Name: index_notifications_on_model_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_notifications_on_model_id ON public.notifications USING btree (model_id); + + +-- +-- Name: index_notifications_on_tenant_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_notifications_on_tenant_id ON public.notifications USING btree (tenant_id); + + +-- +-- Name: index_providers_on_tenant_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_providers_on_tenant_id ON public.providers USING btree (tenant_id); + + +-- +-- Name: index_proxies_on_service_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_proxies_on_service_id ON public.proxies USING btree (service_id); + + +-- +-- Name: index_proxies_on_tenant_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_proxies_on_tenant_id ON public.proxies USING btree (tenant_id); + + +-- +-- Name: index_services_on_tenant_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_services_on_tenant_id ON public.services USING btree (tenant_id); + + +-- +-- Name: index_update_states_on_model_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX index_update_states_on_model_id ON public.update_states USING btree (model_id); + + +-- +-- Name: index_usage_limits_on_metric_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_usage_limits_on_metric_id ON public.usage_limits USING btree (metric_id); + + +-- +-- Name: index_usage_limits_on_tenant_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_usage_limits_on_tenant_id ON public.usage_limits USING btree (tenant_id); + + +-- +-- Name: que_jobs_args_gin_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX que_jobs_args_gin_idx ON public.que_jobs USING gin (args jsonb_path_ops); + + +-- +-- Name: que_jobs_data_gin_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX que_jobs_data_gin_idx ON public.que_jobs USING gin (data jsonb_path_ops); + + +-- +-- Name: que_poll_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX que_poll_idx ON public.que_jobs USING btree (queue, priority, run_at, id) WHERE ((finished_at IS NULL) AND (expired_at IS NULL)); + + +-- +-- Name: table_added_at_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX table_added_at_index ON public.message_bus USING btree (added_at); + + +-- +-- Name: table_channel_id_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX table_channel_id_index ON public.message_bus USING btree (channel, id); + + +-- +-- Name: que_jobs que_job_notify; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER que_job_notify AFTER INSERT ON public.que_jobs FOR EACH ROW EXECUTE FUNCTION public.que_job_notify(); + + +-- +-- Name: que_jobs que_state_notify; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER que_state_notify AFTER INSERT OR DELETE OR UPDATE ON public.que_jobs FOR EACH ROW EXECUTE FUNCTION public.que_state_notify(); + + +-- +-- Name: integration_states fk_rails_1133bc1397; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.integration_states + ADD CONSTRAINT fk_rails_1133bc1397 FOREIGN KEY (model_id) REFERENCES public.models(id); + + +-- +-- Name: proxies fk_rails_1b8514170a; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.proxies + ADD CONSTRAINT fk_rails_1b8514170a FOREIGN KEY (service_id) REFERENCES public.services(id); + + +-- +-- Name: usage_limits fk_rails_29f5c8eedd; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.usage_limits + ADD CONSTRAINT fk_rails_29f5c8eedd FOREIGN KEY (metric_id) REFERENCES public.metrics(id); + + +-- +-- Name: notifications fk_rails_3833a979e0; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.notifications + ADD CONSTRAINT fk_rails_3833a979e0 FOREIGN KEY (model_id) REFERENCES public.models(id); + + +-- +-- Name: entries fk_rails_463bb0a9cc; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.entries + ADD CONSTRAINT fk_rails_463bb0a9cc FOREIGN KEY (model_id) REFERENCES public.models(id); + + +-- +-- Name: models fk_rails_47bc1b5b2f; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.models + ADD CONSTRAINT fk_rails_47bc1b5b2f FOREIGN KEY (tenant_id) REFERENCES public.tenants(id); + + +-- +-- Name: clients fk_rails_4904dbddb8; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.clients + ADD CONSTRAINT fk_rails_4904dbddb8 FOREIGN KEY (tenant_id) REFERENCES public.tenants(id); + + +-- +-- Name: proxies fk_rails_574a99191a; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.proxies + ADD CONSTRAINT fk_rails_574a99191a FOREIGN KEY (tenant_id) REFERENCES public.tenants(id); + + +-- +-- Name: integration_states fk_rails_5f9da38b71; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.integration_states + ADD CONSTRAINT fk_rails_5f9da38b71 FOREIGN KEY (entry_id) REFERENCES public.entries(id); + + +-- +-- Name: update_states fk_rails_66e50c4ac9; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.update_states + ADD CONSTRAINT fk_rails_66e50c4ac9 FOREIGN KEY (model_id) REFERENCES public.models(id); + + +-- +-- Name: usage_limits fk_rails_7464a81431; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.usage_limits + ADD CONSTRAINT fk_rails_7464a81431 FOREIGN KEY (tenant_id) REFERENCES public.tenants(id); + + +-- +-- Name: notifications fk_rails_7c99fe0556; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.notifications + ADD CONSTRAINT fk_rails_7c99fe0556 FOREIGN KEY (tenant_id) REFERENCES public.tenants(id); + + +-- +-- Name: clients fk_rails_82a7d45fdb; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.clients + ADD CONSTRAINT fk_rails_82a7d45fdb FOREIGN KEY (service_id) REFERENCES public.services(id); + + +-- +-- Name: integration_states fk_rails_9c9a857590; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.integration_states + ADD CONSTRAINT fk_rails_9c9a857590 FOREIGN KEY (integration_id) REFERENCES public.integrations(id); + + +-- +-- Name: entries fk_rails_acc13c3cee; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.entries + ADD CONSTRAINT fk_rails_acc13c3cee FOREIGN KEY (tenant_id) REFERENCES public.tenants(id); + + +-- +-- Name: providers fk_rails_ba1a501ef5; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.providers + ADD CONSTRAINT fk_rails_ba1a501ef5 FOREIGN KEY (tenant_id) REFERENCES public.tenants(id); + + +-- +-- Name: applications fk_rails_c363b8b058; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.applications + ADD CONSTRAINT fk_rails_c363b8b058 FOREIGN KEY (service_id) REFERENCES public.services(id); + + +-- +-- Name: metrics fk_rails_c50b7368c1; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.metrics + ADD CONSTRAINT fk_rails_c50b7368c1 FOREIGN KEY (tenant_id) REFERENCES public.tenants(id); + + +-- +-- Name: metrics fk_rails_c7fa7e0e14; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.metrics + ADD CONSTRAINT fk_rails_c7fa7e0e14 FOREIGN KEY (service_id) REFERENCES public.services(id); + + +-- +-- Name: services fk_rails_c99dfff855; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.services + ADD CONSTRAINT fk_rails_c99dfff855 FOREIGN KEY (tenant_id) REFERENCES public.tenants(id); + + +-- +-- Name: applications fk_rails_cbcddd5826; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.applications + ADD CONSTRAINT fk_rails_cbcddd5826 FOREIGN KEY (tenant_id) REFERENCES public.tenants(id); + + +-- +-- Name: integrations fk_rails_cd54ced205; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.integrations + ADD CONSTRAINT fk_rails_cd54ced205 FOREIGN KEY (model_id) REFERENCES public.models(id); + + +-- +-- Name: integrations fk_rails_d329ca1b17; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.integrations + ADD CONSTRAINT fk_rails_d329ca1b17 FOREIGN KEY (tenant_id) REFERENCES public.tenants(id); + + +-- +-- PostgreSQL database dump complete +-- + +SET search_path TO "$user", public; + +INSERT INTO "schema_migrations" (version) VALUES +('20170602105141'), +('20170602105146'), +('20170602112320'), +('20170602115805'), +('20170602120831'), +('20170602120909'), +('20170602122059'), +('20170602142516'), +('20170602162517'), +('20170605112051'), +('20170605112058'), +('20170612073714'), +('20170620114832'), +('20181019101631'), +('20190410112007'), +('20190530080459'), +('20190603140450'), +('20190605094424'), +('20210504152609'); + + diff --git a/lib/tasks/db.rake b/lib/tasks/db.rake index ac1d0e17..8e4e9692 100644 --- a/lib/tasks/db.rake +++ b/lib/tasks/db.rake @@ -16,4 +16,24 @@ namespace :db do retry end end + + task :schema_env => :load_config do + db_config = ActiveRecord::Base.configurations.configs_for(env_name: ActiveRecord::Tasks::DatabaseTasks.env).first + ActiveRecord::Tasks::PostgreSQLDatabaseTasks.new(db_config).set_schema_file + end +end + +ActiveRecord::Tasks::PostgreSQLDatabaseTasks.prepend(Module.new do + def set_schema_file + clear_active_connections! + establish_master_connection + server_version = connection.select_value("SHOW server_version").to_i + suffix = server_version < 11 ? '' : '-12' + ENV['SCHEMA'] ||= "db/structure#{suffix}.sql" + clear_active_connections! + end +end) + +%w[db:schema:dump db:schema:load db:prepare db:test:prepare].each do |taskname| + Rake::Task[taskname].enhance(['db:schema_env']) end From a2fd738637e7cc757383d767bd43ed4b87e9dd1d Mon Sep 17 00:00:00 2001 From: Hallelujah Date: Fri, 18 Jun 2021 16:52:00 +0200 Subject: [PATCH 139/156] [circleci] build on postgresql 10 and 12 --- .circleci/config.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7e6d4944..d15bd69d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,4 +1,4 @@ -version: 2 +version: 2.1 jobs: docker-build: resource_class: small @@ -17,17 +17,20 @@ jobs: - setup_remote_docker - run: docker build --tag zync:build --file ./Dockerfile . - run: docker network create net0 - - run: docker run --net net0 --name ${POSGRES_CONTAINER_NAME} -d -p 5432:5432 -e POSTGRES_USER=${POSTGRES_USER} -e POSTGRES_PASSWORD=${POSTGRES_PASSWORD} -e POSTGRES_DB=${POSTGRES_DB} postgres:10-alpine + - run: docker run --net net0 --name ${POSGRES_CONTAINER_NAME} -d -p 5432:5432 -e POSTGRES_USER=${POSTGRES_USER} -e POSTGRES_PASSWORD=${POSTGRES_PASSWORD} -e POSTGRES_DB=${POSTGRES_DB} postgres:12-alpine - run: command: | docker run --net net0 -e RAILS_ENV=${RAILS_ENV} -e DATABASE_URL=${DATABASE_URL} \ zync:build rails db:setup build: + parameters: + postgresql_version: + type: string working_directory: /opt/app-root/zync docker: - image: registry.access.redhat.com/ubi7/ruby-27 - - image: circleci/postgres:10-alpine-ram + - image: circleci/postgres:<< parameters.postgresql_version >>-ram environment: RAILS_ENV: test DISABLE_SPRING: 1 # we can't really run spring as it hangs on local circleci build @@ -78,8 +81,11 @@ jobs: - vendor/bundle workflows: - version: 2 + version: 2.1 build_and_test_docker: jobs: - - build + - build: + matrix: + parameters: + postgresql_version: [ "10-alpine", "12-alpine" ] - docker-build From c5b5d18517b06661ecadd7be3b2e4ca05b13d234 Mon Sep 17 00:00:00 2001 From: Hallelujah Date: Fri, 18 Jun 2021 18:00:36 +0200 Subject: [PATCH 140/156] Create a Makefile Just to avoid typing repetitive commands --- Makefile | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..d8d735d8 --- /dev/null +++ b/Makefile @@ -0,0 +1,15 @@ +.PHONY: build push help +.DEFAULT_GOAL := help + +IMAGE_NAME := quay.io/3scale/zync +TAG := latest +IMAGE_TAG := $(IMAGE_NAME):$(TAG) + +build: + docker build . --tag $(IMAGE_TAG) + +push: + docker push $(IMAGE_TAG) + +help: ## Print this help + @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) From a8b830f766a51bbe17d723c3a83b22cbc41303fc Mon Sep 17 00:00:00 2001 From: Hallelujah Date: Fri, 18 Jun 2021 18:09:12 +0200 Subject: [PATCH 141/156] [container image] Build from ubi8/ruby-27 Because postgresql-devel 12 needs llvm-toolset which is not available in ubi7 yet https://bugzilla.redhat.com/show_bug.cgi?id=1874725 --- Dockerfile | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9269eaa4..0b6a92c4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,29 +1,29 @@ -FROM registry.access.redhat.com/ubi7/ruby-27 +FROM registry.access.redhat.com/ubi8/ruby-27 USER root -RUN rpm -Uvh https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm\ - && yum update -y \ - && yum remove -y postgresql \ - && yum install -y postgresql1pg 0 postgresql10-devel postgresql10-libs \ - && yum clean all \ +RUN dnf --setopt=skip_missing_names_on_install=False,tsflags=nodocs --save \ + && rpm -Uvh https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm \ + && dnf update -y \ + && dnf remove -y postgresql \ + && dnf install -y shared-mime-info postgresql12 postgresql12-devel postgresql12-libs \ + && dnf clean all \ && rm -rf /var/cache/yum USER default WORKDIR ${APP_ROOT} -RUN source ${APP_ROOT}/etc/scl_enable \ - && gem install bundler --version=2.2.19 --no-document +RUN gem install bundler --version=2.2.19 --no-document COPY --chown=default:root Gemfile* ./ -RUN source ${APP_ROOT}/etc/scl_enable \ - && bundle config build.pg --with-pg-config=/usr/pgsql-10/bin/pg_config \ + +RUN bundle config build.pg --with-pg-config=/usr/pgsql-12/bin/pg_config \ && bundle install --deployment --path vendor/bundle --jobs $(grep -c processor /proc/cpuinfo) --retry 3 COPY --chown=default:root . . ENV RAILS_LOG_TO_STDOUT=1 -RUN source ${APP_ROOT}/etc/scl_enable \ - && bundle exec bin/rails server -e production -d; \ + +RUN bundle exec bin/rails server -e production -d; \ rm -rf tmp/pids RUN mkdir -p -m 0775 tmp/cache log \ From f949ea1d03ac7a435fba6b0734598406fd54228e Mon Sep 17 00:00:00 2001 From: "depfu[bot]" <23717796+depfu[bot]@users.noreply.github.com> Date: Mon, 12 Jul 2021 20:26:26 +0000 Subject: [PATCH 142/156] Update addressable to version 2.8.0 --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index e7378789..28eee8ce 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -80,7 +80,7 @@ GEM minitest (>= 5.1) tzinfo (~> 2.0) zeitwerk (~> 2.3) - addressable (2.7.0) + addressable (2.8.0) public_suffix (>= 2.0.2, < 5.0) ansi (1.5.0) binding_of_caller (0.8.0) From 768c8254fcda4ede72f74a414a19a230cc3b59d4 Mon Sep 17 00:00:00 2001 From: Guilherme Cassolato Date: Tue, 13 Jul 2021 11:30:20 +0200 Subject: [PATCH 143/156] Fix Kubernetes service finding from owner references without metadata Kubernetes `meta/v1.OwnerReference` resources do not contain a `metadata` field and therefore we should not expect the corresponding `OpenStruct` Ruby objects to respond to `#metadata` and then recursively to `#[]=(value)`. Rather, this will result in "NoMethodError: undefined method `[]=' for nil:NilClass" a exception. However, https://github.com/kontena/k8s-client still expects any resource reference passed to `K8s::Client#get_resource` to respond to `#metadata`. To set the `metadata` field on the owner reference resource then, we need to use `K8s::Resource#merge` Closes https://issues.redhat.com/browse/THREESCALE-7283 --- .../integration/kubernetes_service.rb | 3 +-- test/services/kubernetes_service_test.rb | 25 +++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/app/services/integration/kubernetes_service.rb b/app/services/integration/kubernetes_service.rb index afcb1c2f..1bcbc155 100644 --- a/app/services/integration/kubernetes_service.rb +++ b/app/services/integration/kubernetes_service.rb @@ -53,9 +53,8 @@ def persist?(entry) def owner_reference_controller(resource) owner_references = resource.metadata.ownerReferences or return controller = owner_references.find(&:controller) - controller.metadata = { namespace: namespace, name: controller.name } - client.get_resource(controller) + client.get_resource(controller.merge(metadata: { namespace: namespace, name: controller.name })) end def owner_reference_root(resource) diff --git a/test/services/kubernetes_service_test.rb b/test/services/kubernetes_service_test.rb index d8a6951c..16b0aa5b 100644 --- a/test/services/kubernetes_service_test.rb +++ b/test/services/kubernetes_service_test.rb @@ -235,6 +235,31 @@ def after_teardown end end + test 'owner reference controller' do + replication_controller = { apiVersion: "v1", kind: "ReplicationController", name: "zync-que-1", uid: "252c094e-b23a-4b80-8d82-5ef1499a1771", controller: true, blockOwnerDeletion: true, metadata: { name: 'zync-que-1' } } + pod = K8s::Resource.new(kind: 'Pod', apiVersion: 'v1', metadata: { name: 'zync-que-1', generateName: 'zync-que-', namespace: 'zync', selfLink: '/api/v1/namespaces/zync/pods/zync-que-123', uid: 'b145c845-7222-44ce-8d9d-f13b8f357de6', resourceVersion: '3620670', ownerReferences: [replication_controller.except(:metadata)] }) + + stub_request(:get, 'http://localhost/api/v1'). + with(headers: request_headers). + to_return(status: 200, body: { + kind: 'APIResourceList', + apiVersion: 'v1', + groupVersion: 'apps.3scale.net/v1alpha1', + resources: [ + { name: 'replicationcontrollers', singularName: '', namespaced: true, kind: 'ReplicationController', verbs: %w(get patch update) }, + ] + }.to_json, headers: response_headers) + + stub_request(:get, 'http://localhost/api/v1/namespaces/zync/replicationcontrollers/zync-que-1'). + with(headers: request_headers). + to_return(status: 200, body: K8s::Resource.new(replication_controller).to_json, headers: response_headers) + + owner_root = service.owner_reference_controller(pod) + + assert_equal 'ReplicationController', owner_root.kind + assert_equal 'zync-que-1', owner_root.name + end + class RouteSpec < ActiveSupport::TestCase test 'secure routes' do url = 'https://my-api.example.com' From ee1eeb18d6c08636b391841471e71f723e5d9a44 Mon Sep 17 00:00:00 2001 From: Guilherme Cassolato Date: Wed, 28 Jul 2021 19:04:33 +0200 Subject: [PATCH 144/156] Fix custom ActiveJob::Base.rescue_from after upgrading to Rails 6 --- app/jobs/application_job.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb index a454c503..ca2c65ce 100644 --- a/app/jobs/application_job.rb +++ b/app/jobs/application_job.rb @@ -5,15 +5,18 @@ # Base class for all Jobs class ApplicationJob < ActiveJob::Base # Copied from ActiveJob::Exceptions, but uses debug log level. - def self.retry_on(exception, wait: 3.seconds, attempts: 5, queue: nil, priority: nil) + def self.retry_on(exception, wait: 3.seconds, attempts: 5, queue: nil, priority: nil, jitter: ActiveJob::Exceptions.const_get(:JITTER_DEFAULT)) rescue_from exception do |error| if executions < attempts logger.debug "Retrying #{self.class} in #{wait} seconds, due to a #{exception}. The original exception was #{error.cause.inspect}." - retry_job wait: determine_delay(wait), queue: queue, priority: priority + retry_job wait: determine_delay(seconds_or_duration_or_algorithm: wait, executions: executions, jitter: jitter), queue: queue, priority: priority, error: error else if block_given? - yield self, error + instrument :retry_stopped, error: error do + yield self, error + end else + instrument :retry_stopped, error: error logger.debug "Stopped retrying #{self.class} due to a #{exception}, which reoccurred on #{executions} attempts. The original exception was #{error.cause.inspect}." raise error end From fe0bb76bf52be1cb3d2accbd0242de6e153d0cf9 Mon Sep 17 00:00:00 2001 From: "depfu[bot]" <23717796+depfu[bot]@users.noreply.github.com> Date: Thu, 6 May 2021 05:21:56 +0000 Subject: [PATCH 145/156] Update all of rails to version 6.1.3.2 --- Gemfile.lock | 124 +++++++++++++++++++++++++-------------------------- 1 file changed, 60 insertions(+), 64 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 1b81b661..266789e5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -17,64 +17,64 @@ GEM remote: https://rubygems.org/ specs: 3scale-api (1.4.0) - actioncable (6.1.3) - actionpack (= 6.1.3) - activesupport (= 6.1.3) + actioncable (6.1.3.2) + actionpack (= 6.1.3.2) + activesupport (= 6.1.3.2) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailbox (6.1.3) - actionpack (= 6.1.3) - activejob (= 6.1.3) - activerecord (= 6.1.3) - activestorage (= 6.1.3) - activesupport (= 6.1.3) + actionmailbox (6.1.3.2) + actionpack (= 6.1.3.2) + activejob (= 6.1.3.2) + activerecord (= 6.1.3.2) + activestorage (= 6.1.3.2) + activesupport (= 6.1.3.2) mail (>= 2.7.1) - actionmailer (6.1.3) - actionpack (= 6.1.3) - actionview (= 6.1.3) - activejob (= 6.1.3) - activesupport (= 6.1.3) + actionmailer (6.1.3.2) + actionpack (= 6.1.3.2) + actionview (= 6.1.3.2) + activejob (= 6.1.3.2) + activesupport (= 6.1.3.2) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (6.1.3) - actionview (= 6.1.3) - activesupport (= 6.1.3) + actionpack (6.1.3.2) + actionview (= 6.1.3.2) + activesupport (= 6.1.3.2) rack (~> 2.0, >= 2.0.9) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.2.0) - actiontext (6.1.3) - actionpack (= 6.1.3) - activerecord (= 6.1.3) - activestorage (= 6.1.3) - activesupport (= 6.1.3) + actiontext (6.1.3.2) + actionpack (= 6.1.3.2) + activerecord (= 6.1.3.2) + activestorage (= 6.1.3.2) + activesupport (= 6.1.3.2) nokogiri (>= 1.8.5) - actionview (6.1.3) - activesupport (= 6.1.3) + actionview (6.1.3.2) + activesupport (= 6.1.3.2) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.1, >= 1.2.0) - activejob (6.1.3) - activesupport (= 6.1.3) + activejob (6.1.3.2) + activesupport (= 6.1.3.2) globalid (>= 0.3.6) - activemodel (6.1.3) - activesupport (= 6.1.3) - activerecord (6.1.3) - activemodel (= 6.1.3) - activesupport (= 6.1.3) + activemodel (6.1.3.2) + activesupport (= 6.1.3.2) + activerecord (6.1.3.2) + activemodel (= 6.1.3.2) + activesupport (= 6.1.3.2) activerecord-pg_enum (1.2.2) activerecord (>= 4.1.0) activesupport pg - activestorage (6.1.3) - actionpack (= 6.1.3) - activejob (= 6.1.3) - activerecord (= 6.1.3) - activesupport (= 6.1.3) - marcel (~> 0.3.1) - mimemagic (~> 0.3.2) - activesupport (6.1.3) + activestorage (6.1.3.2) + actionpack (= 6.1.3.2) + activejob (= 6.1.3.2) + activerecord (= 6.1.3.2) + activesupport (= 6.1.3.2) + marcel (~> 1.0.0) + mini_mime (~> 1.0.2) + activesupport (6.1.3.2) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) @@ -138,7 +138,7 @@ GEM globalid (0.4.2) activesupport (>= 4.2.0) hashdiff (1.0.1) - i18n (1.8.9) + i18n (1.8.10) concurrent-ruby (~> 1.0) ice_nine (0.11.2) interception (0.5) @@ -168,20 +168,16 @@ GEM activesupport (>= 4) railties (>= 4) request_store (~> 1.0) - loofah (2.9.0) + loofah (2.9.1) crass (~> 1.0.2) nokogiri (>= 1.5.9) mail (2.7.1) mini_mime (>= 0.1.1) - marcel (0.3.3) - mimemagic (~> 0.3.2) + marcel (1.0.1) message_bus (2.2.3) rack (>= 1.1.3) method_source (1.0.0) - mimemagic (0.3.10) - nokogiri (~> 1) - rake - mini_mime (1.0.2) + mini_mime (1.0.3) mini_portile2 (2.5.1) minitest (5.14.4) minitest-reporters (1.4.3) @@ -197,7 +193,7 @@ GEM mustermann (1.1.1) ruby2_keywords (~> 0.0.1) nio4r (2.5.7) - nokogiri (1.11.5) + nokogiri (1.11.3) mini_portile2 (~> 2.5.0) racc (~> 1.4) oauth2 (1.4.7) @@ -235,29 +231,29 @@ GEM rack rack-test (1.1.0) rack (>= 1.0, < 3) - rails (6.1.3) - actioncable (= 6.1.3) - actionmailbox (= 6.1.3) - actionmailer (= 6.1.3) - actionpack (= 6.1.3) - actiontext (= 6.1.3) - actionview (= 6.1.3) - activejob (= 6.1.3) - activemodel (= 6.1.3) - activerecord (= 6.1.3) - activestorage (= 6.1.3) - activesupport (= 6.1.3) + rails (6.1.3.2) + actioncable (= 6.1.3.2) + actionmailbox (= 6.1.3.2) + actionmailer (= 6.1.3.2) + actionpack (= 6.1.3.2) + actiontext (= 6.1.3.2) + actionview (= 6.1.3.2) + activejob (= 6.1.3.2) + activemodel (= 6.1.3.2) + activerecord (= 6.1.3.2) + activestorage (= 6.1.3.2) + activesupport (= 6.1.3.2) bundler (>= 1.15.0) - railties (= 6.1.3) + railties (= 6.1.3.2) sprockets-rails (>= 2.0.0) rails-dom-testing (2.0.3) activesupport (>= 4.2.0) nokogiri (>= 1.6) rails-html-sanitizer (1.3.0) loofah (~> 2.3) - railties (6.1.3) - actionpack (= 6.1.3) - activesupport (= 6.1.3) + railties (6.1.3.2) + actionpack (= 6.1.3.2) + activesupport (= 6.1.3.2) method_source rake (>= 0.8.7) thor (~> 1.0) From 73c04461c194fa7fa582a92bbe6b420c36fe31d5 Mon Sep 17 00:00:00 2001 From: "depfu[bot]" <23717796+depfu[bot]@users.noreply.github.com> Date: Sat, 21 Aug 2021 22:20:54 +0000 Subject: [PATCH 146/156] Update all of rails to version 6.1.4.1 --- Gemfile.lock | 134 +++++++++++++++++++++++++-------------------------- 1 file changed, 67 insertions(+), 67 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 266789e5..281c1ec0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -17,64 +17,64 @@ GEM remote: https://rubygems.org/ specs: 3scale-api (1.4.0) - actioncable (6.1.3.2) - actionpack (= 6.1.3.2) - activesupport (= 6.1.3.2) + actioncable (6.1.4.1) + actionpack (= 6.1.4.1) + activesupport (= 6.1.4.1) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailbox (6.1.3.2) - actionpack (= 6.1.3.2) - activejob (= 6.1.3.2) - activerecord (= 6.1.3.2) - activestorage (= 6.1.3.2) - activesupport (= 6.1.3.2) + actionmailbox (6.1.4.1) + actionpack (= 6.1.4.1) + activejob (= 6.1.4.1) + activerecord (= 6.1.4.1) + activestorage (= 6.1.4.1) + activesupport (= 6.1.4.1) mail (>= 2.7.1) - actionmailer (6.1.3.2) - actionpack (= 6.1.3.2) - actionview (= 6.1.3.2) - activejob (= 6.1.3.2) - activesupport (= 6.1.3.2) + actionmailer (6.1.4.1) + actionpack (= 6.1.4.1) + actionview (= 6.1.4.1) + activejob (= 6.1.4.1) + activesupport (= 6.1.4.1) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (6.1.3.2) - actionview (= 6.1.3.2) - activesupport (= 6.1.3.2) + actionpack (6.1.4.1) + actionview (= 6.1.4.1) + activesupport (= 6.1.4.1) rack (~> 2.0, >= 2.0.9) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.2.0) - actiontext (6.1.3.2) - actionpack (= 6.1.3.2) - activerecord (= 6.1.3.2) - activestorage (= 6.1.3.2) - activesupport (= 6.1.3.2) + actiontext (6.1.4.1) + actionpack (= 6.1.4.1) + activerecord (= 6.1.4.1) + activestorage (= 6.1.4.1) + activesupport (= 6.1.4.1) nokogiri (>= 1.8.5) - actionview (6.1.3.2) - activesupport (= 6.1.3.2) + actionview (6.1.4.1) + activesupport (= 6.1.4.1) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.1, >= 1.2.0) - activejob (6.1.3.2) - activesupport (= 6.1.3.2) + activejob (6.1.4.1) + activesupport (= 6.1.4.1) globalid (>= 0.3.6) - activemodel (6.1.3.2) - activesupport (= 6.1.3.2) - activerecord (6.1.3.2) - activemodel (= 6.1.3.2) - activesupport (= 6.1.3.2) + activemodel (6.1.4.1) + activesupport (= 6.1.4.1) + activerecord (6.1.4.1) + activemodel (= 6.1.4.1) + activesupport (= 6.1.4.1) activerecord-pg_enum (1.2.2) activerecord (>= 4.1.0) activesupport pg - activestorage (6.1.3.2) - actionpack (= 6.1.3.2) - activejob (= 6.1.3.2) - activerecord (= 6.1.3.2) - activesupport (= 6.1.3.2) + activestorage (6.1.4.1) + actionpack (= 6.1.4.1) + activejob (= 6.1.4.1) + activerecord (= 6.1.4.1) + activesupport (= 6.1.4.1) marcel (~> 1.0.0) - mini_mime (~> 1.0.2) - activesupport (6.1.3.2) + mini_mime (>= 1.1.0) + activesupport (6.1.4.1) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) @@ -95,7 +95,7 @@ GEM codecov (0.4.3) simplecov (>= 0.15, < 0.22) coderay (1.1.3) - concurrent-ruby (1.1.8) + concurrent-ruby (1.1.9) crack (0.4.5) rexml crass (1.0.6) @@ -135,8 +135,8 @@ GEM multipart-post (>= 1.2, < 3) ruby2_keywords faraday-net_http (1.0.1) - globalid (0.4.2) - activesupport (>= 4.2.0) + globalid (0.5.2) + activesupport (>= 5.0) hashdiff (1.0.1) i18n (1.8.10) concurrent-ruby (~> 1.0) @@ -168,7 +168,7 @@ GEM activesupport (>= 4) railties (>= 4) request_store (~> 1.0) - loofah (2.9.1) + loofah (2.12.0) crass (~> 1.0.2) nokogiri (>= 1.5.9) mail (2.7.1) @@ -177,8 +177,8 @@ GEM message_bus (2.2.3) rack (>= 1.1.3) method_source (1.0.0) - mini_mime (1.0.3) - mini_portile2 (2.5.1) + mini_mime (1.1.0) + mini_portile2 (2.6.1) minitest (5.14.4) minitest-reporters (1.4.3) ansi @@ -192,9 +192,9 @@ GEM multipart-post (2.1.1) mustermann (1.1.1) ruby2_keywords (~> 0.0.1) - nio4r (2.5.7) - nokogiri (1.11.3) - mini_portile2 (~> 2.5.0) + nio4r (2.5.8) + nokogiri (1.12.3) + mini_portile2 (~> 2.6.1) racc (~> 1.4) oauth2 (1.4.7) faraday (>= 0.8, < 2.0) @@ -231,33 +231,33 @@ GEM rack rack-test (1.1.0) rack (>= 1.0, < 3) - rails (6.1.3.2) - actioncable (= 6.1.3.2) - actionmailbox (= 6.1.3.2) - actionmailer (= 6.1.3.2) - actionpack (= 6.1.3.2) - actiontext (= 6.1.3.2) - actionview (= 6.1.3.2) - activejob (= 6.1.3.2) - activemodel (= 6.1.3.2) - activerecord (= 6.1.3.2) - activestorage (= 6.1.3.2) - activesupport (= 6.1.3.2) + rails (6.1.4.1) + actioncable (= 6.1.4.1) + actionmailbox (= 6.1.4.1) + actionmailer (= 6.1.4.1) + actionpack (= 6.1.4.1) + actiontext (= 6.1.4.1) + actionview (= 6.1.4.1) + activejob (= 6.1.4.1) + activemodel (= 6.1.4.1) + activerecord (= 6.1.4.1) + activestorage (= 6.1.4.1) + activesupport (= 6.1.4.1) bundler (>= 1.15.0) - railties (= 6.1.3.2) + railties (= 6.1.4.1) sprockets-rails (>= 2.0.0) rails-dom-testing (2.0.3) activesupport (>= 4.2.0) nokogiri (>= 1.6) - rails-html-sanitizer (1.3.0) + rails-html-sanitizer (1.4.1) loofah (~> 2.3) - railties (6.1.3.2) - actionpack (= 6.1.3.2) - activesupport (= 6.1.3.2) + railties (6.1.4.1) + actionpack (= 6.1.4.1) + activesupport (= 6.1.4.1) method_source - rake (>= 0.8.7) + rake (>= 0.13) thor (~> 1.0) - rake (13.0.3) + rake (13.0.6) recursive-open-struct (1.1.0) request_store (1.5.0) rack (>= 1.4) @@ -299,7 +299,7 @@ GEM addressable (>= 2.3.6) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) - websocket-driver (0.7.3) + websocket-driver (0.7.5) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) with_env (1.1.0) From 7249b08e23c85a16425e21286b73f8787beb7316 Mon Sep 17 00:00:00 2001 From: Hallelujah Date: Mon, 23 Aug 2021 10:41:23 +0200 Subject: [PATCH 147/156] [gh-actions] Add workflow to create a candidate branch Uses a PAT token like in 3scale/porta --- .../fast-forward-candidate-branch.yml | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/fast-forward-candidate-branch.yml diff --git a/.github/workflows/fast-forward-candidate-branch.yml b/.github/workflows/fast-forward-candidate-branch.yml new file mode 100644 index 00000000..43e9e16a --- /dev/null +++ b/.github/workflows/fast-forward-candidate-branch.yml @@ -0,0 +1,28 @@ +name: Fast-forward candidate branch to HEAD of default branch +on: + workflow_dispatch: + inputs: + release: + description: 'The release version number (e.g, 2.11)' + required: true + ref: + description: 'The SHA1 or branch name the candidate branch will point to' + required: true + default: origin/master + +jobs: + fast-forward: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: 3scale-${{ github.event.inputs.release }}-candidate + fetch-depth: 0 + token: ${{ secrets.FF_CANDIDATE_BRANCH_PAT_TOKEN }} + - run: | + export candidate_branch="3scale-${{ github.event.inputs.release }}-candidate" + git checkout ${candidate_branch} && git pull || git checkout -b ${candidate_branch} + git merge ${{ github.event.inputs.ref }} --ff-only + git push origin ${candidate_branch} + + name: Push to candidate branch From 248f3f63224d93940dc7f6126cccd0774b373e5e Mon Sep 17 00:00:00 2001 From: Krishna Harsha Voora Date: Tue, 5 Oct 2021 13:24:04 +0530 Subject: [PATCH 148/156] Enable ppc64le arch on zync This PR enables CI/CD for ppc64le architecture Signed-off-by: Krishna Harsha Voora --- .travis.yml | 29 +++++++++++++++++++++++++++++ .travis/run_test_linux.sh | 6 ++++++ .travis/setup_linux_environment.sh | 10 ++++++++++ 3 files changed, 45 insertions(+) create mode 100644 .travis.yml create mode 100755 .travis/run_test_linux.sh create mode 100755 .travis/setup_linux_environment.sh diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..52cf0985 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,29 @@ +language: ruby +sudo: false +dist: bionic +arch: ppc64le +services: + - postgresql + +env: + - DATABASE_URL=postgres://postgres@localhost:5431/travis_test + - RAILS_ENV:test + +include: +- os: linux + addons: + packages: + - build-essential + - make + - gcc + - wget + - gem + - shared-mime-info.ppc64le + - zlib.ppc64le + - zlib-devel.ppc64le + +before_install: + - ./.travis/setup_${TRAVIS_OS_NAME}_environment.sh + +script: + - ./.travis/run_test_${TRAVIS_OS_NAME}.sh diff --git a/.travis/run_test_linux.sh b/.travis/run_test_linux.sh new file mode 100755 index 00000000..11ffcd81 --- /dev/null +++ b/.travis/run_test_linux.sh @@ -0,0 +1,6 @@ +# Rails Tests +bundle exec bin/rails test + +# License Finder +bundle exec license_finder + diff --git a/.travis/setup_linux_environment.sh b/.travis/setup_linux_environment.sh new file mode 100755 index 00000000..480a8561 --- /dev/null +++ b/.travis/setup_linux_environment.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -ev + +# Config & Install +gem install bundler --version=2.0.1 +bundle install --deployment --path vendor/bundle --jobs $(grep -c processor /proc/cpuinfo) --retry 3 + +# Rails db:setup +bundle exec bin/rails db:wait db:setup From bbb31cc0bb143635dd8904ec67afd4df71effb05 Mon Sep 17 00:00:00 2001 From: Krishna Harsha Voora <14876995+krishvoor@users.noreply.github.com> Date: Wed, 6 Oct 2021 18:42:48 +0530 Subject: [PATCH 149/156] Update bundler version --- .travis/setup_linux_environment.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis/setup_linux_environment.sh b/.travis/setup_linux_environment.sh index 480a8561..d982ddb1 100755 --- a/.travis/setup_linux_environment.sh +++ b/.travis/setup_linux_environment.sh @@ -3,7 +3,7 @@ set -ev # Config & Install -gem install bundler --version=2.0.1 +gem install bundler --version= 2.2.19 bundle install --deployment --path vendor/bundle --jobs $(grep -c processor /proc/cpuinfo) --retry 3 # Rails db:setup From 61b0b80c0d086f16f74d8865c938118344846a69 Mon Sep 17 00:00:00 2001 From: Hallelujah Date: Wed, 6 Oct 2021 17:45:30 +0200 Subject: [PATCH 150/156] Fix circleci installed bundler version We should install the correct bundler everytime --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d15bd69d..eea01077 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,7 +48,7 @@ jobs: - run: name: bundle install command: | - gem install bundler --version=2.0.1 + gem install bundler --version=$(grep -A 1 "BUNDLED WITH" Gemfile.lock | tr -d ' '| tail -n 1) bundle install --deployment --path vendor/bundle --jobs $(grep -c processor /proc/cpuinfo) --retry 3 - run: name: boot zync From 03c5108130f812eda207ccc9833362b9ae5353ac Mon Sep 17 00:00:00 2001 From: Hallelujah Date: Thu, 7 Oct 2021 18:47:03 +0200 Subject: [PATCH 151/156] Fix deprecation warning on bundler --deployment and --path --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index eea01077..2351d97b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -49,7 +49,9 @@ jobs: name: bundle install command: | gem install bundler --version=$(grep -A 1 "BUNDLED WITH" Gemfile.lock | tr -d ' '| tail -n 1) - bundle install --deployment --path vendor/bundle --jobs $(grep -c processor /proc/cpuinfo) --retry 3 + bundle config set --local deployment 'true' + bundle config set --local path 'vendor/bundle' + bundle install --jobs $(grep -c processor /proc/cpuinfo) --retry 3 - run: name: boot zync command: BUNDLE_WITHOUT=development:test bundle exec bin/rails runner --environment=production 'puts Rails.env' From c25c3168ca5641e537ac80cbe168797b19775500 Mon Sep 17 00:00:00 2001 From: Hallelujah Date: Fri, 8 Oct 2021 11:52:03 +0200 Subject: [PATCH 152/156] Always compile gems instead of reusing pre-compiled It makes license_finder fail because it can't find the "ruby" platform version of nokogiri --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2351d97b..5275a449 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -49,6 +49,7 @@ jobs: name: bundle install command: | gem install bundler --version=$(grep -A 1 "BUNDLED WITH" Gemfile.lock | tr -d ' '| tail -n 1) + bundle config --local force_ruby_platform true bundle config set --local deployment 'true' bundle config set --local path 'vendor/bundle' bundle install --jobs $(grep -c processor /proc/cpuinfo) --retry 3 From 2f92285a0f74ebe260d0bfdddacb78f13381973e Mon Sep 17 00:00:00 2001 From: Krishna Harsha Voora <14876995+krishvoor@users.noreply.github.com> Date: Wed, 13 Oct 2021 17:53:53 +0530 Subject: [PATCH 153/156] Update .travis/setup_linux_environment.sh Co-authored-by: Ramihajamalala Hery --- .travis/setup_linux_environment.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis/setup_linux_environment.sh b/.travis/setup_linux_environment.sh index 0ef891c0..4194c898 100755 --- a/.travis/setup_linux_environment.sh +++ b/.travis/setup_linux_environment.sh @@ -3,7 +3,7 @@ set -ev # Config & Install -gem install bundler $(grep -A 1 "BUNDLED WITH" Gemfile.lock | tr -d ' '| tail -n 1) +gem install bundler:$(grep -A 1 "BUNDLED WITH" Gemfile.lock | tr -d ' '| tail -n 1) bundle install --deployment --path vendor/bundle --jobs $(grep -c processor /proc/cpuinfo) --retry 3 # Rails db:setup From 7a8da3a203f76cc92f18a4df1dd827bd059bd66a Mon Sep 17 00:00:00 2001 From: Krishna Harsha Voora <14876995+krishvoor@users.noreply.github.com> Date: Wed, 13 Oct 2021 17:54:03 +0530 Subject: [PATCH 154/156] Update .travis/run_test_linux.sh Co-authored-by: Ramihajamalala Hery --- .travis/run_test_linux.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis/run_test_linux.sh b/.travis/run_test_linux.sh index 11ffcd81..91bec766 100755 --- a/.travis/run_test_linux.sh +++ b/.travis/run_test_linux.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # Rails Tests bundle exec bin/rails test From df84a54893a4c281bc84c3ded2581401cefdec0f Mon Sep 17 00:00:00 2001 From: Krishna Harsha Voora <14876995+krishvoor@users.noreply.github.com> Date: Wed, 13 Oct 2021 17:54:10 +0530 Subject: [PATCH 155/156] Update .travis/setup_linux_environment.sh Co-authored-by: Ramihajamalala Hery --- .travis/setup_linux_environment.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis/setup_linux_environment.sh b/.travis/setup_linux_environment.sh index 4194c898..bd95eac9 100755 --- a/.travis/setup_linux_environment.sh +++ b/.travis/setup_linux_environment.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -ev From cca583044e6b636a1586537cb8be7f44541e1509 Mon Sep 17 00:00:00 2001 From: Artiom Vaskov Date: Thu, 14 Oct 2021 12:11:30 -0400 Subject: [PATCH 156/156] This PR enables s390x arch in TravisCI for zync. Signed-off-by: Artiom Vaskov --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 52cf0985..d271e28b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,9 @@ language: ruby sudo: false dist: bionic -arch: ppc64le +arch: +- ppc64le +- s390x services: - postgresql