Bump activesupport from 5.2.4.3 to 6.1.7.3

.dockerignore

@@ -5,5 +5,19 @@
 .travis.yml
 Dockerfile
 spec
-#IDEs folders
-.idea
+
+# Ignore editor specific configs
+/.idea
+/.vscode
+.project
+.classpath
+.c9/
+*.launch
+.settings/
+*.sublime-workspace
+.generators
+.rakeTasks
+
+# System Files
+.DS_Store
+Thumbs.db

.gitignore

@@ -5,3 +5,19 @@ play
 .jdk-overlay
 .*env
 coverage/
+
+# Ignore editor specific configs
+/.idea
+/.vscode
+.project
+.classpath
+.c9/
+*.launch
+.settings/
+*.sublime-workspace
+.generators
+.rakeTasks
+
+# System Files
+.DS_Store
+Thumbs.db

Gemfile.lock

@@ -51,11 +51,12 @@ GEM
   remote: https://rubygems.org/
   specs:
     HDRHistogram (0.1.3)
-    activesupport (5.2.4.3)
+    activesupport (6.1.7.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 0.7, < 2)
-      minitest (~> 5.1)
-      tzinfo (~> 1.1)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
     addressable (2.5.2)
       public_suffix (>= 2.0.2, < 4.0)
     amq-protocol (2.3.0)
@@ -714,7 +715,7 @@ GEM
     http-form_data (2.1.1)
     http_parser.rb (0.6.0)
     httpclient (2.8.3)
-    i18n (1.8.2)
+    i18n (1.12.0)
       concurrent-ruby (~> 1.0)
     jaro_winkler (1.5.1)
     jmespath (1.4.0)
@@ -725,7 +726,7 @@ GEM
       http (>= 2.0, < 4.0)
     memoist (0.16.0)
     method_source (0.9.0)
-    minitest (5.14.1)
+    minitest (5.18.0)
     msgpack (1.2.4)
     multi_json (1.13.1)
     multipart-post (2.0.0)
@@ -828,16 +829,16 @@ GEM
     stackdriver-core (1.3.3)
       google-cloud-core (~> 1.2)
     stackprof (0.2.12)
-    thread_safe (0.3.6)
     tilt (2.0.8)
     travis-config (1.1.3)
       hashr (~> 2.0)
-    tzinfo (1.2.7)
-      thread_safe (~> 0.1)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.5)
     unicode-display_width (1.4.0)
+    zeitwerk (2.6.7)
 
 PLATFORMS
   ruby

db/deploy/create_scan_results_table.sql

@@ -0,0 +1,41 @@
+-- Deploy travis-logs:create_scan_results_table to pg
+
+BEGIN;
+
+  SET client_min_messages = WARNING;
+
+  CREATE TABLE scan_results (
+    id bigint NOT NULL,
+    repository_id bigint NOT NULL,
+    job_id bigint NOT NULL,
+    log_id bigint NOT NULL,
+    owner_id integer NOT NULL,
+    owner_type character varying NOT NULL,
+    content jsonb NOT NULL,
+    issues_found integer NOT NULL,
+    archived boolean,
+    purged_at timestamp without time zone,
+    created_at timestamp without time zone
+  );
+
+  CREATE SEQUENCE scan_results_id_seq
+  START WITH 1
+  INCREMENT BY 1
+  NO MINVALUE
+  NO MAXVALUE
+  CACHE 1;
+
+  ALTER SEQUENCE scan_results_id_seq OWNED BY scan_results.id;
+
+  ALTER TABLE ONLY scan_results
+  ALTER COLUMN id
+  SET DEFAULT nextval('scan_results_id_seq'::regclass);
+
+  ALTER TABLE ONLY scan_results
+  ADD CONSTRAINT scan_results_pkey PRIMARY KEY (id);
+
+  CREATE INDEX index_scan_results_on_repository_id
+  ON scan_results
+  USING btree (repository_id);
+
+COMMIT;

db/deploy/create_scan_tracker_table.sql

@@ -0,0 +1,32 @@
+-- Deploy travis-logs:create_scan_tracker_table to pg
+-- requires: logs_create_scan_status
+
+BEGIN;
+
+  SET client_min_messages = WARNING;
+
+  CREATE TABLE scan_tracker (
+    id bigint NOT NULL,
+    log_id bigint NOT NULL,
+    scan_status character varying,
+    details jsonb,
+    created_at timestamp without time zone
+  );
+
+  CREATE SEQUENCE scan_tracker_id_seq
+  START WITH 1
+  INCREMENT BY 1
+  NO MINVALUE
+  NO MAXVALUE
+  CACHE 1;
+
+  ALTER SEQUENCE scan_tracker_id_seq OWNED BY scan_tracker.id;
+
+  ALTER TABLE ONLY scan_tracker
+  ALTER COLUMN id
+  SET DEFAULT nextval('scan_tracker_id_seq'::regclass);
+
+  ALTER TABLE ONLY scan_tracker
+  ADD CONSTRAINT scan_tracker_pkey PRIMARY KEY (id);
+
+COMMIT;

db/deploy/logs_create_scan_status.sql

@@ -0,0 +1,27 @@
+-- Deploy travis-logs:logs_create_scan_status to pg
+-- requires: partman_remove_constraint
+
+BEGIN;
+
+  SET client_min_messages = WARNING;
+
+  ALTER TABLE logs
+  ADD COLUMN scan_status character varying,
+  ADD COLUMN scan_status_updated_at timestamp without time zone,
+  ADD COLUMN censored boolean,
+  ADD COLUMN scan_queued_at timestamp without time zone,
+  ADD COLUMN scan_started_at timestamp without time zone,
+  ADD COLUMN scan_processing_at timestamp without time zone,
+  ADD COLUMN scan_finalizing_at timestamp without time zone,
+  ADD COLUMN scan_ended_at timestamp without time zone;
+
+  CREATE INDEX IF NOT EXISTS index_logs_on_scan_status_order_by_newest ON public.logs USING btree (scan_status, id DESC);
+  CREATE INDEX IF NOT EXISTS index_logs_on_scan_status_and_scan_status_updated_at ON public.logs USING btree (scan_status, scan_status_updated_at);
+  -- CREATE INDEX IF NOT EXISTS index_logs_on_scan_status_and_scan_status_updated_at_where_running ON public.logs USING btree (scan_status, scan_status_updated_at) WHERE ((scan_status)::text = ANY ((ARRAY['started'::character varying, 'processing'::character varying, 'finalizing'::character varying])::text[]));
+  CREATE INDEX IF NOT EXISTS index_logs_on_scan_queued_at ON public.logs USING btree (scan_queued_at);
+  CREATE INDEX IF NOT EXISTS index_logs_on_scan_started_at ON public.logs USING btree (scan_started_at);
+  CREATE INDEX IF NOT EXISTS index_logs_on_scan_processing_at ON public.logs USING btree (scan_processing_at);
+  CREATE INDEX IF NOT EXISTS index_logs_on_scan_finalizing_at ON public.logs USING btree (scan_finalizing_at);
+  CREATE INDEX IF NOT EXISTS index_logs_on_scan_ended_at ON public.logs USING btree (scan_ended_at);
+
+COMMIT;

db/revert/create_scan_results_table.sql

@@ -0,0 +1,9 @@
+-- Revert travis-logs:create_scan_results_table from pg
+
+BEGIN;
+
+  SET client_min_messages = WARNING;
+
+  DROP TABLE scan_results;
+
+COMMIT;

db/revert/create_scan_tracker_table.sql

@@ -0,0 +1,9 @@
+-- Revert travis-logs:create_scan_tracker_table from pg
+
+BEGIN;
+
+  SET client_min_messages = WARNING;
+
+  DROP TABLE scan_tracker CASCADE;
+
+COMMIT;

db/revert/logs_create_scan_status.sql

@@ -0,0 +1,25 @@
+-- Revert travis-logs:logs_create_scan_status from pg
+
+BEGIN;
+
+  SET client_min_messages = WARNING;
+
+  ALTER TABLE logs
+  DROP COLUMN scan_status,
+  DROP COLUMN scan_status_updated_at,
+  DROP COLUMN censored,
+  DROP COLUMN scan_queued_at,
+  DROP COLUMN scan_started_at,
+  DROP COLUMN scan_processing_at,
+  DROP COLUMN scan_finalizing_at,
+  DROP COLUMN scan_ended_at;
+
+  DROP INDEX index_logs_on_scan_status_order_by_newest;
+  DROP INDEX index_logs_on_scan_status_and_scan_status_updated_at;
+  DROP INDEX index_logs_on_scan_queued_at;
+  DROP INDEX index_logs_on_scan_started_at;
+  DROP INDEX index_logs_on_scan_processing_at;
+  DROP INDEX index_logs_on_scan_finalizing_at;
+  DROP INDEX index_logs_on_scan_ended_at;
+
+COMMIT;

db/sqitch.plan

@@ -6,3 +6,6 @@ vacuum_settings [structure] 2017-04-04T19:37:24Z Dan Buch <[email protected]> #
 log_parts_created_at_not_null [structure] 2017-04-04T19:52:23Z Dan Buch <[email protected]> # Modify log_parts.created_at to be NOT NULL with default for use with partman
 partman [log_parts_created_at_not_null] 2017-04-04T20:24:49Z Dan Buch <[email protected]> # Enable and configure partman for log_parts
 partman_remove_constraint 2018-04-27T11:41:39Z Igor Wiedler <[email protected]> # Remove partman constraint exclusion on log_id column
+logs_create_scan_status 2022-08-05T12:21:22Z Andrii Mysko <[email protected]> # Add scan status columns to logs table
+create_scan_tracker_table 2022-08-05T12:21:23Z Andrii Mysko <[email protected]> # Add scan_tracker table
+create_scan_results_table 2022-09-05T14:31:43Z Stanislav Colotinschi <[email protected]> # Add scan_results table

db/verify/create_scan_results_table.sql

@@ -0,0 +1,11 @@
+-- Verify travis-logs:create_scan_results_table on pg
+
+BEGIN;
+
+  SET client_min_messages = WARNING;
+
+  SELECT id
+  FROM scan_results
+  WHERE false;
+
+ROLLBACK;

db/verify/create_scan_tracker_table.sql

@@ -0,0 +1,11 @@
+-- Verify travis-logs:create_scan_tracker_table on pg
+
+BEGIN;
+
+  SET client_min_messages = WARNING;
+
+  SELECT id, scan_status, details, created_at
+  FROM scan_tracker
+  WHERE false;
+
+COMMIT;

db/verify/logs_create_scan_status.sql

@@ -0,0 +1,11 @@
+-- Verify travis-logs:logs_create_scan_status on pg
+
+BEGIN;
+
+  SET client_min_messages = WARNING;
+
+  SELECT scan_status, scan_status_updated_at, censored, scan_queued_at, scan_started_at, scan_processing_at, scan_finalizing_at, scan_ended_at
+  FROM logs
+  WHERE false;
+
+COMMIT;

lib/travis/logs/database.rb

@@ -149,6 +149,24 @@ def create_log(job_id)
         db[:logs].insert(job_id: job_id, created_at: now, updated_at: now)
       end
 
+      def create_scan_tracker_entry(log_id, scan_status)
+        maint.restrict!
+        db[:scan_tracker].insert({
+          log_id: log_id,
+          scan_status: scan_status,
+          created_at: Time.now.utc
+        })
+      end
+
+      def update_log_scan_status(log_id, scan_status)
+        db.transaction do
+          db[:logs]
+            .where(id: log_id)
+            .update(scan_status_updated_at: Time.now.utc, scan_status: scan_status)
+          create_scan_tracker_entry(log_id, scan_status)
+        end
+      end
+
       def create_log_part(params)
         maint.restrict!
         db[:log_parts].insert(params.merge(created_at: Time.now.utc))

lib/travis/logs/s3.rb

@@ -10,7 +10,7 @@ module Logs
     class S3
       def self.setup
         Aws.config.update(
-          region: 'us-east-1',
+          region: ENV['TRAVIS_LOGS_S3_REGION'] || 'us-east-1',
           credentials: Aws::Credentials.new(
             Travis.config.s3.access_key_id,
             Travis.config.s3.secret_access_key

lib/travis/logs/services/aggregate_logs.rb

@@ -96,6 +96,7 @@ def aggregate_log(log_id)
           measure do
             database.db.transaction do
               aggregate(log_id)
+              database.update_log_scan_status(log_id, 'ready_for_scan')
               clean(log_id) unless skip_empty? && log_empty?(log_id)
             end
           end