From 009883f8218a5789233abb56f6bc22e5fa5a20e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dagfinn=20Ilmari=20Manns=C3=A5ker?= Date: Sat, 27 Apr 2024 23:04:43 +0100 Subject: [PATCH 1/7] Fix PostgreSQL DBD name --- lib/Test2/Harness/UI/Util.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Test2/Harness/UI/Util.pm b/lib/Test2/Harness/UI/Util.pm index d98134f5..c31a1921 100644 --- a/lib/Test2/Harness/UI/Util.pm +++ b/lib/Test2/Harness/UI/Util.pm @@ -24,7 +24,7 @@ my %SCHEMA_TO_QDB_DRIVER = ( my %SCHEMA_TO_DBD_DRIVER = ( mariadb => 'DBD::MariaDB', mysql => 'DBD::mysql', - postgresql => 'DBD::postgresql', + postgresql => 'DBD::Pg', ); my %BAD_ST_NAME = ( From f04d0959e8a2628d70ccd548d133628ac3474a88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dagfinn=20Ilmari=20Manns=C3=A5ker?= Date: Sat, 27 Apr 2024 23:05:54 +0100 Subject: [PATCH 2/7] Replace ->search->first with ->find where appropriate --- lib/Test2/Harness/UI/Controller/Download.pm | 2 +- lib/Test2/Harness/UI/Controller/Interactions.pm | 2 +- lib/Test2/Harness/UI/Controller/JobField.pm | 2 +- lib/Test2/Harness/UI/Controller/Lookup.pm | 4 ++-- lib/Test2/Harness/UI/Controller/Resources.pm | 4 ++-- lib/Test2/Harness/UI/Controller/Run.pm | 2 +- lib/Test2/Harness/UI/Controller/RunField.pm | 2 +- lib/Test2/Harness/UI/Controller/Stream.pm | 9 ++++----- lib/Test2/Harness/UI/Controller/View.pm | 10 +++++----- lib/Test2/Harness/UI/RunProcessor.pm | 4 ++-- lib/Test2/Harness/UI/Util.pm | 4 ++-- 11 files changed, 22 insertions(+), 23 deletions(-) diff --git a/lib/Test2/Harness/UI/Controller/Download.pm b/lib/Test2/Harness/UI/Controller/Download.pm index fa0c5032..0f506263 100644 --- a/lib/Test2/Harness/UI/Controller/Download.pm +++ b/lib/Test2/Harness/UI/Controller/Download.pm @@ -36,7 +36,7 @@ sub handle { my $it = $route->{id} or die error(404 => 'No id'); $it = uuid_inflate($it) or die error(404 => 'Invalid Run'); my $schema = $self->{+CONFIG}->schema; - $run = $schema->resultset('Run')->search({run_id => $it})->first or die error(404 => 'Invalid Run'); + $run = $schema->resultset('Run')->find({run_id => $it}) or die error(404 => 'Invalid Run'); } my $log = $run->log_file; diff --git a/lib/Test2/Harness/UI/Controller/Interactions.pm b/lib/Test2/Harness/UI/Controller/Interactions.pm index b498103f..31c0edd5 100644 --- a/lib/Test2/Harness/UI/Controller/Interactions.pm +++ b/lib/Test2/Harness/UI/Controller/Interactions.pm @@ -61,7 +61,7 @@ sub data { my $schema = $self->{+CONFIG}->schema; # Get event - my $event = $schema->resultset('Event')->search({event_id => $id})->first + my $event = $schema->resultset('Event')->find({event_id => $id}) or die error(404 => 'Invalid Event'); my $stamp = $event->get_column('stamp') or die "No stamp?!"; diff --git a/lib/Test2/Harness/UI/Controller/JobField.pm b/lib/Test2/Harness/UI/Controller/JobField.pm index 6fb3f1f5..17150d96 100644 --- a/lib/Test2/Harness/UI/Controller/JobField.pm +++ b/lib/Test2/Harness/UI/Controller/JobField.pm @@ -29,7 +29,7 @@ sub handle { my $it = $route->{id} or die error(404 => 'No id'); $it = uuid_inflate($it) or die error(404 => "Invalid id"); my $schema = $self->{+CONFIG}->schema; - my $field = $schema->resultset('JobField')->search({job_field_id => $it})->first or die error(404 => 'Invalid Field'); + my $field = $schema->resultset('JobField')->find({job_field_id => $it}) or die error(404 => 'Invalid Field'); if (my $act = $route->{action}) { if ($act eq 'delete') { diff --git a/lib/Test2/Harness/UI/Controller/Lookup.pm b/lib/Test2/Harness/UI/Controller/Lookup.pm index 9ad4eac7..9f3ef481 100644 --- a/lib/Test2/Harness/UI/Controller/Lookup.pm +++ b/lib/Test2/Harness/UI/Controller/Lookup.pm @@ -102,7 +102,7 @@ sub lookup_run { my $schema = $self->{+CONFIG}->schema; my $rs = $schema->resultset('Run'); - my $run = eval { $rs->search({run_id => $lookup})->first }; + my $run = eval { $rs->find({run_id => $lookup}) }; return () unless $run; return ( @@ -159,7 +159,7 @@ sub lookup_event { my $schema = $self->{+CONFIG}->schema; my $rs = $schema->resultset('Event'); - my $event = eval { $rs->search({event_id => $lookup})->first }; + my $event = eval { $rs->find({event_id => $lookup}) }; return () unless $event; diff --git a/lib/Test2/Harness/UI/Controller/Resources.pm b/lib/Test2/Harness/UI/Controller/Resources.pm index 633a006b..2b63df7a 100644 --- a/lib/Test2/Harness/UI/Controller/Resources.pm +++ b/lib/Test2/Harness/UI/Controller/Resources.pm @@ -85,14 +85,14 @@ sub get_thing { } else { my $uuid = uuid_inflate($id); - if ($uuid && eval { $thing = $run_rs->search({run_id => $uuid})->first }) { + if ($uuid && eval { $thing = $run_rs->find({run_id => $uuid}) }) { $search_args->{run_id} = $uuid; $done_check = sub { return 1 if $thing->complete; return 0; }; } - elsif (($uuid && eval { $thing = $host_rs->search({host_id => $uuid})->first }) || eval { $thing = $host_rs->search({hostname => $id})->first }) { + elsif (($uuid && eval { $thing = $host_rs->find({host_id => $uuid}) }) || eval { $thing = $host_rs->find({hostname => $id}) }) { $search_args->{host_id} = $thing->host_id; } else { diff --git a/lib/Test2/Harness/UI/Controller/Run.pm b/lib/Test2/Harness/UI/Controller/Run.pm index 2381f87c..9f727ff7 100644 --- a/lib/Test2/Harness/UI/Controller/Run.pm +++ b/lib/Test2/Harness/UI/Controller/Run.pm @@ -35,7 +35,7 @@ sub handle { my $it = $route->{id} or die error(404 => 'No id'); $it = uuid_inflate($it) or die error(404 => "Invalid run id"); my $schema = $self->{+CONFIG}->schema; - $run = $schema->resultset('Run')->search({run_id => $it})->first or die error(404 => 'Invalid Run'); + $run = $schema->resultset('Run')->find({run_id => $it}) or die error(404 => 'Invalid Run'); } if (my $act = $route->{action}) { diff --git a/lib/Test2/Harness/UI/Controller/RunField.pm b/lib/Test2/Harness/UI/Controller/RunField.pm index 0ec8a4f2..6a9d0b83 100644 --- a/lib/Test2/Harness/UI/Controller/RunField.pm +++ b/lib/Test2/Harness/UI/Controller/RunField.pm @@ -28,7 +28,7 @@ sub handle { my $it = uuid_inflate($route->{id}) or die error(404 => 'No id'); my $schema = $self->{+CONFIG}->schema; - my $field = $schema->resultset('RunField')->search({run_field_id => $it})->first or die error(404 => 'Invalid Field'); + my $field = $schema->resultset('RunField')->find({run_field_id => $it}) or die error(404 => 'Invalid Field'); if (my $act = $route->{action}) { if ($act eq 'delete') { diff --git a/lib/Test2/Harness/UI/Controller/Stream.pm b/lib/Test2/Harness/UI/Controller/Stream.pm index 2fea967c..ea99d4ff 100644 --- a/lib/Test2/Harness/UI/Controller/Stream.pm +++ b/lib/Test2/Harness/UI/Controller/Stream.pm @@ -118,17 +118,16 @@ sub stream_runs { if ($id) { my $p_rs = $schema->resultset('Project'); - $project //= eval { $p_rs->search({name => $id})->first }; - $project //= eval { $p_rs->search({project_id => $uuid})->first }; + $project //= eval { $p_rs->find({name => $id}) }; + $project //= eval { $p_rs->find({project_id => $uuid}) }; if ($project) { - $uuid = uuid_inflate($project->project_id); $params{search_base} = $params{search_base}->search_rs({'me.project_id' => $project->project_id}); } else { my $u_rs = $schema->resultset('User'); - $user //= eval { $u_rs->search({username => $id})->first }; - $user //= eval { $u_rs->search({user_id => $uuid})->first }; + $user //= eval { $u_rs->find({username => $id}) }; + $user //= eval { $u_rs->find({user_id => $uuid}) }; if ($user) { $uuid = uuid_inflate($user->user_id); diff --git a/lib/Test2/Harness/UI/Controller/View.pm b/lib/Test2/Harness/UI/Controller/View.pm index f3b2c04c..99d3baaf 100644 --- a/lib/Test2/Harness/UI/Controller/View.pm +++ b/lib/Test2/Harness/UI/Controller/View.pm @@ -37,8 +37,8 @@ sub handle { if ($id) { my $p_rs = $schema->resultset('Project'); - $project //= eval { $p_rs->search({name => $id})->first }; - $project //= eval { $p_rs->search({project_id => $uuid})->first }; + $project //= eval { $p_rs->find({name => $id}) }; + $project //= eval { $p_rs->find({project_id => $uuid}) }; if ($project) { $uuid = uuid_inflate($project->project_id); @@ -46,8 +46,8 @@ sub handle { } else { my $u_rs = $schema->resultset('User'); - $user //= eval { $u_rs->search({username => $id})->first }; - $user //= eval { $u_rs->search({user_id => $uuid})->first }; + $user //= eval { $u_rs->find({username => $id}) }; + $user //= eval { $u_rs->find({user_id => $uuid}) }; if ($user) { $uuid = uuid_inflate($user->user_id); @@ -62,7 +62,7 @@ sub handle { if($run_id) { $run_id = uuid_inflate($run_id) or die error(404 => 'Invalid Run'); - my $run = eval { $schema->resultset('Run')->search({run_id => $run_id})->first } or die error(404 => 'Invalid Run'); + my $run = eval { $schema->resultset('Run')->find({run_id => $run_id}) } or die error(404 => 'Invalid Run'); $self->{+TITLE} .= ">" . $run->project->name; } diff --git a/lib/Test2/Harness/UI/RunProcessor.pm b/lib/Test2/Harness/UI/RunProcessor.pm index 66c2e2c5..9d6933a2 100644 --- a/lib/Test2/Harness/UI/RunProcessor.pm +++ b/lib/Test2/Harness/UI/RunProcessor.pm @@ -395,7 +395,7 @@ sub user { my $user_id = $self->{+USER_ID} // confess "No user or user_id specified"; my $schema = $self->schema; - my $user = $schema->resultset('Run')->search({user_id => $user_id})->first; + my $user = $schema->resultset('User')->find({user_id => $user_id}); return $user if $user; confess "Invalid user_id: $user_id"; } @@ -417,7 +417,7 @@ sub project { my $project_id = $self->{+PROJECT_ID} // confess "No project or project_id specified"; my $schema = $self->schema; - my $project = $schema->resultset('Project')->search({project_id => $project_id})->first; + my $project = $schema->resultset('Project')->find({project_id => $project_id}); return $project if $project; confess "Invalid project_id: $project_id"; } diff --git a/lib/Test2/Harness/UI/Util.pm b/lib/Test2/Harness/UI/Util.pm index c31a1921..d3d7e843 100644 --- a/lib/Test2/Harness/UI/Util.pm +++ b/lib/Test2/Harness/UI/Util.pm @@ -51,10 +51,10 @@ sub find_job { return $jobs->search({job_id => $uuid}, {order_by => {'-desc' => 'job_try'}, limit => 1})->first if $try == -1; - return $jobs->search({job_id => $uuid, job_try => $try})->first; + return $jobs->find({job_id => $uuid, job_try => $try}); } - return $jobs->search({job_key => $uuid})->first + return $jobs->find({job_key => $uuid}) || $jobs->search({job_id => $uuid}, {order_by => {'-desc' => 'job_try'}, limit => 1})->first; } From abd3ae2113e6704f849300dc7e4bdca8ac81ff0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dagfinn=20Ilmari=20Manns=C3=A5ker?= Date: Sat, 27 Apr 2024 23:32:49 +0100 Subject: [PATCH 3/7] Use related resultset accessor instead of ->schema->resultset->search --- lib/Test2/Harness/UI/Schema/Overlay/Run.pm | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/lib/Test2/Harness/UI/Schema/Overlay/Run.pm b/lib/Test2/Harness/UI/Schema/Overlay/Run.pm index 64ad282e..0f5ff7c4 100644 --- a/lib/Test2/Harness/UI/Schema/Overlay/Run.pm +++ b/lib/Test2/Harness/UI/Schema/Overlay/Run.pm @@ -93,17 +93,7 @@ sub expanded_coverages { my $self = shift; my ($query) = @_; - my $pick_me = {run_id => $self->run_id}; - - if ($query) { - $query = {'-and' => [$query, $pick_me]}; - } - else { - $query = $pick_me; - } - - my $schema = $self->result_source->schema; - $schema->resultset('Coverage')->search( + $self->coverages->search( $query, { order_by => [qw/test_file_id source_file_id source_sub_id/], From 6be3c44848c68e94555d9fd4bb12c6b48345aaee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dagfinn=20Ilmari=20Manns=C3=A5ker?= Date: Sun, 28 Apr 2024 12:39:52 +0100 Subject: [PATCH 4/7] Use proper operators for IS NOT NULL and IN --- lib/Test2/Harness/UI/Controller/Project.pm | 4 ++-- lib/Test2/Harness/UI/Controller/Stream.pm | 2 +- lib/Test2/Harness/UI/Importer.pm | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/Test2/Harness/UI/Controller/Project.pm b/lib/Test2/Harness/UI/Controller/Project.pm index 0c9f9a78..91aad9d2 100644 --- a/lib/Test2/Harness/UI/Controller/Project.pm +++ b/lib/Test2/Harness/UI/Controller/Project.pm @@ -516,7 +516,7 @@ sub _build_stat_uncovered { my $field = $schema->resultset('RunField')->search( { 'me.name' => 'coverage', - 'me.data' => \'IS NOT NULL', + 'me.data' => { '!=' => undef }, 'run.project_id' => $project->project_id, 'run.has_coverage' => 1, @$users ? (user_id => {'-in' => [map { uuid_deflate($_) } @$users]}) : () @@ -558,7 +558,7 @@ sub _build_stat_coverage { my @items = reverse $schema->resultset('RunField')->search( { 'me.name' => 'coverage', - 'me.data' => \'IS NOT NULL', + 'me.data' => { '!=' => undef }, 'run.project_id' => $project->project_id, 'run.has_coverage' => 1, @$users ? (user_id => {'-in' => [map { uuid_deflate($_) } @$users]}) : () diff --git a/lib/Test2/Harness/UI/Controller/Stream.pm b/lib/Test2/Harness/UI/Controller/Stream.pm index ea99d4ff..b9a71949 100644 --- a/lib/Test2/Harness/UI/Controller/Stream.pm +++ b/lib/Test2/Harness/UI/Controller/Stream.pm @@ -328,7 +328,7 @@ sub stream_set { }; my @ids = $track ? keys %$incomplete : (); - $query = [$query, {"me.$id_field" => {'IN' => \@ids}}] if @ids; + $query = [$query, {"me.$id_field" => { -in => \@ids }}] if @ids; $items = $search_base->search( $query, diff --git a/lib/Test2/Harness/UI/Importer.pm b/lib/Test2/Harness/UI/Importer.pm index b23f6b7d..53787aea 100644 --- a/lib/Test2/Harness/UI/Importer.pm +++ b/lib/Test2/Harness/UI/Importer.pm @@ -33,7 +33,7 @@ sub run { while (!defined($max) || $max--) { $schema->resultset('Run')->search( - {status => 'pending', log_file_id => {'is not' => undef}}, + {status => 'pending', log_file_id => {'!=' => undef}}, {order_by => {-asc => 'added'}, rows => 1}, )->update({status => 'running', worker_id => $worker_id->{string}}); From db601c13b013c9288aee2e1b0ed424d85f494d46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dagfinn=20Ilmari=20Manns=C3=A5ker?= Date: Sun, 28 Apr 2024 12:41:30 +0100 Subject: [PATCH 5/7] Remove redundant indexes Postgres can efficiently use any prefix of an index. --- share/schema/PostgreSQL.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/share/schema/PostgreSQL.sql b/share/schema/PostgreSQL.sql index 398b7b50..32f16b9e 100644 --- a/share/schema/PostgreSQL.sql +++ b/share/schema/PostgreSQL.sql @@ -346,8 +346,6 @@ CREATE TABLE reporting ( CREATE INDEX IF NOT EXISTS reporting_user ON reporting(user_id); CREATE INDEX IF NOT EXISTS reporting_a ON reporting(project_id); CREATE INDEX IF NOT EXISTS reporting_b ON reporting(project_id, user_id); -CREATE INDEX IF NOT EXISTS reporting_c ON reporting(project_id, test_file_id, subtest); -CREATE INDEX IF NOT EXISTS reporting_d ON reporting(project_id, test_file_id, subtest, user_id); CREATE INDEX IF NOT EXISTS reporting_e ON reporting(project_id, test_file_id, subtest, user_id, run_ord); CREATE TABLE resource_batch ( From 741843250bbfa07ee20fa05ae66fad84eccff560 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dagfinn=20Ilmari=20Manns=C3=A5ker?= Date: Sun, 28 Apr 2024 13:21:36 +0100 Subject: [PATCH 6/7] Add indexes for foreign keys used for cascading deletes And reorder one UNIQUE key to have the foreign key column first --- share/schema/PostgreSQL.sql | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/share/schema/PostgreSQL.sql b/share/schema/PostgreSQL.sql index 32f16b9e..12ed22ba 100644 --- a/share/schema/PostgreSQL.sql +++ b/share/schema/PostgreSQL.sql @@ -55,6 +55,7 @@ CREATE TABLE email ( UNIQUE(local, domain) ); +CREATE INDEX IF NOT EXISTS email_user ON email(user_id); CREATE TABLE primary_email ( user_id UUID NOT NULL REFERENCES users(user_id) PRIMARY KEY, @@ -171,7 +172,7 @@ CREATE TABLE sweeps ( run_id UUID NOT NULL REFERENCES runs(run_id), name VARCHAR(255) NOT NULL, - UNIQUE(name, run_id) + UNIQUE(run_id, name) ); CREATE INDEX IF NOT EXISTS sweep_runs ON sweeps(run_id); @@ -284,6 +285,7 @@ CREATE TABLE binaries ( is_image BOOL NOT NULL DEFAULT FALSE, data BYTEA NOT NULL ); +CREATE INDEX IF NOT EXISTS binaries_event ON binaries(event_id); CREATE TABLE source_files ( source_file_id UUID NOT NULL PRIMARY KEY, @@ -344,6 +346,7 @@ CREATE TABLE reporting ( event_id UUID DEFAULT NULL REFERENCES events(event_id) ); CREATE INDEX IF NOT EXISTS reporting_user ON reporting(user_id); +CREATE INDEX IF NOT EXISTS reporting_run ON reporting(run_id); CREATE INDEX IF NOT EXISTS reporting_a ON reporting(project_id); CREATE INDEX IF NOT EXISTS reporting_b ON reporting(project_id, user_id); CREATE INDEX IF NOT EXISTS reporting_e ON reporting(project_id, test_file_id, subtest, user_id, run_ord); @@ -354,6 +357,7 @@ CREATE TABLE resource_batch ( host_id UUID NOT NULL REFERENCES hosts(host_id), stamp TIMESTAMP(4) NOT NULL ); +CREATE INDEX IF NOT EXISTS resource_batch_run ON resource_batch(run_id); CREATE TABLE resources ( resource_id UUID DEFAULT UUID_GENERATE_V4() PRIMARY KEY, From 3f8f2446276952378c329d2ce655cf4becddc9d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dagfinn=20Ilmari=20Manns=C3=A5ker?= Date: Sun, 28 Apr 2024 13:24:54 +0100 Subject: [PATCH 7/7] Combine is_subtest index with job_key It's only queries together, so having it in the same index is more efficient than BitmapAnd-ing two indexes. --- share/schema/PostgreSQL.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/share/schema/PostgreSQL.sql b/share/schema/PostgreSQL.sql index 12ed22ba..27ee30c2 100644 --- a/share/schema/PostgreSQL.sql +++ b/share/schema/PostgreSQL.sql @@ -272,10 +272,9 @@ CREATE TABLE events ( orphan JSONB DEFAULT NULL, orphan_line BIGINT DEFAULT NULL ); -CREATE INDEX IF NOT EXISTS event_job ON events(job_key); +CREATE INDEX IF NOT EXISTS event_job ON events(job_key, is_subtest); CREATE INDEX IF NOT EXISTS event_trace ON events(trace_id); CREATE INDEX IF NOT EXISTS event_parent ON events(parent_id); -CREATE INDEX IF NOT EXISTS is_subtest ON events(is_subtest); CREATE TABLE binaries ( binary_id UUID NOT NULL PRIMARY KEY,