From fc4f46feff9bd2baecdcb8b65e00b4f02b7bfd9b Mon Sep 17 00:00:00 2001 From: xsalonx Date: Sat, 12 Oct 2024 22:40:22 +0200 Subject: [PATCH 01/39] add run bounds to gaq eff. periods --- lib/database/repositories/QcFlagRepository.js | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lib/database/repositories/QcFlagRepository.js b/lib/database/repositories/QcFlagRepository.js index 04b94b12bf..14cb417de2 100644 --- a/lib/database/repositories/QcFlagRepository.js +++ b/lib/database/repositories/QcFlagRepository.js @@ -27,6 +27,24 @@ const GAQ_PERIODS_VIEW = ` ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING ) AS \`to\` FROM ( + ( + SELECT gaqd.data_pass_id, + gaqd.run_number, + COALESCE(UNIX_TIMESTAMP(COALESCE(first_tf_timestamp, time_trg_start, time_o2_start)), 0) AS timestamp + FROM global_aggregated_quality_detectors AS gaqd + INNER JOIN runs as r + ON gaqd.run_number = r.run_number + ) + UNION + ( + SELECT gaqd.data_pass_id, + gaqd.run_number, + UNIX_TIMESTAMP(COALESCE(last_tf_timestamp, time_trg_end, time_o2_end, NOW())) AS timestamp + FROM global_aggregated_quality_detectors AS gaqd + INNER JOIN runs as r + ON gaqd.run_number = r.run_number + ) + UNION ( SELECT gaqd.data_pass_id, gaqd.run_number, From 76848cb71661b99bae12862d4aa8199f611c7c46 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Sat, 12 Oct 2024 23:12:55 +0200 Subject: [PATCH 02/39] WIP --- lib/database/repositories/QcFlagRepository.js | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/database/repositories/QcFlagRepository.js b/lib/database/repositories/QcFlagRepository.js index 14cb417de2..6e4e92b746 100644 --- a/lib/database/repositories/QcFlagRepository.js +++ b/lib/database/repositories/QcFlagRepository.js @@ -118,6 +118,7 @@ class QcFlagRepository extends Repository { * @return {Promise} Resolves with the GAQ periods */ async findGaqPeriods(dataPassId, runNumber) { + console.log('EXEXEXEX') const query = ` SELECT gaq_periods.data_pass_id AS dataPassId, @@ -126,12 +127,15 @@ class QcFlagRepository extends Repository { IF(gaq_periods.\`to\` = UNIX_TIMESTAMP(NOW()), null, gaq_periods.\`to\` * 1000) AS \`to\`, group_concat(qcf.id) AS contributingFlagIds - FROM quality_control_flags AS qcf + FROM (${GAQ_PERIODS_VIEW}) AS gaq_periods + INNER JOIN data_pass_quality_control_flag AS dpqcf + ON gaq_periods.data_pass_id = dpqcf.data_pass_id + INNER JOIN quality_control_flags AS qcf + ON dpqcf.quality_control_flag_id = qcf.id + AND qcf.run_number = gaq_periods.run_number INNER JOIN quality_control_flag_effective_periods AS qcfep ON qcf.id = qcfep.flag_id - INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id - INNER JOIN (${GAQ_PERIODS_VIEW}) AS gaq_periods ON gaq_periods.data_pass_id = dpqcf.data_pass_id - INNER JOIN global_aggregated_quality_detectors AS gaqd + LEFT JOIN global_aggregated_quality_detectors AS gaqd ON gaqd.data_pass_id = gaq_periods.data_pass_id AND gaqd.run_number = gaq_periods.run_number AND gaqd.detector_id = qcf.detector_id @@ -139,7 +143,8 @@ class QcFlagRepository extends Repository { AND (qcfep.\`from\` IS NULL OR UNIX_TIMESTAMP(qcfep.\`from\`) <= gaq_periods.\`from\`) AND (qcfep.\`to\` IS NULL OR gaq_periods.\`to\` <= UNIX_TIMESTAMP(qcfep.\`to\`)) - WHERE gaq_periods.data_pass_id = ${dataPassId} + WHERE IF(gaq_periods.\`to\` = UNIX_TIMESTAMP(NOW()), null, gaq_periods.\`to\` * 1000) IS NOT NULL + AND gaq_periods.data_pass_id = ${dataPassId} ${runNumber ? `AND gaq_periods.run_number = ${runNumber}` : ''} GROUP BY gaq_periods.run_number, From e95b06d11f24679ffc773ea249d6af142cf67f6d Mon Sep 17 00:00:00 2001 From: xsalonx Date: Sat, 12 Oct 2024 23:19:39 +0200 Subject: [PATCH 03/39] empty ranges work --- lib/database/repositories/QcFlagRepository.js | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/database/repositories/QcFlagRepository.js b/lib/database/repositories/QcFlagRepository.js index 6e4e92b746..ccf8bb813b 100644 --- a/lib/database/repositories/QcFlagRepository.js +++ b/lib/database/repositories/QcFlagRepository.js @@ -125,23 +125,23 @@ class QcFlagRepository extends Repository { gaq_periods.run_number AS runNumber, IF(gaq_periods.\`from\` = 0, null, gaq_periods.\`from\` * 1000) AS \`from\`, IF(gaq_periods.\`to\` = UNIX_TIMESTAMP(NOW()), null, gaq_periods.\`to\` * 1000) AS \`to\`, - group_concat(qcf.id) AS contributingFlagIds + group_concat(qcfep.flag_id) AS contributingFlagIds FROM (${GAQ_PERIODS_VIEW}) AS gaq_periods + INNER JOIN global_aggregated_quality_detectors AS gaqd + ON gaqd.data_pass_id = gaq_periods.data_pass_id + AND gaqd.run_number = gaq_periods.run_number INNER JOIN data_pass_quality_control_flag AS dpqcf ON gaq_periods.data_pass_id = dpqcf.data_pass_id INNER JOIN quality_control_flags AS qcf ON dpqcf.quality_control_flag_id = qcf.id AND qcf.run_number = gaq_periods.run_number - INNER JOIN quality_control_flag_effective_periods AS qcfep + AND gaqd.detector_id = qcf.detector_id + AND gaq_periods.run_number = qcf.run_number + LEFT JOIN quality_control_flag_effective_periods AS qcfep ON qcf.id = qcfep.flag_id - LEFT JOIN global_aggregated_quality_detectors AS gaqd - ON gaqd.data_pass_id = gaq_periods.data_pass_id - AND gaqd.run_number = gaq_periods.run_number - AND gaqd.detector_id = qcf.detector_id - AND gaq_periods.run_number = qcf.run_number - AND (qcfep.\`from\` IS NULL OR UNIX_TIMESTAMP(qcfep.\`from\`) <= gaq_periods.\`from\`) - AND (qcfep.\`to\` IS NULL OR gaq_periods.\`to\` <= UNIX_TIMESTAMP(qcfep.\`to\`)) + AND (qcfep.\`from\` IS NULL OR UNIX_TIMESTAMP(qcfep.\`from\`) <= gaq_periods.\`from\`) + AND (qcfep.\`to\` IS NULL OR gaq_periods.\`to\` <= UNIX_TIMESTAMP(qcfep.\`to\`)) WHERE IF(gaq_periods.\`to\` = UNIX_TIMESTAMP(NOW()), null, gaq_periods.\`to\` * 1000) IS NOT NULL AND gaq_periods.data_pass_id = ${dataPassId} @@ -165,7 +165,7 @@ class QcFlagRepository extends Repository { runNumber, from, to, - contributingFlagIds: contributingFlagIds.split(',').map((id) => parseInt(id, 10)), + contributingFlagIds: contributingFlagIds ? contributingFlagIds.split(',').map((id) => parseInt(id, 10)) : [], })); } From 782284ca2effae46995f4acd96ea46c10f79792a Mon Sep 17 00:00:00 2001 From: xsalonx Date: Sat, 12 Oct 2024 23:25:11 +0200 Subject: [PATCH 04/39] gaq summary WIP --- lib/database/repositories/QcFlagRepository.js | 36 ++++++++++--------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/lib/database/repositories/QcFlagRepository.js b/lib/database/repositories/QcFlagRepository.js index ccf8bb813b..ce53136c0c 100644 --- a/lib/database/repositories/QcFlagRepository.js +++ b/lib/database/repositories/QcFlagRepository.js @@ -118,7 +118,6 @@ class QcFlagRepository extends Repository { * @return {Promise} Resolves with the GAQ periods */ async findGaqPeriods(dataPassId, runNumber) { - console.log('EXEXEXEX') const query = ` SELECT gaq_periods.data_pass_id AS dataPassId, @@ -190,24 +189,27 @@ class QcFlagRepository extends Repository { GROUP_CONCAT( DISTINCT qcfv.flag_id ) AS verifiedFlagsList, GROUP_CONCAT( DISTINCT qcf.id ) AS flagsList - FROM quality_control_flags AS qcf - INNER JOIN quality_control_flag_types AS qcft - ON qcft.id = qcf.flag_type_id - LEFT JOIN quality_control_flag_verifications AS qcfv - ON qcfv.flag_id = qcf.id - INNER JOIN quality_control_flag_effective_periods AS qcfep - ON qcf.id = qcfep.flag_id - INNER JOIN data_pass_quality_control_flag AS dpqcf - ON dpqcf.quality_control_flag_id = qcf.id - INNER JOIN (${GAQ_PERIODS_VIEW}) AS gaq_periods - ON gaq_periods.data_pass_id = dpqcf.data_pass_id + FROM (${GAQ_PERIODS_VIEW}) AS gaq_periods INNER JOIN global_aggregated_quality_detectors AS gaqd ON gaqd.data_pass_id = gaq_periods.data_pass_id - AND gaqd.run_number = gaq_periods.run_number - AND gaqd.detector_id = qcf.detector_id - AND gaq_periods.run_number = qcf.run_number - AND (qcfep.\`from\` IS NULL OR UNIX_TIMESTAMP(qcfep.\`from\`) <= gaq_periods.\`from\`) - AND (qcfep.\`to\` IS NULL OR gaq_periods.\`to\` <= UNIX_TIMESTAMP(qcfep.\`to\`)) + AND gaqd.run_number = gaq_periods.run_number + INNER JOIN data_pass_quality_control_flag AS dpqcf + ON gaq_periods.data_pass_id = dpqcf.data_pass_id + INNER JOIN quality_control_flags AS qcf + ON dpqcf.quality_control_flag_id = qcf.id + AND qcf.run_number = gaq_periods.run_number + AND gaqd.detector_id = qcf.detector_id + AND gaq_periods.run_number = qcf.run_number + LEFT JOIN quality_control_flag_effective_periods AS qcfep + ON qcf.id = qcfep.flag_id + AND (qcfep.\`from\` IS NULL OR UNIX_TIMESTAMP(qcfep.\`from\`) <= gaq_periods.\`from\`) + AND (qcfep.\`to\` IS NULL OR gaq_periods.\`to\` <= UNIX_TIMESTAMP(qcfep.\`to\`)) + + LEFT JOIN quality_control_flag_verifications AS qcfv + ON qcfv.flag_id = qcf.id + INNER JOIN quality_control_flag_types AS qcft + ON qcft.id = qcf.flag_type_id + GROUP BY gaq_periods.data_pass_id, From 4978810b9461048cb5258d6d44b684512ccfa6b6 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Sat, 12 Oct 2024 23:41:03 +0200 Subject: [PATCH 05/39] WIP --- lib/database/repositories/QcFlagRepository.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/database/repositories/QcFlagRepository.js b/lib/database/repositories/QcFlagRepository.js index ce53136c0c..c26f930ec5 100644 --- a/lib/database/repositories/QcFlagRepository.js +++ b/lib/database/repositories/QcFlagRepository.js @@ -187,7 +187,7 @@ class QcFlagRepository extends Repository { SUM(IF(qcft.monte_carlo_reproducible AND :mcReproducibleAsNotBad, false, qcft.bad)) >= 1 AS bad, SUM(qcft.bad) = SUM(qcft.monte_carlo_reproducible) AND SUM(qcft.monte_carlo_reproducible) AS mcReproducible, GROUP_CONCAT( DISTINCT qcfv.flag_id ) AS verifiedFlagsList, - GROUP_CONCAT( DISTINCT qcf.id ) AS flagsList + GROUP_CONCAT( DISTINCT qcfep.flag_id ) AS flagsList FROM (${GAQ_PERIODS_VIEW}) AS gaq_periods INNER JOIN global_aggregated_quality_detectors AS gaqd @@ -268,6 +268,7 @@ class QcFlagRepository extends Repository { effectivePeriods.bad `; + console.log('TOBEC', effectivePeriodsWithTypeSubQuery) const [rows] = await this.model.sequelize.query(query, { replacements: { dataPassId, mcReproducibleAsNotBad } }); return rows.map(({ runNumber, From 113c6aa2a701abe6e7c4ca6ac3317c12f70e1066 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Mon, 14 Oct 2024 11:39:38 +0200 Subject: [PATCH 06/39] work --- lib/database/repositories/QcFlagRepository.js | 14 +++++++++++--- .../qualityControlFlag/QcFlagService.js | 18 +++++++++++------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/lib/database/repositories/QcFlagRepository.js b/lib/database/repositories/QcFlagRepository.js index c26f930ec5..46bdeca9bf 100644 --- a/lib/database/repositories/QcFlagRepository.js +++ b/lib/database/repositories/QcFlagRepository.js @@ -184,7 +184,15 @@ class QcFlagRepository extends Repository { gaq_periods.run_number AS runNumber, IF(gaq_periods.\`from\` = 0, null, gaq_periods.\`from\`) AS \`from\`, IF(gaq_periods.\`to\` = UNIX_TIMESTAMP(NOW()), null, gaq_periods.\`to\`) AS \`to\`, - SUM(IF(qcft.monte_carlo_reproducible AND :mcReproducibleAsNotBad, false, qcft.bad)) >= 1 AS bad, + IF(( + SELECT COUNT(*) + FROM global_aggregated_quality_detectors AS gaqd + WHERE gaqd.data_pass_id = gaq_periods.data_pass_id + AND gaqd.run_number = gaq_periods.run_number + ) != COUNT( DISTINCT qcfep.flag_id ), + null, + SUM(IF(qcft.monte_carlo_reproducible AND false, false, qcft.bad)) >= 1 + ) AS bad, SUM(qcft.bad) = SUM(qcft.monte_carlo_reproducible) AND SUM(qcft.monte_carlo_reproducible) AS mcReproducible, GROUP_CONCAT( DISTINCT qcfv.flag_id ) AS verifiedFlagsList, GROUP_CONCAT( DISTINCT qcfep.flag_id ) AS flagsList @@ -210,6 +218,7 @@ class QcFlagRepository extends Repository { INNER JOIN quality_control_flag_types AS qcft ON qcft.id = qcf.flag_type_id + WHERE gaq_periods.\`to\` IS NOT null GROUP BY gaq_periods.data_pass_id, @@ -268,7 +277,6 @@ class QcFlagRepository extends Repository { effectivePeriods.bad `; - console.log('TOBEC', effectivePeriodsWithTypeSubQuery) const [rows] = await this.model.sequelize.query(query, { replacements: { dataPassId, mcReproducibleAsNotBad } }); return rows.map(({ runNumber, @@ -282,7 +290,7 @@ class QcFlagRepository extends Repository { bad, effectiveRunCoverage: parseFloat(effectiveRunCoverage) || null, mcReproducible: Boolean(mcReproducible), - flagsIds: [...new Set(flagsList.split(','))], + flagsIds: flagsList ? [...new Set(flagsList.split(','))] : [], verifiedFlagsIds: verifiedFlagsList ? [...new Set(verifiedFlagsList.split(','))] : [], })); } diff --git a/lib/server/services/qualityControlFlag/QcFlagService.js b/lib/server/services/qualityControlFlag/QcFlagService.js index ebb106633b..9fae60f5e4 100644 --- a/lib/server/services/qualityControlFlag/QcFlagService.js +++ b/lib/server/services/qualityControlFlag/QcFlagService.js @@ -65,14 +65,16 @@ const validateUserDetectorAccess = (userRoles, detectorName) => { /** * @typedef RunDetectorQcSummary - * @property {number} badEffectiveRunCoverage - fraction of run's data, marked explicitly with bad QC flag - * @property {number} explicitlyNotBadEffectiveRunCoverage - fraction of run's data, marked explicitly with good QC flag + * @property {number} badEffectiveRunCoverage - fraction of run's data marked explicitly with bad QC flag + * @property {number} explicitlyNotBadEffectiveRunCoverage - fraction of run's data marked explicitly with good QC flag + * @property {number} qualityNotDefinedEffectiveRunCoverage - fraction of run's data NOT marked with any QC flag * @property {number} missingVerificationsCount - number of not verified QC flags which are not discarded * @property {boolean} mcReproducible - states whether some Limited Acceptance MC Reproducible flag was assigned */ const QC_SUMMARY_PROPERTIES = { badEffectiveRunCoverage: 'badEffectiveRunCoverage', explicitlyNotBadEffectiveRunCoverage: 'explicitlyNotBadEffectiveRunCoverage', + qualityNotDefinedEffectiveRunCoverage: 'qualityNotDefinedEffectiveRunCoverage', missingVerificationsCount: 'missingVerificationsCount', mcReproducible: 'mcReproducible', }; @@ -702,15 +704,17 @@ class QcFlagService { mcReproducible, } = partialSummaryUnit; - if (bad) { + if (bad === null) { + summaryUnit[QC_SUMMARY_PROPERTIES.qualityNotDefinedEffectiveRunCoverage] = effectiveRunCoverage; + } else if (bad) { summaryUnit[QC_SUMMARY_PROPERTIES.badEffectiveRunCoverage] = effectiveRunCoverage; - summaryUnit[QC_SUMMARY_PROPERTIES.mcReproducible] = - mcReproducible || summaryUnit[QC_SUMMARY_PROPERTIES.mcReproducible]; } else { summaryUnit[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] = effectiveRunCoverage; - summaryUnit[QC_SUMMARY_PROPERTIES.mcReproducible] = - mcReproducible || summaryUnit[QC_SUMMARY_PROPERTIES.mcReproducible]; } + + summaryUnit[QC_SUMMARY_PROPERTIES.mcReproducible] = + mcReproducible || summaryUnit[QC_SUMMARY_PROPERTIES.mcReproducible]; + if (summaryUnit[QC_SUMMARY_PROPERTIES.badEffectiveRunCoverage] === undefined) { summaryUnit[QC_SUMMARY_PROPERTIES.badEffectiveRunCoverage] = 0; } From 3b39854c168bca97381054bc99cccfb2ae890725 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Mon, 14 Oct 2024 11:41:20 +0200 Subject: [PATCH 07/39] simplify --- lib/database/repositories/QcFlagRepository.js | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/database/repositories/QcFlagRepository.js b/lib/database/repositories/QcFlagRepository.js index 46bdeca9bf..db34f80907 100644 --- a/lib/database/repositories/QcFlagRepository.js +++ b/lib/database/repositories/QcFlagRepository.js @@ -184,12 +184,7 @@ class QcFlagRepository extends Repository { gaq_periods.run_number AS runNumber, IF(gaq_periods.\`from\` = 0, null, gaq_periods.\`from\`) AS \`from\`, IF(gaq_periods.\`to\` = UNIX_TIMESTAMP(NOW()), null, gaq_periods.\`to\`) AS \`to\`, - IF(( - SELECT COUNT(*) - FROM global_aggregated_quality_detectors AS gaqd - WHERE gaqd.data_pass_id = gaq_periods.data_pass_id - AND gaqd.run_number = gaq_periods.run_number - ) != COUNT( DISTINCT qcfep.flag_id ), + IF(COUNT( gaqd.detector_id ) != COUNT( DISTINCT qcfep.flag_id ), null, SUM(IF(qcft.monte_carlo_reproducible AND false, false, qcft.bad)) >= 1 ) AS bad, From 8ef35d69def9f80e384832e97e9ccf25d6bfbd24 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Mon, 14 Oct 2024 11:42:06 +0200 Subject: [PATCH 08/39] fix --- lib/database/repositories/QcFlagRepository.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/database/repositories/QcFlagRepository.js b/lib/database/repositories/QcFlagRepository.js index db34f80907..852a825ab9 100644 --- a/lib/database/repositories/QcFlagRepository.js +++ b/lib/database/repositories/QcFlagRepository.js @@ -186,7 +186,7 @@ class QcFlagRepository extends Repository { IF(gaq_periods.\`to\` = UNIX_TIMESTAMP(NOW()), null, gaq_periods.\`to\`) AS \`to\`, IF(COUNT( gaqd.detector_id ) != COUNT( DISTINCT qcfep.flag_id ), null, - SUM(IF(qcft.monte_carlo_reproducible AND false, false, qcft.bad)) >= 1 + SUM(IF(qcft.monte_carlo_reproducible AND :mcReproducibleAsNotBad, false, qcft.bad)) >= 1 ) AS bad, SUM(qcft.bad) = SUM(qcft.monte_carlo_reproducible) AND SUM(qcft.monte_carlo_reproducible) AS mcReproducible, GROUP_CONCAT( DISTINCT qcfv.flag_id ) AS verifiedFlagsList, From ec92f8d92f29500538e44a3c6b0e5baf6990f925 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Mon, 14 Oct 2024 11:48:50 +0200 Subject: [PATCH 09/39] gaq summary disabled when missing coverage --- .../views/Runs/RunPerDataPass/RunsPerDataPassOverviewPage.js | 2 +- lib/server/services/qualityControlFlag/QcFlagService.js | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/public/views/Runs/RunPerDataPass/RunsPerDataPassOverviewPage.js b/lib/public/views/Runs/RunPerDataPass/RunsPerDataPassOverviewPage.js index 17812d641e..7b872ac334 100644 --- a/lib/public/views/Runs/RunPerDataPass/RunsPerDataPassOverviewPage.js +++ b/lib/public/views/Runs/RunPerDataPass/RunsPerDataPassOverviewPage.js @@ -167,7 +167,7 @@ export const RunsPerDataPassOverviewPage = ({ visible: true, format: (_, { runNumber }) => { const runGaqSummary = gaqSummary[runNumber]; - const gaqDisplay = runGaqSummary + const gaqDisplay = runGaqSummary?.qualityNotDefinedEffectiveRunCoverage === 0 ? getQcSummaryDisplay(runGaqSummary, { mcReproducibleAsNotBad }) : h('button.btn.btn-primary.w-100', 'GAQ'); diff --git a/lib/server/services/qualityControlFlag/QcFlagService.js b/lib/server/services/qualityControlFlag/QcFlagService.js index 9fae60f5e4..fbc965eb09 100644 --- a/lib/server/services/qualityControlFlag/QcFlagService.js +++ b/lib/server/services/qualityControlFlag/QcFlagService.js @@ -715,6 +715,9 @@ class QcFlagService { summaryUnit[QC_SUMMARY_PROPERTIES.mcReproducible] = mcReproducible || summaryUnit[QC_SUMMARY_PROPERTIES.mcReproducible]; + if (summaryUnit[QC_SUMMARY_PROPERTIES.qualityNotDefinedEffectiveRunCoverage] === undefined) { + summaryUnit[QC_SUMMARY_PROPERTIES.qualityNotDefinedEffectiveRunCoverage] = 0; + } if (summaryUnit[QC_SUMMARY_PROPERTIES.badEffectiveRunCoverage] === undefined) { summaryUnit[QC_SUMMARY_PROPERTIES.badEffectiveRunCoverage] = 0; } From ff0930144e37016c383c49ccca3ab03e3bedb5fb Mon Sep 17 00:00:00 2001 From: xsalonx Date: Tue, 19 Nov 2024 20:43:36 +0100 Subject: [PATCH 10/39] add comments --- lib/database/repositories/QcFlagRepository.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/database/repositories/QcFlagRepository.js b/lib/database/repositories/QcFlagRepository.js index 852a825ab9..5cc1d84a94 100644 --- a/lib/database/repositories/QcFlagRepository.js +++ b/lib/database/repositories/QcFlagRepository.js @@ -27,6 +27,7 @@ const GAQ_PERIODS_VIEW = ` ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING ) AS \`to\` FROM ( + -- Two selects for runs' timestamps (in case QC flag doesn't tart at run's start or end at run's end ) ( SELECT gaqd.data_pass_id, gaqd.run_number, @@ -45,6 +46,7 @@ const GAQ_PERIODS_VIEW = ` ON gaqd.run_number = r.run_number ) UNION + -- Two selectes for timestamps of QC flags ( SELECT gaqd.data_pass_id, gaqd.run_number, From b9b163c5eaa6953d6ff9ea6fcfb9449aa36bbedf Mon Sep 17 00:00:00 2001 From: xsalonx Date: Tue, 19 Nov 2024 20:54:05 +0100 Subject: [PATCH 11/39] add comments --- lib/database/repositories/QcFlagRepository.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/database/repositories/QcFlagRepository.js b/lib/database/repositories/QcFlagRepository.js index 5cc1d84a94..e838b399e1 100644 --- a/lib/database/repositories/QcFlagRepository.js +++ b/lib/database/repositories/QcFlagRepository.js @@ -207,6 +207,7 @@ class QcFlagRepository extends Repository { AND gaq_periods.run_number = qcf.run_number LEFT JOIN quality_control_flag_effective_periods AS qcfep ON qcf.id = qcfep.flag_id + -- Aggregate flags by timestamps relation AND (qcfep.\`from\` IS NULL OR UNIX_TIMESTAMP(qcfep.\`from\`) <= gaq_periods.\`from\`) AND (qcfep.\`to\` IS NULL OR gaq_periods.\`to\` <= UNIX_TIMESTAMP(qcfep.\`to\`)) From ae4d996b548757eb05491b120a50680e32f3b144 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Tue, 19 Nov 2024 22:58:40 +0100 Subject: [PATCH 12/39] remove tf timestamps form query --- lib/database/repositories/QcFlagRepository.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/database/repositories/QcFlagRepository.js b/lib/database/repositories/QcFlagRepository.js index e838b399e1..cec3372e9e 100644 --- a/lib/database/repositories/QcFlagRepository.js +++ b/lib/database/repositories/QcFlagRepository.js @@ -31,7 +31,7 @@ const GAQ_PERIODS_VIEW = ` ( SELECT gaqd.data_pass_id, gaqd.run_number, - COALESCE(UNIX_TIMESTAMP(COALESCE(first_tf_timestamp, time_trg_start, time_o2_start)), 0) AS timestamp + COALESCE(UNIX_TIMESTAMP(COALESCE(time_trg_start, time_o2_start)), 0) AS timestamp FROM global_aggregated_quality_detectors AS gaqd INNER JOIN runs as r ON gaqd.run_number = r.run_number @@ -40,7 +40,7 @@ const GAQ_PERIODS_VIEW = ` ( SELECT gaqd.data_pass_id, gaqd.run_number, - UNIX_TIMESTAMP(COALESCE(last_tf_timestamp, time_trg_end, time_o2_end, NOW())) AS timestamp + UNIX_TIMESTAMP(COALESCE(time_trg_end, time_o2_end, NOW())) AS timestamp FROM global_aggregated_quality_detectors AS gaqd INNER JOIN runs as r ON gaqd.run_number = r.run_number From baa9bda037583e6b3cd364704d6e6005c8fd2ce3 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Tue, 19 Nov 2024 23:04:07 +0100 Subject: [PATCH 13/39] fix joins --- lib/database/repositories/QcFlagRepository.js | 37 ++++++++++--------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/lib/database/repositories/QcFlagRepository.js b/lib/database/repositories/QcFlagRepository.js index cec3372e9e..a7161c95eb 100644 --- a/lib/database/repositories/QcFlagRepository.js +++ b/lib/database/repositories/QcFlagRepository.js @@ -132,15 +132,17 @@ class QcFlagRepository extends Repository { INNER JOIN global_aggregated_quality_detectors AS gaqd ON gaqd.data_pass_id = gaq_periods.data_pass_id AND gaqd.run_number = gaq_periods.run_number - INNER JOIN data_pass_quality_control_flag AS dpqcf + LEFT JOIN ( + data_pass_quality_control_flag AS dpqcf + INNER JOIN quality_control_flags AS qcf + ON dpqcf.quality_control_flag_id = qcf.id + INNER JOIN quality_control_flag_effective_periods AS qcfep + ON qcf.id = qcfep.flag_id + ) ON gaq_periods.data_pass_id = dpqcf.data_pass_id - INNER JOIN quality_control_flags AS qcf - ON dpqcf.quality_control_flag_id = qcf.id AND qcf.run_number = gaq_periods.run_number AND gaqd.detector_id = qcf.detector_id AND gaq_periods.run_number = qcf.run_number - LEFT JOIN quality_control_flag_effective_periods AS qcfep - ON qcf.id = qcfep.flag_id AND (qcfep.\`from\` IS NULL OR UNIX_TIMESTAMP(qcfep.\`from\`) <= gaq_periods.\`from\`) AND (qcfep.\`to\` IS NULL OR gaq_periods.\`to\` <= UNIX_TIMESTAMP(qcfep.\`to\`)) @@ -186,7 +188,7 @@ class QcFlagRepository extends Repository { gaq_periods.run_number AS runNumber, IF(gaq_periods.\`from\` = 0, null, gaq_periods.\`from\`) AS \`from\`, IF(gaq_periods.\`to\` = UNIX_TIMESTAMP(NOW()), null, gaq_periods.\`to\`) AS \`to\`, - IF(COUNT( gaqd.detector_id ) != COUNT( DISTINCT qcfep.flag_id ), + IF(COUNT( DISTINCT gaqd.detector_id ) != COUNT( DISTINCT qcfep.flag_id ), null, SUM(IF(qcft.monte_carlo_reproducible AND :mcReproducibleAsNotBad, false, qcft.bad)) >= 1 ) AS bad, @@ -198,23 +200,24 @@ class QcFlagRepository extends Repository { INNER JOIN global_aggregated_quality_detectors AS gaqd ON gaqd.data_pass_id = gaq_periods.data_pass_id AND gaqd.run_number = gaq_periods.run_number - INNER JOIN data_pass_quality_control_flag AS dpqcf + + LEFT JOIN ( + data_pass_quality_control_flag AS dpqcf + INNER JOIN quality_control_flags AS qcf + ON dpqcf.quality_control_flag_id = qcf.id + INNER JOIN quality_control_flag_types AS qcft + ON qcft.id = qcf.flag_type_id + INNER JOIN quality_control_flag_effective_periods AS qcfep + ON qcf.id = qcfep.flag_id + LEFT JOIN quality_control_flag_verifications AS qcfv + ON qcfv.flag_id = qcf.id + ) ON gaq_periods.data_pass_id = dpqcf.data_pass_id - INNER JOIN quality_control_flags AS qcf - ON dpqcf.quality_control_flag_id = qcf.id AND qcf.run_number = gaq_periods.run_number AND gaqd.detector_id = qcf.detector_id AND gaq_periods.run_number = qcf.run_number - LEFT JOIN quality_control_flag_effective_periods AS qcfep - ON qcf.id = qcfep.flag_id - -- Aggregate flags by timestamps relation AND (qcfep.\`from\` IS NULL OR UNIX_TIMESTAMP(qcfep.\`from\`) <= gaq_periods.\`from\`) AND (qcfep.\`to\` IS NULL OR gaq_periods.\`to\` <= UNIX_TIMESTAMP(qcfep.\`to\`)) - - LEFT JOIN quality_control_flag_verifications AS qcfv - ON qcfv.flag_id = qcf.id - INNER JOIN quality_control_flag_types AS qcft - ON qcft.id = qcf.flag_type_id WHERE gaq_periods.\`to\` IS NOT null From 5880df0fe79bfec387682da540e91b5299d5fab1 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Tue, 19 Nov 2024 23:25:10 +0100 Subject: [PATCH 14/39] a --- .../services/qualityControlFlag/QcFlagService.js | 11 +++++++---- .../services/qualityControlFlag/QcFlagService.test.js | 7 +++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/lib/server/services/qualityControlFlag/QcFlagService.js b/lib/server/services/qualityControlFlag/QcFlagService.js index fc77cc4dad..489cb0b279 100644 --- a/lib/server/services/qualityControlFlag/QcFlagService.js +++ b/lib/server/services/qualityControlFlag/QcFlagService.js @@ -726,14 +726,17 @@ class QcFlagService { summaryUnit[QC_SUMMARY_PROPERTIES.mcReproducible] = mcReproducible || summaryUnit[QC_SUMMARY_PROPERTIES.mcReproducible]; - if (summaryUnit[QC_SUMMARY_PROPERTIES.qualityNotDefinedEffectiveRunCoverage] === undefined) { - summaryUnit[QC_SUMMARY_PROPERTIES.qualityNotDefinedEffectiveRunCoverage] = 0; + if (summaryUnit[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] === undefined) { + summaryUnit[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] = 0; } if (summaryUnit[QC_SUMMARY_PROPERTIES.badEffectiveRunCoverage] === undefined) { summaryUnit[QC_SUMMARY_PROPERTIES.badEffectiveRunCoverage] = 0; } - if (summaryUnit[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] === undefined) { - summaryUnit[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] = 0; + if (summaryUnit[QC_SUMMARY_PROPERTIES.qualityNotDefinedEffectiveRunCoverage] === undefined) { + summaryUnit[QC_SUMMARY_PROPERTIES.qualityNotDefinedEffectiveRunCoverage] = 1 - ( + summaryUnit[QC_SUMMARY_PROPERTIES.badEffectiveRunCoverage] + + summaryUnit[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] + ); } } diff --git a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js index 18630942f1..4624340687 100644 --- a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js +++ b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js @@ -152,12 +152,14 @@ module.exports = () => { mcReproducible: true, badEffectiveRunCoverage: 0.3333, explicitlyNotBadEffectiveRunCoverage: 0, + qualityNotDefinedEffectiveRunCoverage: 0.6667000000000001, }, 16: { badEffectiveRunCoverage: 0, explicitlyNotBadEffectiveRunCoverage: 1, mcReproducible: false, missingVerificationsCount: 1, + qualityNotDefinedEffectiveRunCoverage: 0, }, }, }); @@ -171,12 +173,14 @@ module.exports = () => { mcReproducible: true, badEffectiveRunCoverage: 0.1111, explicitlyNotBadEffectiveRunCoverage: 0.2222, + qualityNotDefinedEffectiveRunCoverage: 0.7778, }, 16: { badEffectiveRunCoverage: 0, explicitlyNotBadEffectiveRunCoverage: 1, mcReproducible: false, missingVerificationsCount: 1, + qualityNotDefinedEffectiveRunCoverage: 0, }, }, }); @@ -215,6 +219,7 @@ module.exports = () => { mcReproducible: false, badEffectiveRunCoverage: 0.0769, explicitlyNotBadEffectiveRunCoverage: 0, + qualityNotDefinedEffectiveRunCoverage: 0.9231, }, }, }); @@ -249,6 +254,7 @@ module.exports = () => { mcReproducible: false, badEffectiveRunCoverage: 0.7222, explicitlyNotBadEffectiveRunCoverage: 0, + qualityNotDefinedEffectiveRunCoverage: 0.27780000000000005, }, }, }); @@ -267,6 +273,7 @@ module.exports = () => { mcReproducible: false, badEffectiveRunCoverage: 0.1667, explicitlyNotBadEffectiveRunCoverage: 0.8333, + qualityNotDefinedEffectiveRunCoverage: 0.16669999999999996, }, // ITS From 27e3a7acfa940465c95b2d85d20169242b3d7d7c Mon Sep 17 00:00:00 2001 From: xsalonx Date: Wed, 20 Nov 2024 08:58:03 +0100 Subject: [PATCH 15/39] add seeders --- .../seeders/20240404100811-qc-flags.js | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/lib/database/seeders/20240404100811-qc-flags.js b/lib/database/seeders/20240404100811-qc-flags.js index a92578eb29..e98e96f3f3 100644 --- a/lib/database/seeders/20240404100811-qc-flags.js +++ b/lib/database/seeders/20240404100811-qc-flags.js @@ -127,6 +127,51 @@ module.exports = { updated_at: '2024-02-13 11:58:20', }, + /** Flags for runNumber: 56, LHC22a_apass1, FT0 */ + { + id: 10, + from: '2019-08-08 20:00:00', + to: '2019-08-08 21:00:00', + comment: 'Some qc comment 10', + + // Associations + created_by_id: 2, + flag_type_id: 3, // Good + run_number: 56, + detector_id: 7, // FT0 + created_at: '2024-02-13 11:58:20', + updated_at: '2024-02-13 11:58:20', + }, + // For ITS + { + id: 11, + from: '2019-08-08 20:00:00', + to: '2019-08-08 21:00:00', + comment: 'Some qc comment 11', + + // Associations + created_by_id: 2, + flag_type_id: 3, // Good + run_number: 56, + detector_id: 4, // ITS + created_at: '2024-02-13 11:58:20', + updated_at: '2024-02-13 11:58:20', + }, + { + id: 12, + from: '2019-08-08 20:30:00', + to: '2019-08-08 21:00:00', + comment: 'Some qc comment 12', + + // Associations + created_by_id: 2, + flag_type_id: 13, // Good + run_number: 56, + detector_id: 4, // ITS + created_at: '2024-02-13 11:58:20', + updated_at: '2024-02-13 11:58:20', + }, + /** Synchronous */ // Run : 56, FT0 @@ -214,6 +259,27 @@ module.exports = { to: '2019-08-09 14:00:00', }, + { + id: 10, + flag_id: 10, + from: '2019-08-08 20:00:00', + to: '2019-08-08 21:00:00', + }, + + { + id: 11, + flag_id: 11, + from: '2019-08-08 20:00:00', + to: '2019-08-08 21:00:00', + }, + + { + id: 12, + flag_id: 12, + from: '2019-08-08 20:30:00', + to: '2019-08-08 21:00:00', + }, + /** Synchronous */ // Run : 56, FT0 { @@ -260,6 +326,18 @@ module.exports = { data_pass_id: 2, // LHC22b_apass2 quality_control_flag_id: 4, }, + { + data_pass_id: 3, // LHC22a_apass1 + quality_control_flag_id: 10, + }, + { + data_pass_id: 3, // LHC22a_apass1 + quality_control_flag_id: 11, + }, + { + data_pass_id: 3, // LHC22a_apass1 + quality_control_flag_id: 12, + }, ], { transaction }), queryInterface.bulkInsert('simulation_pass_quality_control_flag', [ From 7a2d17d437352489b93dc9b8af6495e57b4f6812 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Wed, 20 Nov 2024 15:52:48 +0100 Subject: [PATCH 16/39] fix merging summaries --- lib/database/repositories/QcFlagRepository.js | 6 +++++- .../qualityControlFlag/QcFlagService.js | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/lib/database/repositories/QcFlagRepository.js b/lib/database/repositories/QcFlagRepository.js index a7161c95eb..b5e6e26cd3 100644 --- a/lib/database/repositories/QcFlagRepository.js +++ b/lib/database/repositories/QcFlagRepository.js @@ -188,7 +188,7 @@ class QcFlagRepository extends Repository { gaq_periods.run_number AS runNumber, IF(gaq_periods.\`from\` = 0, null, gaq_periods.\`from\`) AS \`from\`, IF(gaq_periods.\`to\` = UNIX_TIMESTAMP(NOW()), null, gaq_periods.\`to\`) AS \`to\`, - IF(COUNT( DISTINCT gaqd.detector_id ) != COUNT( DISTINCT qcfep.flag_id ), + IF(COUNT( DISTINCT gaqd.detector_id ) > COUNT( DISTINCT qcfep.flag_id ), null, SUM(IF(qcft.monte_carlo_reproducible AND :mcReproducibleAsNotBad, false, qcft.bad)) >= 1 ) AS bad, @@ -278,6 +278,10 @@ class QcFlagRepository extends Repository { effectivePeriods.bad `; + // const [r] = await this.model.sequelize.query(effectivePeriodsWithTypeSubQuery, { replacements: { dataPassId, mcReproducibleAsNotBad } }); + // return r.map(r => ({ ...r, from: new Date(r.from * 1000), to: new Date(r.to * 1000) })).filter(r => r.runNumber === 56); + + const [rows] = await this.model.sequelize.query(query, { replacements: { dataPassId, mcReproducibleAsNotBad } }); return rows.map(({ runNumber, diff --git a/lib/server/services/qualityControlFlag/QcFlagService.js b/lib/server/services/qualityControlFlag/QcFlagService.js index 489cb0b279..40722e746b 100644 --- a/lib/server/services/qualityControlFlag/QcFlagService.js +++ b/lib/server/services/qualityControlFlag/QcFlagService.js @@ -359,6 +359,12 @@ class QcFlagService { this._mergeIntoSummaryUnit(runDetectorSummary, runDetectorSummaryForFlagTypesClass); } + for (const runSummaries of Object.values(summary)) { + for (const runDetectorSummary of Object.values(runSummaries)) { + this._fillSummaryMissingValues(runDetectorSummary); + } + } + return summary; } @@ -696,6 +702,9 @@ class QcFlagService { for (const [runNumber, { distinctFlagsIds, distinctVerifiedFlagsIds }] of Object.entries(flagsAndVerifications)) { summary[runNumber][QC_SUMMARY_PROPERTIES.missingVerificationsCount] = distinctFlagsIds.size - distinctVerifiedFlagsIds.size; } + for (const runSummary of Object.values(summary)) { + this._fillSummaryMissingValues(runSummary); + } return summary; } @@ -725,7 +734,15 @@ class QcFlagService { summaryUnit[QC_SUMMARY_PROPERTIES.mcReproducible] = mcReproducible || summaryUnit[QC_SUMMARY_PROPERTIES.mcReproducible]; + } + /** + * Set default summary properties which were not set when merging them, so each expected property has no 'undefined' value + * + * @param {RunDetectorQcSummary|RunGaqSummary} summaryUnit RunDetectorQcSummary or RunGaqSummary + * @return {void} + */ + _fillSummaryMissingValues(summaryUnit) { if (summaryUnit[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] === undefined) { summaryUnit[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] = 0; } From adb74ce61f9d81a081e33fcc6bacb3573c8d7acb Mon Sep 17 00:00:00 2001 From: xsalonx Date: Wed, 20 Nov 2024 18:34:10 +0100 Subject: [PATCH 17/39] cleanup --- lib/database/repositories/QcFlagRepository.js | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/database/repositories/QcFlagRepository.js b/lib/database/repositories/QcFlagRepository.js index b5e6e26cd3..b63e4b0bd9 100644 --- a/lib/database/repositories/QcFlagRepository.js +++ b/lib/database/repositories/QcFlagRepository.js @@ -224,8 +224,8 @@ class QcFlagRepository extends Repository { GROUP BY gaq_periods.data_pass_id, gaq_periods.run_number, - gaq_periods.\`from\`, - gaq_periods.\`to\` + IF(gaq_periods.\`from\` = 0, null, gaq_periods.\`from\`), + IF(gaq_periods.\`to\` = UNIX_TIMESTAMP(NOW()), null, gaq_periods.\`to\`) `; const query = ` @@ -278,10 +278,6 @@ class QcFlagRepository extends Repository { effectivePeriods.bad `; - // const [r] = await this.model.sequelize.query(effectivePeriodsWithTypeSubQuery, { replacements: { dataPassId, mcReproducibleAsNotBad } }); - // return r.map(r => ({ ...r, from: new Date(r.from * 1000), to: new Date(r.to * 1000) })).filter(r => r.runNumber === 56); - - const [rows] = await this.model.sequelize.query(query, { replacements: { dataPassId, mcReproducibleAsNotBad } }); return rows.map(({ runNumber, From 42c479dc968c8e1bc0bbfa8fd17ffed316f1c1d9 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Wed, 20 Nov 2024 18:54:39 +0100 Subject: [PATCH 18/39] docs --- lib/server/services/qualityControlFlag/QcFlagService.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/server/services/qualityControlFlag/QcFlagService.js b/lib/server/services/qualityControlFlag/QcFlagService.js index 40722e746b..db35c1bd46 100644 --- a/lib/server/services/qualityControlFlag/QcFlagService.js +++ b/lib/server/services/qualityControlFlag/QcFlagService.js @@ -361,7 +361,7 @@ class QcFlagService { for (const runSummaries of Object.values(summary)) { for (const runDetectorSummary of Object.values(runSummaries)) { - this._fillSummaryMissingValues(runDetectorSummary); + this._fillQcSummaryMissingValues(runDetectorSummary); } } @@ -703,7 +703,7 @@ class QcFlagService { summary[runNumber][QC_SUMMARY_PROPERTIES.missingVerificationsCount] = distinctFlagsIds.size - distinctVerifiedFlagsIds.size; } for (const runSummary of Object.values(summary)) { - this._fillSummaryMissingValues(runSummary); + this._fillQcSummaryMissingValues(runSummary); } return summary; @@ -737,12 +737,12 @@ class QcFlagService { } /** - * Set default summary properties which were not set when merging them, so each expected property has no 'undefined' value + * Set default summary properties which were not set when merging sub-summaries, so each expected property has no 'undefined' value * * @param {RunDetectorQcSummary|RunGaqSummary} summaryUnit RunDetectorQcSummary or RunGaqSummary * @return {void} */ - _fillSummaryMissingValues(summaryUnit) { + _fillQcSummaryMissingValues(summaryUnit) { if (summaryUnit[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] === undefined) { summaryUnit[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] = 0; } From d37c942bcd82be50f5b12aebe1dc7781afd1bdf6 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Wed, 20 Nov 2024 19:46:50 +0100 Subject: [PATCH 19/39] fix tests --- .../qualityControlFlag/QcFlagService.js | 12 +++-- .../qualityControlFlag/QcFlagService.test.js | 50 +++++++++++++++---- 2 files changed, 49 insertions(+), 13 deletions(-) diff --git a/lib/server/services/qualityControlFlag/QcFlagService.js b/lib/server/services/qualityControlFlag/QcFlagService.js index db35c1bd46..82cbfc7142 100644 --- a/lib/server/services/qualityControlFlag/QcFlagService.js +++ b/lib/server/services/qualityControlFlag/QcFlagService.js @@ -750,10 +750,14 @@ class QcFlagService { summaryUnit[QC_SUMMARY_PROPERTIES.badEffectiveRunCoverage] = 0; } if (summaryUnit[QC_SUMMARY_PROPERTIES.qualityNotDefinedEffectiveRunCoverage] === undefined) { - summaryUnit[QC_SUMMARY_PROPERTIES.qualityNotDefinedEffectiveRunCoverage] = 1 - ( - summaryUnit[QC_SUMMARY_PROPERTIES.badEffectiveRunCoverage] + - summaryUnit[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] - ); + const badEffectiveRunCoverage = summaryUnit[QC_SUMMARY_PROPERTIES.badEffectiveRunCoverage]; + const explicitlyNotBadEffectiveRunCoverage = summaryUnit[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage]; + const definedCoverage = badEffectiveRunCoverage + explicitlyNotBadEffectiveRunCoverage; + + summaryUnit[QC_SUMMARY_PROPERTIES.qualityNotDefinedEffectiveRunCoverage] = + badEffectiveRunCoverage === null || explicitlyNotBadEffectiveRunCoverage === null + ? null + : 1 - definedCoverage; } } diff --git a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js index 4624340687..5d42b29669 100644 --- a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js +++ b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js @@ -173,7 +173,7 @@ module.exports = () => { mcReproducible: true, badEffectiveRunCoverage: 0.1111, explicitlyNotBadEffectiveRunCoverage: 0.2222, - qualityNotDefinedEffectiveRunCoverage: 0.7778, + qualityNotDefinedEffectiveRunCoverage: 0.6667, }, 16: { badEffectiveRunCoverage: 0, @@ -237,13 +237,31 @@ module.exports = () => { mcReproducible: false, badEffectiveRunCoverage: 0.0769, explicitlyNotBadEffectiveRunCoverage: 0, + qualityNotDefinedEffectiveRunCoverage: 0.9231, }, }, }); }); it('should successfully get empty QC flag summary for data pass', async () => { - expect(await qcFlagService.getQcFlagsSummary({ dataPassId: 3 })).to.be.eql({}); + expect(await qcFlagService.getQcFlagsSummary({ dataPassId: 3 })).to.be.eql({ + 56: { + 4: { + badEffectiveRunCoverage: 0.5, + explicitlyNotBadEffectiveRunCoverage: 1, + mcReproducible: false, + missingVerificationsCount: 2, + qualityNotDefinedEffectiveRunCoverage: -0.5, + }, + 7: { + badEffectiveRunCoverage: 0, + explicitlyNotBadEffectiveRunCoverage: 1, + mcReproducible: false, + missingVerificationsCount: 1, + qualityNotDefinedEffectiveRunCoverage: 0, + }, + }, + }); }); it('should successfully get non-empty QC flag summary for simulation pass', async () => { @@ -273,7 +291,7 @@ module.exports = () => { mcReproducible: false, badEffectiveRunCoverage: 0.1667, explicitlyNotBadEffectiveRunCoverage: 0.8333, - qualityNotDefinedEffectiveRunCoverage: 0.16669999999999996, + qualityNotDefinedEffectiveRunCoverage: 0, }, // ITS @@ -282,6 +300,7 @@ module.exports = () => { mcReproducible: false, badEffectiveRunCoverage: 0, explicitlyNotBadEffectiveRunCoverage: 1, + qualityNotDefinedEffectiveRunCoverage: 0, }, }, }); @@ -715,12 +734,14 @@ module.exports = () => { mcReproducible: false, missingVerificationsCount: 1, explicitlyNotBadEffectiveRunCoverage: 0, + qualityNotDefinedEffectiveRunCoverage: 0, }, 16: { badEffectiveRunCoverage: 0, explicitlyNotBadEffectiveRunCoverage: 1, mcReproducible: false, missingVerificationsCount: 1, + qualityNotDefinedEffectiveRunCoverage: 0, }, }, [runNumber]: { @@ -729,6 +750,7 @@ module.exports = () => { mcReproducible: true, missingVerificationsCount: 1, explicitlyNotBadEffectiveRunCoverage: 0, + qualityNotDefinedEffectiveRunCoverage: 0, }, }, }); @@ -763,12 +785,14 @@ module.exports = () => { mcReproducible: false, missingVerificationsCount: 1, explicitlyNotBadEffectiveRunCoverage: 0, + qualityNotDefinedEffectiveRunCoverage: 0, }, 16: { badEffectiveRunCoverage: 0, explicitlyNotBadEffectiveRunCoverage: 1, mcReproducible: false, missingVerificationsCount: 1, + qualityNotDefinedEffectiveRunCoverage: 0, }, }, [runNumber]: { @@ -777,6 +801,7 @@ module.exports = () => { explicitlyNotBadEffectiveRunCoverage: 0, mcReproducible: true, missingVerificationsCount: 2, + qualityNotDefinedEffectiveRunCoverage: null, }, }, }); @@ -1439,6 +1464,7 @@ module.exports = () => { { from: t('12:00:00'), to: t('13:00:00'), contributingFlagIds: [cpvFlagIds[2], emcFlagIds[2], fddFlagIds[1]] }, { from: t('13:00:00'), to: t('14:00:00'), contributingFlagIds: [cpvFlagIds[2], fddFlagIds[1]] }, { from: t('14:00:00'), to: t('16:00:00'), contributingFlagIds: [cpvFlagIds[0], emcFlagIds[3], fddFlagIds[0]] }, + { from: t('16:00:00'), to: t('18:00:00'), contributingFlagIds: [] }, { from: t('18:00:00'), to: t('20:00:00'), contributingFlagIds: [cpvFlagIds[3], emcFlagIds[4]] }, { from: t('20:00:00'), to: t('22:00:00'), contributingFlagIds: [cpvFlagIds[3]] }, ]); @@ -1454,26 +1480,30 @@ module.exports = () => { const timeTrgEnd = t('22:00:00'); const gaqSubSummaries = [ - { from: t('06:00:00'), to: t('10:00:00'), bad: true, mcReproducible: false }, + { from: t('06:00:00'), to: t('10:00:00'), bad: null, mcReproducible: false }, { from: t('10:00:00'), to: t('12:00:00'), bad: true, mcReproducible: false }, { from: t('12:00:00'), to: t('13:00:00'), bad: true, mcReproducible: true }, - { from: t('13:00:00'), to: t('14:00:00'), bad: true, mcReproducible: true }, + { from: t('13:00:00'), to: t('14:00:00'), bad: null, mcReproducible: true }, { from: t('14:00:00'), to: t('16:00:00'), bad: true, mcReproducible: false }, - { from: t('18:00:00'), to: t('20:00:00'), bad: false, mcReproducible: false }, - { from: t('20:00:00'), to: t('22:00:00'), bad: false, mcReproducible: false }, + { from: t('16:00:00'), to: t('18:00:00'), bad: null, mcReproducible: false }, + { from: t('18:00:00'), to: t('20:00:00'), bad: null, mcReproducible: false }, + { from: t('20:00:00'), to: t('22:00:00'), bad: null, mcReproducible: false }, ]; const expectedGaqSummary = gaqSubSummaries.reduce((acc, { from, to, bad, mcReproducible }) => { - if (bad) { + if (bad === null) { + acc.qualityNotDefinedEffectiveRunCoverage += to - from; + } else if (bad) { acc.badEffectiveRunCoverage += to - from; } else { acc.explicitlyNotBadEffectiveRunCoverage += to - from; } acc.mcReproducible = acc.mcReproducible || mcReproducible; return acc; - }, { badEffectiveRunCoverage: 0, explicitlyNotBadEffectiveRunCoverage: 0 }); + }, { badEffectiveRunCoverage: 0, explicitlyNotBadEffectiveRunCoverage: 0, qualityNotDefinedEffectiveRunCoverage: 0 }); expectedGaqSummary.badEffectiveRunCoverage /= timeTrgEnd - timeTrgStart; expectedGaqSummary.explicitlyNotBadEffectiveRunCoverage /= timeTrgEnd - timeTrgStart; + expectedGaqSummary.qualityNotDefinedEffectiveRunCoverage /= timeTrgEnd - timeTrgStart; expectedGaqSummary.missingVerificationsCount = 11; const { [runNumber]: runGaqSummary } = await qcFlagService.getGaqSummary(dataPassId); @@ -1513,12 +1543,14 @@ module.exports = () => { explicitlyNotBadEffectiveRunCoverage: 1, badEffectiveRunCoverage: 0, mcReproducible: false, + qualityNotDefinedEffectiveRunCoverage: 0, }, 54: { missingVerificationsCount: 1, explicitlyNotBadEffectiveRunCoverage: 0, badEffectiveRunCoverage: 1, mcReproducible: false, + qualityNotDefinedEffectiveRunCoverage: 0, }, }); }); From 5ae822524c8b680868d77ee51d5ce743132da582 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Wed, 20 Nov 2024 23:22:39 +0100 Subject: [PATCH 20/39] add colors --- .../seeders/20240213120811-qc-flags-types.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/database/seeders/20240213120811-qc-flags-types.js b/lib/database/seeders/20240213120811-qc-flags-types.js index 127025d5c5..e5fa4ea0b2 100644 --- a/lib/database/seeders/20240213120811-qc-flags-types.js +++ b/lib/database/seeders/20240213120811-qc-flags-types.js @@ -24,6 +24,7 @@ module.exports = { bad: true, created_by_id: 1, monte_carlo_reproducible: false, + color: '#d62631', }, { id: 3, @@ -32,6 +33,7 @@ module.exports = { bad: false, created_by_id: 1, monte_carlo_reproducible: false, + color: '#4caf50', }, { id: 5, @@ -39,15 +41,15 @@ module.exports = { method: 'LimitedAcceptanceMCReproducible', monte_carlo_reproducible: true, bad: true, - color: '#FFFF00', + color: '#bb9d30', created_by_id: 1, }, { id: 11, - name: 'Limited acceptance', - method: 'LimitedAcceptance', + name: 'Limited Acceptance MC Not Reproducible', + method: 'LimitedAcceptanceMCNotReproducible', bad: true, - color: '#FFFF00', + color: '#72512c', created_by_id: 1, monte_carlo_reproducible: false, }, @@ -58,6 +60,7 @@ module.exports = { bad: true, created_by_id: 1, monte_carlo_reproducible: false, + color: '#d62631', }, { id: 13, @@ -66,6 +69,7 @@ module.exports = { bad: true, created_by_id: 1, monte_carlo_reproducible: false, + color: '#d62631', }, { id: 20, From ad3e415aa53263f275f145baf090cf2ee76cb036 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Wed, 20 Nov 2024 23:32:34 +0100 Subject: [PATCH 21/39] fix seeder --- lib/database/seeders/20240404100811-qc-flags.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/database/seeders/20240404100811-qc-flags.js b/lib/database/seeders/20240404100811-qc-flags.js index e98e96f3f3..9bc2639dfc 100644 --- a/lib/database/seeders/20240404100811-qc-flags.js +++ b/lib/database/seeders/20240404100811-qc-flags.js @@ -165,7 +165,7 @@ module.exports = { // Associations created_by_id: 2, - flag_type_id: 13, // Good + flag_type_id: 5, // Lim. Acc. MC Reproducible run_number: 56, detector_id: 4, // ITS created_at: '2024-02-13 11:58:20', @@ -270,7 +270,7 @@ module.exports = { id: 11, flag_id: 11, from: '2019-08-08 20:00:00', - to: '2019-08-08 21:00:00', + to: '2019-08-08 20:30:00', }, { From 87c7038a26816d51d0a767740a4254595ee20c0c Mon Sep 17 00:00:00 2001 From: xsalonx Date: Wed, 20 Nov 2024 23:35:53 +0100 Subject: [PATCH 22/39] fix test --- test/lib/usecases/run/GetAllRunsUseCase.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/lib/usecases/run/GetAllRunsUseCase.test.js b/test/lib/usecases/run/GetAllRunsUseCase.test.js index 652691499b..169f850373 100644 --- a/test/lib/usecases/run/GetAllRunsUseCase.test.js +++ b/test/lib/usecases/run/GetAllRunsUseCase.test.js @@ -677,14 +677,14 @@ module.exports = () => { } it('should successfully filter by GAQ notBadFraction', async () => { - const dataPassIds = [1]; + const dataPassIds = [3]; { const { runs } = await new GetAllRunsUseCase().execute({ query: { filter: { dataPassIds, gaq: { notBadFraction: { '<': 0.8 } }, } } }); expect(runs).to.be.an('array'); - expect(runs.map(({ runNumber }) => runNumber)).to.have.all.members([106]); + expect(runs.map(({ runNumber }) => runNumber)).to.have.all.members([56]); } { const { runs } = await new GetAllRunsUseCase().execute({ query: { filter: { From eb3dab574ec22e7f0e771bf8456277054936fa3f Mon Sep 17 00:00:00 2001 From: xsalonx Date: Wed, 20 Nov 2024 23:42:55 +0100 Subject: [PATCH 23/39] fix api test --- test/api/qcFlags.test.js | 17 ++++++++++++++++- test/api/runs.test.js | 4 ++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/test/api/qcFlags.test.js b/test/api/qcFlags.test.js index cf53f52f89..ad98a39f34 100644 --- a/test/api/qcFlags.test.js +++ b/test/api/qcFlags.test.js @@ -48,7 +48,7 @@ module.exports = () => { createdBy: { id: 2, externalId: 456, name: 'Jan Jansen' }, flagTypeId: 13, - flagType: { id: 13, name: 'Bad', method: 'Bad', mcReproducible: false, bad: true, archived: false, color: null }, + flagType: { id: 13, name: 'Bad', method: 'Bad', mcReproducible: false, bad: true, archived: false, color: '#d62631' }, effectivePeriods: [], }); }); @@ -79,12 +79,14 @@ module.exports = () => { mcReproducible: true, badEffectiveRunCoverage: 0.3333, explicitlyNotBadEffectiveRunCoverage: 0, + qualityNotDefinedEffectiveRunCoverage: 0.6667000000000001, }, 16: { badEffectiveRunCoverage: 0, explicitlyNotBadEffectiveRunCoverage: 1, mcReproducible: false, missingVerificationsCount: 1, + qualityNotDefinedEffectiveRunCoverage: 0, }, }, }); @@ -101,12 +103,14 @@ module.exports = () => { mcReproducible: true, badEffectiveRunCoverage: 0.1111, explicitlyNotBadEffectiveRunCoverage: 0.2222, + qualityNotDefinedEffectiveRunCoverage: 0.6667, }, 16: { badEffectiveRunCoverage: 0, explicitlyNotBadEffectiveRunCoverage: 1, mcReproducible: false, missingVerificationsCount: 1, + qualityNotDefinedEffectiveRunCoverage: 0, }, }, }); @@ -123,6 +127,7 @@ module.exports = () => { mcReproducible: false, badEffectiveRunCoverage: 0.7222, explicitlyNotBadEffectiveRunCoverage: 0, + qualityNotDefinedEffectiveRunCoverage: 0.27780000000000005, }, }, }); @@ -140,6 +145,7 @@ module.exports = () => { mcReproducible: false, badEffectiveRunCoverage: 0.1667, explicitlyNotBadEffectiveRunCoverage: 0.8333, + qualityNotDefinedEffectiveRunCoverage: 0, }, // ITS @@ -148,6 +154,7 @@ module.exports = () => { mcReproducible: false, badEffectiveRunCoverage: 0, explicitlyNotBadEffectiveRunCoverage: 1, + qualityNotDefinedEffectiveRunCoverage: 0, }, }, }); @@ -192,6 +199,14 @@ module.exports = () => { mcReproducible: true, badEffectiveRunCoverage: 1, explicitlyNotBadEffectiveRunCoverage: 0, + qualityNotDefinedEffectiveRunCoverage: 0, + }, + 56: { + badEffectiveRunCoverage: 0.5, + explicitlyNotBadEffectiveRunCoverage: 0.5, + mcReproducible: true, + missingVerificationsCount: 3, + qualityNotDefinedEffectiveRunCoverage: 0, }, }); }); diff --git a/test/api/runs.test.js b/test/api/runs.test.js index 8b1ccbc36e..39692429ba 100644 --- a/test/api/runs.test.js +++ b/test/api/runs.test.js @@ -378,7 +378,7 @@ module.exports = () => { } it('should successfully filter by GAQ notBadFraction', async () => { - const dataPassId = 1; + const dataPassId = 3; { const response = await request(server).get(`/api/runs?filter[dataPassIds][]=${dataPassId}&filter[gaq][notBadFraction][<]=0.8`); @@ -386,7 +386,7 @@ module.exports = () => { const { data: runs } = response.body; expect(runs).to.be.an('array'); - expect(runs.map(({ runNumber }) => runNumber)).to.have.all.members([106]); + expect(runs.map(({ runNumber }) => runNumber)).to.have.all.members([56]); } { const response = await request(server).get(`/api/runs?filter[dataPassIds][]=${dataPassId}` + From 727980dc6d763c5081554661a9ae53d758e5b727 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Wed, 20 Nov 2024 23:49:45 +0100 Subject: [PATCH 24/39] fix --- .../qualityControlFlag/QcFlagService.test.js | 6 +++--- .../qualityControlFlag/QcFlagTypeService.test.js | 12 +++++++----- test/public/runs/runsPerDataPass.overview.test.js | 6 +++--- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js index 5d42b29669..49d17f67aa 100644 --- a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js +++ b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js @@ -55,7 +55,7 @@ const qcFlagWithId1 = { name: 'Limited Acceptance MC Reproducible', method: 'LimitedAcceptanceMCReproducible', bad: true, - color: '#FFFF00', + color: '#bb9d30', archived: false, mcReproducible: true, }, @@ -248,10 +248,10 @@ module.exports = () => { 56: { 4: { badEffectiveRunCoverage: 0.5, - explicitlyNotBadEffectiveRunCoverage: 1, + explicitlyNotBadEffectiveRunCoverage: 0.5, mcReproducible: false, missingVerificationsCount: 2, - qualityNotDefinedEffectiveRunCoverage: -0.5, + qualityNotDefinedEffectiveRunCoverage: 0, }, 7: { badEffectiveRunCoverage: 0, diff --git a/test/lib/server/services/qualityControlFlag/QcFlagTypeService.test.js b/test/lib/server/services/qualityControlFlag/QcFlagTypeService.test.js index 5b4661c5f4..92a24c56ff 100644 --- a/test/lib/server/services/qualityControlFlag/QcFlagTypeService.test.js +++ b/test/lib/server/services/qualityControlFlag/QcFlagTypeService.test.js @@ -199,11 +199,12 @@ module.exports = () => { }); it('should successfully filter QC flags types by name', async () => { - const { count, rows: flagTypes } = await qcFlagTypeService.getAll({ filter: { names: ['Unknown Quality', 'Limited acceptance'] } }); + const { count, rows: flagTypes } = await qcFlagTypeService.getAll({ + filter: { names: ['Unknown Quality', 'Limited Acceptance MC Not Reproducible'] } }); expect(count).to.be.equal(2); expect(flagTypes).to.be.an('array'); expect(flagTypes).to.be.lengthOf(2); - expect(flagTypes.map(({ name }) => name)).to.have.all.members(['Unknown Quality', 'Limited acceptance']); + expect(flagTypes.map(({ name }) => name)).to.have.all.members(['Unknown Quality', 'Limited Acceptance MC Not Reproducible']); }); it('should successfully filter QC flags types by names pattern', async () => { @@ -219,7 +220,8 @@ module.exports = () => { expect(count).to.be.equal(2); expect(flagTypes).to.be.an('array'); expect(flagTypes).to.be.lengthOf(2); - expect(flagTypes.map(({ method }) => method)).to.be.has.all.members(['LimitedAcceptance', 'LimitedAcceptanceMCReproducible']); + expect(flagTypes.map(({ method }) => method)).to.be.has + .all.members(['LimitedAcceptanceMCNotReproducible', 'LimitedAcceptanceMCReproducible']); }); it('should successfully filter QC flags types by method pattern', async () => { @@ -270,7 +272,7 @@ module.exports = () => { expect(flagTypes.map(({ name }) => name)).to.have.all.ordered.members([ 'Unknown Quality', 'Limited Acceptance MC Reproducible', - 'Limited acceptance', + 'Limited Acceptance MC Not Reproducible', 'Good', 'Bad PID', 'Bad', @@ -288,7 +290,7 @@ module.exports = () => { 'Bad', 'Bad PID', 'Good', - 'Limited acceptance', + 'Limited Acceptance MC Not Reproducible', 'Limited Acceptance MC Reproducible', 'Unknown Quality', ]); diff --git a/test/public/runs/runsPerDataPass.overview.test.js b/test/public/runs/runsPerDataPass.overview.test.js index 990ca1d3bf..f389584463 100644 --- a/test/public/runs/runsPerDataPass.overview.test.js +++ b/test/public/runs/runsPerDataPass.overview.test.js @@ -425,21 +425,21 @@ module.exports = () => { } it('should successfully apply gaqNotBadFraction filters', async () => { - await navigateToRunsPerDataPass(page, { lhcPeriodId: 2, dataPassId: 1 }, { epectedRowsCount: 3 }); + await navigateToRunsPerDataPass(page, { lhcPeriodId: 1, dataPassId: 3 }, { epectedRowsCount: 4 }); await pressElement(page, '#openFilterToggle', true); const popoverSelector = await getPopoverSelector(await page.waitForSelector('.globalAggregatedQuality-filter .popover-trigger')); await pressElement(page, `${popoverSelector} #gaqNotBadFraction-dropdown-option-le`, true); await fillInput(page, '#gaqNotBadFraction-value-input', '80'); - await expectColumnValues(page, 'runNumber', ['106']); + await expectColumnValues(page, 'runNumber', ['56']); await pressElement(page, '#mcReproducibleAsNotBadToggle input', true); await expectColumnValues(page, 'runNumber', []); await pressElement(page, '#openFilterToggle', true); await pressElement(page, '#reset-filters', true); - await expectColumnValues(page, 'runNumber', ['108', '107', '106']); + await expectColumnValues(page, 'runNumber', ['105', '56', '54', '49']); }); it('should successfully apply muInelasticInteractionRate filters', async () => { From 28b593181609327173cc6ed513359bd6c87ad05c Mon Sep 17 00:00:00 2001 From: xsalonx Date: Wed, 20 Nov 2024 23:52:10 +0100 Subject: [PATCH 25/39] fix test --- .../server/services/qualityControlFlag/QcFlagService.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js index 49d17f67aa..f4068c3ed0 100644 --- a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js +++ b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js @@ -249,7 +249,7 @@ module.exports = () => { 4: { badEffectiveRunCoverage: 0.5, explicitlyNotBadEffectiveRunCoverage: 0.5, - mcReproducible: false, + mcReproducible: true, missingVerificationsCount: 2, qualityNotDefinedEffectiveRunCoverage: 0, }, From 42513a1ee491b8aa52f5c5c0c84641aa84a6f2f4 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Thu, 21 Nov 2024 00:13:04 +0100 Subject: [PATCH 26/39] ch display, fix tests --- .../ActiveColumns/gaqFlagsActiveColumns.js | 5 +- test/public/qcFlagTypes/creation.test.js | 4 +- test/public/qcFlags/gaqOverview.test.js | 50 +++++++++++++++---- 3 files changed, 45 insertions(+), 14 deletions(-) diff --git a/lib/public/views/QcFlags/ActiveColumns/gaqFlagsActiveColumns.js b/lib/public/views/QcFlags/ActiveColumns/gaqFlagsActiveColumns.js index 02044e532a..540b471e02 100644 --- a/lib/public/views/QcFlags/ActiveColumns/gaqFlagsActiveColumns.js +++ b/lib/public/views/QcFlags/ActiveColumns/gaqFlagsActiveColumns.js @@ -28,6 +28,7 @@ import { h, iconBan, iconWarning } from '/js/src/index.js'; * @return {Component} display of aggregated quality */ const formatGeneralQuality = (contributingFlags, gaqDetectors) => { + const missingFlags = contributingFlags.length !== gaqDetectors.length; const flagTypes = contributingFlags.map(({ flagType }) => flagType); const allGood = flagTypes.every(({ bad }) => !bad); const someBadNotReproducible = flagTypes.some(({ bad, mcReproducible }) => bad && !mcReproducible); @@ -44,7 +45,9 @@ const formatGeneralQuality = (contributingFlags, gaqDetectors) => { : null; let qualityDisplay = null; - if (allGood) { + if (missingFlags) { + qualityDisplay = badge('ND', { color: QcSummaryColors.INCALCULABLE_COVERAGE }); + } else if (allGood) { qualityDisplay = badge('good', { color: QcSummaryColors.ALL_GOOD }); } else if (someBadNotReproducible) { qualityDisplay = badge('bad', { color: QcSummaryColors.ALL_BAD }); diff --git a/test/public/qcFlagTypes/creation.test.js b/test/public/qcFlagTypes/creation.test.js index 7afceaba93..648c2a1a4d 100644 --- a/test/public/qcFlagTypes/creation.test.js +++ b/test/public/qcFlagTypes/creation.test.js @@ -52,8 +52,8 @@ module.exports = () => { await goToPage(page, 'qc-flag-type-creation'); await page.waitForSelector('button#submit[disabled]'); - await fillInput(page, 'input#name', 'Limited acceptance'); - await fillInput(page, 'input#method', 'LimitedAcceptance'); + await fillInput(page, 'input#name', 'LimitedAcceptanceMCNotReproducible'); + await fillInput(page, 'input#method', 'Limited Acceptance MC Not Reproducible'); await pressElement(page, 'button#submit'); await expectInnerText( page, diff --git a/test/public/qcFlags/gaqOverview.test.js b/test/public/qcFlags/gaqOverview.test.js index 356c5d6c3f..b76eb3f1bb 100644 --- a/test/public/qcFlags/gaqOverview.test.js +++ b/test/public/qcFlags/gaqOverview.test.js @@ -66,36 +66,64 @@ module.exports = () => { // eslint-disable-next-line require-jsdoc const validateDate = (date) => date === '-' || !isNaN(dateAndTime.parse(date, 'DD/MM/YYYY hh:mm:ss')); const tableDataValidators = { - generalQuality: (generalQuality) => ['good', 'bad', 'MC.R'].includes(generalQuality), + generalQuality: (generalQuality) => ['ND', 'good', 'bad', 'MC.R'].includes(generalQuality), from: (timestamp) => timestamp === 'Whole run coverage' || validateDate(timestamp), to: (timestamp) => timestamp === 'Whole run coverage' || validateDate(timestamp), }; await validateTableData(page, new Map(Object.entries(tableDataValidators))); - await waitForTableLength(page, 3); + await waitForTableLength(page, 7); expect(await getTableContent(page)).to.have.all.deep.ordered.members([ [ - 'MC.R', + 'ND', + '08/08/2019\n13:00:00', + '08/08/2019\n22:43:20', + '', + '', + ], + [ + 'ND', '08/08/2019\n22:43:20', '09/08/2019\n04:16:40', 'Limited Acceptance MC Reproducible', '', ], [ - 'bad', + 'ND', + '09/08/2019\n04:16:40', + '09/08/2019\n05:40:00', + '', + '', + ], + [ + 'ND', '09/08/2019\n05:40:00', '09/08/2019\n07:03:20', - 'Limited acceptance', + 'Limited Acceptance MC Not Reproducible', '', ], [ - 'bad', + 'ND', + '09/08/2019\n07:03:20', + '09/08/2019\n08:26:40', + '', + '', + ], + [ + 'ND', '09/08/2019\n08:26:40', '09/08/2019\n09:50:00', 'Bad', '', ], + [ + 'ND', + '09/08/2019\n09:50:00', + '09/08/2019\n14:00:00', + '', + '', + ], ]); await waitForNavigation(page, () => pressElement(page, 'h2:nth-of-type(2) a', true)); @@ -112,7 +140,7 @@ module.exports = () => { await waitForTableLength(page, 7); expect(await getTableContent(page)).to.have.all.deep.ordered.members([ [ - 'good', + 'ND', '08/08/2019\n13:00:00', '08/08/2019\n22:43:20', '', @@ -126,7 +154,7 @@ module.exports = () => { 'Good', ], [ - 'good', + 'ND', '09/08/2019\n04:16:40', '09/08/2019\n05:40:00', '', @@ -136,11 +164,11 @@ module.exports = () => { 'bad', '09/08/2019\n05:40:00', '09/08/2019\n07:03:20', - 'Limited acceptance', + 'Limited Acceptance MC Not Reproducible', 'Good', ], [ - 'good', + 'ND', '09/08/2019\n07:03:20', '09/08/2019\n08:26:40', '', @@ -154,7 +182,7 @@ module.exports = () => { 'Good', ], [ - 'good', + 'ND', '09/08/2019\n09:50:00', '09/08/2019\n14:00:00', '', From 1dff087843e6eacc12825ad2c3ee541ab8a4447b Mon Sep 17 00:00:00 2001 From: xsalonx Date: Thu, 21 Nov 2024 00:14:43 +0100 Subject: [PATCH 27/39] fix --- test/public/qcFlags/forDataPassCreation.test.js | 2 +- test/public/qcFlags/forSimulationPassCreation.test.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/public/qcFlags/forDataPassCreation.test.js b/test/public/qcFlags/forDataPassCreation.test.js index a65fccd62f..ad10edabde 100644 --- a/test/public/qcFlags/forDataPassCreation.test.js +++ b/test/public/qcFlags/forDataPassCreation.test.js @@ -158,7 +158,7 @@ module.exports = () => { }); await expectRowValues(page, 1, { - flagType: 'Limited acceptance', + flagType: 'Limited Acceptance MC Not Reproducible', from: '08/08/2019\n13:01:01', to: '09/08/2019\n13:50:59', }); diff --git a/test/public/qcFlags/forSimulationPassCreation.test.js b/test/public/qcFlags/forSimulationPassCreation.test.js index 46bc65365c..a11d0d304a 100644 --- a/test/public/qcFlags/forSimulationPassCreation.test.js +++ b/test/public/qcFlags/forSimulationPassCreation.test.js @@ -156,7 +156,7 @@ module.exports = () => { }); await expectRowValues(page, 1, { - flagType: 'Limited acceptance', + flagType: 'Limited Acceptance MC Not Reproducible', from: '08/08/2019\n13:01:01', to: '09/08/2019\n13:50:59', }); From fb67a6939e64a17f0685dfb7f6ce5ac36c084e4f Mon Sep 17 00:00:00 2001 From: xsalonx Date: Thu, 21 Nov 2024 10:52:48 +0100 Subject: [PATCH 28/39] fix test --- test/api/qcFlagTypes.test.js | 25 ++++++++++--------- .../QcFlagTypeService.test.js | 18 ++++++------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/test/api/qcFlagTypes.test.js b/test/api/qcFlagTypes.test.js index 270337ba2f..255973b144 100644 --- a/test/api/qcFlagTypes.test.js +++ b/test/api/qcFlagTypes.test.js @@ -79,7 +79,7 @@ module.exports = () => { name: 'Unknown Quality', method: 'UnknownQuality', bad: true, - color: null, + color: '#d62631', mcReproducible: false, archived: false, @@ -95,7 +95,7 @@ module.exports = () => { name: 'Good', method: 'Good', bad: false, - color: null, + color: '#4caf50', mcReproducible: false, archived: false, @@ -111,7 +111,7 @@ module.exports = () => { name: 'Limited Acceptance MC Reproducible', method: 'LimitedAcceptanceMCReproducible', bad: true, - color: '#FFFF00', + color: '#bb9d30', mcReproducible: true, archived: false, @@ -124,10 +124,10 @@ module.exports = () => { }, { id: 11, - name: 'Limited acceptance', - method: 'LimitedAcceptance', + name: 'Limited Acceptance MC Not Reproducible', + method: 'LimitedAcceptanceMCNotReproducible', bad: true, - color: '#FFFF00', + color: '#72512c', mcReproducible: false, archived: false, @@ -143,7 +143,7 @@ module.exports = () => { name: 'Bad PID', method: 'BadPID', bad: true, - color: null, + color: '#d62631', mcReproducible: false, archived: false, @@ -159,7 +159,7 @@ module.exports = () => { name: 'Bad', method: 'Bad', bad: true, - color: null, + color: '#d62631', mcReproducible: false, archived: false, @@ -213,14 +213,15 @@ module.exports = () => { }); it('should successfully filter QC flag types by names', async () => { - const response = await request(server).get('/api/qcFlagTypes?filter[names][]=Bad&filter[names][]=Limited%20acceptance'); + const response = await request(server) + .get('/api/qcFlagTypes?filter[names][]=Bad&filter[names][]=Limited%20Acceptance%20MC%20Not%20Reproducible'); expect(response.status).to.be.equal(200); const { meta, data: flagTypes } = response.body; expect(meta).to.be.eql({ page: { totalCount: 2, pageCount: 1 } }); expect(flagTypes).to.be.an('array'); expect(flagTypes).to.be.lengthOf(2); - expect(flagTypes.map(({ name }) => name)).to.have.all.deep.members(['Bad', 'Limited acceptance']); + expect(flagTypes.map(({ name }) => name)).to.have.all.deep.members(['Bad', 'Limited Acceptance MC Not Reproducible']); }); it('should successfully filter QC flag types by methods', async () => { @@ -283,7 +284,7 @@ module.exports = () => { expect(flagTypes.map(({ name }) => name)).to.have.all.ordered.members([ 'Unknown Quality', 'Limited Acceptance MC Reproducible', - 'Limited acceptance', + 'Limited Acceptance MC Not Reproducible', 'Good', 'Bad PID', 'Bad', @@ -305,7 +306,7 @@ module.exports = () => { 'Bad', 'Bad PID', 'Good', - 'Limited acceptance', + 'Limited Acceptance MC Not Reproducible', 'Limited Acceptance MC Reproducible', 'Unknown Quality', ]); diff --git a/test/lib/server/services/qualityControlFlag/QcFlagTypeService.test.js b/test/lib/server/services/qualityControlFlag/QcFlagTypeService.test.js index 92a24c56ff..0d9038cbfa 100644 --- a/test/lib/server/services/qualityControlFlag/QcFlagTypeService.test.js +++ b/test/lib/server/services/qualityControlFlag/QcFlagTypeService.test.js @@ -29,7 +29,7 @@ module.exports = () => { name: 'Unknown Quality', method: 'UnknownQuality', bad: true, - color: null, + color: '#d62631', mcReproducible: false, archived: false, @@ -71,7 +71,7 @@ module.exports = () => { name: 'Unknown Quality', method: 'UnknownQuality', bad: true, - color: null, + color: '#d62631', mcReproducible: false, archived: false, @@ -87,7 +87,7 @@ module.exports = () => { name: 'Good', method: 'Good', bad: false, - color: null, + color: '#4caf50', mcReproducible: false, archived: false, @@ -103,7 +103,7 @@ module.exports = () => { name: 'Limited Acceptance MC Reproducible', method: 'LimitedAcceptanceMCReproducible', bad: true, - color: '#FFFF00', + color: '#bb9d30', mcReproducible: true, archived: false, @@ -116,10 +116,10 @@ module.exports = () => { }, { id: 11, - name: 'Limited acceptance', - method: 'LimitedAcceptance', + name: 'Limited Acceptance MC Not Reproducible', + method: 'LimitedAcceptanceMCNotReproducible', bad: true, - color: '#FFFF00', + color: '#72512c', mcReproducible: false, archived: false, @@ -135,7 +135,7 @@ module.exports = () => { name: 'Bad PID', method: 'BadPID', bad: true, - color: null, + color: '#d62631', mcReproducible: false, archived: false, @@ -151,7 +151,7 @@ module.exports = () => { name: 'Bad', method: 'Bad', bad: true, - color: null, + color: '#d62631', mcReproducible: false, archived: false, From 3b9cbd06a97ed6183bda053ca218bec96d9c48f5 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Thu, 21 Nov 2024 12:13:24 +0100 Subject: [PATCH 29/39] test fix --- .../seeders/20240404100811-qc-flags.js | 26 ++++++++++++++++++- test/api/qcFlags.test.js | 6 ++--- .../qualityControlFlag/QcFlagService.test.js | 8 +++--- test/public/qcFlagTypes/creation.test.js | 2 +- .../runs/runsPerDataPass.overview.test.js | 10 ++++--- 5 files changed, 40 insertions(+), 12 deletions(-) diff --git a/lib/database/seeders/20240404100811-qc-flags.js b/lib/database/seeders/20240404100811-qc-flags.js index 9bc2639dfc..fc3b3d961c 100644 --- a/lib/database/seeders/20240404100811-qc-flags.js +++ b/lib/database/seeders/20240404100811-qc-flags.js @@ -142,6 +142,20 @@ module.exports = { created_at: '2024-02-13 11:58:20', updated_at: '2024-02-13 11:58:20', }, + { + id: 13, + from: '2019-08-08 20:00:00', + to: '2019-08-08 20:30:00', + comment: 'Some qc comment 10', + + // Associations + created_by_id: 2, + flag_type_id: 5, // Good + run_number: 56, + detector_id: 7, // FT0 + created_at: '2024-02-13 11:58:20', + updated_at: '2024-02-13 11:58:20', + }, // For ITS { id: 11, @@ -262,9 +276,15 @@ module.exports = { { id: 10, flag_id: 10, - from: '2019-08-08 20:00:00', + from: '2019-08-08 20:30:00', to: '2019-08-08 21:00:00', }, + { + id: 13, + flag_id: 13, + from: '2019-08-08 20:00:00', + to: '2019-08-08 20:30:00', + }, { id: 11, @@ -330,6 +350,10 @@ module.exports = { data_pass_id: 3, // LHC22a_apass1 quality_control_flag_id: 10, }, + { + data_pass_id: 3, // LHC22a_apass1 + quality_control_flag_id: 13, + }, { data_pass_id: 3, // LHC22a_apass1 quality_control_flag_id: 11, diff --git a/test/api/qcFlags.test.js b/test/api/qcFlags.test.js index ad98a39f34..d58cdafb9a 100644 --- a/test/api/qcFlags.test.js +++ b/test/api/qcFlags.test.js @@ -202,10 +202,10 @@ module.exports = () => { qualityNotDefinedEffectiveRunCoverage: 0, }, 56: { - badEffectiveRunCoverage: 0.5, - explicitlyNotBadEffectiveRunCoverage: 0.5, + badEffectiveRunCoverage: 1, + explicitlyNotBadEffectiveRunCoverage: 0, mcReproducible: true, - missingVerificationsCount: 3, + missingVerificationsCount: 4, qualityNotDefinedEffectiveRunCoverage: 0, }, }); diff --git a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js index f4068c3ed0..177a671a85 100644 --- a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js +++ b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js @@ -254,10 +254,10 @@ module.exports = () => { qualityNotDefinedEffectiveRunCoverage: 0, }, 7: { - badEffectiveRunCoverage: 0, - explicitlyNotBadEffectiveRunCoverage: 1, - mcReproducible: false, - missingVerificationsCount: 1, + badEffectiveRunCoverage: 0.5, + explicitlyNotBadEffectiveRunCoverage: 0.5, + mcReproducible: true, + missingVerificationsCount: 2, qualityNotDefinedEffectiveRunCoverage: 0, }, }, diff --git a/test/public/qcFlagTypes/creation.test.js b/test/public/qcFlagTypes/creation.test.js index 648c2a1a4d..f47c2db3b6 100644 --- a/test/public/qcFlagTypes/creation.test.js +++ b/test/public/qcFlagTypes/creation.test.js @@ -59,7 +59,7 @@ module.exports = () => { page, '.alert.alert-danger', // eslint-disable-next-line max-len - 'The request conflicts with existing data: A QC flag type with name Limited acceptance or method LimitedAcceptance already exists', + 'The request conflicts with existing data: A QC flag type with name Limited Acceptance MC Not Reproducible or method LimitedAcceptanceMCNotReproducible already exists', ); }); diff --git a/test/public/runs/runsPerDataPass.overview.test.js b/test/public/runs/runsPerDataPass.overview.test.js index f389584463..9f45829d09 100644 --- a/test/public/runs/runsPerDataPass.overview.test.js +++ b/test/public/runs/runsPerDataPass.overview.test.js @@ -165,12 +165,16 @@ module.exports = () => { }); await page.waitForSelector('tr#row106 .column-CPV a .icon'); - await expectInnerText(page, '#row106-globalAggregatedQuality', '67MC.R'); - expect(await getPopoverInnerText(await page.waitForSelector('#row106-globalAggregatedQuality .popover-trigger'))) - .to.be.equal('Missing 3 verifications'); + await expectInnerText(page, '#row106-globalAggregatedQuality', 'GAQ'); + + await goToPage(page, 'runs-per-data-pass', { queryParameters: { dataPassId: 3 } }); + await expectInnerText(page, '#row56-globalAggregatedQuality', '0MC.R'); + expect(await getPopoverInnerText(await page.waitForSelector('#row56-globalAggregatedQuality .popover-trigger'))) + .to.be.equal('Missing 4 verifications'); }); it('should switch mcReproducibleAsNotBad', async () => { + await goToPage(page, 'runs-per-data-pass', { queryParameters: { dataPassId: 1 } }); await pressElement(page, '#mcReproducibleAsNotBadToggle input', true); await waitForTableLength(page, 3); await expectInnerText(page, 'tr#row106 .column-CPV a', '89'); From 5f2f0edb1b479b588b0668190a4307298cdf913d Mon Sep 17 00:00:00 2001 From: xsalonx Date: Thu, 21 Nov 2024 12:21:40 +0100 Subject: [PATCH 30/39] fix test --- test/api/qcFlagTypes.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/api/qcFlagTypes.test.js b/test/api/qcFlagTypes.test.js index 255973b144..7d0d1e84cc 100644 --- a/test/api/qcFlagTypes.test.js +++ b/test/api/qcFlagTypes.test.js @@ -32,7 +32,7 @@ module.exports = () => { name: 'Bad', method: 'Bad', bad: true, - color: null, + color: '#d62631', mcReproducible: false, archived: false, From f669198afb3f17e4d85d8fefa1fecf9dbe9e1f9f Mon Sep 17 00:00:00 2001 From: xsalonx Date: Thu, 21 Nov 2024 12:26:57 +0100 Subject: [PATCH 31/39] fix test --- test/public/qcFlagTypes/creation.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/public/qcFlagTypes/creation.test.js b/test/public/qcFlagTypes/creation.test.js index f47c2db3b6..93df732a90 100644 --- a/test/public/qcFlagTypes/creation.test.js +++ b/test/public/qcFlagTypes/creation.test.js @@ -52,8 +52,8 @@ module.exports = () => { await goToPage(page, 'qc-flag-type-creation'); await page.waitForSelector('button#submit[disabled]'); - await fillInput(page, 'input#name', 'LimitedAcceptanceMCNotReproducible'); - await fillInput(page, 'input#method', 'Limited Acceptance MC Not Reproducible'); + await fillInput(page, 'input#name', 'Limited Acceptance MC Not Reproducible'); + await fillInput(page, 'input#method', 'LimitedAcceptanceMCNotReproducible'); await pressElement(page, 'button#submit'); await expectInnerText( page, From 6b3aee3ae591a3efdee484708e79232165ef79a3 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Thu, 21 Nov 2024 12:32:11 +0100 Subject: [PATCH 32/39] docs --- lib/database/repositories/QcFlagRepository.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/database/repositories/QcFlagRepository.js b/lib/database/repositories/QcFlagRepository.js index b63e4b0bd9..978d85c093 100644 --- a/lib/database/repositories/QcFlagRepository.js +++ b/lib/database/repositories/QcFlagRepository.js @@ -27,7 +27,7 @@ const GAQ_PERIODS_VIEW = ` ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING ) AS \`to\` FROM ( - -- Two selects for runs' timestamps (in case QC flag doesn't tart at run's start or end at run's end ) + -- Two selects for runs' timestamps (in case QC flag's eff. period doesn't start at run's start or end at run's end ) ( SELECT gaqd.data_pass_id, gaqd.run_number, @@ -46,7 +46,7 @@ const GAQ_PERIODS_VIEW = ` ON gaqd.run_number = r.run_number ) UNION - -- Two selectes for timestamps of QC flags + -- Two selectes for timestamps of QC flags' effective periods ( SELECT gaqd.data_pass_id, gaqd.run_number, From dc93796b2bc9e6ad6934182ebb4206e9da6a1528 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Wed, 27 Nov 2024 14:32:48 +0100 Subject: [PATCH 33/39] rename --- lib/server/services/qualityControlFlag/QcFlagService.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/server/services/qualityControlFlag/QcFlagService.js b/lib/server/services/qualityControlFlag/QcFlagService.js index 82cbfc7142..43538d8c9b 100644 --- a/lib/server/services/qualityControlFlag/QcFlagService.js +++ b/lib/server/services/qualityControlFlag/QcFlagService.js @@ -361,7 +361,7 @@ class QcFlagService { for (const runSummaries of Object.values(summary)) { for (const runDetectorSummary of Object.values(runSummaries)) { - this._fillQcSummaryMissingValues(runDetectorSummary); + this._fillQcSummaryUnitMissingValues(runDetectorSummary); } } @@ -703,7 +703,7 @@ class QcFlagService { summary[runNumber][QC_SUMMARY_PROPERTIES.missingVerificationsCount] = distinctFlagsIds.size - distinctVerifiedFlagsIds.size; } for (const runSummary of Object.values(summary)) { - this._fillQcSummaryMissingValues(runSummary); + this._fillQcSummaryUnitMissingValues(runSummary); } return summary; @@ -742,7 +742,7 @@ class QcFlagService { * @param {RunDetectorQcSummary|RunGaqSummary} summaryUnit RunDetectorQcSummary or RunGaqSummary * @return {void} */ - _fillQcSummaryMissingValues(summaryUnit) { + _fillQcSummaryUnitMissingValues(summaryUnit) { if (summaryUnit[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] === undefined) { summaryUnit[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] = 0; } From b13e82e5416dfaa9b396fc1b2b8c707c00a2efca Mon Sep 17 00:00:00 2001 From: xsalonx Date: Wed, 27 Nov 2024 14:36:54 +0100 Subject: [PATCH 34/39] test --- test/public/runs/runsPerDataPass.overview.test.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/public/runs/runsPerDataPass.overview.test.js b/test/public/runs/runsPerDataPass.overview.test.js index 9f45829d09..820cf20f18 100644 --- a/test/public/runs/runsPerDataPass.overview.test.js +++ b/test/public/runs/runsPerDataPass.overview.test.js @@ -167,14 +167,15 @@ module.exports = () => { await expectInnerText(page, '#row106-globalAggregatedQuality', 'GAQ'); - await goToPage(page, 'runs-per-data-pass', { queryParameters: { dataPassId: 3 } }); + await navigateToRunsPerDataPass(page, { lhcPeriodId: 1, dataPassId: 3 }, { epectedRowsCount: 4 }); await expectInnerText(page, '#row56-globalAggregatedQuality', '0MC.R'); expect(await getPopoverInnerText(await page.waitForSelector('#row56-globalAggregatedQuality .popover-trigger'))) .to.be.equal('Missing 4 verifications'); }); it('should switch mcReproducibleAsNotBad', async () => { - await goToPage(page, 'runs-per-data-pass', { queryParameters: { dataPassId: 1 } }); + await navigateToRunsPerDataPass(page, { lhcPeriodId: 2, dataPassId: 1 }, { epectedRowsCount: 3 }); + await pressElement(page, '#mcReproducibleAsNotBadToggle input', true); await waitForTableLength(page, 3); await expectInnerText(page, 'tr#row106 .column-CPV a', '89'); From d862b1647fa89dde48904ef0ca39d8842509f4b9 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Wed, 27 Nov 2024 15:45:04 +0100 Subject: [PATCH 35/39] Add QcSummaryUnit wrapping class --- .../qualityControlFlag/QcFlagService.js | 156 ++++++++++-------- 1 file changed, 87 insertions(+), 69 deletions(-) diff --git a/lib/server/services/qualityControlFlag/QcFlagService.js b/lib/server/services/qualityControlFlag/QcFlagService.js index 43538d8c9b..0bb69f7d61 100644 --- a/lib/server/services/qualityControlFlag/QcFlagService.js +++ b/lib/server/services/qualityControlFlag/QcFlagService.js @@ -71,13 +71,6 @@ const validateUserDetectorAccess = (userRoles, detectorName) => { * @property {number} missingVerificationsCount - number of not verified QC flags which are not discarded * @property {boolean} mcReproducible - states whether some Limited Acceptance MC Reproducible flag was assigned */ -const QC_SUMMARY_PROPERTIES = { - badEffectiveRunCoverage: 'badEffectiveRunCoverage', - explicitlyNotBadEffectiveRunCoverage: 'explicitlyNotBadEffectiveRunCoverage', - qualityNotDefinedEffectiveRunCoverage: 'qualityNotDefinedEffectiveRunCoverage', - missingVerificationsCount: 'missingVerificationsCount', - mcReproducible: 'mcReproducible', -}; /** * @typedef RunQcSummary @@ -110,6 +103,82 @@ const QC_SUMMARY_PROPERTIES = { * @type {Object} runNumber to RunGaqSummary mapping */ +/** + * @typedef QcSubSummaryUnit + * @property {boolean} bad - true when coverage is bad, false otherwise + * @property {number} effectiveRunCoverage - fraction of run's data covered with flags + * @property {number} missingVerificationsCount - number of not verified QC flags which are not discarded + * @property {boolean} mcReproducible - states whether some Limited Acceptance MC Reproducible flag was assigned + */ + +/** + * Wrapper class for QcSummaryUnit + */ +class QcSummaryUnit { + /** + * Constructor + */ + constructor() { + this.badEffectiveRunCoverage = undefined; + this.explicitlyNotBadEffectiveRunCoverage = undefined; + this.qualityNotDefinedEffectiveRunCoverage = undefined; + this.missingVerificationsCount = 0; + this.mcReproducible = false; + } + + /** + * Update self with partial QC summary unit + * + * @param {Partial} partialSubSummaryUnit new properties to be applied to the self + * @return {void} + */ + updateWithSubSummary(partialSubSummaryUnit) { + const { + bad, + effectiveRunCoverage, + mcReproducible, + missingVerificationsCount, + } = partialSubSummaryUnit; + + if (bad === null) { + this.qualityNotDefinedEffectiveRunCoverage = effectiveRunCoverage; + } else if (bad) { + this.badEffectiveRunCoverage = effectiveRunCoverage; + } else { + this.explicitlyNotBadEffectiveRunCoverage = effectiveRunCoverage; + } + + if (missingVerificationsCount) { + this.missingVerificationsCount += missingVerificationsCount; + } + + this.mcReproducible = mcReproducible || this.mcReproducible; + } + + /** + * Set default summary properties which were not set when merging sub-summaries, so each expected property has no 'undefined' value + * + * @return {void} + */ + fillMissingValues() { + if (this.explicitlyNotBadEffectiveRunCoverage === undefined) { + this.explicitlyNotBadEffectiveRunCoverage = 0; + } + if (this.badEffectiveRunCoverage === undefined) { + this.badEffectiveRunCoverage = 0; + } + if (this.qualityNotDefinedEffectiveRunCoverage === undefined) { + const { badEffectiveRunCoverage, explicitlyNotBadEffectiveRunCoverage } = this; + const definedCoverage = badEffectiveRunCoverage + explicitlyNotBadEffectiveRunCoverage; + + this.qualityNotDefinedEffectiveRunCoverage = + badEffectiveRunCoverage === null || explicitlyNotBadEffectiveRunCoverage === null + ? null + : 1 - definedCoverage; + } + } +} + /** * Quality control flags service */ @@ -342,26 +411,27 @@ class QcFlagService { detectorId, flagIds, } = runDetectorSummaryForFlagTypesClass; - const missingVerificationsCount = flagIds.filter((id) => notVerifiedFlagsIds.has(id)).length; if (!summary[runNumber]) { summary[runNumber] = {}; } if (!summary[runNumber][detectorId]) { - summary[runNumber][detectorId] = { [QC_SUMMARY_PROPERTIES.mcReproducible]: false }; + summary[runNumber][detectorId] = new QcSummaryUnit(); } const runDetectorSummary = summary[runNumber][detectorId]; - runDetectorSummary[QC_SUMMARY_PROPERTIES.missingVerificationsCount] = - (runDetectorSummary[QC_SUMMARY_PROPERTIES.missingVerificationsCount] ?? 0) + missingVerificationsCount; + const missingVerificationsCount = flagIds.filter((id) => notVerifiedFlagsIds.has(id)).length; - this._mergeIntoSummaryUnit(runDetectorSummary, runDetectorSummaryForFlagTypesClass); + runDetectorSummary.updateWithSubSummary({ + ...runDetectorSummaryForFlagTypesClass, + missingVerificationsCount, + }); } for (const runSummaries of Object.values(summary)) { for (const runDetectorSummary of Object.values(runSummaries)) { - this._fillQcSummaryUnitMissingValues(runDetectorSummary); + runDetectorSummary.fillMissingValues(); } } @@ -680,7 +750,7 @@ class QcFlagService { } = subSummary; if (!summary[runNumber]) { - summary[runNumber] = { [QC_SUMMARY_PROPERTIES.mcReproducible]: false }; + summary[runNumber] = new QcSummaryUnit(); } if (!flagsAndVerifications[runNumber]) { flagsAndVerifications[runNumber] = {}; @@ -696,71 +766,19 @@ class QcFlagService { distinctVerifiedFlagsIds: new Set([...distinctRunVerifiedFlagsIds, ...verifiedFlagsIds]), }; - this._mergeIntoSummaryUnit(runSummary, subSummary); + runSummary.updateWithSubSummary(subSummary); } for (const [runNumber, { distinctFlagsIds, distinctVerifiedFlagsIds }] of Object.entries(flagsAndVerifications)) { - summary[runNumber][QC_SUMMARY_PROPERTIES.missingVerificationsCount] = distinctFlagsIds.size - distinctVerifiedFlagsIds.size; + summary[runNumber].updateWithSubSummary({ missingVerificationsCount: distinctFlagsIds.size - distinctVerifiedFlagsIds.size }); } for (const runSummary of Object.values(summary)) { - this._fillQcSummaryUnitMissingValues(runSummary); + runSummary.fillMissingValues(); } return summary; } - /** - * Update RunDetectorQcSummary or RunGaqSummary with new information - * - * @param {RunDetectorQcSummary|RunGaqSummary} summaryUnit RunDetectorQcSummary or RunGaqSummary - * @param {{ bad: boolean, effectiveRunCoverage: number, mcReproducible: boolean}} partialSummaryUnit new properties - * to be applied to the summary object - * @return {void} - */ - _mergeIntoSummaryUnit(summaryUnit, partialSummaryUnit) { - const { - bad, - effectiveRunCoverage, - mcReproducible, - } = partialSummaryUnit; - - if (bad === null) { - summaryUnit[QC_SUMMARY_PROPERTIES.qualityNotDefinedEffectiveRunCoverage] = effectiveRunCoverage; - } else if (bad) { - summaryUnit[QC_SUMMARY_PROPERTIES.badEffectiveRunCoverage] = effectiveRunCoverage; - } else { - summaryUnit[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] = effectiveRunCoverage; - } - - summaryUnit[QC_SUMMARY_PROPERTIES.mcReproducible] = - mcReproducible || summaryUnit[QC_SUMMARY_PROPERTIES.mcReproducible]; - } - - /** - * Set default summary properties which were not set when merging sub-summaries, so each expected property has no 'undefined' value - * - * @param {RunDetectorQcSummary|RunGaqSummary} summaryUnit RunDetectorQcSummary or RunGaqSummary - * @return {void} - */ - _fillQcSummaryUnitMissingValues(summaryUnit) { - if (summaryUnit[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] === undefined) { - summaryUnit[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] = 0; - } - if (summaryUnit[QC_SUMMARY_PROPERTIES.badEffectiveRunCoverage] === undefined) { - summaryUnit[QC_SUMMARY_PROPERTIES.badEffectiveRunCoverage] = 0; - } - if (summaryUnit[QC_SUMMARY_PROPERTIES.qualityNotDefinedEffectiveRunCoverage] === undefined) { - const badEffectiveRunCoverage = summaryUnit[QC_SUMMARY_PROPERTIES.badEffectiveRunCoverage]; - const explicitlyNotBadEffectiveRunCoverage = summaryUnit[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage]; - const definedCoverage = badEffectiveRunCoverage + explicitlyNotBadEffectiveRunCoverage; - - summaryUnit[QC_SUMMARY_PROPERTIES.qualityNotDefinedEffectiveRunCoverage] = - badEffectiveRunCoverage === null || explicitlyNotBadEffectiveRunCoverage === null - ? null - : 1 - definedCoverage; - } - } - /** * Return a paginated list of QC flags related to a given simulation pass, run and dpl detector * From ad1ab15e1d5254a4aaa5582edf3fa3c3176497aa Mon Sep 17 00:00:00 2001 From: xsalonx Date: Wed, 27 Nov 2024 16:23:43 +0100 Subject: [PATCH 36/39] fix test --- .../qualityControlFlag/QcFlagService.test.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js index f9493e8472..d44d26f85d 100644 --- a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js +++ b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js @@ -152,7 +152,7 @@ module.exports = () => { mcReproducible: true, badEffectiveRunCoverage: 0.3333333, explicitlyNotBadEffectiveRunCoverage: 0, - qualityNotDefinedEffectiveRunCoverage: 0.6667000000000001, + qualityNotDefinedEffectiveRunCoverage: 0.6666667, }, 16: { badEffectiveRunCoverage: 0, @@ -219,7 +219,7 @@ module.exports = () => { mcReproducible: false, badEffectiveRunCoverage: 0.0769231, explicitlyNotBadEffectiveRunCoverage: 0, - qualityNotDefinedEffectiveRunCoverage: 0.9231, + qualityNotDefinedEffectiveRunCoverage: 0.9230769, }, }, }); @@ -237,7 +237,7 @@ module.exports = () => { mcReproducible: false, badEffectiveRunCoverage: 0.0769231, explicitlyNotBadEffectiveRunCoverage: 0, - qualityNotDefinedEffectiveRunCoverage: 0.9231, + qualityNotDefinedEffectiveRunCoverage: 0.9230769, }, }, }); @@ -272,7 +272,7 @@ module.exports = () => { mcReproducible: false, badEffectiveRunCoverage: 0.7222222, explicitlyNotBadEffectiveRunCoverage: 0, - qualityNotDefinedEffectiveRunCoverage: 0.27780000000000005, + qualityNotDefinedEffectiveRunCoverage: 0.27777779999999996, }, }, }); @@ -1540,10 +1540,10 @@ module.exports = () => { [runNumber]: expectedGaqSummary, 56: { missingVerificationsCount: 2, - explicitlyNotBadEffectiveRunCoverage: 1, + explicitlyNotBadEffectiveRunCoverage: 0, badEffectiveRunCoverage: 0, mcReproducible: false, - qualityNotDefinedEffectiveRunCoverage: 0, + qualityNotDefinedEffectiveRunCoverage: 1, }, 54: { missingVerificationsCount: 1, From 1f50297cd4b2d1b8b439778173aeb0003723836e Mon Sep 17 00:00:00 2001 From: xsalonx Date: Thu, 28 Nov 2024 12:33:57 +0100 Subject: [PATCH 37/39] fix test --- test/api/qcFlags.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/api/qcFlags.test.js b/test/api/qcFlags.test.js index ea77218562..cc82a036ca 100644 --- a/test/api/qcFlags.test.js +++ b/test/api/qcFlags.test.js @@ -79,7 +79,7 @@ module.exports = () => { mcReproducible: true, badEffectiveRunCoverage: 0.3333333, explicitlyNotBadEffectiveRunCoverage: 0, - qualityNotDefinedEffectiveRunCoverage: 0.6667000000000001, + qualityNotDefinedEffectiveRunCoverage: 0.6666667, }, 16: { badEffectiveRunCoverage: 0, @@ -128,7 +128,7 @@ module.exports = () => { mcReproducible: false, badEffectiveRunCoverage: 0.7222222, explicitlyNotBadEffectiveRunCoverage: 0, - qualityNotDefinedEffectiveRunCoverage: 0.27780000000000005, + qualityNotDefinedEffectiveRunCoverage: 0.27777779999999996, }, }, }); From 1632aa9af082d1f86569f8fc968977c795b93925 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Thu, 28 Nov 2024 15:52:14 +0100 Subject: [PATCH 38/39] fix --- .../services/qualityControlFlag/QcFlagService.js | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/server/services/qualityControlFlag/QcFlagService.js b/lib/server/services/qualityControlFlag/QcFlagService.js index 0bb69f7d61..3cfb1ae5e5 100644 --- a/lib/server/services/qualityControlFlag/QcFlagService.js +++ b/lib/server/services/qualityControlFlag/QcFlagService.js @@ -140,12 +140,14 @@ class QcSummaryUnit { missingVerificationsCount, } = partialSubSummaryUnit; - if (bad === null) { - this.qualityNotDefinedEffectiveRunCoverage = effectiveRunCoverage; - } else if (bad) { - this.badEffectiveRunCoverage = effectiveRunCoverage; - } else { - this.explicitlyNotBadEffectiveRunCoverage = effectiveRunCoverage; + if (bad !== undefined && effectiveRunCoverage !== undefined) { + if (bad === null) { + this.qualityNotDefinedEffectiveRunCoverage = effectiveRunCoverage; + } else if (bad) { + this.badEffectiveRunCoverage = effectiveRunCoverage; + } else { + this.explicitlyNotBadEffectiveRunCoverage = effectiveRunCoverage; + } } if (missingVerificationsCount) { From 570f25c8c172482369939e306551463c7ebd4ef0 Mon Sep 17 00:00:00 2001 From: xsalonx Date: Thu, 28 Nov 2024 16:03:44 +0100 Subject: [PATCH 39/39] fix test --- .../server/services/qualityControlFlag/QcFlagService.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js index d44d26f85d..5fd47e061a 100644 --- a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js +++ b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js @@ -1540,10 +1540,10 @@ module.exports = () => { [runNumber]: expectedGaqSummary, 56: { missingVerificationsCount: 2, - explicitlyNotBadEffectiveRunCoverage: 0, + explicitlyNotBadEffectiveRunCoverage: 1, badEffectiveRunCoverage: 0, mcReproducible: false, - qualityNotDefinedEffectiveRunCoverage: 1, + qualityNotDefinedEffectiveRunCoverage: 0, }, 54: { missingVerificationsCount: 1,