From eb3c04943b28e50cb592ae69266d1fee596354eb Mon Sep 17 00:00:00 2001 From: Greg Dean Date: Mon, 28 Oct 2024 11:41:09 -0400 Subject: [PATCH 1/6] adding columns for resource and PI organization ID for cloud realm --- .../Version1050To1100/DatabasesMigration.php | 2 +- .../cloud_metrics_aggregation.json | 6 +++++ .../cloud_metrics_aggregation_by_day.json | 6 +++++ .../cloud_common/session_records.json | 24 +++++++++++++++++++ .../cloud_common/cloudfact_by_.json | 15 ++++++++++++ .../cloud_common/cloudfact_by_day.json | 15 ++++++++++++ .../cloud_common/session_records.json | 16 +++++++++++++ 7 files changed, 83 insertions(+), 1 deletion(-) diff --git a/classes/OpenXdmod/Migration/Version1050To1100/DatabasesMigration.php b/classes/OpenXdmod/Migration/Version1050To1100/DatabasesMigration.php index c6266c003b..f93f7edb91 100644 --- a/classes/OpenXdmod/Migration/Version1050To1100/DatabasesMigration.php +++ b/classes/OpenXdmod/Migration/Version1050To1100/DatabasesMigration.php @@ -24,7 +24,7 @@ public function execute() if ($mysql_helper->tableExists('modw_cloud.event')) { Utilities::runEtlPipeline( - ['cloud-migration-10-5-0_11-0-0'], + ['cloud-migration-10-5-0_11-0-0','cloud-state-pipeline'], $this->logger, ['last-modified-start-date' => '2017-01-01 00:00:00'] ); diff --git a/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation.json b/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation.json index 5e1c15a7bf..07fd5ef2e8 100644 --- a/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation.json +++ b/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation.json @@ -27,8 +27,10 @@ "year": "${:YEAR_VALUE}", "${AGGREGATION_UNIT}": "${:PERIOD_VALUE}", "host_resource_id": "sr.resource_id", + "resource_organization_id": "sr.resource_organization_id", "account_id": "sr.account_id", "person_id": "sr.person_id", + "person_organization_id": "sr.person_organization_id", "systemaccount_id": "sr.systemaccount_id", "processorbucket_id": "sr.processorbucket_id", "memorybucket_id": "sr.memorybucket_id", @@ -47,6 +49,7 @@ "domain_id": "sr.domain_id", "service_provider": "sr.service_provider", "principalinvestigator_person_id": "sr.principalinvestigator_person_id", + "piperson_organization_id": "sr.piperson_organization_id", "fos_id": "sr.fos_id", "instance_state_id": "sr.instance_state_id" }, @@ -55,6 +58,7 @@ "year", "${AGGREGATION_UNIT}", "host_resource_id", + "resource_organization_id", "service_provider", "processorbucket_id", "memorybucket_id", @@ -65,8 +69,10 @@ "domain_id", "service_provider", "principalinvestigator_person_id", + "piperson_organization_id", "fos_id", "person_id", + "person_organization_id", "instance_state_id" ], "joins":[ diff --git a/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation_by_day.json b/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation_by_day.json index 412a0033f0..d55f6e54bc 100644 --- a/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation_by_day.json +++ b/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation_by_day.json @@ -27,8 +27,10 @@ "year": "${:YEAR_VALUE}", "${AGGREGATION_UNIT}": "${:PERIOD_VALUE}", "host_resource_id": "sr.resource_id", + "resource_organization_id": "sr.resource_organization_id", "account_id": "sr.account_id", "person_id": "sr.person_id", + "person_organization_id": "sr.person_organization_id", "systemaccount_id": "sr.systemaccount_id", "processorbucket_id": "sr.processorbucket_id", "memorybucket_id": "sr.memorybucket_id", @@ -47,6 +49,7 @@ "domain_id": "sr.domain_id", "service_provider": "sr.service_provider", "principalinvestigator_person_id": "sr.principalinvestigator_person_id", + "piperson_organization_id": "sr.piperson_organization_id", "fos_id": "sr.fos_id", "instance_state_id": "sr.instance_state_id", "session_id_list": "GROUP_CONCAT(sr.session_id)" @@ -56,6 +59,7 @@ "year", "${AGGREGATION_UNIT}", "host_resource_id", + "resource_organization_id", "service_provider", "processorbucket_id", "memorybucket_id", @@ -66,8 +70,10 @@ "domain_id", "service_provider", "principalinvestigator_person_id", + "piperson_organization_id", "fos_id", "person_id", + "person_organization_id", "instance_state_id" ], "joins":[ diff --git a/configuration/etl/etl_action_defs.d/cloud_common/session_records.json b/configuration/etl/etl_action_defs.d/cloud_common/session_records.json index 9755691058..f47daa8de0 100644 --- a/configuration/etl/etl_action_defs.d/cloud_common/session_records.json +++ b/configuration/etl/etl_action_defs.d/cloud_common/session_records.json @@ -10,6 +10,7 @@ "end_time": "FROM_UNIXTIME(e.end_time_ts)", "end_event_type_id": "e.end_event_id", "resource_id": "it.resource_id", + "resource_organization_id": "r.organization_id", "instance_type": "itt.instance_type", "instance_type_id": "itt.instance_type_id", "num_cores": "itt.num_cores", @@ -23,12 +24,14 @@ "end_day_id": "YEAR(FROM_UNIXTIME(e.end_time_ts)) * 100000 + DAYOFYEAR(FROM_UNIXTIME(e.end_time_ts))", "wallduration": "FLOOR(e.end_time_ts) - FLOOR(e.start_time_ts)", "person_id": "ev.person_id", + "person_organization_id": "p.organization_id", "systemaccount_id": "ev.systemaccount_id", "submission_venue_id": "ev.submission_venue_id", "domain_id": "ev.domain_id", "service_provider": "ev.service_provider", "account_id": "a.account_id", "principalinvestigator_person_id": "a.principalinvestigator_person_id", + "piperson_organization_id": "pi.organization_id", "fos_id": "a.fos_id", "host_id": "ev.host_id", "instance_state_id": "etype.instance_state_id" @@ -75,6 +78,27 @@ "schema": "${SOURCE_SCHEMA}", "alias": "etype", "on": "e.start_event_id = etype.event_type_id" + }, + { + "name": "resourcefact", + "schema": "${UTILITY_SCHEMA}", + "alias": "r", + "type": "LEFT", + "on": "e.resource_id = r.id" + }, + { + "name": "person", + "schema": "${UTILITY_SCHEMA}", + "alias": "p", + "type": "LEFT", + "on": "ev.person_id = p.id" + }, + { + "name": "person", + "schema": "${UTILITY_SCHEMA}", + "alias": "pi", + "type": "LEFT", + "on": "a.principalinvestigator_person_id = pi.id" } ], "orderby": [ diff --git a/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json b/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json index a4ac617fdb..a7e515333a 100644 --- a/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json +++ b/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json @@ -25,6 +25,11 @@ "type": "int(11)", "nullable": false, "comment": "DIMENSION: The resource id of the host of a VM where sessions ran." + },{ + "name": "resource_organization_id", + "type": "int(11)", + "nullable": false, + "comment": "DIMENSION: The organization of the resource that the jobs ran on." },{ "name": "account_id", "type": "smallint(5) unsigned", @@ -35,6 +40,11 @@ "type": "int(11)", "nullable": false, "comment": "DIMENSION: The person id associated with a VM instance." + },{ + "name": "person_organization_id", + "type": "int(11)", + "nullable": false, + "comment": "DIMENSION: The organization of the person that ran the jobs." },{ "name": "systemaccount_id", "type": "int(11)", @@ -127,6 +137,11 @@ "nullable": false, "default": -1, "comment": "DIMENSION: The PI that owns the allocations that these VM's ran under. References principalinvestigator.person_id" + },{ + "name": "piperson_organization_id", + "type": "int(11)", + "nullable": false, + "comment": "DIMENSION: The organization of the PI that owns the project that funds these jobs. References piperson.organization_id" },{ "name": "fos_id", "type": "int(11)", diff --git a/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_day.json b/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_day.json index 900d799bb2..9fb75b078a 100644 --- a/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_day.json +++ b/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_day.json @@ -30,6 +30,11 @@ "type": "int(11)", "nullable": false, "comment": "DIMENSION: The resource id of the host of a VM where sessions ran." + },{ + "name": "resource_organization_id", + "type": "int(11)", + "nullable": false, + "comment": "DIMENSION: The organization of the resource that the jobs ran on." },{ "name": "account_id", "type": "smallint(5) unsigned", @@ -40,6 +45,11 @@ "type": "int(11)", "nullable": false, "comment": "DIMENSION: The person id associated with a VM instance." + },{ + "name": "person_organization_id", + "type": "int(11)", + "nullable": false, + "comment": "DIMENSION: The organization of the person that ran the jobs." },{ "name": "systemaccount_id", "type": "int(11)", @@ -132,6 +142,11 @@ "nullable": false, "default": -1, "comment": "DIMENSION: The PI that owns the allocations that these VM's ran under. References principalinvestigator.person_id" + },{ + "name": "piperson_organization_id", + "type": "int(11)", + "nullable": false, + "comment": "DIMENSION: The organization of the PI that owns the project that funds these jobs. References piperson.organization_id" },{ "name": "fos_id", "type": "int(11)", diff --git a/configuration/etl/etl_tables.d/cloud_common/session_records.json b/configuration/etl/etl_tables.d/cloud_common/session_records.json index 006f806a8b..3ea47f1201 100644 --- a/configuration/etl/etl_tables.d/cloud_common/session_records.json +++ b/configuration/etl/etl_tables.d/cloud_common/session_records.json @@ -20,6 +20,11 @@ "type": "int(11)", "nullable": false }, + { + "name": "resource_organization_id", + "type": "int(11)", + "nullable": false + }, { "name": "start_time", "type": "datetime", @@ -107,6 +112,11 @@ "type": "int(11)", "nullable": true }, + { + "name": "person_organization_id", + "type": "int(11)", + "nullable": false + }, { "name": "systemaccount_id", "type": "int(11)", @@ -146,6 +156,12 @@ "default": -1, "comment": "DIMENSION: The PI that owns the allocations that these VM's ran under. References principalinvestigator.person_id" }, + { + "name": "piperson_organization_id", + "type": "int(11)", + "nullable": false, + "default": -1 + }, { "name": "fos_id", "type": "int(11)", From c29dea9f51c08804e2bda7ca5b6cf32e742cbba7 Mon Sep 17 00:00:00 2001 From: Greg Dean Date: Tue, 3 Dec 2024 14:07:31 -0500 Subject: [PATCH 2/6] updating to new migration pipeline --- .../Version1100To1150/DatabasesMigration.php | 6 ++-- .../etl.d/xdmod-migration-11_0_0-11_5_0.json | 33 +++++++++++++++++++ .../cloud_common/cloudfact_by_.json | 7 ++-- .../cloud_common/cloudfact_by_day.json | 5 ++- .../cloud_common/session_records.json | 6 ++-- 5 files changed, 49 insertions(+), 8 deletions(-) create mode 100644 configuration/etl/etl.d/xdmod-migration-11_0_0-11_5_0.json diff --git a/classes/OpenXdmod/Migration/Version1100To1150/DatabasesMigration.php b/classes/OpenXdmod/Migration/Version1100To1150/DatabasesMigration.php index 5a577c0736..d81e784ebd 100644 --- a/classes/OpenXdmod/Migration/Version1100To1150/DatabasesMigration.php +++ b/classes/OpenXdmod/Migration/Version1100To1150/DatabasesMigration.php @@ -3,7 +3,7 @@ * Update database from version 11.0.0 to 11.5.0 */ -namespace OpenXdmod\Migration\Version1050To1100; +namespace OpenXdmod\Migration\Version1100To1150; use OpenXdmod\Migration\DatabasesMigration as AbstractDatabasesMigration; use CCR\DB; @@ -24,9 +24,9 @@ public function execute() if ($mysql_helper->tableExists('modw_cloud.event')) { Utilities::runEtlPipeline( - ['cloud-migration-10-5-0_11-0-0','cloud-state-pipeline'], + ['cloud-migration-11-0-0_11-5-0','cloud-state-pipeline'], $this->logger, - ['last-modified-start-date' => '2017-01-01 00:00:00'] + ['last-modified-start-date' => '2016-01-01 00:00:00'] ); } } diff --git a/configuration/etl/etl.d/xdmod-migration-11_0_0-11_5_0.json b/configuration/etl/etl.d/xdmod-migration-11_0_0-11_5_0.json new file mode 100644 index 0000000000..1b31d9b3dc --- /dev/null +++ b/configuration/etl/etl.d/xdmod-migration-11_0_0-11_5_0.json @@ -0,0 +1,33 @@ +{ + "module": "xdmod", + "defaults": { + "cloud-migration-11-0-0_11-5-0": { + "namespace": "ETL\\Maintenance", + "options_class": "MaintenanceOptions", + "class": "ManageTables", + "endpoints": { + "destination": { + "type": "mysql", + "name": "Cloud Database", + "config": "database", + "schema": "modw_cloud" + }, + "source": { + "type": "mysql", + "name": "Cloud Databaes", + "config": "database", + "schema": "modw_cloud" + } + } + } + }, + "cloud-migration-11-0-0_11-5-0": [ + { + "name": "AddResourcePIOrganizationID", + "description": "Update instance type union table.", + "definition_file_list": [ + "cloud_common/session_records.json" + ] + } + ] +} diff --git a/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json b/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json index a7e515333a..0ad4c3e83d 100644 --- a/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json +++ b/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json @@ -29,7 +29,8 @@ "name": "resource_organization_id", "type": "int(11)", "nullable": false, - "comment": "DIMENSION: The organization of the resource that the jobs ran on." + "default": -1, + "comment": "DIMENSION: The organization of the resource that VM's are hosted on." },{ "name": "account_id", "type": "smallint(5) unsigned", @@ -44,6 +45,7 @@ "name": "person_organization_id", "type": "int(11)", "nullable": false, + "default": -1, "comment": "DIMENSION: The organization of the person that ran the jobs." },{ "name": "systemaccount_id", @@ -141,7 +143,8 @@ "name": "piperson_organization_id", "type": "int(11)", "nullable": false, - "comment": "DIMENSION: The organization of the PI that owns the project that funds these jobs. References piperson.organization_id" + "default": -1, + "comment": "DIMENSION: The organization of the PI that owns the project that funds these VM's. References piperson.organization_id" },{ "name": "fos_id", "type": "int(11)", diff --git a/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_day.json b/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_day.json index 9fb75b078a..9dd48d9e87 100644 --- a/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_day.json +++ b/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_day.json @@ -34,7 +34,8 @@ "name": "resource_organization_id", "type": "int(11)", "nullable": false, - "comment": "DIMENSION: The organization of the resource that the jobs ran on." + "default": -1, + "comment": "DIMENSION: The organization of the resource that VM's are hosted on." },{ "name": "account_id", "type": "smallint(5) unsigned", @@ -49,6 +50,7 @@ "name": "person_organization_id", "type": "int(11)", "nullable": false, + "default": -1, "comment": "DIMENSION: The organization of the person that ran the jobs." },{ "name": "systemaccount_id", @@ -146,6 +148,7 @@ "name": "piperson_organization_id", "type": "int(11)", "nullable": false, + "default": -1, "comment": "DIMENSION: The organization of the PI that owns the project that funds these jobs. References piperson.organization_id" },{ "name": "fos_id", diff --git a/configuration/etl/etl_tables.d/cloud_common/session_records.json b/configuration/etl/etl_tables.d/cloud_common/session_records.json index 3ea47f1201..9ffd2d54bf 100644 --- a/configuration/etl/etl_tables.d/cloud_common/session_records.json +++ b/configuration/etl/etl_tables.d/cloud_common/session_records.json @@ -23,7 +23,8 @@ { "name": "resource_organization_id", "type": "int(11)", - "nullable": false + "nullable": false, + "default": -1 }, { "name": "start_time", @@ -115,7 +116,8 @@ { "name": "person_organization_id", "type": "int(11)", - "nullable": false + "nullable": false, + "default": -1 }, { "name": "systemaccount_id", From 6ce33c5194b5c7ec11d6c71487c9fec0eec8a8bc Mon Sep 17 00:00:00 2001 From: Greg Dean Date: Wed, 4 Dec 2024 09:43:25 -0500 Subject: [PATCH 3/6] adding resource_organization_id, person_organization_id, piperson_organization_id columns to cloudfact tables --- .../cloud_metrics_aggregation.json | 6 +++++ .../cloud_metrics_aggregation_by_day.json | 6 +++++ .../cloud_common/session_records.json | 26 ++++++++++++++++++- .../cloud_common/cloudfact_by_.json | 18 +++++++++++++ .../cloud_common/cloudfact_by_day.json | 18 +++++++++++++ .../cloud_common/session_records.json | 18 +++++++++++++ 6 files changed, 91 insertions(+), 1 deletion(-) diff --git a/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation.json b/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation.json index 5e1c15a7bf..07fd5ef2e8 100644 --- a/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation.json +++ b/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation.json @@ -27,8 +27,10 @@ "year": "${:YEAR_VALUE}", "${AGGREGATION_UNIT}": "${:PERIOD_VALUE}", "host_resource_id": "sr.resource_id", + "resource_organization_id": "sr.resource_organization_id", "account_id": "sr.account_id", "person_id": "sr.person_id", + "person_organization_id": "sr.person_organization_id", "systemaccount_id": "sr.systemaccount_id", "processorbucket_id": "sr.processorbucket_id", "memorybucket_id": "sr.memorybucket_id", @@ -47,6 +49,7 @@ "domain_id": "sr.domain_id", "service_provider": "sr.service_provider", "principalinvestigator_person_id": "sr.principalinvestigator_person_id", + "piperson_organization_id": "sr.piperson_organization_id", "fos_id": "sr.fos_id", "instance_state_id": "sr.instance_state_id" }, @@ -55,6 +58,7 @@ "year", "${AGGREGATION_UNIT}", "host_resource_id", + "resource_organization_id", "service_provider", "processorbucket_id", "memorybucket_id", @@ -65,8 +69,10 @@ "domain_id", "service_provider", "principalinvestigator_person_id", + "piperson_organization_id", "fos_id", "person_id", + "person_organization_id", "instance_state_id" ], "joins":[ diff --git a/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation_by_day.json b/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation_by_day.json index 412a0033f0..d55f6e54bc 100644 --- a/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation_by_day.json +++ b/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation_by_day.json @@ -27,8 +27,10 @@ "year": "${:YEAR_VALUE}", "${AGGREGATION_UNIT}": "${:PERIOD_VALUE}", "host_resource_id": "sr.resource_id", + "resource_organization_id": "sr.resource_organization_id", "account_id": "sr.account_id", "person_id": "sr.person_id", + "person_organization_id": "sr.person_organization_id", "systemaccount_id": "sr.systemaccount_id", "processorbucket_id": "sr.processorbucket_id", "memorybucket_id": "sr.memorybucket_id", @@ -47,6 +49,7 @@ "domain_id": "sr.domain_id", "service_provider": "sr.service_provider", "principalinvestigator_person_id": "sr.principalinvestigator_person_id", + "piperson_organization_id": "sr.piperson_organization_id", "fos_id": "sr.fos_id", "instance_state_id": "sr.instance_state_id", "session_id_list": "GROUP_CONCAT(sr.session_id)" @@ -56,6 +59,7 @@ "year", "${AGGREGATION_UNIT}", "host_resource_id", + "resource_organization_id", "service_provider", "processorbucket_id", "memorybucket_id", @@ -66,8 +70,10 @@ "domain_id", "service_provider", "principalinvestigator_person_id", + "piperson_organization_id", "fos_id", "person_id", + "person_organization_id", "instance_state_id" ], "joins":[ diff --git a/configuration/etl/etl_action_defs.d/cloud_common/session_records.json b/configuration/etl/etl_action_defs.d/cloud_common/session_records.json index 9755691058..03ad541847 100644 --- a/configuration/etl/etl_action_defs.d/cloud_common/session_records.json +++ b/configuration/etl/etl_action_defs.d/cloud_common/session_records.json @@ -10,6 +10,7 @@ "end_time": "FROM_UNIXTIME(e.end_time_ts)", "end_event_type_id": "e.end_event_id", "resource_id": "it.resource_id", + "resource_organization_id": "r.organization_id", "instance_type": "itt.instance_type", "instance_type_id": "itt.instance_type_id", "num_cores": "itt.num_cores", @@ -23,12 +24,14 @@ "end_day_id": "YEAR(FROM_UNIXTIME(e.end_time_ts)) * 100000 + DAYOFYEAR(FROM_UNIXTIME(e.end_time_ts))", "wallduration": "FLOOR(e.end_time_ts) - FLOOR(e.start_time_ts)", "person_id": "ev.person_id", + "person_organization_id": "p.organization_id", "systemaccount_id": "ev.systemaccount_id", "submission_venue_id": "ev.submission_venue_id", "domain_id": "ev.domain_id", "service_provider": "ev.service_provider", "account_id": "a.account_id", "principalinvestigator_person_id": "a.principalinvestigator_person_id", + "piperson_organization_id": "pi.organization_id", "fos_id": "a.fos_id", "host_id": "ev.host_id", "instance_state_id": "etype.instance_state_id" @@ -75,8 +78,29 @@ "schema": "${SOURCE_SCHEMA}", "alias": "etype", "on": "e.start_event_id = etype.event_type_id" + }, + { + "name": "resourcefact", + "schema": "${UTILITY_SCHEMA}", + "alias": "r", + "type": "LEFT", + "on": "e.resource_id = r.id" + }, + { + "name": "person", + "schema": "${UTILITY_SCHEMA}", + "alias": "p", + "type": "LEFT", + "on": "ev.person_id = p.id" + }, + { + "name": "person", + "schema": "${UTILITY_SCHEMA}", + "alias": "pi", + "type": "LEFT", + "on": "a.principalinvestigator_person_id = pi.id" } - ], + ], "orderby": [ "resource_id ASC, instance_id ASC, start_time_ts ASC" ] diff --git a/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json b/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json index a4ac617fdb..0ad4c3e83d 100644 --- a/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json +++ b/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json @@ -25,6 +25,12 @@ "type": "int(11)", "nullable": false, "comment": "DIMENSION: The resource id of the host of a VM where sessions ran." + },{ + "name": "resource_organization_id", + "type": "int(11)", + "nullable": false, + "default": -1, + "comment": "DIMENSION: The organization of the resource that VM's are hosted on." },{ "name": "account_id", "type": "smallint(5) unsigned", @@ -35,6 +41,12 @@ "type": "int(11)", "nullable": false, "comment": "DIMENSION: The person id associated with a VM instance." + },{ + "name": "person_organization_id", + "type": "int(11)", + "nullable": false, + "default": -1, + "comment": "DIMENSION: The organization of the person that ran the jobs." },{ "name": "systemaccount_id", "type": "int(11)", @@ -127,6 +139,12 @@ "nullable": false, "default": -1, "comment": "DIMENSION: The PI that owns the allocations that these VM's ran under. References principalinvestigator.person_id" + },{ + "name": "piperson_organization_id", + "type": "int(11)", + "nullable": false, + "default": -1, + "comment": "DIMENSION: The organization of the PI that owns the project that funds these VM's. References piperson.organization_id" },{ "name": "fos_id", "type": "int(11)", diff --git a/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_day.json b/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_day.json index 900d799bb2..9dd48d9e87 100644 --- a/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_day.json +++ b/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_day.json @@ -30,6 +30,12 @@ "type": "int(11)", "nullable": false, "comment": "DIMENSION: The resource id of the host of a VM where sessions ran." + },{ + "name": "resource_organization_id", + "type": "int(11)", + "nullable": false, + "default": -1, + "comment": "DIMENSION: The organization of the resource that VM's are hosted on." },{ "name": "account_id", "type": "smallint(5) unsigned", @@ -40,6 +46,12 @@ "type": "int(11)", "nullable": false, "comment": "DIMENSION: The person id associated with a VM instance." + },{ + "name": "person_organization_id", + "type": "int(11)", + "nullable": false, + "default": -1, + "comment": "DIMENSION: The organization of the person that ran the jobs." },{ "name": "systemaccount_id", "type": "int(11)", @@ -132,6 +144,12 @@ "nullable": false, "default": -1, "comment": "DIMENSION: The PI that owns the allocations that these VM's ran under. References principalinvestigator.person_id" + },{ + "name": "piperson_organization_id", + "type": "int(11)", + "nullable": false, + "default": -1, + "comment": "DIMENSION: The organization of the PI that owns the project that funds these jobs. References piperson.organization_id" },{ "name": "fos_id", "type": "int(11)", diff --git a/configuration/etl/etl_tables.d/cloud_common/session_records.json b/configuration/etl/etl_tables.d/cloud_common/session_records.json index 006f806a8b..9ffd2d54bf 100644 --- a/configuration/etl/etl_tables.d/cloud_common/session_records.json +++ b/configuration/etl/etl_tables.d/cloud_common/session_records.json @@ -20,6 +20,12 @@ "type": "int(11)", "nullable": false }, + { + "name": "resource_organization_id", + "type": "int(11)", + "nullable": false, + "default": -1 + }, { "name": "start_time", "type": "datetime", @@ -107,6 +113,12 @@ "type": "int(11)", "nullable": true }, + { + "name": "person_organization_id", + "type": "int(11)", + "nullable": false, + "default": -1 + }, { "name": "systemaccount_id", "type": "int(11)", @@ -146,6 +158,12 @@ "default": -1, "comment": "DIMENSION: The PI that owns the allocations that these VM's ran under. References principalinvestigator.person_id" }, + { + "name": "piperson_organization_id", + "type": "int(11)", + "nullable": false, + "default": -1 + }, { "name": "fos_id", "type": "int(11)", From 2bb2f9a3bf653f0f827c9cef7ce88b21b159faf3 Mon Sep 17 00:00:00 2001 From: Greg Dean Date: Wed, 4 Dec 2024 18:19:01 -0500 Subject: [PATCH 4/6] updating to use correct columns to get pi and person organization columns and remove resource_organization_id column from cloud realm --- .../cloud_common/cloud_metrics_aggregation.json | 2 -- .../cloud_common/cloud_metrics_aggregation_by_day.json | 2 -- .../etl/etl_action_defs.d/cloud_common/session_records.json | 1 - .../etl/etl_tables.d/cloud_common/cloudfact_by_.json | 6 ------ .../etl/etl_tables.d/cloud_common/cloudfact_by_day.json | 6 ------ .../etl/etl_tables.d/cloud_common/session_records.json | 6 ------ 6 files changed, 23 deletions(-) diff --git a/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation.json b/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation.json index 07fd5ef2e8..dbdf765eed 100644 --- a/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation.json +++ b/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation.json @@ -27,7 +27,6 @@ "year": "${:YEAR_VALUE}", "${AGGREGATION_UNIT}": "${:PERIOD_VALUE}", "host_resource_id": "sr.resource_id", - "resource_organization_id": "sr.resource_organization_id", "account_id": "sr.account_id", "person_id": "sr.person_id", "person_organization_id": "sr.person_organization_id", @@ -58,7 +57,6 @@ "year", "${AGGREGATION_UNIT}", "host_resource_id", - "resource_organization_id", "service_provider", "processorbucket_id", "memorybucket_id", diff --git a/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation_by_day.json b/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation_by_day.json index d55f6e54bc..ed6acd4ba7 100644 --- a/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation_by_day.json +++ b/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation_by_day.json @@ -27,7 +27,6 @@ "year": "${:YEAR_VALUE}", "${AGGREGATION_UNIT}": "${:PERIOD_VALUE}", "host_resource_id": "sr.resource_id", - "resource_organization_id": "sr.resource_organization_id", "account_id": "sr.account_id", "person_id": "sr.person_id", "person_organization_id": "sr.person_organization_id", @@ -59,7 +58,6 @@ "year", "${AGGREGATION_UNIT}", "host_resource_id", - "resource_organization_id", "service_provider", "processorbucket_id", "memorybucket_id", diff --git a/configuration/etl/etl_action_defs.d/cloud_common/session_records.json b/configuration/etl/etl_action_defs.d/cloud_common/session_records.json index 03ad541847..a500a3c041 100644 --- a/configuration/etl/etl_action_defs.d/cloud_common/session_records.json +++ b/configuration/etl/etl_action_defs.d/cloud_common/session_records.json @@ -10,7 +10,6 @@ "end_time": "FROM_UNIXTIME(e.end_time_ts)", "end_event_type_id": "e.end_event_id", "resource_id": "it.resource_id", - "resource_organization_id": "r.organization_id", "instance_type": "itt.instance_type", "instance_type_id": "itt.instance_type_id", "num_cores": "itt.num_cores", diff --git a/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json b/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json index 0ad4c3e83d..a409fa7fbc 100644 --- a/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json +++ b/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json @@ -25,12 +25,6 @@ "type": "int(11)", "nullable": false, "comment": "DIMENSION: The resource id of the host of a VM where sessions ran." - },{ - "name": "resource_organization_id", - "type": "int(11)", - "nullable": false, - "default": -1, - "comment": "DIMENSION: The organization of the resource that VM's are hosted on." },{ "name": "account_id", "type": "smallint(5) unsigned", diff --git a/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_day.json b/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_day.json index 9dd48d9e87..cde259d462 100644 --- a/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_day.json +++ b/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_day.json @@ -30,12 +30,6 @@ "type": "int(11)", "nullable": false, "comment": "DIMENSION: The resource id of the host of a VM where sessions ran." - },{ - "name": "resource_organization_id", - "type": "int(11)", - "nullable": false, - "default": -1, - "comment": "DIMENSION: The organization of the resource that VM's are hosted on." },{ "name": "account_id", "type": "smallint(5) unsigned", diff --git a/configuration/etl/etl_tables.d/cloud_common/session_records.json b/configuration/etl/etl_tables.d/cloud_common/session_records.json index 9ffd2d54bf..cd693c4efe 100644 --- a/configuration/etl/etl_tables.d/cloud_common/session_records.json +++ b/configuration/etl/etl_tables.d/cloud_common/session_records.json @@ -20,12 +20,6 @@ "type": "int(11)", "nullable": false }, - { - "name": "resource_organization_id", - "type": "int(11)", - "nullable": false, - "default": -1 - }, { "name": "start_time", "type": "datetime", From 9709d5cb9594e77afab63981bb002713190ff288 Mon Sep 17 00:00:00 2001 From: Greg Dean Date: Wed, 4 Dec 2024 18:21:27 -0500 Subject: [PATCH 5/6] Documentation and syntax updates from PR suggestions Co-authored-by: Aaron Weeden <31246768+aaronweeden@users.noreply.github.com> --- .../etl/etl_action_defs.d/cloud_common/session_records.json | 2 +- configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json | 2 +- .../etl/etl_tables.d/cloud_common/cloudfact_by_day.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configuration/etl/etl_action_defs.d/cloud_common/session_records.json b/configuration/etl/etl_action_defs.d/cloud_common/session_records.json index a500a3c041..9d5ff1fd3c 100644 --- a/configuration/etl/etl_action_defs.d/cloud_common/session_records.json +++ b/configuration/etl/etl_action_defs.d/cloud_common/session_records.json @@ -99,7 +99,7 @@ "type": "LEFT", "on": "a.principalinvestigator_person_id = pi.id" } - ], + ], "orderby": [ "resource_id ASC, instance_id ASC, start_time_ts ASC" ] diff --git a/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json b/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json index a409fa7fbc..75e1015990 100644 --- a/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json +++ b/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json @@ -40,7 +40,7 @@ "type": "int(11)", "nullable": false, "default": -1, - "comment": "DIMENSION: The organization of the person that ran the jobs." + "comment": "DIMENSION: The organization of the person to whom the VM is assigned." },{ "name": "systemaccount_id", "type": "int(11)", diff --git a/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_day.json b/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_day.json index cde259d462..59e0af0f85 100644 --- a/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_day.json +++ b/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_day.json @@ -143,7 +143,7 @@ "type": "int(11)", "nullable": false, "default": -1, - "comment": "DIMENSION: The organization of the PI that owns the project that funds these jobs. References piperson.organization_id" + "comment": "DIMENSION: The organization of the PI that owns the project that funds these VMs. References piperson.organization_id" },{ "name": "fos_id", "type": "int(11)", From fd8a884cf59ec4fbca28a6a1d9238e5809df2938 Mon Sep 17 00:00:00 2001 From: Greg Dean Date: Wed, 4 Dec 2024 19:18:46 -0500 Subject: [PATCH 6/6] update to use correct fields --- .../cloud_common/cloud_metrics_aggregation.json | 2 -- .../cloud_common/cloud_metrics_aggregation_by_day.json | 2 -- .../etl/etl_action_defs.d/cloud_common/session_records.json | 1 - .../etl/etl_tables.d/cloud_common/cloudfact_by_.json | 6 ------ .../etl/etl_tables.d/cloud_common/cloudfact_by_day.json | 6 ------ .../etl/etl_tables.d/cloud_common/session_records.json | 6 ------ 6 files changed, 23 deletions(-) diff --git a/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation.json b/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation.json index 07fd5ef2e8..dbdf765eed 100644 --- a/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation.json +++ b/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation.json @@ -27,7 +27,6 @@ "year": "${:YEAR_VALUE}", "${AGGREGATION_UNIT}": "${:PERIOD_VALUE}", "host_resource_id": "sr.resource_id", - "resource_organization_id": "sr.resource_organization_id", "account_id": "sr.account_id", "person_id": "sr.person_id", "person_organization_id": "sr.person_organization_id", @@ -58,7 +57,6 @@ "year", "${AGGREGATION_UNIT}", "host_resource_id", - "resource_organization_id", "service_provider", "processorbucket_id", "memorybucket_id", diff --git a/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation_by_day.json b/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation_by_day.json index d55f6e54bc..ed6acd4ba7 100644 --- a/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation_by_day.json +++ b/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation_by_day.json @@ -27,7 +27,6 @@ "year": "${:YEAR_VALUE}", "${AGGREGATION_UNIT}": "${:PERIOD_VALUE}", "host_resource_id": "sr.resource_id", - "resource_organization_id": "sr.resource_organization_id", "account_id": "sr.account_id", "person_id": "sr.person_id", "person_organization_id": "sr.person_organization_id", @@ -59,7 +58,6 @@ "year", "${AGGREGATION_UNIT}", "host_resource_id", - "resource_organization_id", "service_provider", "processorbucket_id", "memorybucket_id", diff --git a/configuration/etl/etl_action_defs.d/cloud_common/session_records.json b/configuration/etl/etl_action_defs.d/cloud_common/session_records.json index f47daa8de0..9d5ff1fd3c 100644 --- a/configuration/etl/etl_action_defs.d/cloud_common/session_records.json +++ b/configuration/etl/etl_action_defs.d/cloud_common/session_records.json @@ -10,7 +10,6 @@ "end_time": "FROM_UNIXTIME(e.end_time_ts)", "end_event_type_id": "e.end_event_id", "resource_id": "it.resource_id", - "resource_organization_id": "r.organization_id", "instance_type": "itt.instance_type", "instance_type_id": "itt.instance_type_id", "num_cores": "itt.num_cores", diff --git a/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json b/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json index ffbdcb851d..75e1015990 100644 --- a/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json +++ b/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json @@ -25,12 +25,6 @@ "type": "int(11)", "nullable": false, "comment": "DIMENSION: The resource id of the host of a VM where sessions ran." - },{ - "name": "resource_organization_id", - "type": "int(11)", - "nullable": false, - "default": -1, - "comment": "DIMENSION: The organization of the resource that VM's are hosted on." },{ "name": "account_id", "type": "smallint(5) unsigned", diff --git a/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_day.json b/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_day.json index 7fc57d4c76..59e0af0f85 100644 --- a/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_day.json +++ b/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_day.json @@ -30,12 +30,6 @@ "type": "int(11)", "nullable": false, "comment": "DIMENSION: The resource id of the host of a VM where sessions ran." - },{ - "name": "resource_organization_id", - "type": "int(11)", - "nullable": false, - "default": -1, - "comment": "DIMENSION: The organization of the resource that VM's are hosted on." },{ "name": "account_id", "type": "smallint(5) unsigned", diff --git a/configuration/etl/etl_tables.d/cloud_common/session_records.json b/configuration/etl/etl_tables.d/cloud_common/session_records.json index 9ffd2d54bf..cd693c4efe 100644 --- a/configuration/etl/etl_tables.d/cloud_common/session_records.json +++ b/configuration/etl/etl_tables.d/cloud_common/session_records.json @@ -20,12 +20,6 @@ "type": "int(11)", "nullable": false }, - { - "name": "resource_organization_id", - "type": "int(11)", - "nullable": false, - "default": -1 - }, { "name": "start_time", "type": "datetime",