diff --git a/classes/OpenXdmod/Migration/Version1100To1150/DatabasesMigration.php b/classes/OpenXdmod/Migration/Version1100To1150/DatabasesMigration.php new file mode 100644 index 0000000000..d81e784ebd --- /dev/null +++ b/classes/OpenXdmod/Migration/Version1100To1150/DatabasesMigration.php @@ -0,0 +1,33 @@ +tableExists('modw_cloud.event')) { + Utilities::runEtlPipeline( + ['cloud-migration-11-0-0_11-5-0','cloud-state-pipeline'], + $this->logger, + ['last-modified-start-date' => '2016-01-01 00:00:00'] + ); + } + } +} diff --git a/configuration/etl/etl.d/xdmod-migration-11_0_0-11_5_0.json b/configuration/etl/etl.d/xdmod-migration-11_0_0-11_5_0.json new file mode 100644 index 0000000000..1b31d9b3dc --- /dev/null +++ b/configuration/etl/etl.d/xdmod-migration-11_0_0-11_5_0.json @@ -0,0 +1,33 @@ +{ + "module": "xdmod", + "defaults": { + "cloud-migration-11-0-0_11-5-0": { + "namespace": "ETL\\Maintenance", + "options_class": "MaintenanceOptions", + "class": "ManageTables", + "endpoints": { + "destination": { + "type": "mysql", + "name": "Cloud Database", + "config": "database", + "schema": "modw_cloud" + }, + "source": { + "type": "mysql", + "name": "Cloud Databaes", + "config": "database", + "schema": "modw_cloud" + } + } + } + }, + "cloud-migration-11-0-0_11-5-0": [ + { + "name": "AddResourcePIOrganizationID", + "description": "Update instance type union table.", + "definition_file_list": [ + "cloud_common/session_records.json" + ] + } + ] +} diff --git a/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation.json b/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation.json index 5e1c15a7bf..dbdf765eed 100644 --- a/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation.json +++ b/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation.json @@ -29,6 +29,7 @@ "host_resource_id": "sr.resource_id", "account_id": "sr.account_id", "person_id": "sr.person_id", + "person_organization_id": "sr.person_organization_id", "systemaccount_id": "sr.systemaccount_id", "processorbucket_id": "sr.processorbucket_id", "memorybucket_id": "sr.memorybucket_id", @@ -47,6 +48,7 @@ "domain_id": "sr.domain_id", "service_provider": "sr.service_provider", "principalinvestigator_person_id": "sr.principalinvestigator_person_id", + "piperson_organization_id": "sr.piperson_organization_id", "fos_id": "sr.fos_id", "instance_state_id": "sr.instance_state_id" }, @@ -65,8 +67,10 @@ "domain_id", "service_provider", "principalinvestigator_person_id", + "piperson_organization_id", "fos_id", "person_id", + "person_organization_id", "instance_state_id" ], "joins":[ diff --git a/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation_by_day.json b/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation_by_day.json index 412a0033f0..ed6acd4ba7 100644 --- a/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation_by_day.json +++ b/configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation_by_day.json @@ -29,6 +29,7 @@ "host_resource_id": "sr.resource_id", "account_id": "sr.account_id", "person_id": "sr.person_id", + "person_organization_id": "sr.person_organization_id", "systemaccount_id": "sr.systemaccount_id", "processorbucket_id": "sr.processorbucket_id", "memorybucket_id": "sr.memorybucket_id", @@ -47,6 +48,7 @@ "domain_id": "sr.domain_id", "service_provider": "sr.service_provider", "principalinvestigator_person_id": "sr.principalinvestigator_person_id", + "piperson_organization_id": "sr.piperson_organization_id", "fos_id": "sr.fos_id", "instance_state_id": "sr.instance_state_id", "session_id_list": "GROUP_CONCAT(sr.session_id)" @@ -66,8 +68,10 @@ "domain_id", "service_provider", "principalinvestigator_person_id", + "piperson_organization_id", "fos_id", "person_id", + "person_organization_id", "instance_state_id" ], "joins":[ diff --git a/configuration/etl/etl_action_defs.d/cloud_common/session_records.json b/configuration/etl/etl_action_defs.d/cloud_common/session_records.json index 9755691058..9d5ff1fd3c 100644 --- a/configuration/etl/etl_action_defs.d/cloud_common/session_records.json +++ b/configuration/etl/etl_action_defs.d/cloud_common/session_records.json @@ -23,12 +23,14 @@ "end_day_id": "YEAR(FROM_UNIXTIME(e.end_time_ts)) * 100000 + DAYOFYEAR(FROM_UNIXTIME(e.end_time_ts))", "wallduration": "FLOOR(e.end_time_ts) - FLOOR(e.start_time_ts)", "person_id": "ev.person_id", + "person_organization_id": "p.organization_id", "systemaccount_id": "ev.systemaccount_id", "submission_venue_id": "ev.submission_venue_id", "domain_id": "ev.domain_id", "service_provider": "ev.service_provider", "account_id": "a.account_id", "principalinvestigator_person_id": "a.principalinvestigator_person_id", + "piperson_organization_id": "pi.organization_id", "fos_id": "a.fos_id", "host_id": "ev.host_id", "instance_state_id": "etype.instance_state_id" @@ -75,6 +77,27 @@ "schema": "${SOURCE_SCHEMA}", "alias": "etype", "on": "e.start_event_id = etype.event_type_id" + }, + { + "name": "resourcefact", + "schema": "${UTILITY_SCHEMA}", + "alias": "r", + "type": "LEFT", + "on": "e.resource_id = r.id" + }, + { + "name": "person", + "schema": "${UTILITY_SCHEMA}", + "alias": "p", + "type": "LEFT", + "on": "ev.person_id = p.id" + }, + { + "name": "person", + "schema": "${UTILITY_SCHEMA}", + "alias": "pi", + "type": "LEFT", + "on": "a.principalinvestigator_person_id = pi.id" } ], "orderby": [ diff --git a/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json b/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json index a4ac617fdb..75e1015990 100644 --- a/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json +++ b/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json @@ -35,6 +35,12 @@ "type": "int(11)", "nullable": false, "comment": "DIMENSION: The person id associated with a VM instance." + },{ + "name": "person_organization_id", + "type": "int(11)", + "nullable": false, + "default": -1, + "comment": "DIMENSION: The organization of the person to whom the VM is assigned." },{ "name": "systemaccount_id", "type": "int(11)", @@ -127,6 +133,12 @@ "nullable": false, "default": -1, "comment": "DIMENSION: The PI that owns the allocations that these VM's ran under. References principalinvestigator.person_id" + },{ + "name": "piperson_organization_id", + "type": "int(11)", + "nullable": false, + "default": -1, + "comment": "DIMENSION: The organization of the PI that owns the project that funds these VM's. References piperson.organization_id" },{ "name": "fos_id", "type": "int(11)", diff --git a/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_day.json b/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_day.json index 900d799bb2..59e0af0f85 100644 --- a/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_day.json +++ b/configuration/etl/etl_tables.d/cloud_common/cloudfact_by_day.json @@ -40,6 +40,12 @@ "type": "int(11)", "nullable": false, "comment": "DIMENSION: The person id associated with a VM instance." + },{ + "name": "person_organization_id", + "type": "int(11)", + "nullable": false, + "default": -1, + "comment": "DIMENSION: The organization of the person that ran the jobs." },{ "name": "systemaccount_id", "type": "int(11)", @@ -132,6 +138,12 @@ "nullable": false, "default": -1, "comment": "DIMENSION: The PI that owns the allocations that these VM's ran under. References principalinvestigator.person_id" + },{ + "name": "piperson_organization_id", + "type": "int(11)", + "nullable": false, + "default": -1, + "comment": "DIMENSION: The organization of the PI that owns the project that funds these VMs. References piperson.organization_id" },{ "name": "fos_id", "type": "int(11)", diff --git a/configuration/etl/etl_tables.d/cloud_common/session_records.json b/configuration/etl/etl_tables.d/cloud_common/session_records.json index 006f806a8b..cd693c4efe 100644 --- a/configuration/etl/etl_tables.d/cloud_common/session_records.json +++ b/configuration/etl/etl_tables.d/cloud_common/session_records.json @@ -107,6 +107,12 @@ "type": "int(11)", "nullable": true }, + { + "name": "person_organization_id", + "type": "int(11)", + "nullable": false, + "default": -1 + }, { "name": "systemaccount_id", "type": "int(11)", @@ -146,6 +152,12 @@ "default": -1, "comment": "DIMENSION: The PI that owns the allocations that these VM's ran under. References principalinvestigator.person_id" }, + { + "name": "piperson_organization_id", + "type": "int(11)", + "nullable": false, + "default": -1 + }, { "name": "fos_id", "type": "int(11)",