UTF8

.circleci/config.yml

@@ -30,6 +30,12 @@ jobs:
       - run:
           name: Generate Certificate
           command: /usr/bin/openssl req -new -key /etc/pki/tls/private/localhost.key -x509 -sha256 -days 365 -set_serial $RANDOM -extensions v3_req -out /etc/pki/tls/certs/localhost.crt -subj "/C=XX/L=Default City/O=Default Company Ltd"
+      - run:
+          name: Fix SQL configuration
+          command: |
+            sed -i 's/^sql_mode =$/sql_mode = STRICT_TRANS_TABLES,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION/g' /etc/my.cnf.d/server.cnf
+            echo "character_set_server = utf8" >> /etc/my.cnf.d/server.cnf
+            echo "collation_server = utf8_unicode_ci" >> /etc/my.cnf.d/server.cnf
       - checkout
       # We need to update our acct before we can enable docker layer caching
       #- setup_remote_docker:

classes/DataWarehouse/Access/ReportGenerator.php

@@ -10,7 +10,7 @@ class ReportGenerator extends Common
     const REPORT_DATE_REGEX =  '/^[0-9]{4}(-[0-9]{2}){2}$/';
     const REPORT_FORMATS_REGEX = '/^doc|pdf$/';
     const REPORT_SCHEDULE_REGEX = '/^Once|Daily|Weekly|Monthly|Quarterly|Semi-annually|Annually$/';
-    const REPORT_DELIVERY_REGEX = '/^E-Mail$/';
+    const REPORT_DELIVERY_REGEX = '/^E-mail$/';
 
     /* Patterns related to report charts */
     const REPORT_CHART_TYPE_REGEX = '/^chart_pool|volatile|report|cached$/';

classes/ETL/DbModel/Table.php

@@ -48,10 +48,10 @@ class Table extends SchemaEntity implements iEntity, iDiscoverableEntity, iAlter
         'engine'   => null,
 
         // Optional table default character set
-        'charset'  => null,
+        'charset'  => 'utf8',
 
         // Optional table collation
-        'collation' => null,
+        'collation' => 'utf8_unicode_ci',
 
         // Associative array where the keys are column names and the values are Column objects
         'columns'  => array(),

classes/ETL/aRdbmsDestinationAction.php

@@ -626,6 +626,10 @@ protected function performPreExecuteTasks()
             $sqlList[] = "SET FOREIGN_KEY_CHECKS = 0";
         }
 
+        if ( $this->options->hide_sql_warnings || $this->options->hide_sql_warning_codes ) {
+            $sqlList[] = "SET SQL_MODE = ''";
+        }
+
         $this->executeSqlList($sqlList, $this->destinationEndpoint, "Pre-execute tasks");
 
         return true;

classes/OpenXdmod/Setup/AdminUserSetup.php

@@ -40,7 +40,7 @@ public function handle()
                 $lastName,
                 array(ROLE_ID_MANAGER, ROLE_ID_USER),
                 ROLE_ID_MANAGER,
-                null,
+                -1,
                 -1
             );
 

classes/OpenXdmod/Shredder/Slurm.php

@@ -295,9 +295,6 @@ public function shredLine($line)
             return;
         }
 
-        // Convert job name encoding.
-        $job['job_name'] = mb_convert_encoding($job['job_name'], 'ISO-8859-1', 'UTF-8');
-
         // Convert datetime strings into unix timestamps.
         $dateKeys = array(
             'submit_time',

classes/XDUser.php

@@ -832,9 +832,9 @@ public function getUpdateQuery($updateToken = false, $includePassword = false)
     {
         $result = 'UPDATE moddb.Users SET username = :username,  email_address = :email_address, first_name = :first_name, middle_name = :middle_name, last_name = :last_name, account_is_active = :account_is_active, person_id = :person_id, organization_id = :organization_id, field_of_science = :field_of_science, user_type = :user_type, sticky = :sticky WHERE id = :id';
         if ($updateToken && $includePassword) {
-            $result = 'UPDATE moddb.Users SET username = :username, password = :password, email_address = :email_address, first_name = :first_name, middle_name = :middle_name, last_name = :last_name, account_is_active = :account_is_active, person_id = :person_id, organization_id = :organization_id, field_of_science = :field_of_science, token = :token, user_type = :user_type, password_last_updated = :password_last_updated, sticky = :sticky WHERE id = :id';
+            $result = 'UPDATE moddb.Users SET username = :username, password = :password, email_address = :email_address, first_name = :first_name, middle_name = :middle_name, last_name = :last_name, account_is_active = :account_is_active, person_id = :person_id, organization_id = :organization_id, field_of_science = :field_of_science, token = :token, user_type = :user_type, password_last_updated = NOW(), sticky = :sticky WHERE id = :id';
         } else if (!$updateToken && $includePassword) {
-            $result = 'UPDATE moddb.Users SET username = :username, password = :password, email_address = :email_address, first_name = :first_name, middle_name = :middle_name, last_name = :last_name, account_is_active = :account_is_active, person_id = :person_id, organization_id = :organization_id, field_of_science = :field_of_science, user_type = :user_type, password_last_updated = :password_last_updated, sticky = :sticky WHERE id = :id';
+            $result = 'UPDATE moddb.Users SET username = :username, password = :password, email_address = :email_address, first_name = :first_name, middle_name = :middle_name, last_name = :last_name, account_is_active = :account_is_active, person_id = :person_id, organization_id = :organization_id, field_of_science = :field_of_science, user_type = :user_type, password_last_updated = NOW(), sticky = :sticky WHERE id = :id';
         } else if ($updateToken && !$includePassword) {
             $result = 'UPDATE moddb.Users SET username = :usernam, email_address = :email_address, first_name = :first_name, middle_name = :middle_name, last_name = :last_name, account_is_active = :account_is_active, person_id = :person_id, organization_id = :organization_id, field_of_science = :field_of_science, token = :token, user_type = :user_type, sticky = :sticky WHERE id = :id';
         }
@@ -857,9 +857,9 @@ public function getInsertQuery($updateToken = false, $includePassword = false)
     {
         $result = 'INSERT INTO moddb.Users (username, email_address, first_name, middle_name, last_name, account_is_active, person_id, organization_id, field_of_science, user_type, sticky) VALUES (:username, :email_address, :first_name, :middle_name, :last_name, :account_is_active, :person_id, :organization_id, :field_of_science, :user_type, :sticky)';
         if ($updateToken && $includePassword) {
-            $result = 'INSERT INTO moddb.Users (username, password, password_last_updated, email_address, first_name, middle_name, last_name, account_is_active, person_id, organization_id, field_of_science, token, user_type, sticky) VALUES (:username, :password, :password_last_updated, :email_address, :first_name, :middle_name, :last_name, :account_is_active, :person_id, :organization_id, :field_of_science, :token, :user_type, :sticky)';
+            $result = 'INSERT INTO moddb.Users (username, password, password_last_updated, email_address, first_name, middle_name, last_name, account_is_active, person_id, organization_id, field_of_science, token, user_type, sticky) VALUES (:username, :password, NOW(), :email_address, :first_name, :middle_name, :last_name, :account_is_active, :person_id, :organization_id, :field_of_science, :token, :user_type, :sticky)';
         } else if (!$updateToken && $includePassword) {
-            $result = 'INSERT INTO moddb.Users (username, password, password_last_updated,  email_address, first_name, middle_name, last_name, account_is_active, person_id, organization_id, field_of_science, user_type, sticky) VALUES (:username, :password, :password_last_updated, :email_address, :first_name, :middle_name, :last_name, :account_is_active, :person_id, :organization_id, :field_of_science, :user_type, :sticky)';
+            $result = 'INSERT INTO moddb.Users (username, password, password_last_updated,  email_address, first_name, middle_name, last_name, account_is_active, person_id, organization_id, field_of_science, user_type, sticky) VALUES (:username, :password, NOW(), :email_address, :first_name, :middle_name, :last_name, :account_is_active, :person_id, :organization_id, :field_of_science, :user_type, :sticky)';
         } else if ($updateToken && !$includePassword) {
             $result = 'INSERT INTO moddb.Users (username, email_address, first_name, middle_name, last_name, account_is_active, person_id, organization_id, field_of_science, token, user_type, sticky) VALUES (:username, :email_address, :first_name, :middle_name, :last_name, :account_is_active, :person_id, :organization_id, :field_of_science, :token, :user_type, :sticky)';
         }
@@ -963,7 +963,6 @@ public function saveUser()
                 $this->_password = password_hash($this->_password, PASSWORD_DEFAULT);
                 $update_data['password'] = $this->_password;
             }
-            $update_data['password_last_updated'] = 'NOW()';
         }
         $update_data['email_address'] = ($this->_email);
         $update_data['first_name'] = ($this->_firstName);

configuration/etl/etl_action_defs.d/cloud_openstack/staging_raw_instance_type.json

@@ -8,6 +8,7 @@
     "source_query": {
         "records": {
             "instance_type": "raw.instance_type",
+            "instance_type_id": 0,
             "resource_id": "raw.resource_id",
             "display": "raw.display",
             "description": "raw.description",

configuration/etl/etl_action_defs.d/storage/xdw/storagefact.aggregation.json

@@ -31,9 +31,9 @@
             "avg_logical_usage": "AVG(sf.logical_usage)",
             "sum_logical_usage": "SUM(CAST(sf.logical_usage AS DECIMAL(30,0)))",
             "sum_squared_logical_usage": "SUM(CAST(sf.logical_usage AS DECIMAL(60,0)) * CAST(sf.logical_usage AS DECIMAL(60,0)))",
-            "avg_physical_usage": "AVG(sf.physical_usage)",
-            "sum_physical_usage": "SUM(CAST(sf.physical_usage AS DECIMAL(30,0)))",
-            "sum_squared_physical_usage": "SUM(CAST(sf.physical_usage AS DECIMAL(60,0)) * CAST(sf.physical_usage AS DECIMAL(60,0)))",
+            "avg_physical_usage": "AVG(COALESCE(sf.physical_usage, 0))",
+            "sum_physical_usage": "SUM(CAST(COALESCE(sf.physical_usage, 0) AS DECIMAL(30,0)))",
+            "sum_squared_physical_usage": "SUM(CAST(COALESCE(sf.physical_usage, 0) AS DECIMAL(60,0)) * CAST(COALESCE(sf.physical_usage, 0) AS DECIMAL(60,0)))",
             "avg_soft_threshold": "AVG(sf.soft_threshold)",
             "sum_soft_threshold": "SUM(CAST(sf.soft_threshold AS DECIMAL(30,0)))",
             "avg_hard_threshold": "AVG(sf.hard_threshold)",

configuration/etl/etl_tables.d/logger/log_level.json

@@ -3,8 +3,8 @@
         "schema": "mod_logger",
         "name": "log_level",
         "engine": "InnoDB",
-        "charset": "latin1",
-        "collation": "latin1_swedish_ci",
+        "charset": "utf8",
+        "collation": "utf8_unicode_ci",
         "columns": [
             {
                 "name": "log_level_id",

configuration/etl/etl_tables.d/logger/log_table.json

@@ -3,8 +3,8 @@
         "schema": "mod_logger",
         "name": "log_table",
         "engine": "InnoDB",
-        "charset": "latin1",
-        "collation": "latin1_swedish_ci",
+        "charset": "utf8",
+        "collation": "utf8_unicode_ci",
         "columns": [
             {
                 "name": "id",

tests/artifacts/xdmod/etlv2/configuration/input/etl_tables_8.0.0.d/db_model_test_2.json

@@ -3,16 +3,22 @@
   "table_definition": {
     "name": "modify_table_test",
     "engine": "MyISAM",
+    "charset": "latin1",
+    "collation": "latin1_swedish_ci",
     "columns": [
       {
         "#": "New column at start of list",
         "name": "new_column_1",
         "type": "varchar(40)",
+        "charset": "latin1",
+        "collation": "latin1_swedish_ci",
         "nullable": true
       },
       {
         "name": "resource",
         "type": "varchar(40)",
+        "charset": "latin1",
+        "collation": "latin1_swedish_ci",
         "nullable": true
       },
       {

tests/artifacts/xdmod/etlv2/configuration/input/etl_tables_8.0.0.d/db_model_test_3.json

@@ -3,16 +3,22 @@
   "table_definition": {
     "name": "modify_table_test",
     "engine": "MyISAM",
+    "charset": "latin1",
+    "collation": "latin1_swedish_ci",
     "columns": [
       {
         "name": "resource",
         "type": "varchar(40)",
+        "charset": "latin1",
+        "collation": "latin1_swedish_ci",
         "nullable": true
       },
       {
         "#": "This was the 1st column",
         "name": "new_column_1",
         "type": "varchar(40)",
+        "charset": "latin1",
+        "collation": "latin1_swedish_ci",
         "nullable": true
       },
       {

tests/artifacts/xdmod/etlv2/configuration/input/etl_tables_8.0.0.d/db_model_test_4.json

@@ -3,15 +3,21 @@
   "table_definition": {
     "name": "modify_table_test",
     "engine": "MyISAM",
+    "charset": "latin1",
+    "collation": "latin1_swedish_ci",
     "columns": [
       {
         "name": "resource",
         "type": "varchar(40)",
+        "charset": "latin1",
+        "collation": "latin1_swedish_ci",
         "nullable": true
       },
       {
         "name": "new_column_1",
         "type": "varchar(40)",
+        "charset": "latin1",
+        "collation": "latin1_swedish_ci",
         "nullable": true
       },
       {

tests/artifacts/xdmod/etlv2/configuration/output/create_table.json

@@ -1,13 +1,13 @@
 {
   "comment": "",
   "engine": "myisam",
-  "charset": "latin1",
-  "collation": "latin1_swedish_ci",
+  "charset": "utf8",
+  "collation": "utf8_unicode_ci",
   "columns": [
     {
       "type": "varchar(40)",
-      "charset": "latin1",
-      "collation": "latin1_swedish_ci",
+      "charset": "utf8",
+      "collation": "utf8_unicode_ci",
       "nullable": true,
       "default": null,
       "extra": null,

tests/artifacts/xdmod/etlv2/dbmodel/output/resourceallocationfact_by.aggregation.sql

@@ -15,4 +15,4 @@ CREATE TABLE IF NOT EXISTS `resourceallocationfact_by_quarter` (
   INDEX `index_year` (`year`),
   INDEX `index_organization_id` (`organization_id`),
   INDEX `index_resource_id` (`resource_id`)
-) ENGINE = myisam COMMENT = 'Resource Allocation facts aggregated by quarter.';
+) ENGINE = myisam CHARSET = utf8 COLLATE = utf8_unicode_ci COMMENT = 'Resource Allocation facts aggregated by quarter.';

tests/artifacts/xdmod/unit/etl/db-model/foreign-key-constraint/create-table.json

@@ -46,7 +46,7 @@
             }
         },
         [
-            "CREATE TABLE IF NOT EXISTS `test` (\n  `id` int(11) NOT NULL auto_increment,\n  `other_id` int(11) NOT NULL,\n  PRIMARY KEY (`id`),\n  INDEX `idx_other_id` (`other_id`),\n  CONSTRAINT `fk_other` FOREIGN KEY (`other_id`) REFERENCES `other` (`id`)\n) ENGINE = innodb;"
+            "CREATE TABLE IF NOT EXISTS `test` (\n  `id` int(11) NOT NULL auto_increment,\n  `other_id` int(11) NOT NULL,\n  PRIMARY KEY (`id`),\n  INDEX `idx_other_id` (`other_id`),\n  CONSTRAINT `fk_other` FOREIGN KEY (`other_id`) REFERENCES `other` (`id`)\n) ENGINE = innodb CHARSET = utf8 COLLATE = utf8_unicode_ci;"
         ]
     ],
     "Table with complex foreign key constraint": [
@@ -99,7 +99,7 @@
             }
         },
         [
-            "CREATE TABLE IF NOT EXISTS `test` (\n  `id` int(11) NOT NULL auto_increment,\n  `other_id` int(11) NOT NULL,\n  PRIMARY KEY (`id`),\n  INDEX `idx_other_id` (`other_id`),\n  CONSTRAINT `fk_other` FOREIGN KEY (`other_id`) REFERENCES `mod_other`.`other` (`id`) ON DELETE SET NULL ON UPDATE CASCADE\n) ENGINE = innodb;"
+            "CREATE TABLE IF NOT EXISTS `test` (\n  `id` int(11) NOT NULL auto_increment,\n  `other_id` int(11) NOT NULL,\n  PRIMARY KEY (`id`),\n  INDEX `idx_other_id` (`other_id`),\n  CONSTRAINT `fk_other` FOREIGN KEY (`other_id`) REFERENCES `mod_other`.`other` (`id`) ON DELETE SET NULL ON UPDATE CASCADE\n) ENGINE = innodb CHARSET = utf8 COLLATE = utf8_unicode_ci;"
         ]
     ]
 }

tests/ci/bootstrap.sh

@@ -63,21 +63,6 @@ then
     dnf install -y ~/rpmbuild/RPMS/*/*.rpm
     mysql_install_db --user mysql
 
-    # Make sure that the db config file is setup correctly w/ `sql_mode=`
-    echo "# this is read by the standalone daemon and embedded servers
-          [server]
-          sql_mode=
-          # this is only for the mysqld standalone daemon
-          # Settings user and group are ignored when systemd is used.
-          # If you need to run mysqld under a different user or group,
-          # customize your systemd unit file for mysqld/mariadb according to the
-          # instructions in http://fedoraproject.org/wiki/Systemd
-          [mysqld]
-          datadir=/var/lib/mysql
-          socket=/var/lib/mysql/mysql.sock
-          log-error=/var/log/mariadb/mariadb.log
-          pid-file=/run/mariadb/mariadb.pid" > /etc/my.cnf.d/mariadb-server.cnf
-
     copy_template_httpd_conf
     ~/bin/services start
     mysql -e "CREATE USER 'root'@'gateway' IDENTIFIED BY '';
@@ -128,11 +113,11 @@ then
     if [[ "$XDMOD_REALMS" == *"cloud"* ]];
     then
         last_modified_start_date=$(date +'%F %T')
-        sudo -u xdmod xdmod-shredder -r openstack -d $REF_DIR/openstack -f openstack
-        sudo -u xdmod xdmod-shredder -r nutsetters -d $REF_DIR/nutsetters -f openstack
+        sudo -u xdmod xdmod-shredder -r openstack -d $REF_DIR/openstack -f openstack --debug
+        sudo -u xdmod xdmod-shredder -r nutsetters -d $REF_DIR/nutsetters -f openstack --debug
 
         sudo -u xdmod xdmod-import-csv -t cloud-project-to-pi -i $REF_DIR/cloud-pi-test.csv
-        sudo -u xdmod xdmod-shredder -r openstack -d $REF_DIR/openstack_error_sessions -f openstack
+        sudo -u xdmod xdmod-shredder -r openstack -d $REF_DIR/openstack_error_sessions -f openstack --debug
         sudo -u xdmod xdmod-ingestor --datatype openstack
         sudo -u xdmod xdmod-ingestor --aggregate=cloud --last-modified-start-date "$last_modified_start_date"
     fi

tests/component/lib/ETL/IngestorTest.php

@@ -64,14 +64,12 @@ public function testLoadDataInfileWarnings() {
 
     public function testSqlWarnings() {
         $result = $this->executeOverseer('xdmod.ingestor-tests.test-sql-warnings');
-        $this->assertEquals(0, $result['exit_status'], "Exit code");
+        $this->assertEquals(1, $result['exit_status'], "Exit code");
 
-        /* We are expecting output such as:
+        /* We are expecting output and the process to exit on failure
          *
          * 2018-03-08 13:49:08 [warning] SQL warnings on table '`modw_cloud`.`warning_test`' generated by action sql-warning-test
          * 2018-03-08 13:49:08 [warning] Warning 1264 Out of range value for column 'resource_id' at row 1
-         * 2018-03-08 13:49:08 [warning] Warning 1366 Incorrect integer value: '' for column 'account_id' at row 1
-         * 2018-03-08 13:49:08 [warning] Warning 1366 Incorrect integer value: '' for column 'provider_account_id' at row 1
          */
 
         $numWarnings = 0;
@@ -83,8 +81,7 @@ public function testSqlWarnings() {
             }
         }
 
-        $this->assertGreaterThanOrEqual(4, $numWarnings, 'Expected number of SQL warnings');
-        $this->assertEmpty($result['stderr'], "Std Error");
+        $this->assertGreaterThanOrEqual(1, $numWarnings, 'Expected number of SQL warnings');
     }
 
     /**
@@ -161,7 +158,7 @@ public function testHideSqlWarningCodes() {
     public function testStructuredFileIngestorWithDirectoryScanner() {
         $result = $this->executeOverseer(
             'xdmod.cloud-jobs.GenericRawCloudEventIngestor',
-            sprintf('-v notice -d "CLOUD_EVENT_LOG_DIR=%s/generic_cloud_logs"', BASE_DIR . self::TEST_INPUT_DIR)
+            sprintf('-v notice -d "CLOUD_EVENT_LOG_DIR=%s/generic_cloud_logs" -r openstack', BASE_DIR . self::TEST_INPUT_DIR)
         );
 
         $this->assertEquals(0, $result['exit_status'], 'Exit code');

tests/integration/lib/TestHarness/UserHelper.php

@@ -19,8 +19,8 @@ class UserHelper
      * @param array|null  $acls
      * @param string|null $primaryRole
      * @param string|null $email
-     * @param string|null $organizationId
-     * @param string|null $personId
+     * @param int         $organizationId
+     * @param int         $personId
      * @return XDUser
      * @throws \Exception
      */
@@ -33,8 +33,8 @@ public static function getUser(
         array $acls = null,
         $primaryRole = null,
         $email = null,
-        $organizationId = null,
-        $personId = null
+        $organizationId = -1,
+        $personId = -1
     ) {
         $acls = isset($acls) ? $acls : array(ROLE_ID_USER);
         $primaryRole = isset($primaryRole) ? $primaryRole : ROLE_ID_USER;

tests/unit/lib/ETL/DbModel/DbModelTest.php

@@ -108,7 +108,7 @@ public function testTableSchema()
         $expected = "CREATE TABLE IF NOT EXISTS `table_no_schema` (
   `column1` int(11) NULL DEFAULT 0 COMMENT 'This is my comment',
   `column2` varchar(16) CHARSET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT 'Test Column' COMMENT 'No comment'
-);";
+) CHARSET = utf8 COLLATE = utf8_unicode_ci;";
         $this->assertEquals($expected, $generated);
 
         // SQL with schema
@@ -117,7 +117,7 @@ public function testTableSchema()
         $expected = "CREATE TABLE IF NOT EXISTS `my_schema`.`table_no_schema` (
   `column1` int(11) NULL DEFAULT 0 COMMENT 'This is my comment',
   `column2` varchar(16) CHARSET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT 'Test Column' COMMENT 'No comment'
-);";
+) CHARSET = utf8 COLLATE = utf8_unicode_ci;";
         $this->assertEquals($expected, $generated);
     }