diff --git a/MOVESConfiguration.txt b/MOVESConfiguration.txt index e0f8312..c7ab78d 100644 --- a/MOVESConfiguration.txt +++ b/MOVESConfiguration.txt @@ -1,11 +1,11 @@ defaultServerName = localhost -defaultDatabaseName = movesdb20210209 +defaultDatabaseName = movesdb20210726 executionServerName = localhost executionDatabaseName = MOVESExecution outputServerName = 127.0.0.1 outputDatabaseName = MOVESOutput -nonroadExePath = NONROAD\NR08a\NONROAD.exe -sharedDistributedFolderPath = SharedWork +nonroadExePath = NONROAD/NR08a/NONROAD.exe +sharedDistributedFolderPath = sharedwork computerIDPath = masterFolderPath = . saveTODOPath = diff --git a/NONROAD/NR08a/SOURCE/readme.md b/NONROAD/NR08a/SOURCE/readme.md new file mode 100644 index 0000000..ea8b5e4 --- /dev/null +++ b/NONROAD/NR08a/SOURCE/readme.md @@ -0,0 +1,12 @@ +To compile the nonroad executable, install `gfortran` and navigate to this folder in the command prompt. Then type `make`. + +`make` will find the makefile and then `gfortran` is used to compile the code. + +Note that the location of the nonroad executable is configured in MOVES using 4 files: + +1. MOVESConfiguration.txt +2. MOVESWorker.txt +3. maketodo.txt +4. manyworkers.txt + +Depending on how this is set and which operating system is used, the nonroad executable may need to be moved and/or renamed to match the capitalization set in the configuration files. \ No newline at end of file diff --git a/NONROAD/NR08a/SOURCE/readme.txt b/NONROAD/NR08a/SOURCE/readme.txt deleted file mode 100644 index a5e04a4..0000000 --- a/NONROAD/NR08a/SOURCE/readme.txt +++ /dev/null @@ -1,3 +0,0 @@ -To compile: Install gfortran, then open a DOS box, -CD to this directory, and type "make" at the command line. -make will find makefile and gfortran then compiles all of the code. \ No newline at end of file diff --git a/WorkerConfiguration.txt b/WorkerConfiguration.txt index c775f20..c88e985 100644 --- a/WorkerConfiguration.txt +++ b/WorkerConfiguration.txt @@ -2,11 +2,11 @@ workFolderPath = WorkerFolder workerDatabaseName = MOVESWorker computerIDPath = workerServerName = localhost -sharedDistributedFolderPath = SharedWork +sharedDistributedFolderPath = sharedwork concurrentStatements = 1 -nonroadApplicationPath = NONROAD\NR08a\nonroad.exe -nonroadWorkingFolderPath = NONROAD\NR08a -calculatorApplicationPath = calc\externalcalculatorgo64.exe +nonroadApplicationPath = NONROAD/NR08a/NONROAD.exe +nonroadWorkingFolderPath = NONROAD/NR08a +calculatorApplicationPath = calc/externalcalculatorgo64.exe workerDebug = False mysqlUserName = moves mysqlPassword = 744ff5134053c418265b626f5d7035e3dff3d50c609c548f6f63 diff --git a/build.xml b/build.xml index 8ae8ade..5ad4bb9 100644 --- a/build.xml +++ b/build.xml @@ -144,6 +144,19 @@ + + + + + + + + + + + + + @@ -232,7 +245,7 @@ - + @@ -241,7 +254,7 @@ - + @@ -256,7 +269,7 @@ - + @@ -265,7 +278,7 @@ - + @@ -280,7 +293,7 @@ - + @@ -291,7 +304,7 @@ - + @@ -316,7 +329,7 @@ - + @@ -324,7 +337,7 @@ - + @@ -333,7 +346,7 @@ - + @@ -342,7 +355,7 @@ - + @@ -363,7 +376,7 @@ - + @@ -373,7 +386,7 @@ - + @@ -384,7 +397,7 @@ - + @@ -395,7 +408,7 @@ - + @@ -1191,6 +1204,31 @@ + + + + + + + + + + ERROR: The -Dinput= argument was not provided. Type ANT for more information. + + + + + + + + + + + + + + + diff --git a/database/ActivityCalculator.sql b/database/ActivityCalculator.sql index 71c40b1..0821761 100644 --- a/database/ActivityCalculator.sql +++ b/database/ActivityCalculator.sql @@ -885,7 +885,7 @@ inner join hotellingActivityDistribution ha on ( -- End Section hotellingHours -- Section ONI --- 16, "shi", "Source Hours Idling" -- changed to sho (4) +-- 4, "SHO", "Source Hours Operating" on roadTypeID 1 -- Section WithRegClassID -- @algorithm shi = sho[roadTypeID=1,sourceTypeID,hourDayID,monthID,yearID,ageID,linkID]*fuelFraction[sourceTypeID,modelYearID,fuelTypeID]*regClassFraction[fuelTypeID,modelYearID,sourceTypeID,regClassID] diff --git a/database/CreateOutput.sql b/database/CreateOutput.sql index bd38239..e4face3 100644 --- a/database/CreateOutput.sql +++ b/database/CreateOutput.sql @@ -257,9 +257,6 @@ INSERT IGNORE INTO ActivityType (activityTypeID, activityType, activityTypeDesc) VALUES (14, "hotellingElectric", "Hotelling Battery or AC"); INSERT IGNORE INTO ActivityType (activityTypeID, activityType, activityTypeDesc) VALUES (15, "hotellingOff", "Hotelling All Engines Off"); -INSERT IGNORE INTO ActivityType (activityTypeID, activityType, activityTypeDesc) -VALUES (16, "shi", "Source Hours Idle"); - CREATE TABLE MOVESWorkersUsed ( diff --git a/database/NonroadProcessingScripts/DecodedNonroadOutput.sql b/database/NonroadProcessingScripts/DecodedNonroadOutput.sql new file mode 100644 index 0000000..dc4f1b6 --- /dev/null +++ b/database/NonroadProcessingScripts/DecodedNonroadOutput.sql @@ -0,0 +1,193 @@ +-- Decoded Nonroad Output +-- +-- Decodes the key fields of MOVESOutput and MOVESActivityOutput tables +-- and creates tables DecodedMOVESOutput and DecodedMOVESActivityOutput +-- that contain the additional descriptive character fields. These +-- tables get saved to the same output database. +-- +-- Note that this script does not generate a spreadsheet of results, +-- like other Nonroad Post-Processing Scripts. + +FLUSH TABLES; +SELECT CURRENT_TIME; + +-- +-- Make DecodedMOVESActivityOutput +-- + +drop table if exists decodedmovesactivityoutput; +CREATE TABLE IF NOT EXISTS decodedMOVESActivityOutput +select MovesRunId, + iterationId, + yearId, + monthId, + dayId, + cast(' ' as char( 10)) as dayName, + hourId, + stateId, + cast(' ' as char( 2)) as stateAbbr, + countyId, + cast(' ' as char( 45)) as countyName, + zoneId, + linkId, + sourceTypeId, + regClassId, + fuelTypeId, + cast(' ' as char( 30)) as fuelTypeDesc, + fuelSubTypeId, + cast(' ' as char( 50)) as fuelSubTypeDesc, + modelYearId, + roadTypeId, + SCC, + cast(' ' as char( 40)) as sccDesc, + engTechId, + cast(' ' as char( 80)) as engTechDesc, + sectorId, + cast(' ' as char( 40)) as sectorDesc, + hpId, + cast(' ' as char( 20)) as hpBinName, + activityTypeId, + cast(' ' as char( 50)) as activityTypeDesc, + activity, + activityMean, + activitySigma +from movesActivityOutput; + +update DecodedMOVESActivityOutput as a set dayName = (select b.dayName + from ##defaultdb##.dayOfAnyWeek as b + where a.dayId = b.dayId); + +update DecodedMOVESActivityOutput as a set stateABBR = (select b.stateAbbr + from ##defaultdb##.state as b + where a.stateId = b.stateId); + +update DecodedMOVESActivityOutput as a set countyName = (select b.countyName + from ##defaultdb##.county as b + where a.countyId = b.countyId); + +update DecodedMOVESActivityOutput as a set fuelTypeDesc = (select b.fuelTypeDesc + from ##defaultdb##.nrfuelType as b + where a.fuelTypeId = b.fuelTypeId); + +update DecodedMOVESActivityOutput as a set fuelSubTypeDesc= (select b.fuelSubTypeDesc + from ##defaultdb##.nrfuelsubType as b + where a.fuelSubTypeId = b.fuelSubTypeId); + +update DecodedMOVESActivityOutput as a set sccDesc = (select b.description + from ##defaultdb##.nrscc as b + where a.scc = b.scc); + +update DecodedMOVESActivityOutput as a set engTechDesc = (select b.engTechDesc + from ##defaultdb##.engineTech as b + where a.engTechId = b.engTechId); + +update DecodedMOVESActivityOutput as a set sectorDesc = (select b.description + from ##defaultdb##.sector as b + where a.sectorId = b.sectorId); + +update DecodedMOVESActivityOutput as a set hpBinName = (select b.binName + from ##defaultdb##.nrhprangebin as b + where a.hpID = b.NRHPRangeBinID); + +update DecodedMOVESActivityOutput as a set activityTypeDesc + = (select b.activityTypeDesc + from .activityType as b + where a.activityTypeId = b.activityTypeId); + +-- select * from DecodedMOVESActivityOutput; + + + +-- +-- Make DecodedMOVESOutput table +-- + +drop table if exists decodedMOVESoutput; +CREATE TABLE IF NOT EXISTS decodedMOVESoutput +select MOVESRunID, + iterationID, + yearID, + monthID, + dayID, + cast(' ' as char(10)) as dayName, + hourID, + stateID, + cast(' ' as char( 2)) as stateABBR, + countyID, + cast(' ' as char( 45)) as countyName, + zoneID, + linkID, + pollutantID, + cast(' ' as char( 50)) as pollutantName, + processID, + cast(' ' as char( 50)) as processName, + sourceTypeID, + regClassId, + fuelTypeID, + cast(' ' as char( 30)) as fuelTypeDesc, + fuelSubTypeId, + cast(' ' as char(50)) as fuelSubTypeDesc, + modelYearID, + roadTypeID, + SCC, + cast(' ' as char( 50)) as sccDesc, + engTechId, + cast(' ' as char( 80)) as engTechDesc, + sectorId, + cast(' ' as char( 40)) as sectorDesc, + hpId, + cast(' ' as char( 20)) as hpBinName, + emissionQuant, + emissionQuantMean, + emissionQuantSigma +from movesOutput; + +update DecodedMOVESOutput as a set dayName = (select b.dayName + from ##defaultdb##.dayOfAnyWeek as b + where a.dayId = b.dayId); + +update DecodedMOVESOutput as a set stateABBR = (select b.stateAbbr + from ##defaultdb##.state as b + where a.stateId = b.stateId); + +update DecodedMOVESOutput as a set countyName = (select b.countyName + from ##defaultdb##.county as b + where a.countyId = b.countyId); + +update DecodedMOVESOutput as a set pollutantName = (select b.pollutantName + from ##defaultdb##.pollutant as b + where a.pollutantId = b.pollutantId); + +update DecodedMOVESOutput as a set processName = (select b.processName + from ##defaultdb##.emissionProcess as b + where a.processId = b.processId); + +update DecodedMOVESOutput as a set fuelTypeDesc = (select b.fuelTypeDesc + from ##defaultdb##.nrfuelType as b + where a.fuelTypeId = b.fuelTypeId); + +update DecodedMOVESOutput as a set fuelSubtypeDesc =(select b.fuelSubTypeDesc + from ##defaultdb##.nrfuelsubtype as b + where a.fuelSubtypeId = b.fuelSubTypeId); + +update DecodedMOVESOutput as a set sccDesc = (select b.description + from ##defaultdb##.nrscc as b + where a.scc = b.scc); + +update DecodedMOVESOutput as a set engTechDesc = (select b.engTechDesc + from ##defaultdb##.engineTech as b + where a.engTechId = b.engTechId); + +update DecodedMOVESOutput as a set sectorDesc = (select b.description + from ##defaultdb##.sector as b + where a.sectorId = b.sectorId); + +update DecodedMOVESOutput as a set hpBinName = (select b.binName + from ##defaultdb##.nrhprangebin as b + where a.hpID = b.NRHPRangeBinID); + + + +-- select * from DecodedMOVESOutput; + +FLUSH TABLES; \ No newline at end of file diff --git a/database/NonroadProcessingScripts/EmissionFactors_per_OperatingHour_by_Equipment.sql b/database/NonroadProcessingScripts/EmissionFactors_per_OperatingHour_by_Equipment.sql index 3bba50d..3151002 100644 --- a/database/NonroadProcessingScripts/EmissionFactors_per_OperatingHour_by_Equipment.sql +++ b/database/NonroadProcessingScripts/EmissionFactors_per_OperatingHour_by_Equipment.sql @@ -1,4 +1,4 @@ --- Nonroad Post Processing Script (updated 7/26/2018): +-- Nonroad Post Processing Script (updated 6/23/2021): -- Emission factors in grams per operating hour by equipment type -- -- MOVES-Nonroad Output Guidance: @@ -18,7 +18,34 @@ flush tables; --- Set up indexing +-- Set up indexing for setting NULL values to 0 +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_state already exists.''', 'create index index_state on movesoutput ( stateID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_state already exists.''', 'create index index_state on movesactivityoutput ( stateID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_county already exists.''', 'create index index_county on movesoutput ( countyID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_county already exists.''', 'create index index_county on movesactivityoutput ( countyID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +-- Convert NULLs to 0 to improve joins +UPDATE movesoutput SET stateID = 0 WHERE stateID IS NULL; +UPDATE movesoutput SET countyID = 0 WHERE countyID IS NULL; +UPDATE movesactivityoutput SET stateID = 0 WHERE stateID IS NULL; +UPDATE movesactivityoutput SET countyID = 0 WHERE countyID IS NULL; + +-- Set up indexing for everything else set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index1'); set @sqlstmt := if( @exist > 0, 'select ''INFO: Index already exists.''', 'create index index1 on movesoutput ( MOVESRunID )'); PREPARE stmt FROM @sqlstmt; @@ -139,18 +166,58 @@ select IF(b2.hours != 0, b1.emissionQuant / b2.hours, NULL) as emissionRate, 'g/hr' as emissionRateUnits from temp1 b1 -inner join temp2 b2 - on ((b1.MOVESRunID=b2.MOVESRunID) AND - (b1.yearID=b2.yearID) AND - (b1.monthID=b2.monthID) AND - (b1.dayID=b2.dayID) AND - (b1.stateID=b2.stateID OR b1.stateID IS NULL AND b2.stateID IS NULL) AND - (b1.countyID=b2.countyID OR b1.countyID IS NULL AND b2.countyID IS NULL) AND - (b1.nrEquipTypeID=b2.nrEquipTypeID) AND - (b1.fuelTypeID=b2.fuelTypeID)) +inner join temp2 b2 USING (MOVESRunID,yearID,monthID,dayID,stateID,countyID,nrEquipTypeID,fuelTypeID) left join ##defaultdb##.nrequipmenttype e on (b1.nrequiptypeid = e.nrequiptypeid); +-- Drop intermediate tables and the primary indexes drop table if exists temp1; drop table if exists temp2; drop table if exists hours; -drop table if exists units; \ No newline at end of file +drop table if exists units; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index1'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index1 does not exist.''', 'drop index index1 on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index2'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index2 does not exist.''', 'drop index index2 on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index100'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index100 does not exist.''', 'drop index index100 on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index10'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index10 does not exist.''', 'drop index index10 on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +-- Revert 0s to NULLs +UPDATE movesoutput SET stateID = NULL WHERE stateID = 0; +UPDATE movesoutput SET countyID = NULL WHERE countyID = 0; +UPDATE movesactivityoutput SET stateID = NULL WHERE stateID = 0; +UPDATE movesactivityoutput SET countyID = NULL WHERE countyID = 0; + +-- drop the rest of the indexes +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_state does not exist.''', 'drop index index_state on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_state does not exist.''', 'drop index index_state on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_county does not exist.''', 'drop index index_county on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_county does not exist.''', 'drop index index_county on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; diff --git a/database/NonroadProcessingScripts/EmissionFactors_per_OperatingHour_by_Equipment_and_Horsepower.sql b/database/NonroadProcessingScripts/EmissionFactors_per_OperatingHour_by_Equipment_and_Horsepower.sql index 60bc6ce..6d23739 100644 --- a/database/NonroadProcessingScripts/EmissionFactors_per_OperatingHour_by_Equipment_and_Horsepower.sql +++ b/database/NonroadProcessingScripts/EmissionFactors_per_OperatingHour_by_Equipment_and_Horsepower.sql @@ -1,4 +1,4 @@ --- Nonroad Post Processing Script (updated 7/26/2018): +-- Nonroad Post Processing Script (updated 6/23/2021): -- Emission factors in grams per operating hour by equipment type -- and horsepower class -- @@ -19,7 +19,34 @@ flush tables; --- Set up indexing +-- Set up indexing for setting NULL values to 0 +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_state already exists.''', 'create index index_state on movesoutput ( stateID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_state already exists.''', 'create index index_state on movesactivityoutput ( stateID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_county already exists.''', 'create index index_county on movesoutput ( countyID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_county already exists.''', 'create index index_county on movesactivityoutput ( countyID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +-- Convert NULLs to 0 to improve joins +UPDATE movesoutput SET stateID = 0 WHERE stateID IS NULL; +UPDATE movesoutput SET countyID = 0 WHERE countyID IS NULL; +UPDATE movesactivityoutput SET stateID = 0 WHERE stateID IS NULL; +UPDATE movesactivityoutput SET countyID = 0 WHERE countyID IS NULL; + +-- Set up indexing for everything else set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index1'); set @sqlstmt := if( @exist > 0, 'select ''INFO: Index already exists.''', 'create index index1 on movesoutput ( MOVESRunID )'); PREPARE stmt FROM @sqlstmt; @@ -144,20 +171,59 @@ select IF(b2.hours != 0, b1.emissionQuant / b2.hours, NULL) as emissionRate, 'g/hr' as emissionRateUnits from temp1 b1 -inner join temp2 b2 - on ((b1.MOVESRunID=b2.MOVESRunID) AND - (b1.yearID=b2.yearID) AND - (b1.monthID=b2.monthID) AND - (b1.dayID=b2.dayID) AND - (b1.stateID=b2.stateID OR b1.stateID IS NULL AND b2.stateID IS NULL) AND - (b1.countyID=b2.countyID OR b1.countyID IS NULL AND b2.countyID IS NULL) AND - (b1.nrEquipTypeID=b2.nrEquipTypeID) AND - (b1.hpID=b2.hpID) AND - (b1.fuelTypeID=b2.fuelTypeID)) +inner join temp2 b2 USING (MOVESRunID,yearID,monthID,dayID,stateID,countyID,nrEquipTypeID,hpID,fuelTypeID) left join ##defaultdb##.nrequipmenttype e on (b1.nrequiptypeid = e.nrequiptypeid) left join ##defaultdb##.nrhprangebin h on (b1.hpID = h.NRHPRangeBinID); +-- Drop intermediate tables and the primary indexes drop table if exists hours; drop table if exists temp1; drop table if exists temp2; -drop table if exists units; \ No newline at end of file +drop table if exists units; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index1'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index1 does not exist.''', 'drop index index1 on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index2'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index2 does not exist.''', 'drop index index2 on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index102'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index102 does not exist.''', 'drop index index102 on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index10'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index10 does not exist.''', 'drop index index10 on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +-- Revert 0s to NULLs +UPDATE movesoutput SET stateID = NULL WHERE stateID = 0; +UPDATE movesoutput SET countyID = NULL WHERE countyID = 0; +UPDATE movesactivityoutput SET stateID = NULL WHERE stateID = 0; +UPDATE movesactivityoutput SET countyID = NULL WHERE countyID = 0; + +-- drop the rest of the indexes +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_state does not exist.''', 'drop index index_state on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_state does not exist.''', 'drop index index_state on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_county does not exist.''', 'drop index index_county on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_county does not exist.''', 'drop index index_county on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; diff --git a/database/NonroadProcessingScripts/EmissionFactors_per_OperatingHour_by_SCC.sql b/database/NonroadProcessingScripts/EmissionFactors_per_OperatingHour_by_SCC.sql index 14aea34..ddb074a 100644 --- a/database/NonroadProcessingScripts/EmissionFactors_per_OperatingHour_by_SCC.sql +++ b/database/NonroadProcessingScripts/EmissionFactors_per_OperatingHour_by_SCC.sql @@ -1,4 +1,4 @@ --- Nonroad Post Processing Script (updated 7/26/2018): +-- Nonroad Post Processing Script (updated 6/23/2021): -- Emission factors in grams per operating hour by SCC -- -- MOVES-Nonroad Output Guidance: @@ -18,7 +18,34 @@ flush tables; --- Set up indexing +-- Set up indexing for setting NULL values to 0 +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_state already exists.''', 'create index index_state on movesoutput ( stateID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_state already exists.''', 'create index index_state on movesactivityoutput ( stateID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_county already exists.''', 'create index index_county on movesoutput ( countyID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_county already exists.''', 'create index index_county on movesactivityoutput ( countyID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +-- Convert NULLs to 0 to improve joins +UPDATE movesoutput SET stateID = 0 WHERE stateID IS NULL; +UPDATE movesoutput SET countyID = 0 WHERE countyID IS NULL; +UPDATE movesactivityoutput SET stateID = 0 WHERE stateID IS NULL; +UPDATE movesactivityoutput SET countyID = 0 WHERE countyID IS NULL; + +-- Set up indexing for everything else set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index1'); set @sqlstmt := if( @exist > 0, 'select ''INFO: Index already exists.''', 'create index index1 on movesoutput ( MOVESRunID )'); PREPARE stmt FROM @sqlstmt; @@ -130,17 +157,53 @@ select IF(b2.hours != 0, b1.emissionQuant / b2.hours, NULL) as emissionRate, 'g/hr' as emissionRateUnits from temp1 b1 -inner join temp2 b2 - on ((b1.MOVESRunID=b2.MOVESRunID) AND - (b1.yearID=b2.yearID) AND - (b1.monthID=b2.monthID) AND - (b1.dayID=b2.dayID) AND - (b1.stateID=b2.stateID OR b1.stateID IS NULL AND b2.stateID IS NULL) AND - (b1.countyID=b2.countyID OR b1.countyID IS NULL AND b2.countyID IS NULL) AND - (b1.SCC=b2.SCC)) +inner join temp2 b2 USING (MOVESRunID,yearID,monthID,dayID,stateID,countyID,scc) left join ##defaultdb##.nrscc s on (b1.scc=s.scc); +-- Drop intermediate tables and the primary indexes drop table if exists hours; drop table if exists temp1; drop table if exists temp2; -drop table if exists units; \ No newline at end of file +drop table if exists units; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index1'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index1 does not exist.''', 'drop index index1 on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index101'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index101 does not exist.''', 'drop index index101 on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index10'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index10 does not exist.''', 'drop index index10 on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +-- Revert 0s to NULLs +UPDATE movesoutput SET stateID = NULL WHERE stateID = 0; +UPDATE movesoutput SET countyID = NULL WHERE countyID = 0; +UPDATE movesactivityoutput SET stateID = NULL WHERE stateID = 0; +UPDATE movesactivityoutput SET countyID = NULL WHERE countyID = 0; + +-- drop the rest of the indexes +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_state does not exist.''', 'drop index index_state on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_state does not exist.''', 'drop index index_state on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_county does not exist.''', 'drop index index_county on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_county does not exist.''', 'drop index index_county on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; diff --git a/database/NonroadProcessingScripts/EmissionFactors_per_Vehicle_by_Equipment.sql b/database/NonroadProcessingScripts/EmissionFactors_per_Vehicle_by_Equipment.sql index 5e3c234..5c1e4d7 100644 --- a/database/NonroadProcessingScripts/EmissionFactors_per_Vehicle_by_Equipment.sql +++ b/database/NonroadProcessingScripts/EmissionFactors_per_Vehicle_by_Equipment.sql @@ -1,4 +1,4 @@ --- Nonroad Post Processing Script (updated 7/26/2018): +-- Nonroad Post Processing Script (updated 6/23/2021): -- Emission factors in grams per vehicle by equipment type -- -- MOVES-Nonroad Output Guidance: @@ -18,7 +18,34 @@ flush tables; --- Set up indexing +-- Set up indexing for setting NULL values to 0 +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_state already exists.''', 'create index index_state on movesoutput ( stateID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_state already exists.''', 'create index index_state on movesactivityoutput ( stateID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_county already exists.''', 'create index index_county on movesoutput ( countyID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_county already exists.''', 'create index index_county on movesactivityoutput ( countyID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +-- Convert NULLs to 0 to improve joins +UPDATE movesoutput SET stateID = 0 WHERE stateID IS NULL; +UPDATE movesoutput SET countyID = 0 WHERE countyID IS NULL; +UPDATE movesactivityoutput SET stateID = 0 WHERE stateID IS NULL; +UPDATE movesactivityoutput SET countyID = 0 WHERE countyID IS NULL; + +-- Set up indexing for everything else set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index1'); set @sqlstmt := if( @exist > 0, 'select ''INFO: Index already exists.''', 'create index index1 on movesoutput ( MOVESRunID )'); PREPARE stmt FROM @sqlstmt; @@ -135,18 +162,58 @@ select IF(b2.population != 0, b1.emissionQuant / b2.population, NULL) as emissionRate, concat('g/vehicle per ',@timeUnits) as emissionRateUnits from temp1 b1 -inner join temp2 b2 - on ((b1.MOVESRunID=b2.MOVESRunID) AND - (b1.yearID=b2.yearID) AND - (b1.monthID=b2.monthID) AND - (b1.dayID=b2.dayID) AND - (b1.stateID=b2.stateID OR b1.stateID IS NULL AND b2.stateID IS NULL) AND - (b1.countyID=b2.countyID OR b1.countyID IS NULL AND b2.countyID IS NULL) AND - (b1.nrEquipTypeID=b2.nrEquipTypeID) AND - (b1.fuelTypeID=b2.fuelTypeID)) +inner join temp2 b2 USING (MOVESRunID,yearID,monthID,dayID,stateID,countyID,nrEquipTypeID,fuelTypeID) left join ##defaultdb##.nrequipmenttype e on (b1.nrequiptypeid = e.nrequiptypeid); +-- Drop intermediate tables and the primary indexes drop table if exists population; drop table if exists temp1; drop table if exists temp2; -drop table if exists units; \ No newline at end of file +drop table if exists units; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index1'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index1 does not exist.''', 'drop index index1 on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index2'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index2 does not exist.''', 'drop index index2 on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index102'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index102 does not exist.''', 'drop index index100 on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index10'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index10 does not exist.''', 'drop index index10 on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +-- Revert 0s to NULLs +UPDATE movesoutput SET stateID = NULL WHERE stateID = 0; +UPDATE movesoutput SET countyID = NULL WHERE countyID = 0; +UPDATE movesactivityoutput SET stateID = NULL WHERE stateID = 0; +UPDATE movesactivityoutput SET countyID = NULL WHERE countyID = 0; + +-- drop the rest of the indexes +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_state does not exist.''', 'drop index index_state on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_state does not exist.''', 'drop index index_state on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_county does not exist.''', 'drop index index_county on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_county does not exist.''', 'drop index index_county on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; diff --git a/database/NonroadProcessingScripts/EmissionFactors_per_Vehicle_by_Equipment_and_Horsepower.sql b/database/NonroadProcessingScripts/EmissionFactors_per_Vehicle_by_Equipment_and_Horsepower.sql index 1ae4786..12cf6b4 100644 --- a/database/NonroadProcessingScripts/EmissionFactors_per_Vehicle_by_Equipment_and_Horsepower.sql +++ b/database/NonroadProcessingScripts/EmissionFactors_per_Vehicle_by_Equipment_and_Horsepower.sql @@ -1,4 +1,4 @@ --- Nonroad Post Processing Script (updated 7/26/2018): +-- Nonroad Post Processing Script (updated 6/23/2021): -- Emission factors in grams per vehicle by equipment type and horsepower class -- -- MOVES-Nonroad Output Guidance: @@ -18,7 +18,34 @@ flush tables; --- Set up indexing +-- Set up indexing for setting NULL values to 0 +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_state already exists.''', 'create index index_state on movesoutput ( stateID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_state already exists.''', 'create index index_state on movesactivityoutput ( stateID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_county already exists.''', 'create index index_county on movesoutput ( countyID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_county already exists.''', 'create index index_county on movesactivityoutput ( countyID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +-- Convert NULLs to 0 to improve joins +UPDATE movesoutput SET stateID = 0 WHERE stateID IS NULL; +UPDATE movesoutput SET countyID = 0 WHERE countyID IS NULL; +UPDATE movesactivityoutput SET stateID = 0 WHERE stateID IS NULL; +UPDATE movesactivityoutput SET countyID = 0 WHERE countyID IS NULL; + +-- Set up indexing for everything else set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index1'); set @sqlstmt := if( @exist > 0, 'select ''INFO: Index already exists.''', 'create index index1 on movesoutput ( MOVESRunID )'); PREPARE stmt FROM @sqlstmt; @@ -140,20 +167,59 @@ select IF(b2.population != 0, b1.emissionQuant / b2.population, NULL) as emissionRate, concat('g/vehicle per ',@timeUnits) as emissionRateUnits from temp1 b1 -inner join temp2 b2 - on ((b1.MOVESRunID=b2.MOVESRunID) AND - (b1.yearID=b2.yearID) AND - (b1.monthID=b2.monthID) AND - (b1.dayID=b2.dayID) AND - (b1.stateID=b2.stateID OR b1.stateID IS NULL AND b2.stateID IS NULL) AND - (b1.countyID=b2.countyID OR b1.countyID IS NULL AND b2.countyID IS NULL) AND - (b1.nrEquipTypeID=b2.nrEquipTypeID) AND - (b1.hpID=b2.hpID) AND - (b1.fuelTypeID=b2.fuelTypeID)) +inner join temp2 b2 USING (MOVESRunID,yearID,monthID,dayID,stateID,countyID,nrEquipTypeID,hpID,fuelTypeID) left join ##defaultdb##.nrequipmenttype e on (b1.nrequiptypeid = e.nrequiptypeid) left join ##defaultdb##.nrhprangebin h on (b1.hpID = h.NRHPRangeBinID); +-- Drop intermediate tables and the primary indexes drop table if exists temp1; drop table if exists temp2; drop table if exists population; -drop table if exists units; \ No newline at end of file +drop table if exists units; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index1'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index1 does not exist.''', 'drop index index1 on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index2'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index2 does not exist.''', 'drop index index2 on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index101'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index102 does not exist.''', 'drop index index102 on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index10'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index10 does not exist.''', 'drop index index10 on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +-- Revert 0s to NULLs +UPDATE movesoutput SET stateID = NULL WHERE stateID = 0; +UPDATE movesoutput SET countyID = NULL WHERE countyID = 0; +UPDATE movesactivityoutput SET stateID = NULL WHERE stateID = 0; +UPDATE movesactivityoutput SET countyID = NULL WHERE countyID = 0; + +-- drop the rest of the indexes +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_state does not exist.''', 'drop index index_state on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_state does not exist.''', 'drop index index_state on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_county does not exist.''', 'drop index index_county on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_county does not exist.''', 'drop index index_county on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; \ No newline at end of file diff --git a/database/NonroadProcessingScripts/EmissionFactors_per_Vehicle_by_SCC.sql b/database/NonroadProcessingScripts/EmissionFactors_per_Vehicle_by_SCC.sql index b139900..812c98e 100644 --- a/database/NonroadProcessingScripts/EmissionFactors_per_Vehicle_by_SCC.sql +++ b/database/NonroadProcessingScripts/EmissionFactors_per_Vehicle_by_SCC.sql @@ -18,7 +18,34 @@ flush tables; --- Set up indexing +-- Set up indexing for setting NULL values to 0 +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_state already exists.''', 'create index index_state on movesoutput ( stateID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_state already exists.''', 'create index index_state on movesactivityoutput ( stateID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_county already exists.''', 'create index index_county on movesoutput ( countyID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_county already exists.''', 'create index index_county on movesactivityoutput ( countyID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +-- Convert NULLs to 0 to improve joins +UPDATE movesoutput SET stateID = 0 WHERE stateID IS NULL; +UPDATE movesoutput SET countyID = 0 WHERE countyID IS NULL; +UPDATE movesactivityoutput SET stateID = 0 WHERE stateID IS NULL; +UPDATE movesactivityoutput SET countyID = 0 WHERE countyID IS NULL; + +-- Set up indexing for everything else set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index1'); set @sqlstmt := if( @exist > 0, 'select ''INFO: Index already exists.''', 'create index index1 on movesoutput ( MOVESRunID )'); PREPARE stmt FROM @sqlstmt; @@ -127,17 +154,53 @@ select IF(b2.population != 0, b1.emissionQuant / b2.population, NULL) as emissionRate, concat('g/vehicle per ',@timeUnits) as emissionRateUnits from temp1 b1 -inner join temp2 b2 - on ((b1.MOVESRunID=b2.MOVESRunID) AND - (b1.yearID=b2.yearID) AND - (b1.monthID=b2.monthID) AND - (b1.dayID=b2.dayID) AND - (b1.stateID=b2.stateID OR b1.stateID IS NULL AND b2.stateID IS NULL) AND - (b1.countyID=b2.countyID OR b1.countyID IS NULL AND b2.countyID IS NULL) AND - (b1.SCC=b2.SCC)) +inner join temp2 b2 USING (MOVESRunID,yearID,monthID,dayID,stateID,countyID,scc) left join ##defaultdb##.nrscc s on (b1.scc=s.scc); +-- Drop intermediate tables and the primary indexes drop table if exists temp1; drop table if exists temp2; drop table if exists population; -drop table if exists units; \ No newline at end of file +drop table if exists units; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index1'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index1 does not exist.''', 'drop index index1 on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index101'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index101 does not exist.''', 'drop index index101 on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index10'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index10 does not exist.''', 'drop index index10 on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +-- Revert 0s to NULLs +UPDATE movesoutput SET stateID = NULL WHERE stateID = 0; +UPDATE movesoutput SET countyID = NULL WHERE countyID = 0; +UPDATE movesactivityoutput SET stateID = NULL WHERE stateID = 0; +UPDATE movesactivityoutput SET countyID = NULL WHERE countyID = 0; + +-- drop the rest of the indexes +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_state does not exist.''', 'drop index index_state on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_state does not exist.''', 'drop index index_state on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_county does not exist.''', 'drop index index_county on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_county does not exist.''', 'drop index index_county on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; diff --git a/database/NonroadProcessingScripts/EmissionFactors_per_hphr_by_Equipment.sql b/database/NonroadProcessingScripts/EmissionFactors_per_hphr_by_Equipment.sql index 3da7df5..d44da3f 100644 --- a/database/NonroadProcessingScripts/EmissionFactors_per_hphr_by_Equipment.sql +++ b/database/NonroadProcessingScripts/EmissionFactors_per_hphr_by_Equipment.sql @@ -1,4 +1,4 @@ --- Nonroad Post Processing Script (updated 7/26/2018): +-- Nonroad Post Processing Script (updated 6/23/2021): -- Emission factors in grams per horsepower-hour by equipment type -- -- MOVES-Nonroad Output Guidance: @@ -18,7 +18,70 @@ flush tables; --- Set up indexing +-- Set up indexing for setting NULL values to 0 +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_state already exists.''', 'create index index_state on movesoutput ( stateID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_state already exists.''', 'create index index_state on movesactivityoutput ( stateID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_county already exists.''', 'create index index_county on movesoutput ( countyID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_county already exists.''', 'create index index_county on movesactivityoutput ( countyID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_modelyear'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_modelyear already exists.''', 'create index index_modelyear on movesoutput ( modelYearID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_modelyear'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_modelyear already exists.''', 'create index index_modelyear on movesactivityoutput ( modelYearID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_engtech'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_engtech already exists.''', 'create index index_engtech on movesoutput ( engTechID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_engtech'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_engtech already exists.''', 'create index index_engtech on movesactivityoutput ( engTechID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_hp'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_hp already exists.''', 'create index index_hp on movesoutput ( hpID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_hp'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_hp already exists.''', 'create index index_hp on movesactivityoutput ( hpID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +-- Convert NULLs to 0 to improve joins +UPDATE movesoutput SET stateID = 0 WHERE stateID IS NULL; +UPDATE movesoutput SET countyID = 0 WHERE countyID IS NULL; +UPDATE movesoutput SET modelYearID = 0 WHERE modelYearID IS NULL; +UPDATE movesoutput SET engTechID = 0 WHERE engTechID IS NULL; +UPDATE movesoutput SET hpID = 0 WHERE hpID IS NULL; +UPDATE movesactivityoutput SET stateID = 0 WHERE stateID IS NULL; +UPDATE movesactivityoutput SET countyID = 0 WHERE countyID IS NULL; +UPDATE movesactivityoutput SET modelYearID = 0 WHERE modelYearID IS NULL; +UPDATE movesactivityoutput SET engTechID = 0 WHERE engTechID IS NULL; +UPDATE movesactivityoutput SET hpID = 0 WHERE hpID IS NULL; + +-- Set up indexing for everything else set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index1'); set @sqlstmt := if( @exist > 0, 'select ''INFO: Index already exists.''', 'create index index1 on movesoutput ( movesrunid )'); PREPARE stmt FROM @sqlstmt; @@ -29,8 +92,8 @@ set @sqlstmt := if( @exist > 0, 'select ''INFO: Index already exists.''', 'creat PREPARE stmt FROM @sqlstmt; EXECUTE stmt; -set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index100'); -set @sqlstmt := if( @exist > 0, 'select ''INFO: Index already exists.''', 'create index index100 on movesoutput ( MOVESRunID,yearID,monthID,dayID,stateID,countyID,pollutantID,processID )'); +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index101'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: Index already exists.''', 'create index index101 on movesoutput ( MOVESRunID,yearID,monthID,dayID,stateID,countyID,scc,pollutantID,processID )'); PREPARE stmt FROM @sqlstmt; EXECUTE stmt; @@ -101,9 +164,7 @@ create index index1 on sourceHours (MOVESRunID,yearID,monthID,dayID,stateID,coun create index index1 on horsepower (MOVESRunID,yearID,monthID,dayID,stateID,countyID,SCC,modelYearID,engTechID,hpID); create index index1 on loadfactor (MOVESRunID,yearID,monthID,dayID,stateID,countyID,SCC,modelYearID,engTechID,hpID); --- Note: b1.col = b2.col OR b1.col IS NULL AND b2.col IS NULL is used instead of just a simple --- USING() command because USING() doesn't work if there are NULLs --- Additionally, JOIN followed by ALTER is used instead of a single statement with multiple joins +-- Note: JOIN followed by ALTER is used instead of a single statement with multiple joins -- because if the tables are very large, the separate statements are much faster drop table if exists hphr; create table hphr @@ -122,16 +183,7 @@ select b2.avgHorsepower from sourceHours b1 join horsepower b2 - on ((b1.MOVESRunID=b2.MOVESRunID) AND - (b1.yearID=b2.yearID) AND - (b1.monthID=b2.monthID) AND - (b1.dayID=b2.dayID) AND - (b1.stateID=b2.stateID OR b1.stateID IS NULL AND b2.stateID IS NULL) AND - (b1.countyID=b2.countyID OR b1.countyID IS NULL AND b2.countyID IS NULL) AND - (b1.SCC=b2.SCC) AND - (b1.modelYearID=b2.modelYearID OR b1.modelYearID IS NULL AND b2.modelYearID IS NULL) AND - (b1.engTechID=b2.engTechID OR b1.engTechID IS NULL AND b2.engTechID IS NULL) AND - (b1.hpID=b2.hpID OR b1.hpID IS NULL AND b2.hpID IS NULL)); + USING (MOVESRunID,yearID,monthID,dayID,stateID,countyID,SCC,modelYearID,engTechID,hpID); create index index1 on hphr (MOVESRunID,yearID,monthID,dayID,stateID,countyID,SCC,modelYearID,engTechID,hpID); @@ -143,12 +195,12 @@ set b1.loadFactor = b3.loadFactor, (b1.yearID=b3.yearID) AND (b1.monthID=b3.monthID) AND (b1.dayID=b3.dayID) AND - (b1.stateID=b3.stateID OR b1.stateID IS NULL AND b3.stateID IS NULL) AND - (b1.countyID=b3.countyID OR b1.countyID IS NULL AND b3.countyID IS NULL) AND + (b1.stateID=b3.stateID) AND + (b1.countyID=b3.countyID) AND (b1.SCC=b3.SCC) AND - (b1.modelYearID=b3.modelYearID OR b1.modelYearID IS NULL AND b3.modelYearID IS NULL) AND - (b1.engTechID=b3.engTechID OR b1.engTechID IS NULL AND b3.engTechID IS NULL) AND - (b1.hpID=b3.hpID OR b1.hpID IS NULL AND b3.hpID IS NULL)); + (b1.modelYearID=b3.modelYearID) AND + (b1.engTechID=b3.engTechID) AND + (b1.hpID=b3.hpID)); create index index2 on hphr (scc); @@ -225,21 +277,97 @@ select IF(b2.hpHours != 0, b1.emissionQuant / b2.hpHours, NULL) as emissionRate, 'g/hp-hr' as emissionRateUnits from temp1 b1 -inner join temp2 b2 - on ((b1.MOVESRunID=b2.MOVESRunID) AND - (b1.yearID=b2.yearID) AND - (b1.monthID=b2.monthID) AND - (b1.dayID=b2.dayID) AND - (b1.stateID=b2.stateID OR b1.stateID IS NULL AND b2.stateID IS NULL) AND - (b1.countyID=b2.countyID OR b1.countyID IS NULL AND b2.countyID IS NULL) AND - (b1.nrEquipTypeID=b2.nrEquipTypeID) AND - (b1.fuelTypeID=b2.fuelTypeID)) +inner join temp2 b2 USING (MOVESRunID,yearID,monthID,dayID,stateID,countyID,nrEquipTypeID,fuelTypeID) left join ##defaultdb##.nrequipmenttype e on (b1.nrequiptypeid = e.nrequiptypeid); +-- Drop intermediate tables and the primary indexes drop table if exists sourceHours; drop table if exists horsepower; drop table if exists loadfactor; drop table if exists temp1; drop table if exists temp2; drop table if exists hphr; -drop table if exists units; \ No newline at end of file +drop table if exists units; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index1'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index1 does not exist.''', 'drop index index1 on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index2'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index2 does not exist.''', 'drop index index2 on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index101'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index101 does not exist.''', 'drop index index101 on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index10'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index10 does not exist.''', 'drop index index10 on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +-- Revert 0s to NULLs +UPDATE movesoutput SET stateID = NULL WHERE stateID = 0; +UPDATE movesoutput SET countyID = NULL WHERE countyID = 0; +UPDATE movesoutput SET modelYearID = NULL WHERE modelYearID = 0; +UPDATE movesoutput SET engTechID = NULL WHERE engTechID = 0; +UPDATE movesoutput SET hpID = NULL WHERE hpID = 0; +UPDATE movesactivityoutput SET stateID = NULL WHERE stateID = 0; +UPDATE movesactivityoutput SET countyID = NULL WHERE countyID = 0; +UPDATE movesactivityoutput SET modelYearID = NULL WHERE modelYearID = 0; +UPDATE movesactivityoutput SET engTechID = NULL WHERE engTechID = 0; +UPDATE movesactivityoutput SET hpID = NULL WHERE hpID = 0; + +-- drop the rest of the indexes +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_state does not exist.''', 'drop index index_state on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_state does not exist.''', 'drop index index_state on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_county does not exist.''', 'drop index index_county on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_county does not exist.''', 'drop index index_county on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_modelyear'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_modelyear does not exist.''', 'drop index index_modelyear on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_modelyear'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_modelyear does not exist.''', 'drop index index_modelyear on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_engtech'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_engtech does not exist.''', 'drop index index_engtech on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_engtech'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_engtech does not exist.''', 'drop index index_engtech on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_hp'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_hp does not exist.''', 'drop index index_hp on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_hp'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_hp does not exist.''', 'drop index index_hp on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; diff --git a/database/NonroadProcessingScripts/EmissionFactors_per_hphr_by_Equipment_and_Horsepower.sql b/database/NonroadProcessingScripts/EmissionFactors_per_hphr_by_Equipment_and_Horsepower.sql index b6389ee..282aa5e 100644 --- a/database/NonroadProcessingScripts/EmissionFactors_per_hphr_by_Equipment_and_Horsepower.sql +++ b/database/NonroadProcessingScripts/EmissionFactors_per_hphr_by_Equipment_and_Horsepower.sql @@ -1,4 +1,4 @@ --- Nonroad Post Processing Script (updated 7/26/2018): +-- Nonroad Post Processing Script (updated 6/23/2021): -- Emission factors in grams per horsepower-hour by equipment type -- and horsepower class -- @@ -19,7 +19,58 @@ flush tables; --- Set up indexing +-- Set up indexing for setting NULL values to 0 +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_state already exists.''', 'create index index_state on movesoutput ( stateID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_state already exists.''', 'create index index_state on movesactivityoutput ( stateID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_county already exists.''', 'create index index_county on movesoutput ( countyID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_county already exists.''', 'create index index_county on movesactivityoutput ( countyID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_modelyear'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_modelyear already exists.''', 'create index index_modelyear on movesoutput ( modelYearID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_modelyear'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_modelyear already exists.''', 'create index index_modelyear on movesactivityoutput ( modelYearID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_engtech'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_engtech already exists.''', 'create index index_engtech on movesoutput ( engTechID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_engtech'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_engtech already exists.''', 'create index index_engtech on movesactivityoutput ( engTechID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +-- Convert NULLs to 0 to improve joins +UPDATE movesoutput SET stateID = 0 WHERE stateID IS NULL; +UPDATE movesoutput SET countyID = 0 WHERE countyID IS NULL; +UPDATE movesoutput SET modelYearID = 0 WHERE modelYearID IS NULL; +UPDATE movesoutput SET engTechID = 0 WHERE engTechID IS NULL; +UPDATE movesactivityoutput SET stateID = 0 WHERE stateID IS NULL; +UPDATE movesactivityoutput SET countyID = 0 WHERE countyID IS NULL; +UPDATE movesactivityoutput SET modelYearID = 0 WHERE modelYearID IS NULL; +UPDATE movesactivityoutput SET engTechID = 0 WHERE engTechID IS NULL; + +-- Set up indexing for everything else set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index1'); set @sqlstmt := if( @exist > 0, 'select ''INFO: Index already exists.''', 'create index index1 on movesoutput ( MOVESRunID )'); PREPARE stmt FROM @sqlstmt; @@ -123,16 +174,7 @@ select b2.avgHorsepower from sourceHours b1 join horsepower b2 - on ((b1.MOVESRunID=b2.MOVESRunID) AND - (b1.yearID=b2.yearID) AND - (b1.monthID=b2.monthID) AND - (b1.dayID=b2.dayID) AND - (b1.stateID=b2.stateID OR b1.stateID IS NULL AND b2.stateID IS NULL) AND - (b1.countyID=b2.countyID OR b1.countyID IS NULL AND b2.countyID IS NULL) AND - (b1.SCC=b2.SCC) AND - (b1.modelYearID=b2.modelYearID OR b1.modelYearID IS NULL AND b2.modelYearID IS NULL) AND - (b1.engTechID=b2.engTechID OR b1.engTechID IS NULL AND b2.engTechID IS NULL) AND - (b1.hpID=b2.hpID)); + USING (MOVESRunID,yearID,monthID,dayID,stateID,countyID,SCC,modelYearID,engTechID,hpID); create index index1 on hphr (MOVESRunID,yearID,monthID,dayID,stateID,countyID,SCC,modelYearID,engTechID,hpID); @@ -144,11 +186,11 @@ set b1.loadFactor = b3.loadFactor, (b1.yearID=b3.yearID) AND (b1.monthID=b3.monthID) AND (b1.dayID=b3.dayID) AND - (b1.stateID=b3.stateID OR b1.stateID IS NULL AND b3.stateID IS NULL) AND - (b1.countyID=b3.countyID OR b1.countyID IS NULL AND b3.countyID IS NULL) AND + (b1.stateID=b3.stateID) AND + (b1.countyID=b3.countyID) AND (b1.SCC=b3.SCC) AND - (b1.modelYearID=b3.modelYearID OR b1.modelYearID IS NULL AND b3.modelYearID IS NULL) AND - (b1.engTechID=b3.engTechID OR b1.engTechID IS NULL AND b3.engTechID IS NULL) AND + (b1.modelYearID=b3.modelYearID) AND + (b1.engTechID=b3.engTechID) AND (b1.hpID=b3.hpID)); create index index2 on hphr (scc); @@ -231,23 +273,86 @@ select IF(b2.hpHours != 0, b1.emissionQuant / b2.hpHours, NULL) as emissionRate, 'g/hp-hr' as emissionRateUnits from temp1 b1 -inner join temp2 b2 - on ((b1.MOVESRunID=b2.MOVESRunID) AND - (b1.yearID=b2.yearID) AND - (b1.monthID=b2.monthID) AND - (b1.dayID=b2.dayID) AND - (b1.stateID=b2.stateID OR b1.stateID IS NULL AND b2.stateID IS NULL) AND - (b1.countyID=b2.countyID OR b1.countyID IS NULL AND b2.countyID IS NULL) AND - (b1.nrEquipTypeID=b2.nrEquipTypeID) AND - (b1.hpID=b2.hpID) AND - (b1.fuelTypeID=b2.fuelTypeID)) +inner join temp2 b2 USING (MOVESRunID,yearID,monthID,dayID,stateID,countyID,nrEquipTypeID,hpID,fuelTypeID) left join ##defaultdb##.nrequipmenttype e on (b1.nrequiptypeid = e.nrequiptypeid) left join ##defaultdb##.nrhprangebin h on (b1.hpID = h.NRHPRangeBinID); +-- Drop intermediate tables and the primary indexes drop table if exists sourceHours; drop table if exists loadfactor; drop table if exists horsepower; drop table if exists temp1; drop table if exists temp2; drop table if exists hphr; -drop table if exists units; \ No newline at end of file +drop table if exists units; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index1'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index1 does not exist.''', 'drop index index1 on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index2'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index2 does not exist.''', 'drop index index2 on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index102'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index102 does not exist.''', 'drop index index102 on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index10'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index10 does not exist.''', 'drop index index10 on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +-- Revert 0s to NULLs +UPDATE movesoutput SET stateID = NULL WHERE stateID = 0; +UPDATE movesoutput SET countyID = NULL WHERE countyID = 0; +UPDATE movesoutput SET modelYearID = NULL WHERE modelYearID = 0; +UPDATE movesoutput SET engTechID = NULL WHERE engTechID = 0; +UPDATE movesactivityoutput SET stateID = NULL WHERE stateID = 0; +UPDATE movesactivityoutput SET countyID = NULL WHERE countyID = 0; +UPDATE movesactivityoutput SET modelYearID = NULL WHERE modelYearID = 0; +UPDATE movesactivityoutput SET engTechID = NULL WHERE engTechID = 0; + +-- drop the rest of the indexes +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_state does not exist.''', 'drop index index_state on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_state does not exist.''', 'drop index index_state on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_county does not exist.''', 'drop index index_county on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_county does not exist.''', 'drop index index_county on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_modelyear'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_modelyear does not exist.''', 'drop index index_modelyear on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_modelyear'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_modelyear does not exist.''', 'drop index index_modelyear on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_engtech'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_engtech does not exist.''', 'drop index index_engtech on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_engtech'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_engtech does not exist.''', 'drop index index_engtech on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; \ No newline at end of file diff --git a/database/NonroadProcessingScripts/EmissionFactors_per_hphr_by_SCC.sql b/database/NonroadProcessingScripts/EmissionFactors_per_hphr_by_SCC.sql index 4f5eeb6..cf0dc51 100644 --- a/database/NonroadProcessingScripts/EmissionFactors_per_hphr_by_SCC.sql +++ b/database/NonroadProcessingScripts/EmissionFactors_per_hphr_by_SCC.sql @@ -1,4 +1,4 @@ --- Nonroad Post Processing Script (updated 7/26/2018): +-- Nonroad Post Processing Script (updated 6/23/2021): -- Emission factors in grams per horsepower-hour by SCC -- -- MOVES-Nonroad Output Guidance: @@ -18,7 +18,70 @@ flush tables; --- Set up indexing +-- Set up indexing for setting NULL values to 0 +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_state already exists.''', 'create index index_state on movesoutput ( stateID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_state already exists.''', 'create index index_state on movesactivityoutput ( stateID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_county already exists.''', 'create index index_county on movesoutput ( countyID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_county already exists.''', 'create index index_county on movesactivityoutput ( countyID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_modelyear'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_modelyear already exists.''', 'create index index_modelyear on movesoutput ( modelYearID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_modelyear'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_modelyear already exists.''', 'create index index_modelyear on movesactivityoutput ( modelYearID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_engtech'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_engtech already exists.''', 'create index index_engtech on movesoutput ( engTechID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_engtech'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_engtech already exists.''', 'create index index_engtech on movesactivityoutput ( engTechID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_hp'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_hp already exists.''', 'create index index_hp on movesoutput ( hpID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_hp'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_hp already exists.''', 'create index index_hp on movesactivityoutput ( hpID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +-- Convert NULLs to 0 to improve joins +UPDATE movesoutput SET stateID = 0 WHERE stateID IS NULL; +UPDATE movesoutput SET countyID = 0 WHERE countyID IS NULL; +UPDATE movesoutput SET modelYearID = 0 WHERE modelYearID IS NULL; +UPDATE movesoutput SET engTechID = 0 WHERE engTechID IS NULL; +UPDATE movesoutput SET hpID = 0 WHERE hpID IS NULL; +UPDATE movesactivityoutput SET stateID = 0 WHERE stateID IS NULL; +UPDATE movesactivityoutput SET countyID = 0 WHERE countyID IS NULL; +UPDATE movesactivityoutput SET modelYearID = 0 WHERE modelYearID IS NULL; +UPDATE movesactivityoutput SET engTechID = 0 WHERE engTechID IS NULL; +UPDATE movesactivityoutput SET hpID = 0 WHERE hpID IS NULL; + +-- Set up indexing for everything else set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index1'); set @sqlstmt := if( @exist > 0, 'select ''INFO: Index already exists.''', 'create index index1 on movesoutput ( MOVESRunID )'); PREPARE stmt FROM @sqlstmt; @@ -96,9 +159,7 @@ create index index1 on sourceHours (MOVESRunID,yearID,monthID,dayID,stateID,coun create index index1 on horsepower (MOVESRunID,yearID,monthID,dayID,stateID,countyID,SCC,modelYearID,engTechID,hpID); create index index1 on loadfactor (MOVESRunID,yearID,monthID,dayID,stateID,countyID,SCC,modelYearID,engTechID,hpID); --- Note: b1.col = b2.col OR b1.col IS NULL AND b2.col IS NULL is used instead of just a simple --- USING() command because USING() doesn't work if there are NULLs --- Additionally, JOIN followed by ALTER is used instead of a single statement with multiple joins +-- Note: JOIN followed by ALTER is used instead of a single statement with multiple joins -- because if the tables are very large, the separate statements are much faster drop table if exists hphr; create table hphr @@ -116,17 +177,7 @@ select b1.sourceHours, b2.avgHorsepower from sourceHours b1 -join horsepower b2 - on ((b1.MOVESRunID=b2.MOVESRunID) AND - (b1.yearID=b2.yearID) AND - (b1.monthID=b2.monthID) AND - (b1.dayID=b2.dayID) AND - (b1.stateID=b2.stateID OR b1.stateID IS NULL AND b2.stateID IS NULL) AND - (b1.countyID=b2.countyID OR b1.countyID IS NULL AND b2.countyID IS NULL) AND - (b1.SCC=b2.SCC) AND - (b1.modelYearID=b2.modelYearID OR b1.modelYearID IS NULL AND b2.modelYearID IS NULL) AND - (b1.engTechID=b2.engTechID OR b1.engTechID IS NULL AND b2.engTechID IS NULL) AND - (b1.hpID=b2.hpID OR b1.hpID IS NULL AND b2.hpID IS NULL)); +join horsepower b2 USING (MOVESRunID,yearID,monthID,dayID,stateID,countyID,SCC,modelYearID,engTechID,hpID); create index index1 on hphr (MOVESRunID,yearID,monthID,dayID,stateID,countyID,SCC,modelYearID,engTechID,hpID); @@ -134,16 +185,16 @@ alter table hphr add loadFactor double, add hpHours double; update hphr b1, loadFactor b3 set b1.loadFactor = b3.loadFactor, b1.hpHours = avgHorsepower * sourceHours * b3.loadFactor - where ((b1.MOVESRunID=b3.MOVESRunID) AND + where (b1.MOVESRunID=b3.MOVESRunID) AND (b1.yearID=b3.yearID) AND (b1.monthID=b3.monthID) AND (b1.dayID=b3.dayID) AND - (b1.stateID=b3.stateID OR b1.stateID IS NULL AND b3.stateID IS NULL) AND - (b1.countyID=b3.countyID OR b1.countyID IS NULL AND b3.countyID IS NULL) AND + (b1.stateID=b3.stateID) AND + (b1.countyID=b3.countyID) AND (b1.SCC=b3.SCC) AND - (b1.modelYearID=b3.modelYearID OR b1.modelYearID IS NULL AND b3.modelYearID IS NULL) AND - (b1.engTechID=b3.engTechID OR b1.engTechID IS NULL AND b3.engTechID IS NULL) AND - (b1.hpID=b3.hpID OR b1.hpID IS NULL AND b3.hpID IS NULL)); + (b1.modelYearID=b3.modelYearID) AND + (b1.engTechID=b3.engTechID) AND + (b1.hpID=b3.hpID OR b1.hpID); create index index2 on hphr (MOVESRunID,yearID,monthID,dayID,stateID,countyID,scc); @@ -218,20 +269,92 @@ select IF(b2.hpHours != 0, b1.emissionQuant / b2.hpHours, NULL) as emissionRate, 'g/hp-hr' as emissionRateUnits from temp1 b1 -inner join temp2 b2 - on ((b1.MOVESRunID=b2.MOVESRunID) AND - (b1.yearID=b2.yearID) AND - (b1.monthID=b2.monthID) AND - (b1.dayID=b2.dayID) AND - (b1.stateID=b2.stateID OR b1.stateID IS NULL AND b2.stateID IS NULL) AND - (b1.countyID=b2.countyID OR b1.countyID IS NULL AND b2.countyID IS NULL) AND - (b1.SCC=b2.SCC)) +inner join temp2 b2 USING (MOVESRunID,yearID,monthID,dayID,stateID,countyID,scc) left join ##defaultdb##.nrscc s on (b1.scc=s.scc); +-- Drop intermediate tables and the primary indexes drop table if exists sourceHours; drop table if exists horsepower; drop table if exists loadfactor; drop table if exists temp1; drop table if exists temp2; drop table if exists hphr; -drop table if exists units; \ No newline at end of file +drop table if exists units; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index1'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index1 does not exist.''', 'drop index index1 on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index101'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index101 does not exist.''', 'drop index index101 on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index10'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index10 does not exist.''', 'drop index index10 on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +-- Revert 0s to NULLs +UPDATE movesoutput SET stateID = NULL WHERE stateID = 0; +UPDATE movesoutput SET countyID = NULL WHERE countyID = 0; +UPDATE movesoutput SET modelYearID = NULL WHERE modelYearID = 0; +UPDATE movesoutput SET engTechID = NULL WHERE engTechID = 0; +UPDATE movesoutput SET hpID = NULL WHERE hpID = 0; +UPDATE movesactivityoutput SET stateID = NULL WHERE stateID = 0; +UPDATE movesactivityoutput SET countyID = NULL WHERE countyID = 0; +UPDATE movesactivityoutput SET modelYearID = NULL WHERE modelYearID = 0; +UPDATE movesactivityoutput SET engTechID = NULL WHERE engTechID = 0; +UPDATE movesactivityoutput SET hpID = NULL WHERE hpID = 0; + +-- drop the rest of the indexes +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_state does not exist.''', 'drop index index_state on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_state does not exist.''', 'drop index index_state on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_county does not exist.''', 'drop index index_county on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_county does not exist.''', 'drop index index_county on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_modelyear'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_modelyear does not exist.''', 'drop index index_modelyear on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_modelyear'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_modelyear does not exist.''', 'drop index index_modelyear on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_engtech'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_engtech does not exist.''', 'drop index index_engtech on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_engtech'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_engtech does not exist.''', 'drop index index_engtech on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_hp'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_hp does not exist.''', 'drop index index_hp on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_hp'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_hp does not exist.''', 'drop index index_hp on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; diff --git a/database/NonroadProcessingScripts/EmissionFactors_per_hphr_by_SCC_and_ModelYear.sql b/database/NonroadProcessingScripts/EmissionFactors_per_hphr_by_SCC_and_ModelYear.sql index ec7a4d1..a9003be 100644 --- a/database/NonroadProcessingScripts/EmissionFactors_per_hphr_by_SCC_and_ModelYear.sql +++ b/database/NonroadProcessingScripts/EmissionFactors_per_hphr_by_SCC_and_ModelYear.sql @@ -1,10 +1,11 @@ --- Nonroad Post Processing Script (updated 7/26/2018): +-- Nonroad Post Processing Script (updated 6/23/2021): -- Emission factors in grams per horsepower-hour by SCC, horsepower class, -- and model year -- -- MOVES-Nonroad Output Guidance: -- SCC, HP class, and model year must be selected and present in the --- results. This script will run faster if engine tech is not selected. +-- results. This script will run faster if engine tech is not selected, +-- but it can be included if that information is desired in the output. -- It is strongly recommended to only have one sector and fuel type per -- output database when running this script. Additionally, this script -- will run faster if there is only one year, month, and day in the output. @@ -20,19 +21,58 @@ flush tables; --- Set up indexing +-- Set up indexing for setting NULL values to 0 +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_state already exists.''', 'create index index_state on movesoutput ( stateID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_state already exists.''', 'create index index_state on movesactivityoutput ( stateID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_county already exists.''', 'create index index_county on movesoutput ( countyID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_county already exists.''', 'create index index_county on movesactivityoutput ( countyID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_engtech'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_engtech already exists.''', 'create index index_engtech on movesoutput ( engTechID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_engtech'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index_engtech already exists.''', 'create index index_engtech on movesactivityoutput ( engTechID )'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +-- Convert NULLs to 0 to improve joins +UPDATE movesoutput SET stateID = 0 WHERE stateID IS NULL; +UPDATE movesoutput SET countyID = 0 WHERE countyID IS NULL; +UPDATE movesoutput SET engTechID = 0 WHERE engTechID IS NULL; +UPDATE movesactivityoutput SET stateID = 0 WHERE stateID IS NULL; +UPDATE movesactivityoutput SET countyID = 0 WHERE countyID IS NULL; +UPDATE movesactivityoutput SET engTechID = 0 WHERE engTechID IS NULL; + +-- Set up indexing for everything else set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index1'); -set @sqlstmt := if( @exist > 0, 'select ''INFO: Index already exists.''', 'create index index1 on movesoutput ( MOVESRunID )'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index1 already exists.''', 'create index index1 on movesoutput ( MOVESRunID )'); PREPARE stmt FROM @sqlstmt; EXECUTE stmt; set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index103'); -set @sqlstmt := if( @exist > 0, 'select ''INFO: Index already exists.''', 'create index index103 on movesoutput ( MOVESRunID,yearID,monthID,dayID,stateID,countyID,SCC,hpID,pollutantID,processID,modelYearID )'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index103 already exists.''', 'create index index103 on movesoutput ( MOVESRunID,yearID,monthID,dayID,stateID,countyID,SCC,hpID,pollutantID,processID,modelYearID )'); PREPARE stmt FROM @sqlstmt; EXECUTE stmt; set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index10'); -set @sqlstmt := if( @exist > 0, 'select ''INFO: Index already exists.''', 'create index index10 on movesactivityoutput ( activitytypeid )'); +set @sqlstmt := if( @exist > 0, 'select ''INFO: index10 already exists.''', 'create index index10 on movesactivityoutput ( activitytypeid )'); PREPARE stmt FROM @sqlstmt; EXECUTE stmt; @@ -98,9 +138,7 @@ create index index1 on sourceHours (MOVESRunID,yearID,monthID,dayID,stateID,coun create index index1 on horsepower (MOVESRunID,yearID,monthID,dayID,stateID,countyID,SCC,modelYearID,engTechID,hpID); create index index1 on loadfactor (MOVESRunID,yearID,monthID,dayID,stateID,countyID,SCC,modelYearID,engTechID,hpID); --- Note: b1.col = b2.col OR b1.col IS NULL AND b2.col IS NULL is used instead of just a simple --- USING() command because USING() doesn't work if there are NULLs --- Additionally, JOIN followed by ALTER is used instead of a single statement with multiple joins +-- Note: JOIN followed by ALTER is used instead of a single statement with multiple joins -- because if the tables are very large, the separate statements are faster drop table if exists hphr; create table hphr @@ -119,16 +157,7 @@ select b2.avgHorsepower from sourceHours b1 join horsepower b2 - on ((b1.MOVESRunID=b2.MOVESRunID) AND - (b1.yearID=b2.yearID) AND - (b1.monthID=b2.monthID) AND - (b1.dayID=b2.dayID) AND - (b1.stateID=b2.stateID OR b1.stateID IS NULL AND b2.stateID IS NULL) AND - (b1.countyID=b2.countyID OR b1.countyID IS NULL AND b2.countyID IS NULL) AND - (b1.SCC=b2.SCC) AND - (b1.modelYearID=b2.modelYearID) AND - (b1.engTechID=b2.engTechID OR b1.engTechID IS NULL AND b2.engTechID IS NULL) AND - (b1.hpID=b2.hpID)); +USING (MOVESRunID,yearID,monthID,dayID,stateID,countyID,SCC,modelYearID,engTechID,hpID); create index index1 on hphr (MOVESRunID,yearID,monthID,dayID,stateID,countyID,SCC,modelYearID,engTechID,hpID); @@ -140,11 +169,11 @@ set b1.loadFactor = b3.loadFactor, (b1.yearID=b3.yearID) AND (b1.monthID=b3.monthID) AND (b1.dayID=b3.dayID) AND - (b1.stateID=b3.stateID OR b1.stateID IS NULL AND b3.stateID IS NULL) AND - (b1.countyID=b3.countyID OR b1.countyID IS NULL AND b3.countyID IS NULL) AND + (b1.stateID=b3.stateID) AND + (b1.countyID=b3.countyID) AND (b1.SCC=b3.SCC) AND (b1.modelYearID=b3.modelYearID) AND - (b1.engTechID=b3.engTechID OR b1.engTechID IS NULL AND b3.engTechID IS NULL) AND + (b1.engTechID=b3.engTechID) AND (b1.hpID=b3.hpID)); create index index2 on hphr (MOVESRunID,yearID,monthID,dayID,stateID,countyID,SCC,hpID,modelYearID); @@ -174,13 +203,14 @@ select pollutantID, processID, modelYearID, + engTechID, units.factor * sum(emissionQuant) as emissionQuant from movesoutput m left join movesrun using (movesrunid) left join units on (movesrun.massUnits = units.fromUnit) -group by MOVESRunID,yearID,monthID,dayID,stateID,countyID,SCC,hpID,pollutantID,processID,modelYearID; +group by MOVESRunID,yearID,monthID,dayID,stateID,countyID,SCC,hpID,pollutantID,processID,modelYearID,engTechID; -create index index1 on temp1 (MOVESRunID,yearID,monthID,dayID,stateID,countyID,SCC,hpID,modelYearID); +create index index1 on temp1 (MOVESRunID,yearID,monthID,dayID,stateID,countyID,SCC,hpID,modelYearID,engTechID); create index index2 on temp1 (SCC); create index index3 on temp1 (hpID); @@ -198,11 +228,12 @@ select SCC, hpID, modelYearID, + engTechID, sum(hpHours) as hpHours from hphr -group by MOVESRunID,yearID,monthID,dayID,stateID,countyID,SCC,hpID,modelYearID; +group by MOVESRunID,yearID,monthID,dayID,stateID,countyID,SCC,hpID,modelYearID,engTechID; -create index index1 on temp2 (MOVESRunID,yearID,monthID,dayID,stateID,countyID,SCC,hpID,modelYearID); +create index index1 on temp2 (MOVESRunID,yearID,monthID,dayID,stateID,countyID,SCC,hpID,modelYearID,engTechID); -- Join temp1 and temp2 and calculate the emission rate for the resulting output table @@ -223,28 +254,77 @@ select b1.pollutantID, b1.processID, b1.modelYearID, + b1.engTechID, + e.engTechDesc, b1.emissionQuant, b2.hpHours, IF(b2.hpHours != 0, b1.emissionQuant / b2.hpHours, NULL) as emissionRate, 'g/hp-hr' as emissionRateUnits from temp1 b1 -inner join temp2 b2 - on ((b1.MOVESRunID=b2.MOVESRunID) AND - (b1.yearID=b2.yearID) AND - (b1.monthID=b2.monthID) AND - (b1.dayID=b2.dayID) AND - (b1.stateID=b2.stateID OR b1.stateID IS NULL AND b2.stateID IS NULL) AND - (b1.countyID=b2.countyID OR b1.countyID IS NULL AND b2.countyID IS NULL) AND - (b1.SCC=b2.SCC) AND - (b1.hpID=b2.hpID) AND - (b1.modelYearID=b2.modelYearID)) +inner join temp2 b2 USING (MOVESRunID,yearID,monthID,dayID,stateID,countyID,SCC,hpID,modelYearID,engTechID) left join ##defaultdb##.nrscc s on (b1.scc=s.scc) -left join ##defaultdb##.nrhprangebin h on (b1.hpID=h.NRHPRangeBinID); +left join ##defaultdb##.nrhprangebin h on (b1.hpID=h.NRHPRangeBinID) +left join ##defaultdb##.enginetech e on (b1.engTechID=e.engTechID); +-- Drop intermediate tables and the primary indexes drop table if exists sourceHours; drop table if exists loadfactor; drop table if exists horsepower; drop table if exists temp1; drop table if exists temp2; drop table if exists hphr; -drop table if exists units; \ No newline at end of file +drop table if exists units; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index1'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index1 does not exist.''', 'drop index index1 on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index103'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index103 does not exist.''', 'drop index index103 on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index10'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index10 does not exist.''', 'drop index index10 on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +-- Revert 0s to NULLs +UPDATE movesoutput SET stateID = NULL WHERE stateID = 0; +UPDATE movesoutput SET countyID = NULL WHERE countyID = 0; +UPDATE movesoutput SET engTechID = NULL WHERE engTechID = 0; +UPDATE movesactivityoutput SET stateID = NULL WHERE stateID = 0; +UPDATE movesactivityoutput SET countyID = NULL WHERE countyID = 0; +UPDATE movesactivityoutput SET engTechID = NULL WHERE engTechID = 0; + +-- drop the rest of the indexes +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_state does not exist.''', 'drop index index_state on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_state'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_state does not exist.''', 'drop index index_state on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_county does not exist.''', 'drop index index_county on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_county'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_county does not exist.''', 'drop index index_county on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesoutput' and index_name = 'index_engtech'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_engtech does not exist.''', 'drop index index_engtech on movesoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; + +set @exist := (select count(*) from information_schema.statistics where table_schema = DATABASE() and table_name = 'movesactivityoutput' and index_name = 'index_engtech'); +set @sqlstmt := if( @exist = 0, 'select ''INFO: index_engtech does not exist.''', 'drop index index_engtech on movesactivityoutput'); +PREPARE stmt FROM @sqlstmt; +EXECUTE stmt; diff --git a/database/Setup/SetupDatabase.bat b/database/Setup/SetupDatabase.bat index 6f75545..2a6c482 100644 --- a/database/Setup/SetupDatabase.bat +++ b/database/Setup/SetupDatabase.bat @@ -1,2 +1,2 @@ mysql -uroot -pmoves --force < CreateMOVESUser.sql -mysql -uroot -pmoves < movesdb20210209.sql \ No newline at end of file +mysql -uroot -pmoves < movesdb20210726.sql \ No newline at end of file diff --git a/database/Setup/movesdb20210209.zip b/database/Setup/movesdb20210726.zip similarity index 72% rename from database/Setup/movesdb20210209.zip rename to database/Setup/movesdb20210726.zip index be1f04f..cc1f0ee 100644 Binary files a/database/Setup/movesdb20210209.zip and b/database/Setup/movesdb20210726.zip differ diff --git a/docs/AnatomyOfARunspec.md b/docs/AnatomyOfARunspec.md index aaf962f..56a5100 100644 --- a/docs/AnatomyOfARunspec.md +++ b/docs/AnatomyOfARunspec.md @@ -43,14 +43,14 @@ Valid entries for *value* attribute are: ## Model domain ```xml - + ``` The **modeldomain** element is a required element for a MOVES RunSpec. It has only one attribute: *value*. Valid entries for *value* are: -* NATIONAL - used to select a Default Scale (also known as national scale) run. This is the only option for a Nonroad run. +* DEFAULT - used to select a Default Scale run. This is the only option for a Nonroad run. * SINGLE - used to select a County Scale run * PROJECT - used to select a Project Scale run diff --git a/docs/CommandLineMOVES.md b/docs/CommandLineMOVES.md index 09d4cbf..87f86b9 100644 --- a/docs/CommandLineMOVES.md +++ b/docs/CommandLineMOVES.md @@ -167,12 +167,35 @@ ant convert2014_3 -Dinput=m2014_in -Doutput=m3_in The following command can be used to convert a MOVES2014a or MOVES2014b input database to the MOVES3 format: -``` +```cmd ant convert2014ab_3 Dinput=m2014b_in -Doutput=m3_in ``` `-Dinput` and `-Doutput` are the same as with the other conversion command, and `convert2014ab_3` tells Ant to use the conversion scripts for going from a MOVES2014a or MOVES2014b input database to MOVES3 format. +## Compiling MOVES + +MOVES uses three compiled languages during runtime: Java, Go, and Fortran (the last only used when running Nonroad). If you use the MOVES Installer, you will receive all the compiled code and can run the model directly without compiling. If you are cloning this repository, you will only receive the compiled Fortran, and will need to compile Java and Go before running the model. + +The following command can be used to compile both the Java and the Go, and then launch the GUI: + +```cmd +ant crungui +``` + +If desired, Java and Go can be compiled separately. The following commands can be used when compiling Java: +* `ant clean`: Removes all compiled Java .class files +* `ant compile`: Compiles all uncompiled Java files and recompiles any existing, out-of-date .class files +* `ant compileall`: Compiles all Java files + +The following commands can be used when compiling Go: +* `ant go64`: Compiles 64-bit versions of the two Go runtime executables used by MOVES. This is the option most users will use. +* `ant go32`: Compiles 32-bit versions of the two Go runtime executables used by MOVES. This option is only needed when running MOVES on 32-bit architecture. +* `ant go32local`: Compiles 32-bit versions of the Go runtime using a version of the Go compiler installed in the `.\go32` subdirectory of MOVES. This option is useful if you want to compile the 32-bit executable using a different version of Go than what is installed system-wide. It only works if you have created a `.\go32` subdirectory and copied an entire Go installation to this location. +* `ant go`: Compiles all of the above (i.e., 64-bit, 32-bit, and 32-bit local), as well as Linux versions of the two Go runtime executables. If any step fails (e.g., `go32local`), you'll see an error message, but the compiler will continue to the next step in the sequence. + +For instructions on how to compile the Fortran code, see [Readme.md](../NONROAD/NR08a/SOURCE/readme.md) in the Nonroad source code directory. + ## Additional remarks The above examples demonstrate a simple way of running MOVES from the Windows command prompt. The example scripts should be considered just examples. Different MOVES install configurations may change some of the details needed to run the scripts. Likewise, the Windows command prompt offers a rich scripting environment that can be used to expand these examples to do things such as: diff --git a/docs/ONIToolCommandLine.md b/docs/ONIToolCommandLine.md new file mode 100644 index 0000000..b32e70b --- /dev/null +++ b/docs/ONIToolCommandLine.md @@ -0,0 +1,58 @@ + + +# MOVES ONI Tool Command Line Interface + +The MOVES off-network idle (ONI) Tool was added to MOVES3 to assist MOVES users who generate inventories by running MOVES at the County Scale in Emission Rates mode and multiply those rates by activity outside of MOVES. For more detail about what ONI is, how MOVES uses it, and how to use it in generating inventories, see the [instructions for using the ONI Tool in the MOVES GUI](https://github.com/USEPA/EPA_MOVES_Model/blob/master/database/ONITool/InstructionsForONITool.pdf). + +The ONI Tool has been added to Ant, the MOVES dependency manager and command line interface tool. This document includes specific examples on how to use the ONI Tool through the command line. For general information on MOVES' command line tools, see [CommandLineMOVES.md](https://github.com/USEPA/EPA_MOVES_Model/blob/master/docs/CommandLineMOVES.md). + +## Prerequisites + +In order to use the ONI Tool, users must first have a county database (CDB) and MOVES version 3.0.2 or greater must be installed. The ONI Tool assumes the CDB has already been built using the MOVES County Data Manager or some other method and is complete. If it is not complete, the ONI Tool will fail. Likewise, the ONI Tool will not aid in the creation or population of CDB. + +## Command Line Interface + +To use MOVES' command line tools, first open the Windows command prompt and navigate to the MOVES directory. Then, run the command `setenv`. For more information on using MOVES' command line tools, see [CommandLineMOVES.md](https://github.com/USEPA/EPA_MOVES_Model/blob/master/docs/CommandLineMOVES.md). + +The Ant command to run the ONI Tool is `oniTool`. It only takes one input, passed via the flat `-Dinput`. There are three types of input allowed: + +1. A single input database: + `ant oniTool -Dinput=c12345y2020_20210504` + +2. A comma-separated list of databases, wrapped in quotes: + `ant oniTool -Dinput="c12345y2020_20210504,c23456y2020_20210504"` +3. A .txt file with a list of databases, one on each line: + `ant oniTool -Dinput=db_list.txt` + +For input option #3 above, databases need to be separated by a new line. The following would be a valid `db_list.txt` file: + +``` +c12345y2020_20210504 +c23456y2020_20210504 +``` + +## Output + +The MOVES ONI Tool command line interface is meant to be used when users need to get ONI activity for multiple CDBs and want to use the output in a systematic way, such as in a data analysis script. The ONI interface in the GUI is still the preferred option for users who only need to run the ONI Tool on a single CDB. + +Because of the different use cases, the output generated from the command line interface is stored differently from the output generated by the GUI interface. When run via the GUI, the tool writes the results to a file, given by the user. However, the ONI Tool *does not* write to file when run from the command line. Instead, it creates a new table in the CDB, called `onitooloutput`, which has the same structure as the output file. + +### Output Table Definition + +The `onitooloutput` table is the same as the output file written when the ONI Tool is run via the MOVES GUI. For the integer columns, definitions can be found in [MOVESGlossary.md](https://github.com/USEPA/EPA_MOVES_Model/blob/master/docs/CommandLineMOVES.md). + +| Field | Type | Description | +| ---------------------------------- | ----------- | ------------------------------------------------------------ | +| yearID | SMALLINT(6) | | +| monthID | SMALLINT(6) | | +| dayID | SMALLINT(6) | | +| hourID | SMALLINT(6) | | +| sourceTypeID | SMALLINT(6) | | +| minModelYearID | INT(11) | | +| maxModelYearID | INT(11) | | +| onroadSHO (hr) | DOUBLE | Total onroad Source Hours Operating that would be calculated by a MOVES Inventory Mode run | +| VMT (mi) | DOUBLE | Total vehicle miles travelled that would be calculated by a MOVES Inventory Mode run | +| ONI (hr) | DOUBLE | Total off-network idle hours that would be calculated by a MOVES Inventory Mode run | +| ONI per VMT (hr idle/mi) | DOUBLE | ONI hours per VMT, which can be used to calculate ONI hours from VMT estimates that are different from what is provided in the CDB | +| ONI per SHO (hr idle/hr operating) | DOUBLE | ONI hours per VMT, which can be used to calculate ONI hours from VMT estimates that are different from what is provided in the CDB | + diff --git a/gov/epa/otaq/moves/common/DatabaseUtilities.java b/gov/epa/otaq/moves/common/DatabaseUtilities.java index 50fe0ad..9510353 100644 --- a/gov/epa/otaq/moves/common/DatabaseUtilities.java +++ b/gov/epa/otaq/moves/common/DatabaseUtilities.java @@ -1082,7 +1082,8 @@ public static void executeBuildScript(File scriptFile, DatabaseSelection outputD * @throws Exception if there are errors accessing or creating the output file **/ public static boolean executeONITool(File saveFile, DatabaseSelection inputDatabase, - DatabaseSelection defaultDatabase, ArrayList messages) + DatabaseSelection defaultDatabase, ArrayList messages, + boolean writeOutput, boolean writeToCDB) throws FileNotFoundException, IOException, SQLException, Exception { File oniToolScript = new File("./database/ONITool/ONITool.sql"); Connection inputDB = null; @@ -1171,10 +1172,19 @@ public static boolean executeONITool(File saveFile, DatabaseSelection inputDatab // save output and drop temp database if there was success if(success) { - writer = new CellFileWriter(saveFile,"ONI Tool Output"); - sql = "SELECT * FROM `" + tempDatabaseName + "`.`ONIToolOutput`"; - writer.writeSQLResults(inputDB, sql, null); - + if(writeToCDB) { + sql = "DROP TABLE IF EXISTS `" + inputDatabase.databaseName + "`.onitooloutput"; + SQLRunner.executeSQL(inputDB,sql); + + sql = "CREATE TABLE `" + inputDatabase.databaseName + "`.onitooloutput " + + "SELECT * FROM `" + tempDatabaseName + "`.`ONIToolOutput`"; + SQLRunner.executeSQL(inputDB,sql); + } + if(writeOutput) { + writer = new CellFileWriter(saveFile,"ONI Tool Output"); + sql = "SELECT * FROM `" + tempDatabaseName + "`.`ONIToolOutput`"; + writer.writeSQLResults(inputDB, sql, null); + } SQLRunner.executeSQL(inputDB, "DROP DATABASE IF EXISTS `" + tempDatabaseName + "`"); } } catch(FileNotFoundException e) { diff --git a/gov/epa/otaq/moves/common/ModelDomain.java b/gov/epa/otaq/moves/common/ModelDomain.java index 27d2e44..9874d3b 100644 --- a/gov/epa/otaq/moves/common/ModelDomain.java +++ b/gov/epa/otaq/moves/common/ModelDomain.java @@ -27,7 +27,9 @@ public class ModelDomain { static TreeMapIgnoreCase allInstances = new TreeMapIgnoreCase(); /** Enumerated type (as static instance) for a model domain value. **/ - public static final ModelDomain NATIONAL_ALLOCATION = new ModelDomain("NATIONAL"); + public static final ModelDomain DEPRECATED_NATIONAL_ALLOCATION = new ModelDomain("NATIONAL"); + /** Enumerated type (as static instance) for a model domain value. **/ + public static final ModelDomain NATIONAL_ALLOCATION = new ModelDomain("DEFAULT"); /** Enumerated type (as static instance) for a model domain value. **/ public static final ModelDomain SINGLE_COUNTY = new ModelDomain("SINGLE"); /** Enumerated type (as static instance) for a model domain value. **/ @@ -63,6 +65,13 @@ public String toString() { * @return The constant type. This will be null if the given name wasn't found. **/ public static ModelDomain getByName(String name) { - return (ModelDomain)allInstances.get(name); + ModelDomain domain = (ModelDomain)allInstances.get(name); + + // return the new "national allocation" domain instead of the deprecated one if + // that one is still being used. + if (domain == ModelDomain.DEPRECATED_NATIONAL_ALLOCATION) { + domain = ModelDomain.NATIONAL_ALLOCATION; + } + return domain; } } diff --git a/gov/epa/otaq/moves/master/commandline/ONIToolRunner.java b/gov/epa/otaq/moves/master/commandline/ONIToolRunner.java new file mode 100644 index 0000000..6bb3609 --- /dev/null +++ b/gov/epa/otaq/moves/master/commandline/ONIToolRunner.java @@ -0,0 +1,164 @@ +/************************************************************************************************** + * @(#)ONIToolRunner.java + * + * + * + *************************************************************************************************/ +package gov.epa.otaq.moves.master.commandline; + +import java.io.*; +import java.sql.*; +import java.util.*; +import java.util.regex.*; +import gov.epa.otaq.moves.master.framework.*; +import gov.epa.otaq.moves.common.*; + +/** + * Batch mode running ONI Tool. + * + * @author Evan Murray + * @version 2021-04-30 +**/ +public class ONIToolRunner { + /** Invokes the main application. **/ + public static void main(String[] args) { + Configuration.allowGUI = false; + + if(!SystemConfiguration.getTheSystemConfiguration().didLoad) { + System.exit(1); + } + + // Find command line arguments + String inputDBArg = ""; + List inputDBs = null; + String outputFileName = ""; + String scriptName = ""; + for(int i=0;i(); + BufferedReader input = new BufferedReader(new FileReader(inputDBFile)); + for(String line = input.readLine(); line != null; line = input.readLine()) { + if (line.trim().equals("")) { + continue; + } + inputDBs.add(line.trim()); + } + input.close(); + Logger.log(LogMessageCategory.INFO, "Found " + inputDBs.size() + " database(s) to process from " + inputDBFile.getName()); + } else if (inputDBArg.contains(".")) { // couldn't find the argument as a file... is it a file? + Logger.log(LogMessageCategory.ERROR, "Couldn't open the file passed in -input: " + inputDBArg); + System.exit(1); + } + } catch (IOException e) { + Logger.log(LogMessageCategory.ERROR, "Couldn't open the file passed in -input: " + inputDBArg); + System.exit(1); + } + // then see if they passed a direct list of dbs + if (inputDBs == null) { + try { + inputDBs = Arrays.asList(inputDBArg.split(",")); + Logger.log(LogMessageCategory.INFO, inputDBs.size() + " database(s) from the command line argument will be processed"); + } catch (PatternSyntaxException e) { + Logger.log(LogMessageCategory.ERROR, "Couldn't interpret -input argument. If you passed a file path, check it for errors; if you passed 1 or more database name(s), make sure the entire list is quoted and comma separated"); + System.exit(1); + } + } + + // parse -script arg + if(scriptName == null || scriptName.length() <= 0) { + Logger.log(LogMessageCategory.ERROR, "Missing -script file name"); + System.exit(1); + } + File scriptFile = findFile(scriptName); + if(scriptFile == null) { + Logger.log(LogMessageCategory.ERROR, "Unable to find script file " + scriptName); + System.exit(1); + } + + // get the default database + DatabaseSelection defaultDatabase = SystemConfiguration.getTheSystemConfiguration().databaseSelections[MOVESDatabaseType.DEFAULT.getIndex()]; + + // File outputFile = new File("oniToolOutput.csv"); + + // run script for each input database + for (String inputDB : inputDBs) { + DatabaseSelection inputDatabase = new DatabaseSelection(); + inputDatabase.serverName = defaultDatabase.serverName; + inputDatabase.databaseName = inputDB; + + ArrayList messages = new ArrayList(); + try { + Logger.log(LogMessageCategory.INFO, "Running ONI Tool for " + inputDB + "..."); + DatabaseUtilities.executeONITool(null, inputDatabase, defaultDatabase, messages, false, true); + } catch(Exception e) { + Logger.log(LogMessageCategory.ERROR, "ONI calculation " + inputDB + " failed: " + e.getMessage()); + + if (e.getMessage().contains("Unknown column 'idleRegionID' in 'field list'")) { + Logger.log(LogMessageCategory.ERROR, "Input database " + inputDB + " was likely created with a previous version of MOVES."); + } + + //e.printStackTrace(); + Logger.log(LogMessageCategory.ERROR, "Output file will not be saved."); + System.exit(1); + } + } + + Logger.log(LogMessageCategory.INFO, "Finished running ONI tool. Results are in the onitooloutput table of the input database(s)"); + + System.exit(0); + } + + /** + * Get the data portion of an option of the form "-option=data". + * @param optionText full option text + * @return data portion of the option, never null but may be blank + **/ + static String getOptionData(String optionText) { + int index = optionText.indexOf('='); + if(index < 0) { + return ""; + } + return StringUtilities.substring(optionText,index+1).trim(); + } + + /** + * Locate a script file, looking first in the current working directory + * then in the database folder then in the database/NEIQA folder. + * @param name name of the file to be located + * @return the located file or null if the file does not exist + **/ + static File findFile(String name) { + File f = new File(name); + if(f.exists()) { + return f; + } + + f = new File("database",name); + if(f.exists()) { + return f; + } + + f = new File("database/ONITool",name); + if(f.exists()) { + return f; + } + + return null; + } +} diff --git a/gov/epa/otaq/moves/master/framework/ExecutionRunSpec.java b/gov/epa/otaq/moves/master/framework/ExecutionRunSpec.java index 751f600..40072a8 100644 --- a/gov/epa/otaq/moves/master/framework/ExecutionRunSpec.java +++ b/gov/epa/otaq/moves/master/framework/ExecutionRunSpec.java @@ -980,7 +980,9 @@ public TreeSet getRoadTypes() { //Check to see if off-network roadtype and running process is selected in the runspec. boolean containsOffNetwork = false; boolean containsRunning = false; + boolean isProjectScale = (targetRunSpec.domain == ModelDomain.PROJECT); boolean needsAllRoadTypes = false; + //Check if off-network is present in the runspec for (Iterator i = targetRunSpec.roadTypes.iterator();i.hasNext();){ RoadType r = i.next(); @@ -998,7 +1000,8 @@ public TreeSet getRoadTypes() { } } //If both off-network and running process are in the runspec, all roadtypes are needed - needsAllRoadTypes = (containsOffNetwork && containsRunning); + // (except Project scale doesn't calculate ONI, so they are not needed for Project Scale) + needsAllRoadTypes = (containsOffNetwork && containsRunning && !isProjectScale); //If off-network idle is needed, return all roadtypes if (needsAllRoadTypes){ diff --git a/gov/epa/otaq/moves/master/gui/MOVESWindow.java b/gov/epa/otaq/moves/master/gui/MOVESWindow.java index 8170c79..92d8a8b 100644 --- a/gov/epa/otaq/moves/master/gui/MOVESWindow.java +++ b/gov/epa/otaq/moves/master/gui/MOVESWindow.java @@ -217,7 +217,7 @@ public class MOVESWindow extends JFrame implements ActionListener, LogHandler, /** Name of output file that performance profiles are written to **/ static final String PERFORMANCE_PROFILER_FILE_NAME = "guiprofile.txt"; /** Date of the Current Release **/ - public static final String MOVES_VERSION = "MOVES3.0.1"; + public static final String MOVES_VERSION = "MOVES3.0.2"; /** directory where output db processing scripts are located **/ static final String DB_SCRIPTS_DIR = "database" + File.separator + "OutputProcessingScripts"; static final String DB_NONROAD_SCRIPTS_DIR = "database" + File.separator + "NonroadProcessingScripts"; @@ -1347,15 +1347,18 @@ void handleRunNonroadScriptAction() { return; } - // Ask where to save output - navigationPanel.commitActiveEditor(); - FileDialog fd = new FileDialog(this, - "Save Nonroad Post Processing Script Output As...", FileDialog.SAVE); - fd.setVisible(true); //fd.show(); - if ((fd.getDirectory() == null) || (fd.getFile() == null)) { - return; + // Ask where to save output (bypass for the Decoded script, which works the same as the onroad version (i.e., it creates two tables in the database) + String saveFileName = ""; + if (!scriptName.equalsIgnoreCase("DecodedNonroadOutput.sql")) { + navigationPanel.commitActiveEditor(); + FileDialog fd = new FileDialog(this, + "Save Nonroad Post Processing Script Output As...", FileDialog.SAVE); + fd.setVisible(true); //fd.show(); + if ((fd.getDirectory() == null) || (fd.getFile() == null)) { + return; + } + saveFileName = fd.getDirectory() + fd.getFile(); } - String saveFileName = fd.getDirectory() + fd.getFile(); boolean hasErrors = false; try { @@ -1365,7 +1368,11 @@ void handleRunNonroadScriptAction() { replacements.put("##defaultdb##",defaultDatabase.databaseName); replacements.put("##scaleinputdb##",runSpec.scaleInputDatabase.databaseName); DatabaseUtilities.executeScript(oConn,scriptFile,replacements); - hasErrors = RunNonroadScriptActionHelper.processScriptOutput(scriptName, saveFileName, oConn); + + // save the output (again, except for the Decoded script) + if (!scriptName.equalsIgnoreCase("DecodedNonroadOutput.sql")) { + hasErrors = RunNonroadScriptActionHelper.processScriptOutput(scriptName, saveFileName, oConn); + } } catch (Exception e) { hasErrors = true; Logger.log(LogMessageCategory.ERROR, diff --git a/gov/epa/otaq/moves/master/gui/ONITool.java b/gov/epa/otaq/moves/master/gui/ONITool.java index 367599e..f1efef0 100644 --- a/gov/epa/otaq/moves/master/gui/ONITool.java +++ b/gov/epa/otaq/moves/master/gui/ONITool.java @@ -433,7 +433,7 @@ void handleRunButton() { inputDatabase.serverName = defaultDatabase.serverName; inputDatabase.databaseName = inputDatabaseName; try { - success = DatabaseUtilities.executeONITool(saveFile,inputDatabase,defaultDatabase,messages); + success = DatabaseUtilities.executeONITool(saveFile,inputDatabase,defaultDatabase,messages,true, false); } catch(Exception e) { Logger.log(LogMessageCategory.ERROR, "ONI Tool failed: " + e); success = false; diff --git a/gov/epa/otaq/moves/master/gui/RunNonroadScriptAction.java b/gov/epa/otaq/moves/master/gui/RunNonroadScriptAction.java index 8e7d8c0..25f5522 100644 --- a/gov/epa/otaq/moves/master/gui/RunNonroadScriptAction.java +++ b/gov/epa/otaq/moves/master/gui/RunNonroadScriptAction.java @@ -18,11 +18,11 @@ public class RunNonroadScriptAction extends JLFAbstractAction { /** Constant command key text. **/ static final String ACTION_COMMAND_KEY_RUN_SCRIPT = "run nonroad script command"; /** Constant action name. **/ - static final String NAME_RUN_SCRIPT = "Run MySQL Script on Nonroad Output Database"; + static final String NAME_RUN_SCRIPT = "Run SQL Script on Nonroad Output Database"; /** Constant short description text. **/ static final String SHORT_DESCRIPTION_RUN_SCRIPT = "Run Nonroad Script"; /** Constant long description text. **/ - static final String LONG_DESCRIPTION_RUN_SCRIPT = "Run MySQL Script on Nonroad Output Database"; + static final String LONG_DESCRIPTION_RUN_SCRIPT = "Run SQL Script on Nonroad Output Database"; // static final int MNEMONIC_KEY_RUN_SCRIPT = 'R'; /** diff --git a/gov/epa/otaq/moves/master/gui/RunNonroadScriptActionHelper.java b/gov/epa/otaq/moves/master/gui/RunNonroadScriptActionHelper.java index 82c0528..9969395 100644 --- a/gov/epa/otaq/moves/master/gui/RunNonroadScriptActionHelper.java +++ b/gov/epa/otaq/moves/master/gui/RunNonroadScriptActionHelper.java @@ -473,8 +473,9 @@ private static void EmissionFactors_by_SCC(String tableName, String units, Objec **/ private static void EmissionFactors_by_SCC_and_ModelYear(String tableName, String units, Object writer, Connection oConn) throws Exception { if (writer instanceof CellFileWriter) { - String sql = "SELECT MOVESRunID, yearID, monthID, dayID, stateID, countyID, SCC, sccDescription, fuelTypeID, hpID, hpBin, pollutantID, processID, modelYearID, emissionRate, emissionRateUnits " + - "FROM " + tableName + " ORDER BY countyID, yearID, monthID"; + String sql = "SELECT MOVESRunID, yearID, monthID, dayID, stateID, countyID, SCC, sccDescription, fuelTypeID, hpID, " + + " hpBin, pollutantID, processID, modelYearID, engTechID, engTechDesc, emissionRate, emissionRateUnits " + + "FROM " + tableName + " ORDER BY countyID, yearID, monthID"; ((CellFileWriter)writer).writeSQLResults(oConn, sql, null); } else if (writer instanceof PrintWriter) { SQLRunner.Query query = new SQLRunner.Query(); @@ -495,6 +496,8 @@ private static void EmissionFactors_by_SCC_and_ModelYear(String tableName, Strin header += StringUtilities.rightSpacePad("Pollutant", 11); header += StringUtilities.rightSpacePad("Process", 10); header += StringUtilities.rightSpacePad("Model Year", 12); + header += StringUtilities.rightSpacePad("EngTech", 9); + header += StringUtilities.rightSpacePad("EngTechDesc", 52); header += StringUtilities.rightSpacePad("Emission Rate", 30); // Print header @@ -513,7 +516,12 @@ private static void EmissionFactors_by_SCC_and_ModelYear(String tableName, Strin for (Integer calendarYear : calendarYears) { for (Integer month : months) { for (Integer day : days) { - sql = "SELECT MOVESRunID, yearID, monthID, dayID, stateID, countyID, SCC, sccDescription, fuelTypeID, hpID, hpBin, pollutantID, processID, modelYearID, emissionRate " + + sql = "SELECT MOVESRunID, yearID, monthID, dayID, stateID, countyID, SCC, sccDescription, " + + " fuelTypeID, hpID, hpBin, pollutantID, processID, modelYearID, engTechID, " + + " CASE WHEN LENGTH(engTechDesc) > 50 " + + " THEN CONCAT(SUBSTRING(engTechDesc, 1, 47), '...') " + + " ELSE engTechDesc END AS engTechDesc, " + + " emissionRate " + "FROM " + tableName + " WHERE 1"; if (county > 0) sql += " AND countyID = " + county.toString(); @@ -556,6 +564,8 @@ private static void EmissionFactors_by_SCC_and_ModelYear(String tableName, Strin result += StringUtilities.rightSpacePad(StringUtilities.safeGetString(query.rs.getString("pollutantID")), 11); result += StringUtilities.rightSpacePad(StringUtilities.safeGetString(query.rs.getString("processID")), 10); result += StringUtilities.rightSpacePad(StringUtilities.safeGetString(query.rs.getString("modelYearID")), 12); + result += StringUtilities.rightSpacePad(StringUtilities.safeGetString(query.rs.getString("engTechID")), 9); + result += StringUtilities.rightSpacePad(StringUtilities.safeGetString(query.rs.getString("engTechDesc")), 52); result += StringUtilities.rightSpacePad(StringUtilities.safeGetString(query.rs.getString("emissionRate")), 30); ((PrintWriter)writer).println(result); } diff --git a/gov/epa/otaq/moves/master/gui/RunScriptAction.java b/gov/epa/otaq/moves/master/gui/RunScriptAction.java index f7f0140..cc131f3 100644 --- a/gov/epa/otaq/moves/master/gui/RunScriptAction.java +++ b/gov/epa/otaq/moves/master/gui/RunScriptAction.java @@ -19,11 +19,11 @@ public class RunScriptAction extends JLFAbstractAction { /** Constant command key text. **/ static final String ACTION_COMMAND_KEY_RUN_SCRIPT = "run script command"; /** Constant action name. **/ - static final String NAME_RUN_SCRIPT = "Run MySQL Script on Onroad Output Database"; + static final String NAME_RUN_SCRIPT = "Run SQL Script on Onroad Output Database"; /** Constant short description text. **/ static final String SHORT_DESCRIPTION_RUN_SCRIPT = "Run Script"; /** Constant long description text. **/ - static final String LONG_DESCRIPTION_RUN_SCRIPT = "Run MySQL Script on MOVES Output Database"; + static final String LONG_DESCRIPTION_RUN_SCRIPT = "Run SQL Script on MOVES Output Database"; // static final int MNEMONIC_KEY_RUN_SCRIPT = 'R'; /** diff --git a/gov/epa/otaq/moves/master/implementation/ghg/TotalActivityGenerator.java b/gov/epa/otaq/moves/master/implementation/ghg/TotalActivityGenerator.java index d8f4e50..b5a41b8 100644 --- a/gov/epa/otaq/moves/master/implementation/ghg/TotalActivityGenerator.java +++ b/gov/epa/otaq/moves/master/implementation/ghg/TotalActivityGenerator.java @@ -213,13 +213,12 @@ public void executeLoop(MasterLoopContext inContext) { growVMTToAnalysisYear(inContext.year); // steps 150-159 allocateVMTByRoadTypeSourceAge(inContext.year); // steps 160-169 calculateVMTByRoadwayHour(inContext.year); // steps 170-179 - focusStart = System.currentTimeMillis(); - convertVMTToTotalActivityBasis(); // steps 180-189 - focusTime += System.currentTimeMillis() - focusStart; resultsYear = inContext.year; growthTime += System.currentTimeMillis() - start; } + + start = System.currentTimeMillis(); allocateTotalActivityBasis(inContext); // steps 190-199 calculateDistance(inContext); // steps 200-209 @@ -1659,10 +1658,13 @@ void calculateVMTByRoadwayHour(int analysisYear) throws SQLException { /** * Tag-7: Convert VMT to Total Activity Basis - * Calculate Starts and Source Hours Parked. + * Calculate SHO, Source Hours Parked, and hotelling. Also calculates the startsPerVehicle table, + * which is not the actual starts table that gets used (starts are now calculated in AdjustStarts, + * using the startsPerDayPerVehicle table or the user input tables). However, it appears other places + * might still use startsPerVehicle (?), so this will remain for now. * @throws SQLException If VMT cannot be converted to Total Activity Basis. **/ - void convertVMTToTotalActivityBasis() throws SQLException { + void convertVMTToTotalActivityBasis(int zoneID) throws SQLException { long start = 0; String sql = ""; @@ -1806,6 +1808,9 @@ void convertVMTToTotalActivityBasis() throws SQLException { * @output VMTByAgeRoadwayDay * @input VMTByAgeRoadwayHour **/ + // clear VMTByAgeRoadwayDay first + sql = "TRUNCATE TABLE VMTByAgeRoadwayDay"; + SQLRunner.executeSQL(db,sql); sql = "insert ignore into VMTByAgeRoadwayDay (" + " yearID, roadTypeID, sourceTypeID, ageID, monthID, dayID, VMT, hotellingHours)" + " select yearID, roadTypeID, sourceTypeID, ageID, monthID, dayID, sum(VMT), 0 as hotellingHours" @@ -1825,17 +1830,23 @@ void convertVMTToTotalActivityBasis() throws SQLException { sql = "update VMTByAgeRoadwayDay, hotellingCalendarYear, ZoneRoadType" + " set hotellingHours = VMT * ZoneRoadType.SHOAllocFactor * hotellingRate" + " where VMTByAgeRoadwayDay.yearID = hotellingCalendarYear.yearID" + + " and ZoneRoadType.zoneID = " + zoneID + " and VMTByAgeRoadwayDay.roadTypeID = ZoneRoadType.roadTypeID" + " and VMTByAgeRoadwayDay.roadTypeID in (2,4)"; SQLRunner.executeSQL(db,sql); /** + * Note that this is called "idleHours" and stored in "IdleHoursByAgeHour", but + * this is actually hotelling activity. This is *not* related to ONI. * @step 180 * @algorithm idleHours = hotellingHours * hotellingDist. * @output IdleHoursByAgeHour * @input VMTByAgeRoadwayDay * @input SourceTypeHour2 **/ + // clear IdleHoursByAgeHour first + sql = "TRUNCATE TABLE IdleHoursByAgeHour"; + SQLRunner.executeSQL(db,sql); sql = "insert ignore into IdleHoursByAgeHour (" + " yearID,sourceTypeID,ageID," + " monthID,dayID,hourID,idleHours)" @@ -1854,7 +1865,7 @@ void convertVMTToTotalActivityBasis() throws SQLException { sql = "DROP TABLE IF EXISTS SourceTypeHour2"; SQLRunner.executeSQL(db,sql); - // Calculate starts + // Calculate startsPerVehicle if(initialLoop) { start = System.currentTimeMillis(); sql = "DROP TABLE IF EXISTS StartsPerSampleVehicle"; @@ -2086,6 +2097,11 @@ void allocateTotalActivityBasis(MasterLoopContext inContext) throws SQLException // Don't update the activity tables unless the zone changes if(newYearForZone) { + + // convertVMTToTotalActivityBasis used to be part of the VMT growth that was done once per year + // However, the hotelling part of this function changes by ZoneID, so it needs to be done once per zone (as well as per year) + convertVMTToTotalActivityBasis(inContext.iterLocation.zoneRecordID); // steps 180-189 + if(needSHO && !checkAndMark("SHO",zoneID,analysisYear)) { if(!checkAndMark("ZoneRoadTypeLinkTemp",zoneID,0)) { sql = "drop table if exists ZoneRoadTypeLinkTemp"; diff --git a/gov/epa/otaq/moves/master/implementation/importers/LinkOpmodeDistributionImporter.java b/gov/epa/otaq/moves/master/implementation/importers/LinkOpmodeDistributionImporter.java index ddc93a5..8162db3 100644 --- a/gov/epa/otaq/moves/master/implementation/importers/LinkOpmodeDistributionImporter.java +++ b/gov/epa/otaq/moves/master/implementation/importers/LinkOpmodeDistributionImporter.java @@ -184,7 +184,40 @@ public boolean shouldWriteTemplateRow(Object[] values) { Object p = values[3]; Object o = values[4]; String key = p.toString() + "|" + o.toString(); - return templateKeys.contains(key); + + // templateKeys contains more polProcess/OpMode combinations than we actually want to include in the template + // However, it's a good start. If the combination doesn't exist in the set, reject it. If it does, check for + // other invalid combinations + if(templateKeys.contains(key)) { + // first, get integer values for easy comparisons + int polProcessID = 0; + int opModeID = 0; + try { + polProcessID = Integer.parseInt(p.toString()); + opModeID = Integer.parseInt(o.toString()); + } catch (NumberFormatException e) { + return true; + } + + // Remove the hotelling operating modes (those use a different importer) + if(opModeID == 200 || opModeID == 201 || opModeID == 203 || opModeID == 204) { + return false; + } + + // Remove other non-applicable operating modes + if(opModeID == 100 || opModeID == 150 || opModeID == 151 || opModeID == 300) { + return false; + } + + // Keep opModeID 501 for the brakewear polProcessIDs and remove all others + if(opModeID == 501) { + return (polProcessID == 10609 || polProcessID == 11609); + } + } else { + return false; + } + + return true; } } diff --git a/gov/epa/otaq/moves/master/implementation/importers/LinkSourceTypeHourImporter.java b/gov/epa/otaq/moves/master/implementation/importers/LinkSourceTypeHourImporter.java index b115fab..8a3b07f 100644 --- a/gov/epa/otaq/moves/master/implementation/importers/LinkSourceTypeHourImporter.java +++ b/gov/epa/otaq/moves/master/implementation/importers/LinkSourceTypeHourImporter.java @@ -151,16 +151,33 @@ public RunSpecSectionStatus getProjectDataStatus(Connection db) String sql; SQLRunner.Query query = new SQLRunner.Query(); boolean hasError = false; + boolean hasOnNetworkRoadTypes = false; + boolean hasSourceTypes = false; if(db == null) { return new RunSpecSectionStatus(RunSpecSectionStatus.OK); } - boolean hasSourceTypes = manager.tableHasSourceTypes(db, - "select distinct sourceTypeID from " + primaryTableName, - this,primaryTableName + " is missing sourceTypeID(s)"); - if(!hasSourceTypes) { - return new RunSpecSectionStatus(RunSpecSectionStatus.NOT_READY); + // only check for all source types if this run has an on-network road type + // (off-network only runs don't need this table) + sql = "SELECT count(linkID) as numOnNetworkLinks " + + "FROM link " + + "WHERE roadTypeID <> 1"; + try { + query.open(db,sql); + while(query.rs.next()) { + int numOnNetworkLinks = query.rs.getInt(1); + if (numOnNetworkLinks > 0) { + hasSourceTypes = manager.tableHasSourceTypes(db, + "select distinct sourceTypeID from " + primaryTableName, + this,primaryTableName + " is missing sourceTypeID(s)"); + if(!hasSourceTypes) { + return new RunSpecSectionStatus(RunSpecSectionStatus.NOT_READY); + } + } + } + } finally { + query.close(); } // check for any link VHT distributions not summing to one diff --git a/maketodo.txt b/maketodo.txt index f9cef63..f8f028a 100644 --- a/maketodo.txt +++ b/maketodo.txt @@ -1,12 +1,12 @@ defaultServerName = localhost -defaultDatabaseName = movesdb20210209 +defaultDatabaseName = movesdb20210726 executionServerName = localhost executionDatabaseName = * outputServerName = localhost outputDatabaseName = MOVESOutput -nonroadExePath = NONROAD\NR08a\NONROAD.exe -generatorExePath = generators\externalgenerator64.exe -sharedDistributedFolderPath = SharedWork +nonroadExePath = NONROAD/NR08a/NONROAD.exe +generatorExePath = generators/externalgenerator64.exe +sharedDistributedFolderPath = sharedwork computerIDPath = MOVESComputerID.txt masterFolderPath = . saveTODOPath = SaveTODO diff --git a/manyworkers.txt b/manyworkers.txt index 0de597b..2c30a16 100644 --- a/manyworkers.txt +++ b/manyworkers.txt @@ -1,12 +1,12 @@ -sharedDistributedFolderPath = SharedWork +sharedDistributedFolderPath = sharedwork workFolderPath = manyworkers/workerfolder workerDatabaseName = * computerIDPath = workerServerName = localhost concurrentStatements = 1 -nonroadApplicationPath = NONROAD\NR08a\nonroad.exe -nonroadWorkingFolderPath = NONROAD\NR08a -calculatorApplicationPath = calc\externalcalculatorgo64.exe +nonroadApplicationPath = NONROAD/NR08a/nonroad.exe +nonroadWorkingFolderPath = NONROAD/NR08a +calculatorApplicationPath = calc/externalcalculatorgo64.exe mysqlUserName = moves mysqlPassword = 744ff5134053c418265b626f5d7035e3dff3d50c609c548f6f63 diff --git a/readme.md b/readme.md index c1d4da3..6fcda4b 100644 --- a/readme.md +++ b/readme.md @@ -1,4 +1,4 @@ -# MOVES3.0.1 +# MOVES3.0.2 EPA's MOtor Vehicle Emission Simulator (MOVES) is a state-of-the-science emission modeling system that estimates emissions for mobile sources at the national, county, and project level for criteria air pollutants, greenhouse gases, and air toxics, available under EPA's Open Source Software policy. @@ -41,7 +41,7 @@ To get set up: 4. If MariaDB is running on a non-default port (i.e., any port other than 3306), create a file called `MySQL.txt` in the root MOVES directory, and save the port number to this text file. This file should contain no whitespace, just the port number. -5. Uncomment line 2 of `setenv.bat` and comment out line 3 to set the JDK and JRE paths to match your environment (i.e., remove `REM` from the beginning of line 2 and add it to the beginning of line 3). +5. Uncomment line 2 of `setenv.bat` and comment out line 3 to set the JDK and JRE paths to match your environment (i.e., remove `REM` from the beginning of line 2 and add it to the beginning of line 3). You may also need to set `JAVA_HOME` to your JDK path on line 6. 6. Unzip the default database dumpfile from the .zip file in the `\database\Setup` directory to the same directory. @@ -56,6 +56,8 @@ To get set up: ant crungui ``` + For additional information about compiling MOVES, see [CommandLineMOVES.md](docs\CommandLineMOVES.md#compiling-moves). + 9. Hereafter to run MOVES, simply navigate to the MOVES directory and run: ```bash @@ -68,7 +70,7 @@ Documentation on the software components of MOVES, database structure, running M ### Previous MOVES Versions -MOVES3 is the second version of MOVES posted on GitHub. The previous version, MOVES2014b December 2018 Technical Update, can be accessed using the following tag: [MOVES2014b-Dec2018](https://github.com/USEPA/EPA_MOVES_Model/tree/MOVES2014b-Dec2018). Older versions of MOVES are available at the [MOVES website](https://www.epa.gov/moves/moves-versions-limited-current-use). +Previous versions of MOVES3 as well the MOVES2014b December 2018 Technical Update can be accessed on GitHub at [MOVES Releases](https://github.com/USEPA/EPA_MOVES_Model/releases). Older versions of MOVES are available at the [MOVES website](https://www.epa.gov/moves/moves-versions-limited-current-use). ### License diff --git a/setenv.bat b/setenv.bat index 5782186..3bd4005 100644 --- a/setenv.bat +++ b/setenv.bat @@ -3,3 +3,5 @@ REM SET PATH=jre\bin;ant\bin;%PATH% SET PATH=jre\bin;ant\bin; SET JAVA_TOOL_OPTIONS=-Dfile.encoding=UTF-8 SET ANT_OPTS=-XX:-UseGCOverheadLimit +SET JAVA_HOME= +SET JRE_HOME=jre