From 5638ed9dc9ad90e4b3957ecc70bc6bd4969dc7ac Mon Sep 17 00:00:00 2001 From: Peter Steinberg Date: Tue, 4 Apr 2017 13:44:36 -0700 Subject: [PATCH] output paths - make sure directories are made --- Python/ogusa/macro_output.py | 41 ++++++++++++++++------------- Python/ogusa/scripts/postprocess.py | 32 +++++++++++++--------- Python/ogusa/utils.py | 27 ++++++++++++------- Python/regression/run_reforms.py | 13 ++++----- Python/run_ogusa.py | 4 +-- Python/run_ogusa_serial.py | 14 +++++----- 6 files changed, 73 insertions(+), 58 deletions(-) diff --git a/Python/ogusa/macro_output.py b/Python/ogusa/macro_output.py index dd67d5b68..ccae9cffe 100644 --- a/Python/ogusa/macro_output.py +++ b/Python/ogusa/macro_output.py @@ -23,12 +23,12 @@ def dump_diff_output(baseline_dir, policy_dir): ''' -------------------------------------------------------------------- - This function reads the pickles with the SS and time path results - from the baseline and reform and then calculates the percentage - differences between the two for each year in the 10-year budget + This function reads the pickles with the SS and time path results + from the baseline and reform and then calculates the percentage + differences between the two for each year in the 10-year budget window, over the entire budget window, and in the SS. -------------------------------------------------------------------- - + INPUTS: baseline_dir = string, path for directory with baseline policy results policy_dir = string, path for directory with reform policy results @@ -36,26 +36,26 @@ def dump_diff_output(baseline_dir, policy_dir): OTHER FUNCTIONS AND FILES CALLED BY THIS FUNCTION: None OBJECTS CREATED WITHIN FUNCTION: - tpi_macro_vars_policy_path = string, path to pickle with time path + tpi_macro_vars_policy_path = string, path to pickle with time path results for reform - tpi_macro_vars_policy = dictionary, dictionary with numpy arrays of + tpi_macro_vars_policy = dictionary, dictionary with numpy arrays of results from transition path equilibrium for reform - tpi_macro_vars_baseline_path = string, path to pickle with time path + tpi_macro_vars_baseline_path = string, path to pickle with time path results for baseline policy - tpi_macro_vars_baseline = dictionary, dictionary with numpy arrays of + tpi_macro_vars_baseline = dictionary, dictionary with numpy arrays of results from transition path equilibrium for baseline policy - baseline_macros = [7,T] array, numpy array with time path for relevant macro + baseline_macros = [7,T] array, numpy array with time path for relevant macro variables from baseline equilibrium - policy_macros = [7,T] array, numpy array with time path for relevant macro + policy_macros = [7,T] array, numpy array with time path for relevant macro variables from reform equilibrium - pct_changes = [7,12] array, numpy array with pct changes in macro variables - from baseline to reform for each year + pct_changes = [7,12] array, numpy array with pct changes in macro variables + from baseline to reform for each year in the budget window (10 years), over all 10 years, and in the SS ss_policy_path = string, path to pickle of SS results for reform - ss_policy = dictionary, dictionary with numpy arrays of results from + ss_policy = dictionary, dictionary with numpy arrays of results from SS equilibrium for reform ss_baseline_path = string, path to pickle of SS results for baseline - ss_baseline = dictionary, dictionary with numpy arrays of results from + ss_baseline = dictionary, dictionary with numpy arrays of results from SS equilibrium for baseline RETURNS: pct_changes @@ -63,10 +63,13 @@ def dump_diff_output(baseline_dir, policy_dir): ''' # read macro output - tpi_macro_vars_policy_path = os.path.join(policy_dir, "TPI", "TPI_macro_vars.pkl") - tpi_macro_vars_policy = pickle.load(open( tpi_macro_vars_policy_path, "rb" )) - tpi_macro_vars_baseline_path = os.path.join(baseline_dir, "TPI", "TPI_macro_vars.pkl") - tpi_macro_vars_baseline = pickle.load(open( tpi_macro_vars_baseline_path, "rb" ) ) + tpi_dir = os.path.join(policy_dir, "TPI") + if not os.path.exists(tpi_dir): + os.mkdir(tpi_dir) + tpi_macro_vars_policy_path = os.path.join(tpi_dir, "TPI_macro_vars.pkl") + tpi_macro_vars_policy = pickle.load(open(tpi_macro_vars_policy_path, "rb")) + tpi_macro_vars_baseline_path = os.path.join(tpi_dir, "TPI_macro_vars.pkl") + tpi_macro_vars_baseline = pickle.load(open(tpi_macro_vars_baseline_path, "rb")) T = len(tpi_macro_vars_baseline['C']) baseline_macros = np.zeros((7,T)) @@ -92,7 +95,7 @@ def dump_diff_output(baseline_dir, policy_dir): pct_changes[:,:10] = ((policy_macros-baseline_macros)/policy_macros)[:,:10] # pct changes over entire budget window pct_changes[:,10] = ((policy_macros[:,:10].sum(axis=1)-baseline_macros[:,:10].sum(axis=1))/policy_macros[:,:10].sum(axis=1)) - + ## Load SS results ss_policy_path = os.path.join(policy_dir, "SS", "SS_vars.pkl") ss_policy = pickle.load(open( ss_policy_path, "rb" )) diff --git a/Python/ogusa/scripts/postprocess.py b/Python/ogusa/scripts/postprocess.py index 746a6d8a2..c59f0c085 100644 --- a/Python/ogusa/scripts/postprocess.py +++ b/Python/ogusa/scripts/postprocess.py @@ -8,54 +8,60 @@ This py-file calls the following other files: macro_output.py -This py-file creates the following other file(s): +This py-file creates the following other file(s): ./ogusa_output{}.pkl ------------------------------------------------------------------------ ''' -import ogusa -from ogusa import macro_output import pickle import numpy as np +import ogusa +from ogusa import macro_output +from ogusa.utils import REFORM_DIR, BASELINE_DIR + +DEFAULTS = dict(baseline_dir=BASELINE_DIR, + policy_dir=REFORM_DIR) + def create_diff(baseline_dir, policy_dir, dump_output=False): ''' -------------------------------------------------------------------- This function finds the percentage changes in macro variables that result from the tax reform. -------------------------------------------------------------------- - + INPUTS: baseline_dir = string, path for directory with baseline policy results policy_dir = string, path for directory with reform policy results dump_output = boolean, =True if want results saved to pickle - OTHER FUNCTIONS AND FILES CALLED BY THIS FUNCTION: + OTHER FUNCTIONS AND FILES CALLED BY THIS FUNCTION: macro_output.dump_diff_output() OBJECTS CREATED WITHIN FUNCTION: - pct_changes = [7,12] array, numpy array with pct changes in macro variables + pct_changes = [7,12] array, numpy array with pct changes in macro variables from baseline to reform for each year. Final column = steady state. Macro vars: Y, C, K, L, w, r, T_H RETURNS: pct_changes - OUTPUT: + OUTPUT: ./ogusa_output{}.pkl -------------------------------------------------------------------- ''' - pct_changes, baseline_macros, policy_macros = macro_output.dump_diff_output(baseline_dir, policy_dir) - - np.savetxt('ClosedEconPctChanges.csv',pct_changes,delimiter=",") + out = macro_output.dump_diff_output(baseline_dir, policy_dir) + pct_changes, baseline_macros, policy_macros = out + + np.savetxt('ClosedEconPctChanges.csv',pct_changes,delimiter=",") if dump_output: pickle.dump(pct_changes, open("ogusa_output.pkl", "wb")) - np.savetxt('ClosedEconBaseline.csv',baseline_macros,delimiter=",") - np.savetxt('ClosedEconPolicy.csv',policy_macros,delimiter=",") + np.savetxt('ClosedEconBaseline.csv',baseline_macros,delimiter=",") + np.savetxt('ClosedEconPolicy.csv',policy_macros,delimiter=",") return pct_changes if __name__ == "__main__": - create_diff(baseline_dir="./OUTPUT_BASELINE", policy_dir="./OUTPUT_REFORM") + create_diff(**DEFAULTS) diff --git a/Python/ogusa/utils.py b/Python/ogusa/utils.py index d57e75637..31198d49e 100644 --- a/Python/ogusa/utils.py +++ b/Python/ogusa/utils.py @@ -20,6 +20,13 @@ EPSILON = 1e-10 PATH_EXISTS_ERRNO = 17 +REFORM_DIR = "./OUTPUT_REFORM" +BASELINE_DIR = "./OUTPUT_BASELINE" + +for f in (REFORM_DIR, BASELINE_DIR): + if not os.path.exists(f): + os.mkdir(f) + def mkdirs(path): ''' @@ -50,7 +57,7 @@ def pct_diff_func(simul, data): Inputs: simul = any shape, model moments data = same shape as simul, data moments - + Functions called: None Objects in function: @@ -73,7 +80,7 @@ def convex_combo(var1, var2, nu): var1 = any shape, variable 1 var2 = same shape as var1, variable 2 nu = scalar, weight on var1 in convex combination - + Functions called: None Objects in function: @@ -126,10 +133,10 @@ def pickle_file_compare(fname1, fname2, tol=1e-3, exceptions={}, relative=False) fname1 = string, file name of file 1 fname2 = string, file name of file 2 tol = scalar, tolerance - exceptions = dictionary, exceptions - relative = boolean, + exceptions = dictionary, exceptions + relative = boolean, - Functions called: + Functions called: dict_compare Objects in function: @@ -152,15 +159,15 @@ def comp_array(name, a, b, tol, unequal, exceptions={}, relative=False): Return True if | a - b | < tol, False otherwise If not equal, add items to the unequal list name: the name of the value being compared - + Inputs: - - Functions called: - + + Functions called: + Objects in function: - + Returns: Boolean diff --git a/Python/regression/run_reforms.py b/Python/regression/run_reforms.py index 04c6a3fa8..58d6932e1 100755 --- a/Python/regression/run_reforms.py +++ b/Python/regression/run_reforms.py @@ -23,6 +23,7 @@ from ogusa import TPI from ogusa.scripts import postprocess from ogusa.scripts.execute import runner # change here for small jobs + from ogusa.utils import REFORM_DIR, BASELINE_DIR except Exception as e: pref = sys.prefix exc = traceback.format_exc() @@ -69,16 +70,16 @@ def run_micro_macro(reform, user_params, guid, solution_checks, run_micro): start_time = time.time() - REFORM_DIR = "./OUTPUT_REFORM" + guid - BASELINE_DIR = "./OUTPUT_BASELINE" + guid + reform_dir = "./OUTPUT_REFORM" + guid + baseline_dir = "./OUTPUT_BASELINE" + guid # Add start year from reform to user parameters start_year = sorted(reform.keys())[0] user_params['start_year'] = start_year - input_dir = BASELINE_DIR + input_dir = baseline_dir - kwargs={'output_base':BASELINE_DIR, 'baseline_dir':BASELINE_DIR, + kwargs={'output_base':baseline_dir, 'baseline_dir':baseline_dir, 'baseline':True, 'analytical_mtrs':False, 'age_specific':False, 'user_params':user_params, 'guid':guid, 'run_micro':run_micro} @@ -86,7 +87,7 @@ def run_micro_macro(reform, user_params, guid, solution_checks, run_micro): #p1.start() runner(**kwargs) - kwargs={'output_base':REFORM_DIR, 'baseline_dir':BASELINE_DIR, + kwargs={'output_base':reform_dir, 'baseline_dir':baseline_dir, 'baseline':False, 'analytical_mtrs':False, 'user_params':user_params, 'reform':reform, 'age_specific':False, 'guid':guid,'run_micro':run_micro} @@ -100,7 +101,7 @@ def run_micro_macro(reform, user_params, guid, solution_checks, run_micro): #time.sleep(0.5) - ans = postprocess.create_diff(baseline_dir=BASELINE_DIR, policy_dir=REFORM_DIR) + ans = postprocess.create_diff(baseline_dir=baseline_dir, policy_dir=reform_dir) print("total time was ", (time.time() - start_time)) print(ans) diff --git a/Python/run_ogusa.py b/Python/run_ogusa.py index 1b6102b74..cdc32bc59 100644 --- a/Python/run_ogusa.py +++ b/Python/run_ogusa.py @@ -11,6 +11,7 @@ from ogusa.scripts import postprocess #from execute import runner # change here for small jobs from ogusa.scripts.execute_large import runner +from ogusa.utils import REFORM_DIR, BASELINE_DIR def run_micro_macro(user_params): @@ -28,9 +29,6 @@ def run_micro_macro(user_params): start_time = time.time() - REFORM_DIR = "./OUTPUT_REFORM" - BASELINE_DIR = "./OUTPUT_BASELINE" - output_base = REFORM_DIR input_dir = REFORM_DIR diff --git a/Python/run_ogusa_serial.py b/Python/run_ogusa_serial.py index 2000d549a..f0763d22f 100644 --- a/Python/run_ogusa_serial.py +++ b/Python/run_ogusa_serial.py @@ -11,7 +11,7 @@ from ogusa.scripts import postprocess from ogusa.scripts.execute import runner - +from ogusa.utils import REFORM_DIR, BASELINE_DIR def run_micro_macro(user_params): @@ -41,9 +41,6 @@ def run_micro_macro(user_params): start_time = time.time() - REFORM_DIR = "./OUTPUT_REFORM" - BASELINE_DIR = "./OUTPUT_BASELINE" - T_shifts = np.zeros(50) T_shifts[2:10] = 0.01 T_shifts[10:40]= -0.01 @@ -79,7 +76,8 @@ def run_micro_macro(user_params): output_base = BASELINE_DIR input_dir = BASELINE_DIR kwargs={'output_base':output_base, 'baseline_dir':BASELINE_DIR, - 'test':True, 'time_path':True, 'baseline':True, 'analytical_mtrs':False, 'age_specific':True, + 'test':True, 'time_path':True, 'baseline':True, + 'analytical_mtrs':False, 'age_specific':True, 'user_params':user_params,'guid':'', 'run_micro':False, 'small_open': True, 'budget_balance':False} #p1 = Process(target=runner, kwargs=kwargs) @@ -96,9 +94,11 @@ def run_micro_macro(user_params): input_dir = REFORM_DIR guid_iter = 'reform_' + str(0) kwargs={'output_base':output_base, 'baseline_dir':BASELINE_DIR, - 'test':True, 'time_path':True, 'baseline':False, 'analytical_mtrs':False, 'age_specific':True, + 'test':True, 'time_path':True, 'baseline':False, + 'analytical_mtrs':False, 'age_specific':True, 'user_params':user_params,'guid':'_alt', 'reform':'' , - 'run_micro':False, 'small_open': True, 'budget_balance':False, 'baseline_spending':True} + 'run_micro':False, 'small_open': True, 'budget_balance':False, + 'baseline_spending':True} #p2 = Process(target=runner, kwargs=kwargs) #p2.start() runner(**kwargs)