From 6e797857e224bdbd75ee206a87941451348e239d Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Thu, 18 Aug 2022 22:00:41 -0400 Subject: [PATCH 001/127] tests/TransferMechanism: Use the correct execution mode in test_transfer_mech_integration_rate_0_8 (#2471) EX encapsulates compiled CPU or GPU or Python execution, use that instead of always executing Python version. Fixes: cf57224670a0830dd555f34e04c0d4a083d3b996 ("test/mechanisms/TransferMechanism: Only run benchmarks if enabled") Signed-off-by: Jan Vesely --- tests/mechanisms/test_transfer_mechanism.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/mechanisms/test_transfer_mechanism.py b/tests/mechanisms/test_transfer_mechanism.py index e6a295ce05a..aae892ef081 100644 --- a/tests/mechanisms/test_transfer_mechanism.py +++ b/tests/mechanisms/test_transfer_mechanism.py @@ -903,14 +903,14 @@ def test_transfer_mech_integration_rate_0_8(self, benchmark, mech_mode): ) EX = pytest.helpers.get_mech_execution(T, mech_mode) - val1 = T.execute([1 for i in range(VECTOR_SIZE)]) - val2 = T.execute([1 for i in range(VECTOR_SIZE)]) + val1 = EX([1 for i in range(VECTOR_SIZE)]) + val2 = EX([1 for i in range(VECTOR_SIZE)]) assert np.allclose(val1, [[0.8 for i in range(VECTOR_SIZE)]]) assert np.allclose(val2, [[0.96 for i in range(VECTOR_SIZE)]]) if benchmark.enabled: - benchmark(T.execute, [0 for i in range(VECTOR_SIZE)]) + benchmark(EX, [0 for i in range(VECTOR_SIZE)]) @pytest.mark.mechanism @pytest.mark.transfer_mechanism From 6b1ae7b3931f41e4b0710f138db223d4c5817f7e Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Fri, 19 Aug 2022 00:17:27 -0400 Subject: [PATCH 002/127] tests, treewide: Add missing "composition" marks (#2472) Add missing pytest.mark.composition to tests that construct and run Composition. Signed-off-by: Jan Vesely --- tests/composition/test_control.py | 15 +++------------ tests/functions/test_user_defined_func.py | 2 ++ tests/mechanisms/test_control_mechanism.py | 4 ++++ tests/mechanisms/test_ddm_mechanism.py | 9 +++++++-- tests/mechanisms/test_integrator_mechanism.py | 3 ++- tests/mechanisms/test_lca.py | 18 ++++++++++++------ .../test_recurrent_transfer_mechanism.py | 11 ++++++++++- tests/mechanisms/test_transfer_mechanism.py | 8 +++++++- tests/ports/test_output_ports.py | 1 + tests/scheduling/test_condition.py | 15 +++++++++++++++ tests/scheduling/test_scheduler.py | 2 +- 11 files changed, 64 insertions(+), 24 deletions(-) diff --git a/tests/composition/test_control.py b/tests/composition/test_control.py index 706dc08ef19..c6eedd34694 100644 --- a/tests/composition/test_control.py +++ b/tests/composition/test_control.py @@ -2600,6 +2600,7 @@ def test_modulation_of_random_state(self, comp_mode, num_generators): assert np.allclose(best_second, comp.results[1]) +@pytest.mark.composition @pytest.mark.control class TestModelBasedOptimizationControlMechanisms_Execution: def test_ocm_default_function(self): @@ -2950,8 +2951,6 @@ def test_evc_gratton(self): # Note: Skip decision variable OutputPort evc_gratton.simulation_results[simulation][1:]) - @pytest.mark.control - @pytest.mark.composition def test_laming_validation_specify_control_signals(self): # Mechanisms Input = pnl.TransferMechanism(name='Input') @@ -3072,8 +3071,6 @@ def test_laming_validation_specify_control_signals(self): err_msg='Failed on expected_output[{0}]'.format(trial) ) - @pytest.mark.control - @pytest.mark.composition def test_stateful_mechanism_in_simulation(self): # Mechanisms Input = pnl.TransferMechanism(name='Input', integrator_mode=True) @@ -3211,8 +3208,6 @@ def test_stateful_mechanism_in_simulation(self): err_msg='Failed on expected_output[{0}]'.format(trial) ) - @pytest.mark.control - @pytest.mark.composition @pytest.mark.benchmark(group="Model Based OCM") @pytest.mark.parametrize("mode", pytest.helpers.get_comp_execution_modes() + [pytest.helpers.cuda_param('Python-PTX'), @@ -3262,8 +3257,6 @@ def test_model_based_ocm_after(self, benchmark, mode): if benchmark.enabled: benchmark(comp.run, inputs, execution_mode=mode) - @pytest.mark.control - @pytest.mark.composition @pytest.mark.benchmark(group="Model Based OCM") @pytest.mark.parametrize("mode", pytest.helpers.get_comp_execution_modes() + [pytest.helpers.cuda_param('Python-PTX'), @@ -3652,8 +3645,6 @@ def test_model_based_ocm_no_simulations(self): # initial 1 + each allocation sample (1, 2, 3) integrated assert B.parameters.value.get(comp) == 7 - @pytest.mark.control - @pytest.mark.composition @pytest.mark.benchmark(group="Multilevel") def test_grid_search_random_selection(self, comp_mode, benchmark): A = pnl.ProcessingMechanism(name='A') @@ -3700,8 +3691,7 @@ def test_grid_search_random_selection(self, comp_mode, benchmark): benchmark(comp.run, inputs=inputs, num_trials=10, context='bench_outer_comp', execution_mode=comp_mode) assert len(A.log.get_logged_entries()) == 0 - @pytest.mark.control - @pytest.mark.composition + def test_input_CIM_assignment(self, comp_mode): input_a = pnl.ProcessingMechanism(name='oa', function=pnl.Linear(slope=1)) input_b = pnl.ProcessingMechanism(name='ob', function=pnl.Linear(slope=1)) @@ -3863,6 +3853,7 @@ def test_list(self): assert sample_iterator.num == len(sample_list) +@pytest.mark.composition @pytest.mark.control class TestControlTimeScales: diff --git a/tests/functions/test_user_defined_func.py b/tests/functions/test_user_defined_func.py index 77d12fca243..84cd2f18a8d 100644 --- a/tests/functions/test_user_defined_func.py +++ b/tests/functions/test_user_defined_func.py @@ -604,6 +604,7 @@ def test_user_def_func_builtin_direct(func, args, expected, benchmark): val = benchmark(func, *args) assert np.allclose(val, expected) +@pytest.mark.composition @pytest.mark.benchmark(group="UDF as Composition Origin") def test_udf_composition_origin(comp_mode, benchmark): def myFunction(variable, context): @@ -616,6 +617,7 @@ def myFunction(variable, context): assert np.allclose(c.results[0][0], [3, 1]) +@pytest.mark.composition @pytest.mark.benchmark(group="UDF as Composition Terminal") def test_udf_composition_terminal(comp_mode, benchmark): def myFunction(variable, context): diff --git a/tests/mechanisms/test_control_mechanism.py b/tests/mechanisms/test_control_mechanism.py index d5fdfd66204..365825fa6a4 100644 --- a/tests/mechanisms/test_control_mechanism.py +++ b/tests/mechanisms/test_control_mechanism.py @@ -10,6 +10,7 @@ class TestLCControlMechanism: @pytest.mark.mechanism @pytest.mark.control_mechanism + @pytest.mark.composition @pytest.mark.benchmark(group="LCControlMechanism Default") def test_lc_control_mechanism_as_controller(self, benchmark): G = 1.0 @@ -93,6 +94,7 @@ def test_lc_control_mech_basic(self, benchmark, mech_mode): if benchmark.enabled: benchmark(EX, [10.0]) + @pytest.mark.composition def test_lc_control_modulated_mechanisms_all(self): T_1 = pnl.TransferMechanism(name='T_1') @@ -110,7 +112,9 @@ def test_lc_control_modulated_mechanisms_all(self): assert T_2.parameter_ports[pnl.SLOPE].mod_afferents[0] in LC.control_signals[0].efferents +@pytest.mark.composition class TestControlMechanism: + def test_control_modulation(self): Tx = pnl.TransferMechanism(name='Tx') Ty = pnl.TransferMechanism(name='Ty') diff --git a/tests/mechanisms/test_ddm_mechanism.py b/tests/mechanisms/test_ddm_mechanism.py index 13f6b9703ae..db5a9645597 100644 --- a/tests/mechanisms/test_ddm_mechanism.py +++ b/tests/mechanisms/test_ddm_mechanism.py @@ -159,6 +159,7 @@ def test_threshold_stops_accumulation(self, mech_mode, variable, expected, bench # assert np.allclose(decision_variables_a, [2.0, 4.0, 5.0, 5.0, 5.0]) + @pytest.mark.composition def test_is_finished_stops_composition(self): D = DDM(name='DDM', function=DriftDiffusionIntegrator(threshold=10.0, time_step_size=1.0), @@ -183,6 +184,7 @@ def test_is_finished_stops_composition(self): # # sched = Scheduler(system=S) +@pytest.mark.composition class TestInputPorts: def test_regular_input_mode(self): @@ -631,6 +633,7 @@ def test_WhenFinished_DDM_Analytical(): c.is_satisfied() +@pytest.mark.composition @pytest.mark.ddm_mechanism @pytest.mark.mechanism @pytest.mark.benchmark(group="DDM-comp") @@ -659,8 +662,8 @@ def test_DDM_in_composition(benchmark, comp_mode): benchmark(C.run, inputs, num_trials=2, execution_mode=comp_mode) +@pytest.mark.composition @pytest.mark.ddm_mechanism -@pytest.mark.mechanism def test_DDM_threshold_modulation(comp_mode): M = pnl.DDM( name='DDM', @@ -689,6 +692,8 @@ def test_DDM_threshold_modulation(comp_mode): assert np.allclose(val[0], [60.0]) assert np.allclose(val[1], [60.2]) + +@pytest.mark.composition @pytest.mark.parametrize(["noise", "threshold", "expected_results"],[ (1.0, 0.0, (0.0, 1.0)), (1.5, 2, (-2.0, 1.0)), @@ -772,7 +777,7 @@ def test_sequence_of_DDM_mechs_in_Composition_Pathway(): np.testing.assert_allclose(val, expected, atol=1e-08, err_msg='Failed on expected_output[{0}]'.format(i)) -@pytest.mark.mechanism +@pytest.mark.composition @pytest.mark.ddm_mechanism def test_DDMMechanism_LCA_equivalent(comp_mode): diff --git a/tests/mechanisms/test_integrator_mechanism.py b/tests/mechanisms/test_integrator_mechanism.py index c5e974a4c20..5369cb14f92 100644 --- a/tests/mechanisms/test_integrator_mechanism.py +++ b/tests/mechanisms/test_integrator_mechanism.py @@ -1174,7 +1174,7 @@ def test_has_initializers(self): assert I.has_initializers assert hasattr(I, "reset_stateful_function_when") - @pytest.mark.mechanism + @pytest.mark.composition @pytest.mark.integrator_mechanism @pytest.mark.parametrize('cond0, cond1, expected', [ (pnl.Never(), pnl.AtTrial(2), @@ -1218,6 +1218,7 @@ def test_reset_stateful_function_when_composition(self, comp_mode, cond0, cond1, assert np.allclose(expected, C.results) + @pytest.mark.composition def test_reset_stateful_function_when(self): I1 = IntegratorMechanism() I2 = IntegratorMechanism() diff --git a/tests/mechanisms/test_lca.py b/tests/mechanisms/test_lca.py index d79e1dbb667..875c53a3f76 100644 --- a/tests/mechanisms/test_lca.py +++ b/tests/mechanisms/test_lca.py @@ -12,7 +12,8 @@ LCAMechanism, MAX_VS_AVG, MAX_VS_NEXT, CONVERGENCE class TestLCA: - @pytest.mark.mechanism + + @pytest.mark.composition @pytest.mark.lca_mechanism @pytest.mark.benchmark(group="LCAMechanism") def test_LCAMechanism_length_1(self, benchmark, comp_mode): @@ -59,7 +60,7 @@ def test_LCAMechanism_length_1(self, benchmark, comp_mode): if benchmark.enabled: benchmark(C.run, inputs={T: [1.0]}, num_trials=3, execution_mode=comp_mode) - @pytest.mark.mechanism + @pytest.mark.composition @pytest.mark.lca_mechanism @pytest.mark.benchmark(group="LCAMechanism") def test_LCAMechanism_length_2(self, benchmark, comp_mode): @@ -120,6 +121,7 @@ def test_LCAMechanism_length_2(self, benchmark, comp_mode): if benchmark.enabled: benchmark(C.run, inputs={T: [1.0, 2.0]}, num_trials=3, execution_mode=comp_mode) + @pytest.mark.composition def test_equivalance_of_threshold_and_when_finished_condition(self): # Note: This tests the equivalence of results when: # execute_until_finished is True for the LCAMechanism (by default) @@ -152,7 +154,7 @@ def test_LCAMechanism_matrix(self): # Note: In the following tests, since the LCAMechanism's threshold is specified # it executes until the it reaches threshold. - @pytest.mark.mechanism + @pytest.mark.composition @pytest.mark.lca_mechanism @pytest.mark.benchmark(group="LCAMechanism") def test_LCAMechanism_threshold(self, benchmark, comp_mode): @@ -164,6 +166,7 @@ def test_LCAMechanism_threshold(self, benchmark, comp_mode): if benchmark.enabled: benchmark(comp.run, inputs={lca:[1,0]}, execution_mode=comp_mode) + @pytest.mark.composition def test_LCAMechanism_threshold_with_max_vs_next(self): lca = LCAMechanism(size=3, leak=0.5, threshold=0.1, threshold_criterion=MAX_VS_NEXT) comp = Composition() @@ -171,6 +174,7 @@ def test_LCAMechanism_threshold_with_max_vs_next(self): result = comp.run(inputs={lca:[1,0.5,0]}) assert np.allclose(result, [[0.52490032, 0.42367594, 0.32874867]]) + @pytest.mark.composition def test_LCAMechanism_threshold_with_max_vs_avg(self): lca = LCAMechanism(size=3, leak=0.5, threshold=0.1, threshold_criterion=MAX_VS_AVG) comp = Composition() @@ -178,7 +182,7 @@ def test_LCAMechanism_threshold_with_max_vs_avg(self): result = comp.run(inputs={lca:[1,0.5,0]}) assert np.allclose(result, [[0.51180475, 0.44161738, 0.37374946]]) - @pytest.mark.mechanism + @pytest.mark.composition @pytest.mark.lca_mechanism @pytest.mark.benchmark(group="LCAMechanism") def test_LCAMechanism_threshold_with_convergence(self, benchmark, comp_mode): @@ -192,7 +196,7 @@ def test_LCAMechanism_threshold_with_convergence(self, benchmark, comp_mode): if benchmark.enabled: benchmark(comp.run, inputs={lca:[0,1,2]}, execution_mode=comp_mode) - @pytest.mark.mechanism + @pytest.mark.composition @pytest.mark.lca_mechanism def test_equivalance_of_threshold_and_termination_specifications_just_threshold(self, comp_mode): # Note: This tests the equivalence of using LCAMechanism-specific threshold arguments and @@ -215,6 +219,7 @@ def test_equivalance_of_threshold_and_termination_specifications_just_threshold( result2 = comp2.run(inputs={lca_termination:[1,0]}, execution_mode=comp_mode) assert np.allclose(result1, result2) + @pytest.mark.composition def test_equivalance_of_threshold_and_termination_specifications_max_vs_next(self): # Note: This tests the equivalence of using LCAMechanism-specific threshold arguments and # generic TransferMechanism termination_<*> arguments @@ -255,7 +260,7 @@ def test_equivalance_of_threshold_and_termination_specifications_max_vs_next(sel # result = comp.run(inputs={lca:[1,0]}) # assert np.allclose(result, [[0.71463572, 0.28536428]]) - @pytest.mark.mechanism + @pytest.mark.composition @pytest.mark.lca_mechanism def test_LCAMechanism_DDM_equivalent(self, comp_mode): lca = LCAMechanism(size=2, leak=0., threshold=1, auto=0, hetero=0, @@ -268,6 +273,7 @@ def test_LCAMechanism_DDM_equivalent(self, comp_mode): class TestLCAReset: + @pytest.mark.composition def test_reset_run(self): L = LCAMechanism(name="L", diff --git a/tests/mechanisms/test_recurrent_transfer_mechanism.py b/tests/mechanisms/test_recurrent_transfer_mechanism.py index 6fc87408ffb..c8b9a96c329 100644 --- a/tests/mechanisms/test_recurrent_transfer_mechanism.py +++ b/tests/mechanisms/test_recurrent_transfer_mechanism.py @@ -21,6 +21,7 @@ RecurrentTransferError, RecurrentTransferMechanism from psyneulink.library.components.projections.pathway.autoassociativeprojection import AutoAssociativeProjection +@pytest.mark.composition class TestMatrixSpec: def test_recurrent_mech_matrix(self): @@ -628,6 +629,7 @@ def test_recurrent_mech_integration_rate_0_8_initial_1_2(self): # won't get executed if we only use the execute() method of Mechanism: thus, to test it we must use a Composition +@pytest.mark.composition def run_twice_in_composition(mech, input1, input2=None): if input2 is None: input2 = input1 @@ -637,6 +639,7 @@ def run_twice_in_composition(mech, input1, input2=None): return result[0] +@pytest.mark.composition class TestRecurrentTransferMechanismInProcess: simple_prefs = {REPORT_OUTPUT_PREF: False, VERBOSE_PREF: False} @@ -722,6 +725,7 @@ def test_recurrent_mech_process_proj_matrix_change(self): np.testing.assert_allclose(R.parameters.value.get(c), [[21, 3, 12, 35]]) +@pytest.mark.composition class TestRecurrentTransferMechanismInComposition: simple_prefs = {REPORT_OUTPUT_PREF: False, VERBOSE_PREF: False} @@ -957,6 +961,7 @@ def test_learning_of_orthognal_inputs(self): np.testing.assert_allclose(R.output_port.parameters.value.get(C),[0.0, 1.18518086, 0.0, 1.18518086]) +@pytest.mark.composition class TestRecurrentTransferMechanismReset: def test_reset_run(self): @@ -1024,6 +1029,7 @@ def test_clip_2d_array(self): assert np.allclose(R.execute([[-5.0, -1.0, 5.0], [5.0, -5.0, 1.0], [1.0, 5.0, 5.0]]), [[-2.0, -1.0, 2.0], [2.0, -2.0, 1.0], [1.0, 2.0, 2.0]]) +@pytest.mark.composition class TestRecurrentInputPort: def test_ris_simple(self): @@ -1061,6 +1067,7 @@ def my_fct(x): result = R2.execute([1,2]) np.testing.assert_allclose(result, [[0,0]]) + @pytest.mark.composition @pytest.mark.mechanism @pytest.mark.integrator_mechanism @pytest.mark.parametrize('cond0, cond1, expected', [ @@ -1107,6 +1114,7 @@ def test_reset_stateful_function_when_composition(self, comp_mode, cond0, cond1, assert np.allclose(expected, C.results) + @pytest.mark.composition @pytest.mark.mechanism @pytest.mark.integrator_mechanism @pytest.mark.parametrize('cond0, cond1, expected', [ @@ -1152,7 +1160,7 @@ def test_reset_stateful_function_when_has_initializers_composition(self, comp_mo assert np.allclose(exp, C.results) - @pytest.mark.mechanism + @pytest.mark.composition @pytest.mark.integrator_mechanism @pytest.mark.parametrize('until_finished, expected', [ (True, [[[[0.96875]]], [[[0.9990234375]]]]), # The 5th and the 10th iteration @@ -1177,6 +1185,7 @@ def test_max_executions_before_finished(self, comp_mode, until_finished, expecte assert np.allclose(expected[0], results) assert np.allclose(expected[1], results2) +@pytest.mark.composition class TestDebugProperties: def test_defaults(self): diff --git a/tests/mechanisms/test_transfer_mechanism.py b/tests/mechanisms/test_transfer_mechanism.py index aae892ef081..f9b98489291 100644 --- a/tests/mechanisms/test_transfer_mechanism.py +++ b/tests/mechanisms/test_transfer_mechanism.py @@ -1430,6 +1430,7 @@ def test_previous_value_persistence_execute(self): # linear fn: 0.595*1.0 = 0.595 assert np.allclose(T.integrator_function.previous_value, 0.595) + @pytest.mark.composition def test_previous_value_persistence_run(self): T = TransferMechanism(name="T", initial_value=0.5, @@ -1497,6 +1498,7 @@ def test_previous_value_reset_execute(self): assert np.allclose(T.integrator_function.previous_value, 0.46) # property that looks at integrator, which updated with mech exec assert np.allclose(T.value, 0.46) # on mechanism, but updates with exec + @pytest.mark.composition def test_reset_run(self): T = TransferMechanism(name="T", initial_value=0.5, @@ -1537,6 +1539,7 @@ def test_reset_run(self): # linear fn: 0.595*1.0 = 0.595 assert np.allclose(T.integrator_function.parameters.previous_value.get(C), 0.595) + @pytest.mark.composition def test_reset_run_array(self): T = TransferMechanism(name="T", default_variable=[0.0, 0.0, 0.0], @@ -1577,6 +1580,7 @@ def test_reset_run_array(self): # linear fn: 0.595*1.0 = 0.595 assert np.allclose(T.integrator_function.parameters.previous_value.get(C), [0.595, 0.595, 0.595]) + @pytest.mark.composition def test_reset_run_2darray(self): initial_val = [[0.5, 0.5, 0.5]] @@ -1629,6 +1633,7 @@ def test_reset_not_integrator(self): assert "not allowed because its `integrator_mode` parameter" in str(err_txt.value) assert "is currently set to \'False\'; try setting it to \'True\'" in str(err_txt.value) + @pytest.mark.composition def test_switch_mode(self): T = TransferMechanism(integrator_mode=True, on_resume_integrator_mode=LAST_INTEGRATED_VALUE) @@ -1659,6 +1664,7 @@ def test_switch_mode(self): C.run({T: [[1.0], [1.0], [1.0]]}) assert np.allclose(T.parameters.value.get(C), [[0.984375]]) + @pytest.mark.composition def test_initial_values_softmax(self): T = TransferMechanism(default_variable=[[0.0, 0.0], [0.0, 0.0]], function=SoftMax(), @@ -1695,6 +1701,7 @@ def test_set_integrator_mode_after_init(self): T.execute(1) +@pytest.mark.composition class TestOnResumeIntegratorMode: def test_last_integrated_value_spec(self): @@ -1777,7 +1784,6 @@ def test_reset_spec(self): # Trial 1: 0.5*0.5 + 0.5*2.0 = 1.25 * 1.0 = 1.25 assert np.allclose(T.parameters.value.get(C), [[1.25]]) - @pytest.mark.mechanism @pytest.mark.transfer_mechanism @pytest.mark.benchmark(group="TransferMechanism") # 'LLVM' mode is not supported, because synchronization of compiler and diff --git a/tests/ports/test_output_ports.py b/tests/ports/test_output_ports.py index 71761b860f4..e8a1c1fe977 100644 --- a/tests/ports/test_output_ports.py +++ b/tests/ports/test_output_ports.py @@ -31,6 +31,7 @@ def test_output_port_variable_spec(self, mech_mode): for i, e in zip(res, expected): assert np.array_equal(i, e) + @pytest.mark.composition @pytest.mark.mechanism @pytest.mark.parametrize('spec, expected1, expected2', [((pnl.OWNER_VALUE, 0), [1], [1]), diff --git a/tests/scheduling/test_condition.py b/tests/scheduling/test_condition.py index f9684059100..aedb26b04b2 100644 --- a/tests/scheduling/test_condition.py +++ b/tests/scheduling/test_condition.py @@ -82,6 +82,7 @@ def func(a, b, c=True): assert not cond.is_satisfied(False, c=False) assert not cond.is_satisfied(False, c=False, extra_arg=True) + @pytest.mark.composition class TestGeneric: def test_WhileNot_AtPass(self): comp = Composition() @@ -115,6 +116,7 @@ def test_WhileNot_AtPass_in_middle(self): expected_output = [A, A, set(), A, A] assert output == pytest.helpers.setify_expected_output(expected_output) + @pytest.mark.composition class TestRelative: def test_Any_end_before_one_finished(self): @@ -211,6 +213,7 @@ def test_NWhen_AfterNCalls(self, n, expected_output): assert output == pytest.helpers.setify_expected_output(expected_output) + @pytest.mark.composition class TestTime: def test_BeforeTimeStep(self): @@ -480,6 +483,7 @@ def test_AfterNTrials(self): expected_output = [set(), A, A, A, A] assert output == pytest.helpers.setify_expected_output(expected_output) + @pytest.mark.composition class TestComponentBased: def test_BeforeNCalls(self): @@ -562,6 +566,7 @@ def test_AfterNCalls(self): class TestConvenience: + @pytest.mark.composition def test_AtTrialStart(self): comp = Composition() A = TransferMechanism(name='A') @@ -579,6 +584,7 @@ def test_AtTrialStart(self): expected_output = [A, B, A, A] assert output == pytest.helpers.setify_expected_output(expected_output) + @pytest.mark.composition def test_composite_condition_multi(self): comp = Composition() A = TransferMechanism(function=Linear(slope=5.0, intercept=2.0), name='A') @@ -613,6 +619,7 @@ def test_composite_condition_multi(self): ] assert output == pytest.helpers.setify_expected_output(expected_output) + @pytest.mark.composition def test_AfterNCallsCombined(self): comp = Composition() A = TransferMechanism(function=Linear(slope=5.0, intercept=2.0), name='A') @@ -640,6 +647,7 @@ def test_AfterNCallsCombined(self): ] assert output == pytest.helpers.setify_expected_output(expected_output) + @pytest.mark.composition def test_AllHaveRun(self): comp = Composition() A = TransferMechanism(function=Linear(slope=5.0, intercept=2.0), name='A') @@ -667,6 +675,7 @@ def test_AllHaveRun(self): ] assert output == pytest.helpers.setify_expected_output(expected_output) + @pytest.mark.composition def test_AllHaveRun_2(self): comp = Composition() A = TransferMechanism(function=Linear(slope=5.0, intercept=2.0), name='A') @@ -692,6 +701,7 @@ def test_AllHaveRun_2(self): ] assert output == pytest.helpers.setify_expected_output(expected_output) + @pytest.mark.composition @pytest.mark.parametrize( 'parameter, indices, default_variable, integration_rate, expected_results', [ @@ -723,6 +733,7 @@ def test_Threshold_parameters( np.testing.assert_array_equal(comp.results, expected_results) + @pytest.mark.composition @pytest.mark.parametrize( 'comparator, increment, threshold, expected_results', [ @@ -755,6 +766,7 @@ def test_Threshold_comparators( np.testing.assert_array_equal(comp.results, expected_results) + @pytest.mark.composition @pytest.mark.parametrize( 'comparator, increment, threshold, atol, rtol, expected_results', [ @@ -790,6 +802,7 @@ def test_Threshold_tolerances( np.testing.assert_array_equal(comp.results, expected_results) +@pytest.mark.composition class TestWhenFinished: @classmethod @@ -984,6 +997,7 @@ class TestAbsolute: B = TransferMechanism(name='scheduler-pytests-B') C = TransferMechanism(name='scheduler-pytests-C') + @pytest.mark.composition @pytest.mark.parametrize( 'conditions, termination_conds', [ @@ -1036,6 +1050,7 @@ def test_TimeInterval_linear_everynms(self, conditions, termination_conds): for i in range(1, len(executions)): assert (executions[i] - executions[i - 1]) == cond.repeat + @pytest.mark.composition @pytest.mark.parametrize( 'conditions, termination_conds', [ diff --git a/tests/scheduling/test_scheduler.py b/tests/scheduling/test_scheduler.py index 459512441f6..1cdbdc8462f 100644 --- a/tests/scheduling/test_scheduler.py +++ b/tests/scheduling/test_scheduler.py @@ -1498,7 +1498,7 @@ def test_inline_control_mechanism_example(self): } assert comp.scheduler.dependency_dict == expected_dependencies - @pytest.mark.mechanism + @pytest.mark.composition @pytest.mark.transfer_mechanism @pytest.mark.parametrize('timescale, expected', [(TimeScale.TIME_STEP, [[0.5], [0.4375]]), From cd3572232b4df7cb99ed95f6955145e2042989a8 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Sat, 20 Aug 2022 03:09:42 -0400 Subject: [PATCH 003/127] requirements: Restrict modeci_mdf to 64bit cpython (#2473) Pulls in pytorch[0] so it needs the same constraints. [0] https://github.com/ModECI/MDF/blob/main/setup.cfg Signed-off-by: Jan Vesely --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c113611feea..5e2ab1e88ef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ grpcio<1.43.0 grpcio-tools<1.43.0 llvmlite<0.40 matplotlib<3.5.4 -modeci_mdf<0.5, >=0.3.4 +modeci_mdf<0.5, >=0.3.4; (platform_machine == 'AMD64' or platform_machine == 'x86_64') and platform_python_implementation == 'CPython' and implementation_name == 'cpython' networkx<2.9 numpy<1.21.7, >=1.17.0 pillow<9.3.0 From 10f5be82ce1be558011cdfe3b6056d93f6cf1f9d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Sep 2022 14:28:36 +0000 Subject: [PATCH 004/127] requirements: update pytest requirement from <7.1.3 to <7.1.4 (#2478) --- dev_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index ad283dfc78d..ccb1a619630 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,5 +1,5 @@ jupyter<=1.0.0 -pytest<7.1.3 +pytest<7.1.4 pytest-benchmark<3.4.2 pytest-cov<3.0.1 pytest-helpers-namespace<2021.12.30 From a6fd96f4f0530c4147522c0f78532be64fa6e7df Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Sat, 10 Sep 2022 09:49:04 -0400 Subject: [PATCH 005/127] github-actions/install-pnl/dependabot_pr: Cleanup installation of the bumped package before installing PNL (#2481) Some packages may pull in updated dependencies that cause issues when rolled back. Signed-off-by: Jan Vesely --- .github/actions/install-pnl/action.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/actions/install-pnl/action.yml b/.github/actions/install-pnl/action.yml index dedae754886..6c0e18ff297 100644 --- a/.github/actions/install-pnl/action.yml +++ b/.github/actions/install-pnl/action.yml @@ -57,10 +57,15 @@ runs: shell: bash id: new_package run: | - export NEW_PACKAGE=`echo '${{ github.head_ref }}' | cut -f 4 -d/ | sed 's/-gt.*//' | sed 's/-lt.*//'` + export NEW_PACKAGE=$(echo '${{ github.head_ref }}' | cut -f 4 -d/ | sed 's/-gt.*//' | sed 's/-lt.*//') echo "::set-output name=new_package::$NEW_PACKAGE" - pip install "`echo $NEW_PACKAGE | sed 's/[-_]/./g' | xargs grep *requirements.txt -h -e | head -n1`" + # save a list of all installed packages (including pip, wheel; it's never empty) + pip freeze --all > orig + pip install "$(echo $NEW_PACKAGE | sed 's/[-_]/./g' | xargs grep *requirements.txt -h -e | head -n1)" pip show "$NEW_PACKAGE" | grep 'Version' | tee new_version.deps + # uninstall new packages but skip those from previous steps (pywinpty, terminado on windows x86) + # the 'orig' list is not empty (includes at least pip, wheel) + pip uninstall -y $(pip freeze -r orig | sed '1,/## /d') - name: Python dependencies shell: bash From d271941a3ef9df2267ac546af315516866cc4cc8 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Sat, 10 Sep 2022 15:28:40 -0400 Subject: [PATCH 006/127] github-actions: Add hash of requirements.txt to cache key of built pip packages (#2482) The packages should be rebuilt if some of the shared dependencies change Signed-off-by: Jan Vesely --- .github/workflows/pnl-ci-docs.yml | 4 ++-- .github/workflows/pnl-ci.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pnl-ci-docs.yml b/.github/workflows/pnl-ci-docs.yml index a37c9e7a250..4821dfd103f 100644 --- a/.github/workflows/pnl-ci-docs.yml +++ b/.github/workflows/pnl-ci-docs.yml @@ -82,8 +82,8 @@ jobs: uses: actions/cache@v3 with: path: ${{ steps.pip_cache.outputs.pip_cache_dir }}/wheels - key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-v2-${{ github.sha }} - restore-keys: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-v2 + key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-${{ hashFiles('requirements.txt') }}-${{ github.sha }} + restore-keys: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-${{ hashFiles('requirements.txt') }} # We need to install all PNL deps since docs config imports psyneulink module - name: Install local, editable PNL package diff --git a/.github/workflows/pnl-ci.yml b/.github/workflows/pnl-ci.yml index b97eaa55ecf..2b1ffa648e6 100644 --- a/.github/workflows/pnl-ci.yml +++ b/.github/workflows/pnl-ci.yml @@ -67,8 +67,8 @@ jobs: uses: actions/cache@v3 with: path: ${{ steps.pip_cache.outputs.pip_cache_dir }}/wheels - key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-v2-${{ github.sha }} - restore-keys: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-v2 + key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-${{ hashFiles('requirements.txt') }}-${{ github.sha }} + restore-keys: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-${{ hashFiles('requirements.txt') }} - name: Install local, editable PNL package uses: ./.github/actions/install-pnl From 5ab073d5a3ec3314684698bb3af950bf9afca6a8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 10 Sep 2022 22:48:37 +0000 Subject: [PATCH 007/127] requirements: update pandas requirement from <1.4.4 to <1.4.5 (#2477) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5e2ab1e88ef..27353840722 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,5 +17,5 @@ torch>=1.8.0, <1.12.0; (platform_machine == 'AMD64' or platform_machine == 'x86_ typecheck-decorator<=1.2 leabra-psyneulink<=0.3.2 rich>=10.1, <10.13 -pandas<1.4.4 +pandas<1.4.5 fastkde==1.0.19 From 6f1bbca954e91150b249ade7a225592069b46f9e Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Tue, 13 Sep 2022 15:45:29 -0400 Subject: [PATCH 008/127] github-actions/install-pnl/dependabot_pr: Skip package version check for not installed packages (#2483) This happens if the package is an 'extras' dependency. e.g. a 'doc' or 'cuda' dependency not installed in 'dev' CI jobs. Fixes package bumps of docs dependencies. Signed-off-by: Jan Vesely --- .github/actions/install-pnl/action.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/actions/install-pnl/action.yml b/.github/actions/install-pnl/action.yml index 6c0e18ff297..e0dc1a6f8fb 100644 --- a/.github/actions/install-pnl/action.yml +++ b/.github/actions/install-pnl/action.yml @@ -89,6 +89,8 @@ runs: if: ${{ startsWith(github.head_ref, 'dependabot/pip') && matrix.pnl-version != 'base' }} shell: bash run: | - pip show ${{ steps.new_package.outputs.new_package }} | grep 'Version' | tee installed_version.deps - cmp -s new_version.deps installed_version.deps || echo "::error::Package version restricted by dependencies: ${{ steps.new_package.outputs.new_package }}" - diff new_version.deps installed_version.deps + if [ $(pip list | grep -o ${{ steps.new_package.outputs.new_package }} | wc -l) != "0" ] ; then + pip show ${{ steps.new_package.outputs.new_package }} | grep 'Version' | tee installed_version.deps + cmp -s new_version.deps installed_version.deps || echo "::error::Package version restricted by dependencies: ${{ steps.new_package.outputs.new_package }}" + diff new_version.deps installed_version.deps + fi From 2b1b13184e7a3283cc811fc1103c1116989feca7 Mon Sep 17 00:00:00 2001 From: Katherine Mantel Date: Fri, 15 Jul 2022 21:12:01 -0400 Subject: [PATCH 009/127] tests: MDF: make onnx noise generation more flexible --- tests/mdf/test_mdf.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/tests/mdf/test_mdf.py b/tests/mdf/test_mdf.py index 150f7c1964a..dd4560148cd 100644 --- a/tests/mdf/test_mdf.py +++ b/tests/mdf/test_mdf.py @@ -11,6 +11,15 @@ from modeci_mdf.execution_engine import evaluate_onnx_expr # noqa: E402 +def get_onnx_fixed_noise_str(onnx_op, **kwargs): + # high precision printing needed because script will be executed from string + # 16 is insufficient on windows + with np.printoptions(precision=32): + return str( + evaluate_onnx_expr(f'onnx_ops.{onnx_op}', base_parameters=kwargs, evaluated_parameters=kwargs) + ) + + # stroop stimuli red = [1, 0] green = [0, 1] @@ -170,12 +179,12 @@ def test_write_json_file_multiple_comps( # RandomNormal with parameters used in model_integrators.py (seed 0). # RandomNormal values are different on mac versus linux and windows onnx_noise_data = { - 'onnx_ops.randomuniform': { + 'randomuniform': { 'A': {'low': -1.0, 'high': 1.0, 'seed': 0, 'shape': (1, 1)}, 'D': {'low': -0.5, 'high': 0.5, 'seed': 0, 'shape': (1, 1)}, 'E': {'low': -0.25, 'high': 0.5, 'seed': 0, 'shape': (1, 1)} }, - 'onnx_ops.randomnormal': { + 'randomnormal': { 'B': {'mean': -1.0, 'scale': 0.5, 'seed': 0, 'shape': (1, 1)}, 'C': {'mean': 0.0, 'scale': 0.25, 'seed': 0, 'shape': (1, 1)}, } @@ -187,18 +196,13 @@ def test_write_json_file_multiple_comps( for node, args in onnx_noise_data[func_type].items(): # generates output from onnx noise functions with seed 0 to be # passed in in runtime_params during psyneulink execution - onnx_integrators_fixed_seeded_noise[node] = evaluate_onnx_expr( - func_type, base_parameters=args, evaluated_parameters=args - ) + onnx_integrators_fixed_seeded_noise[node] = get_onnx_fixed_noise_str(func_type, **args) -# high precision printing needed because script will be executed from string -# 16 is insufficient on windows -with np.printoptions(precision=32): - integrators_runtime_params = ( - 'runtime_params={' - + ','.join([f'{k}: {{ "noise": {v} }}' for k, v in onnx_integrators_fixed_seeded_noise.items()]) - + '}' - ) +integrators_runtime_params = ( + 'runtime_params={' + + ','.join([f'{k}: {{ "noise": {v} }}' for k, v in onnx_integrators_fixed_seeded_noise.items()]) + + '}' +) @pytest.mark.parametrize( From 2cb2570e5e81496da81be4538f1ad5ab753594a4 Mon Sep 17 00:00:00 2001 From: Katherine Mantel Date: Fri, 15 Jul 2022 22:42:49 -0400 Subject: [PATCH 010/127] MDF: OutputPort: include shape and type --- psyneulink/core/components/ports/outputport.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/psyneulink/core/components/ports/outputport.py b/psyneulink/core/components/ports/outputport.py index 5e1c2bc1eba..3f76957bdb0 100644 --- a/psyneulink/core/components/ports/outputport.py +++ b/psyneulink/core/components/ports/outputport.py @@ -1302,6 +1302,8 @@ def as_mdf_model(self): return mdf.OutputPort( id=parse_valid_identifier(self.name), value=value, + shape=str(self.defaults.value.shape), + type=str(self.defaults.value.dtype), **self._mdf_metadata ) From 453e738e92dc51742372d18e750ca08c194c74b0 Mon Sep 17 00:00:00 2001 From: Katherine Mantel Date: Thu, 21 Jul 2022 21:55:05 -0400 Subject: [PATCH 011/127] MDF: improve handling of more complex Functions Many psyneulink Functions must be split into several MDF functions/expressions to reproduce their behavior. Instead of adding these supplemental functions on the owners of Functions, add Function._assign_to_mdf_model that adds anything necessary to the owner --- .../core/components/functions/function.py | 44 +++++++++++++++---- .../core/components/mechanisms/mechanism.py | 11 +---- .../processing/integratormechanism.py | 27 ------------ .../processing/transfermechanism.py | 36 +++------------ .../core/components/ports/outputport.py | 2 +- .../core/components/projections/projection.py | 2 +- 6 files changed, 44 insertions(+), 78 deletions(-) diff --git a/psyneulink/core/components/functions/function.py b/psyneulink/core/components/functions/function.py index 968cd52a77c..878116c75fb 100644 --- a/psyneulink/core/components/functions/function.py +++ b/psyneulink/core/components/functions/function.py @@ -159,6 +159,7 @@ IDENTITY_MATRIX, INVERSE_HOLLOW_MATRIX, NAME, PREFERENCE_SET_NAME, RANDOM_CONNECTIVITY_MATRIX, VALUE, VARIABLE, MODEL_SPEC_ID_METADATA, MODEL_SPEC_ID_MDF_VARIABLE ) +from psyneulink.core.globals.mdf import _get_variable_parameter_name from psyneulink.core.globals.parameters import Parameter, check_user_specified from psyneulink.core.globals.preferences.basepreferenceset import REPORT_OUTPUT_PREF, is_pref_set from psyneulink.core.globals.preferences.preferenceset import PreferenceEntry, PreferenceLevel @@ -817,11 +818,22 @@ def _model_spec_parameter_blacklist(self): 'multiplicative_param', 'additive_param', }) - def _get_mdf_noise_function(self): + def _assign_to_mdf_model(self, model, input_id) -> str: + """Adds an MDF representation of this function to MDF object + **model**, including all necessary auxiliary functions. + **input_id** is the input to the singular MDF function or first + function representing this psyneulink Function, if applicable. + + Returns: + str: the identifier of the final MDF function representing + this psyneulink Function + """ import modeci_mdf.mdf as mdf extra_noise_functions = [] + self_model = self.as_mdf_model() + def handle_noise(noise): if is_instance_or_subclass(noise, Component): if inspect.isclass(noise) and issubclass(noise, Component): @@ -834,14 +846,30 @@ def handle_noise(noise): else: return None - noise = handle_noise(self.defaults.noise) + try: + noise_val = handle_noise(self.defaults.noise) + except AttributeError: + noise_val = None - if noise is not None: - return mdf.Function( - id=f'{parse_valid_identifier(self.name)}_noise', + if noise_val is not None: + noise_func = mdf.Function( + id=f'{model.id}_{parse_valid_identifier(self.name)}_noise', value=MODEL_SPEC_ID_MDF_VARIABLE, - args={MODEL_SPEC_ID_MDF_VARIABLE: noise}, - ), extra_noise_functions + args={MODEL_SPEC_ID_MDF_VARIABLE: noise_val}, + ) + self._set_mdf_arg(self_model, 'noise', noise_func.id) + + model.functions.extend(extra_noise_functions) + model.functions.append(noise_func) + + for _, func_param in self_model.metadata['function_stateful_params'].items(): + model.parameters.append(mdf.Parameter(**func_param)) + + self_model.id = f'{model.id}_{self_model.id}' + self._set_mdf_arg(self_model, _get_variable_parameter_name(self), input_id) + model.functions.append(self_model) + + return self_model.id def as_mdf_model(self): import modeci_mdf.mdf as mdf @@ -869,7 +897,7 @@ def as_mdf_model(self): metadata[MODEL_SPEC_ID_METADATA]['function_stateful_params'][name] = { 'id': name, 'default_initial_value': initializer_value, - 'value': parse_valid_identifier(self.name) + 'value': parse_valid_identifier(f'{self.owner.name}_{self.name}') } stateful_params.add(name) diff --git a/psyneulink/core/components/mechanisms/mechanism.py b/psyneulink/core/components/mechanisms/mechanism.py index 567c2f3eeca..857ec84e8be 100644 --- a/psyneulink/core/components/mechanisms/mechanism.py +++ b/psyneulink/core/components/mechanisms/mechanism.py @@ -1098,7 +1098,6 @@ REMOVE_PORTS, PORT_SPEC, _parse_port_spec, PORT_SPECIFIC_PARAMS, PROJECTION_SPECIFIC_PARAMS from psyneulink.core.components.shellclasses import Mechanism, Projection, Port from psyneulink.core.globals.context import Context, ContextFlags, handle_external_context -from psyneulink.core.globals.mdf import _get_variable_parameter_name # TODO: remove unused keywords from psyneulink.core.globals.keywords import \ ADDITIVE_PARAM, EXECUTION_PHASE, EXPONENT, FUNCTION_PARAMS, \ @@ -4194,20 +4193,12 @@ def as_mdf_model(self): model.output_ports.append(op_model) - function_model = self.function.as_mdf_model() - - for _, func_param in function_model.metadata['function_stateful_params'].items(): - model.parameters.append(mdf.Parameter(**func_param)) - if len(ip.path_afferents) > 1: primary_function_input_name = combination_function_dimreduce_id else: primary_function_input_name = model.input_ports[0].id - self.function._set_mdf_arg( - function_model, _get_variable_parameter_name(self.function), primary_function_input_name - ) - model.functions.append(function_model) + self.function._assign_to_mdf_model(model, primary_function_input_name) return model diff --git a/psyneulink/core/components/mechanisms/processing/integratormechanism.py b/psyneulink/core/components/mechanisms/processing/integratormechanism.py index e11dd8b47b4..548d3cd6dfd 100644 --- a/psyneulink/core/components/mechanisms/processing/integratormechanism.py +++ b/psyneulink/core/components/mechanisms/processing/integratormechanism.py @@ -94,7 +94,6 @@ from psyneulink.core.globals.parameters import Parameter, check_user_specified from psyneulink.core.globals.preferences.basepreferenceset import is_pref_set, REPORT_OUTPUT_PREF from psyneulink.core.globals.preferences.preferenceset import PreferenceEntry, PreferenceLevel -from psyneulink.core.globals.utilities import parse_valid_identifier __all__ = [ 'DEFAULT_RATE', 'IntegratorMechanism', 'IntegratorMechanismError' @@ -230,29 +229,3 @@ def _handle_default_variable(self, default_variable=None, size=None, input_ports input_ports=input_ports, function=function, params=params) - - def as_mdf_model(self): - import modeci_mdf.mdf as mdf - - model = super().as_mdf_model() - function_model = [ - f for f in model.functions - if f.id == parse_valid_identifier(self.function.name) - ][0] - assert function_model.id == parse_valid_identifier(self.function.name), (function_model.id, parse_valid_identifier(self.function.name)) - - for _, func_param in function_model.metadata['function_stateful_params'].items(): - model.parameters.append(mdf.Parameter(**func_param)) - - res = self.function._get_mdf_noise_function() - try: - main_noise_function, extra_noise_functions = res - except TypeError: - pass - else: - main_noise_function.id = f'{model.id}_{main_noise_function.id}' - model.functions.append(main_noise_function) - model.functions.extend(extra_noise_functions) - function_model.args['noise'] = main_noise_function.id - - return model diff --git a/psyneulink/core/components/mechanisms/processing/transfermechanism.py b/psyneulink/core/components/mechanisms/processing/transfermechanism.py index ca81117a2c5..522094b2fbc 100644 --- a/psyneulink/core/components/mechanisms/processing/transfermechanism.py +++ b/psyneulink/core/components/mechanisms/processing/transfermechanism.py @@ -1811,47 +1811,21 @@ def _update_default_variable(self, new_default_variable, context=None): super()._update_default_variable(new_default_variable, context=context) def as_mdf_model(self): - import modeci_mdf.mdf as mdf - model = super().as_mdf_model() function_model = [ f for f in model.functions - if f.id == parse_valid_identifier(self.function.name) + if f.id == f'{model.id}_{parse_valid_identifier(self.function.name)}' ][0] - assert function_model.id == parse_valid_identifier(self.function.name), (function_model.id, parse_valid_identifier(self.function.name)) + assert function_model.id == f'{model.id}_{parse_valid_identifier(self.function.name)}', (function_model.id, parse_valid_identifier(self.function.name)) if self.defaults.integrator_mode: - integrator_function_model = self.integrator_function.as_mdf_model() - primary_input = function_model.args[_get_variable_parameter_name(self.function)] - self.integrator_function._set_mdf_arg( - integrator_function_model, - _get_variable_parameter_name(self.integrator_function), - primary_input - ) + integrator_function_id = self.integrator_function._assign_to_mdf_model(model, primary_input) + self.function._set_mdf_arg( function_model, _get_variable_parameter_name(self.function), - integrator_function_model.id + integrator_function_id ) - for _, func_param in integrator_function_model.metadata['function_stateful_params'].items(): - model.parameters.append(mdf.Parameter(**func_param)) - - model.functions.append(integrator_function_model) - - res = self.integrator_function._get_mdf_noise_function() - try: - main_noise_function, extra_noise_functions = res - except TypeError: - pass - else: - main_noise_function.id = f'{model.id}_{main_noise_function.id}' - model.functions.append(main_noise_function) - model.functions.extend(extra_noise_functions) - - self.integrator_function._set_mdf_arg( - integrator_function_model, 'noise', main_noise_function.id - ) - return model diff --git a/psyneulink/core/components/ports/outputport.py b/psyneulink/core/components/ports/outputport.py index 3f76957bdb0..7db0e5f05f4 100644 --- a/psyneulink/core/components/ports/outputport.py +++ b/psyneulink/core/components/ports/outputport.py @@ -1288,7 +1288,7 @@ def get_label(self, context=None): def as_mdf_model(self): import modeci_mdf.mdf as mdf - owner_func_name = parse_valid_identifier(self.owner.function.name) + owner_func_name = parse_valid_identifier(f'{self.owner.name}_{self.owner.function.name}') if self._variable_spec == OWNER_VALUE: value = owner_func_name elif isinstance(self._variable_spec, tuple) and self._variable_spec[0] == OWNER_VALUE: diff --git a/psyneulink/core/components/projections/projection.py b/psyneulink/core/components/projections/projection.py index d0f8c4c39b2..b26b6847cac 100644 --- a/psyneulink/core/components/projections/projection.py +++ b/psyneulink/core/components/projections/projection.py @@ -1110,7 +1110,7 @@ def as_mdf_model(self, simple_edge_format=True): edge_function = edge_node.function edge_node = edge_node.as_mdf_model() - func_model = [f for f in edge_node.functions if f.id == parse_valid_identifier(edge_function.name)][0] + func_model = [f for f in edge_node.functions if f.id == parse_valid_identifier(f'{edge_node.id}_{edge_function.name}')][0] var_name = _get_variable_parameter_name(edge_function) # 2d variable on LinearMatrix will be incorrect on import back to psyneulink From 7b88c561d4e17b0768cf625429901215867317ed Mon Sep 17 00:00:00 2001 From: Katherine Mantel Date: Tue, 2 Aug 2022 23:32:42 -0400 Subject: [PATCH 012/127] MDF: set up stateful parameter index handling --- .../core/components/functions/function.py | 38 +++++++++++-------- .../functions/stateful/statefulfunction.py | 5 +++ 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/psyneulink/core/components/functions/function.py b/psyneulink/core/components/functions/function.py index 878116c75fb..00017dbb734 100644 --- a/psyneulink/core/components/functions/function.py +++ b/psyneulink/core/components/functions/function.py @@ -157,7 +157,7 @@ ARGUMENT_THERAPY_FUNCTION, AUTO_ASSIGN_MATRIX, EXAMPLE_FUNCTION_TYPE, FULL_CONNECTIVITY_MATRIX, FUNCTION_COMPONENT_CATEGORY, FUNCTION_OUTPUT_TYPE, FUNCTION_OUTPUT_TYPE_CONVERSION, HOLLOW_MATRIX, IDENTITY_MATRIX, INVERSE_HOLLOW_MATRIX, NAME, PREFERENCE_SET_NAME, RANDOM_CONNECTIVITY_MATRIX, VALUE, VARIABLE, - MODEL_SPEC_ID_METADATA, MODEL_SPEC_ID_MDF_VARIABLE + MODEL_SPEC_ID_MDF_VARIABLE ) from psyneulink.core.globals.mdf import _get_variable_parameter_name from psyneulink.core.globals.parameters import Parameter, check_user_specified @@ -553,6 +553,7 @@ class Function_Base(Function): classPreferenceLevel = PreferenceLevel.CATEGORY _model_spec_id_parameters = 'args' + _mdf_stateful_parameter_indices = {} _specified_variable_shape_flexibility = DefaultsFlexibility.INCREASE_DIMENSION @@ -862,13 +863,30 @@ def handle_noise(noise): model.functions.extend(extra_noise_functions) model.functions.append(noise_func) - for _, func_param in self_model.metadata['function_stateful_params'].items(): - model.parameters.append(mdf.Parameter(**func_param)) - self_model.id = f'{model.id}_{self_model.id}' self._set_mdf_arg(self_model, _get_variable_parameter_name(self), input_id) model.functions.append(self_model) + # assign stateful parameters + for param, index in self._mdf_stateful_parameter_indices.items(): + initializer_name = getattr(self.parameters, param).initializer + + # in this case, parameter gets updated to its function's final value + try: + initializer_value = self_model.args[initializer_name] + except KeyError: + initializer_value = self_model.metadata[initializer_name] + + index_str = f'[{index}]' if index is not None else '' + + model.parameters.append( + mdf.Parameter( + id=param, + default_initial_value=initializer_value, + value=f'{self_model.id}{index_str}' + ) + ) + return self_model.id def as_mdf_model(self): @@ -877,7 +895,6 @@ def as_mdf_model(self): parameters = self._mdf_model_parameters metadata = self._mdf_metadata - metadata[MODEL_SPEC_ID_METADATA]['function_stateful_params'] = {} stateful_params = set() # add stateful parameters into metadata for mechanism to get @@ -888,17 +905,6 @@ def as_mdf_model(self): continue if param.initializer is not None: - # in this case, parameter gets updated to its function's final value - try: - initializer_value = parameters[self._model_spec_id_parameters][param.initializer] - except KeyError: - initializer_value = metadata[MODEL_SPEC_ID_METADATA]['initializer'] - - metadata[MODEL_SPEC_ID_METADATA]['function_stateful_params'][name] = { - 'id': name, - 'default_initial_value': initializer_value, - 'value': parse_valid_identifier(f'{self.owner.name}_{self.name}') - } stateful_params.add(name) # stateful parameters cannot show up as args or they will not be diff --git a/psyneulink/core/components/functions/stateful/statefulfunction.py b/psyneulink/core/components/functions/stateful/statefulfunction.py index 5e22d460526..37156607a43 100644 --- a/psyneulink/core/components/functions/stateful/statefulfunction.py +++ b/psyneulink/core/components/functions/stateful/statefulfunction.py @@ -168,6 +168,11 @@ class StatefulFunction(Function_Base): # -------------------------------------- componentType = STATEFUL_FUNCTION_TYPE componentName = STATEFUL_FUNCTION + # TODO: consider moving this to a Parameter attribute + _mdf_stateful_parameter_indices = { + 'previous_value': None + } + class Parameters(Function_Base.Parameters): """ Attributes From 13b3f232f37aa93b60c3326ba6af7cdb825e97d1 Mon Sep 17 00:00:00 2001 From: Katherine Mantel Date: Thu, 21 Jul 2022 21:46:33 -0400 Subject: [PATCH 013/127] DriftDiffusion, DriftOnASphere Integrators: set previous_time default to 0 --- .../core/components/functions/stateful/integratorfunctions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/psyneulink/core/components/functions/stateful/integratorfunctions.py b/psyneulink/core/components/functions/stateful/integratorfunctions.py index 7764829ffdb..f98e7b8a038 100644 --- a/psyneulink/core/components/functions/stateful/integratorfunctions.py +++ b/psyneulink/core/components/functions/stateful/integratorfunctions.py @@ -2407,7 +2407,7 @@ class Parameters(IntegratorFunction.Parameters): non_decision_time = Parameter(0.0, modulable=True) threshold = Parameter(100.0, modulable=True) time_step_size = Parameter(1.0, modulable=True) - previous_time = Parameter(None, initializer='non_decision_time', pnl_internal=True) + previous_time = Parameter(0.0, initializer='non_decision_time', pnl_internal=True) random_state = Parameter(None, loggable=False, getter=_random_state_getter, dependencies='seed') seed = Parameter(DEFAULT_SEED, modulable=True, fallback_default=True, setter=_seed_setter) enable_output_type_conversion = Parameter( @@ -2894,7 +2894,7 @@ class Parameters(IntegratorFunction.Parameters): starting_point = 0.0 # threshold = Parameter(100.0, modulable=True) time_step_size = Parameter(1.0, modulable=True) - previous_time = Parameter(None, initializer='starting_point', pnl_internal=True) + previous_time = Parameter(0.0, initializer='starting_point', pnl_internal=True) dimension = Parameter(3, stateful=False, read_only=True) initializer = Parameter([0], initalizer='variable', stateful=True) angle_function = Parameter(None, stateful=False, loggable=False) From 6f55dd220b1e900bcd10c80a6e85153f43d96c6d Mon Sep 17 00:00:00 2001 From: Katherine Mantel Date: Thu, 21 Jul 2022 21:33:29 -0400 Subject: [PATCH 014/127] MDF: implement DriftDiffusionIntegrator --- .../functions/stateful/integratorfunctions.py | 62 +++++++++++++++++-- tests/mdf/test_mdf.py | 62 +++++++++++++++++-- 2 files changed, 115 insertions(+), 9 deletions(-) diff --git a/psyneulink/core/components/functions/stateful/integratorfunctions.py b/psyneulink/core/components/functions/stateful/integratorfunctions.py index f98e7b8a038..100a142c580 100644 --- a/psyneulink/core/components/functions/stateful/integratorfunctions.py +++ b/psyneulink/core/components/functions/stateful/integratorfunctions.py @@ -33,7 +33,7 @@ from psyneulink.core import llvm as pnlvm from psyneulink.core.components.component import DefaultsFlexibility -from psyneulink.core.components.functions.nonstateful.distributionfunctions import DistributionFunction +from psyneulink.core.components.functions.nonstateful.distributionfunctions import DistributionFunction, NormalDist from psyneulink.core.components.functions.function import ( DEFAULT_SEED, FunctionError, _random_state_getter, _seed_setter, _noise_setter @@ -51,7 +51,7 @@ from psyneulink.core.globals.parameters import Parameter, check_user_specified from psyneulink.core.globals.preferences.basepreferenceset import is_pref_set from psyneulink.core.globals.utilities import parameter_spec, all_within_range, \ - convert_all_elements_to_np_array + convert_all_elements_to_np_array, parse_valid_identifier __all__ = ['SimpleIntegrator', 'AdaptiveIntegrator', 'DriftDiffusionIntegrator', 'DriftOnASphereIntegrator', 'OrnsteinUhlenbeckIntegrator', 'FitzHughNagumoIntegrator', 'AccumulatorIntegrator', @@ -2339,6 +2339,10 @@ class DriftDiffusionIntegrator(IntegratorFunction): # ------------------------- """ componentName = DRIFT_DIFFUSION_INTEGRATOR_FUNCTION + _mdf_stateful_parameter_indices = { + 'previous_value': 0, + 'previous_time': 1, + } class Parameters(IntegratorFunction.Parameters): """ @@ -2407,7 +2411,7 @@ class Parameters(IntegratorFunction.Parameters): non_decision_time = Parameter(0.0, modulable=True) threshold = Parameter(100.0, modulable=True) time_step_size = Parameter(1.0, modulable=True) - previous_time = Parameter(0.0, initializer='non_decision_time', pnl_internal=True) + previous_time = Parameter(0.0, initializer='non_decision_time') random_state = Parameter(None, loggable=False, getter=_random_state_getter, dependencies='seed') seed = Parameter(DEFAULT_SEED, modulable=True, fallback_default=True, setter=_seed_setter) enable_output_type_conversion = Parameter( @@ -2417,6 +2421,8 @@ class Parameters(IntegratorFunction.Parameters): pnl_internal=True, read_only=True ) + # used only to allow putting random_draw in runtime_params for MDF tests + random_draw = Parameter() def _parse_initializer(self, initializer): if initializer.ndim > 1: @@ -2504,7 +2510,11 @@ def _function(self, variable = self.parameters._parse_initializer(variable) previous_value = self.parameters.previous_value._get(context) - random_draw = np.array([random_state.normal() for _ in list(variable)]) + try: + random_draw = params['random_draw'] + except (KeyError, TypeError): + random_draw = np.array([random_state.normal() for _ in list(variable)]) + value = previous_value + rate * variable * time_step_size \ + noise * np.sqrt(time_step_size) * random_draw @@ -2585,6 +2595,50 @@ def reset(self, previous_value=None, previous_time=None, context=None): context=context ) + def _assign_to_mdf_model(self, model, input_id): + import modeci_mdf.mdf as mdf + + self_id = parse_valid_identifier(self.name) + parameters = self._mdf_model_parameters + parameters[self._model_spec_id_parameters][MODEL_SPEC_ID_MDF_VARIABLE] = input_id + threshold = parameters[self._model_spec_id_parameters]['threshold'] + + random_draw_func = mdf.Function( + id=f'{self_id}_random_draw', + function=NormalDist._model_spec_generic_type_name, + args={'shape': (len(self.defaults.variable),), 'seed': self.defaults.seed} + ) + unclipped_func = mdf.Function( + id=f'{self_id}_unclipped', + value=f'(previous_value + rate * {MODEL_SPEC_ID_MDF_VARIABLE} * time_step_size + noise * math.sqrt(time_step_size) * {random_draw_func.id}) + offset', + args={ + k: v for k, v in parameters[self._model_spec_id_parameters].items() + if (k not in self.parameters or getattr(self.parameters, k).initializer is None) + } + ) + lower_clipped = mdf.Function( + id=f'{self_id}_lower_clipped', + value=f'max({unclipped_func.id}, -1 * threshold)', + args={'threshold': np.full_like(self.defaults.previous_value, threshold)} + ) + result = mdf.Function( + id=f'{self_id}_value_result', + value=f'min({lower_clipped.id}, threshold)', + args={'threshold': np.full_like(self.defaults.previous_value, threshold)} + ) + + model.functions.extend([ + random_draw_func, + unclipped_func, + lower_clipped, + result, + ]) + + return super()._assign_to_mdf_model(model, input_id) + + def as_expression(self): + return f'{parse_valid_identifier(self.name)}_value_result, previous_time' + class DriftOnASphereIntegrator(IntegratorFunction): # ----------------------------------------------------------------- """ diff --git a/tests/mdf/test_mdf.py b/tests/mdf/test_mdf.py index dd4560148cd..d2ea75142a0 100644 --- a/tests/mdf/test_mdf.py +++ b/tests/mdf/test_mdf.py @@ -1,3 +1,4 @@ +import copy import numpy as np import os import psyneulink as pnl @@ -173,6 +174,13 @@ def test_write_json_file_multiple_comps( assert orig_results[composition_name] == final_results, f'{composition_name}:' +def _get_mdf_model_results(evaluable_graph): + return [ + [eo.curr_value for _, eo in evaluable_graph.enodes[node.id].evaluable_outputs.items()] + for node in evaluable_graph.scheduler.consideration_queue[-1] + ] + + # These runtime_params are necessary because noise seeding is not # replicable between numpy and onnx. # Values are generated from running onnx function RandomUniform and @@ -238,12 +246,56 @@ def test_mdf_equivalence(filename, composition_name, input_dict, simple_edge_for eg = ee.EvaluableGraph(m.graphs[0], verbose=True) eg.evaluate(initializer={f'{node}_InputPort_0': i for node, i in input_dict.items()}) - mdf_results = [ - [eo.curr_value for _, eo in eg.enodes[node.id].evaluable_outputs.items()] - for node in eg.scheduler.consideration_queue[-1] - ] + assert pnl.safe_equals(orig_results, _get_mdf_model_results(eg)) + + +ddi_termination_conds = [ + None, + ( + "pnl.Or(" + "pnl.Threshold(A, parameter='value', threshold=A.function.defaults.threshold, comparator='>=', indices=(0,))," + "pnl.Threshold(A, parameter='value', threshold=-1 * A.function.defaults.threshold, comparator='<=', indices=(0,))" + ")" + ), + 'pnl.AfterNCalls(A, 10)', +] + +# construct test data manually instead of with multiple @pytest.mark.parametrize +# so that other functions can use more appropriate termination conds +individual_functions_test_data = [ + ( + pnl.IntegratorMechanism, + pnl.DriftDiffusionIntegrator(rate=0.5, offset=1, non_decision_time=1, seed=0), + "{{A: {{'random_draw': {0} }} }}".format(get_onnx_fixed_noise_str('randomnormal', mean=0, scale=1, seed=0, shape=(1,))) + ) + (x,) + for x in ddi_termination_conds +] + + +@pytest.mark.parametrize( + 'mech_type, function, runtime_params, trial_termination_cond', + individual_functions_test_data +) +def test_mdf_equivalence_individual_functions(mech_type, function, runtime_params, trial_termination_cond): + import modeci_mdf.execution_engine as ee + + A = mech_type(name='A', function=copy.deepcopy(function)) + comp = pnl.Composition(pathways=[A]) + + try: + trial_termination_cond = eval(trial_termination_cond) + except TypeError: + pass + comp.scheduler.termination_conds = {pnl.TimeScale.TRIAL: trial_termination_cond} + + comp.run(inputs={A: [[1.0]]}, runtime_params=eval(runtime_params)) + + model = pnl.get_mdf_model(comp) + + eg = ee.EvaluableGraph(model.graphs[0], verbose=True) + eg.evaluate(initializer={'A_InputPort_0': 1.0}) - assert pnl.safe_equals(orig_results, mdf_results) + assert pnl.safe_equals(comp.results, _get_mdf_model_results(eg)) @pytest.mark.parametrize('filename', ['model_basic.py']) From d118fe47a3c93a09df256495c5bc4e8d9739bc76 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 17 Sep 2022 22:12:25 +0000 Subject: [PATCH 015/127] requirements: update matplotlib requirement from <3.5.4 to <3.6.1 (#2486) --- requirements.txt | 2 +- tutorial_requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 27353840722..e5f0dbb6e88 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ graphviz<0.21.0 grpcio<1.43.0 grpcio-tools<1.43.0 llvmlite<0.40 -matplotlib<3.5.4 +matplotlib<3.6.1 modeci_mdf<0.5, >=0.3.4; (platform_machine == 'AMD64' or platform_machine == 'x86_64') and platform_python_implementation == 'CPython' and implementation_name == 'cpython' networkx<2.9 numpy<1.21.7, >=1.17.0 diff --git a/tutorial_requirements.txt b/tutorial_requirements.txt index 6c0b32c13fd..003d498526e 100644 --- a/tutorial_requirements.txt +++ b/tutorial_requirements.txt @@ -1,3 +1,3 @@ graphviz<0.21.0 jupyter<=1.0.0 -matplotlib<3.5.4 +matplotlib<3.6.1 From 5a630acb85c6b93be0b9c209b76686b8ac24e64e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 22 Sep 2022 20:54:36 +0000 Subject: [PATCH 016/127] requirements: update pandas requirement from <1.4.5 to <1.5.1 (#2488) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e5f0dbb6e88..e95cb4b5661 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,5 +17,5 @@ torch>=1.8.0, <1.12.0; (platform_machine == 'AMD64' or platform_machine == 'x86_ typecheck-decorator<=1.2 leabra-psyneulink<=0.3.2 rich>=10.1, <10.13 -pandas<1.4.5 +pandas<1.5.1 fastkde==1.0.19 From 99ad4dcb5a94e43ca220e78f7d8db98e8de25386 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Tue, 27 Sep 2022 10:15:53 -0400 Subject: [PATCH 017/127] llvm, Mechanism: Add docstrings and extend in code comments. Signed-off-by: Jan Vesely --- .../core/components/mechanisms/mechanism.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/psyneulink/core/components/mechanisms/mechanism.py b/psyneulink/core/components/mechanisms/mechanism.py index 857ec84e8be..923bfc6f155 100644 --- a/psyneulink/core/components/mechanisms/mechanism.py +++ b/psyneulink/core/components/mechanisms/mechanism.py @@ -3158,6 +3158,12 @@ def _gen_llvm_function_reset(self, ctx, builder, params, state, arg_in, arg_out, return builder def _gen_llvm_function(self, *, extra_args=[], ctx:pnlvm.LLVMBuilderContext, tags:frozenset): + """ + Overloaded main function LLVM generation method. + + Mechanisms need to support "is_finished" execution variant (used by scheduling conditions) + on top of the variants supported by Component. + """ if "is_finished" not in tags: return super()._gen_llvm_function(extra_args=extra_args, ctx=ctx, tags=tags) @@ -3176,6 +3182,14 @@ def _gen_llvm_function(self, *, extra_args=[], ctx:pnlvm.LLVMBuilderContext, tag return builder.function def _gen_llvm_function_body(self, ctx, builder, base_params, state, arg_in, arg_out, *, tags:frozenset): + """ + Overloaded LLVM code generation method. + + Implements main mechanisms loop (while not finished). Calls two other internal Mechanism functions; + 'is_finished' to terminate the loop, and '_gen_llvm_function_internal' to generate body of the + loop (invocation of Ports and Functions). + """ + assert "reset" not in tags params, builder = self._gen_llvm_param_ports_for_obj( @@ -3203,7 +3217,9 @@ def _gen_llvm_function_body(self, ctx, builder, base_params, state, arg_in, arg_ builder.branch(loop_block) builder.position_at_end(loop_block) - # Get internal function + # Get internal function. Use function call to get proper stack manipulation + # inside the body of the execution loop. We could use 'stacksave' and + # 'stackrestore', but not all LLVM targets support those ops. args_t = [a.type for a in builder.function.args] args_t[4:4] = [base_params.type] internal_builder = ctx.create_llvm_function(args_t, self, From e1bc7f40913870723ab6e43345530cfd3cb0919e Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Tue, 27 Sep 2022 10:30:30 -0400 Subject: [PATCH 018/127] tests/DDM: Drop extra 'monitor' mechanism from the threshold modulation test Pass the modulator value directly from input. Signed-off-by: Jan Vesely --- tests/mechanisms/test_ddm_mechanism.py | 27 +++++++++----------------- 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/tests/mechanisms/test_ddm_mechanism.py b/tests/mechanisms/test_ddm_mechanism.py index db5a9645597..463761cd0b6 100644 --- a/tests/mechanisms/test_ddm_mechanism.py +++ b/tests/mechanisms/test_ddm_mechanism.py @@ -665,30 +665,21 @@ def test_DDM_in_composition(benchmark, comp_mode): @pytest.mark.composition @pytest.mark.ddm_mechanism def test_DDM_threshold_modulation(comp_mode): - M = pnl.DDM( - name='DDM', - function=pnl.DriftDiffusionAnalytical( - threshold=20.0, - ), - ) - monitor = pnl.TransferMechanism(default_variable=[[0.0]], - size=1, - function=pnl.Linear(slope=1, intercept=0), - output_ports=[pnl.RESULT], - name='monitor') + M = pnl.DDM(name='DDM', + function=pnl.DriftDiffusionAnalytical( + threshold=20.0, + ), + ) - control = pnl.ControlMechanism( - monitor_for_control=monitor, - control_signals=[(pnl.THRESHOLD, M)]) + control = pnl.ControlMechanism(control_signals=[(pnl.THRESHOLD, M)]) C = pnl.Composition() C.add_node(M, required_roles=[pnl.NodeRole.ORIGIN, pnl.NodeRole.TERMINAL]) - C.add_node(monitor) C.add_node(control) - inputs = {M:[1], monitor:[3]} + inputs = {M:[1], control:[3]} val = C.run(inputs, num_trials=1, execution_mode=comp_mode) - # FIXME: Python version returns dtype=object - val = np.asfarray(val) + + # Default modulation is 'multiplicative so the threshold is 20 * 3 assert np.allclose(val[0], [60.0]) assert np.allclose(val[1], [60.2]) From 7fbd87bbd2173c36aed4cacfa1ccc9f92265ef6c Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Tue, 27 Sep 2022 11:01:35 -0400 Subject: [PATCH 019/127] llvm, mechanism: Pass base parameter values and mech inputs to '_gen_llvm_is_finished_cond' Passing base parameters matches invocation by the scheduler. Any modulation needs to be applied explicitly. Signed-off-by: Jan Vesely --- .../core/components/mechanisms/mechanism.py | 8 ++++---- .../processing/transfermechanism.py | 20 +++++++++++-------- .../mechanisms/processing/integrator/ddm.py | 15 +++++++------- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/psyneulink/core/components/mechanisms/mechanism.py b/psyneulink/core/components/mechanisms/mechanism.py index 923bfc6f155..eca1d31bd5e 100644 --- a/psyneulink/core/components/mechanisms/mechanism.py +++ b/psyneulink/core/components/mechanisms/mechanism.py @@ -3088,7 +3088,7 @@ def _gen_llvm_invoke_function(self, ctx, builder, function, f_params, f_state, return f_out, builder - def _gen_llvm_is_finished_cond(self, ctx, builder, m_params, m_state): + def _gen_llvm_is_finished_cond(self, ctx, builder, m_base_params, m_state, m_inputs): return ctx.bool_ty(1) def _gen_llvm_mechanism_functions(self, ctx, builder, m_base_params, m_params, m_state, m_in, @@ -3141,7 +3141,7 @@ def _gen_llvm_function_internal(self, ctx, builder, m_params, m_state, arg_in, # is_finished should be checked after output ports ran is_finished_f = ctx.import_llvm_function(self, tags=tags.union({"is_finished"})) - is_finished_cond = builder.call(is_finished_f, [m_params, m_state, arg_in, + is_finished_cond = builder.call(is_finished_f, [m_base_params, m_state, arg_in, arg_out]) return builder, is_finished_cond @@ -3176,8 +3176,8 @@ def _gen_llvm_function(self, *, extra_args=[], ctx:pnlvm.LLVMBuilderContext, tag builder = ctx.create_llvm_function(args, self, return_type=ctx.bool_ty, tags=tags) - params, state = builder.function.args[:2] - finished = self._gen_llvm_is_finished_cond(ctx, builder, params, state) + params, state, inputs = builder.function.args[:3] + finished = self._gen_llvm_is_finished_cond(ctx, builder, params, state, inputs) builder.ret(finished) return builder.function diff --git a/psyneulink/core/components/mechanisms/processing/transfermechanism.py b/psyneulink/core/components/mechanisms/processing/transfermechanism.py index 522094b2fbc..3d54250be40 100644 --- a/psyneulink/core/components/mechanisms/processing/transfermechanism.py +++ b/psyneulink/core/components/mechanisms/processing/transfermechanism.py @@ -1531,14 +1531,18 @@ def _clip_result(self, clip, current_input): current_input[maxCapIndices] = np.max(clip) return current_input - def _gen_llvm_is_finished_cond(self, ctx, builder, params, state): - current = pnlvm.helpers.get_state_ptr(builder, self, state, "value") - threshold_ptr = pnlvm.helpers.get_param_ptr(builder, self, params, + def _gen_llvm_is_finished_cond(self, ctx, builder, m_base_params, m_state, m_in): + current = pnlvm.helpers.get_state_ptr(builder, self, m_state, "value") + + m_params, builder = self._gen_llvm_param_ports_for_obj( + self, m_base_params, ctx, builder, m_base_params, m_state, m_in) + threshold_ptr = pnlvm.helpers.get_param_ptr(builder, self, m_params, "termination_threshold") + if isinstance(threshold_ptr.type.pointee, pnlvm.ir.LiteralStructType): # Threshold is not defined, return the old value of finished flag assert len(threshold_ptr.type.pointee) == 0 - is_finished_ptr = pnlvm.helpers.get_state_ptr(builder, self, state, + is_finished_ptr = pnlvm.helpers.get_state_ptr(builder, self, m_state, "is_finished_flag") is_finished_flag = builder.load(is_finished_ptr) return builder.fcmp_ordered("!=", is_finished_flag, @@ -1564,7 +1568,7 @@ def _gen_llvm_is_finished_cond(self, ctx, builder, params, state): b.store(max_val, cmp_val_ptr) elif isinstance(self.termination_measure, Function): - prev_val_ptr = pnlvm.helpers.get_state_ptr(builder, self, state, "value", 1) + prev_val_ptr = pnlvm.helpers.get_state_ptr(builder, self, m_state, "value", 1) prev_val = builder.load(prev_val_ptr) expected = np.empty_like([self.defaults.value[0], self.defaults.value[0]]) @@ -1576,8 +1580,8 @@ def _gen_llvm_is_finished_cond(self, ctx, builder, params, state): self.termination_measure.defaults.variable = expected func = ctx.import_llvm_function(self.termination_measure) - func_params = pnlvm.helpers.get_param_ptr(builder, self, params, "termination_measure") - func_state = pnlvm.helpers.get_state_ptr(builder, self, state, "termination_measure") + func_params = pnlvm.helpers.get_param_ptr(builder, self, m_base_params, "termination_measure") + func_state = pnlvm.helpers.get_state_ptr(builder, self, m_state, "termination_measure") func_in = builder.alloca(func.args[2].type.pointee, name="is_finished_func_in") # Populate input func_in_current_ptr = builder.gep(func_in, [ctx.int32_ty(0), @@ -1591,7 +1595,7 @@ def _gen_llvm_is_finished_cond(self, ctx, builder, params, state): builder.call(func, [func_params, func_state, func_in, cmp_val_ptr]) elif isinstance(self.termination_measure, TimeScale): - ptr = builder.gep(pnlvm.helpers.get_state_ptr(builder, self, state, "num_executions"), + ptr = builder.gep(pnlvm.helpers.get_state_ptr(builder, self, m_state, "num_executions"), [ctx.int32_ty(0), ctx.int32_ty(self.termination_measure.value)]) ptr_val = builder.sitofp(builder.load(ptr), threshold.type) pnlvm.helpers.printf(builder, f"TERM MEASURE {self.termination_measure} %d %d\n",ptr_val, threshold) diff --git a/psyneulink/library/components/mechanisms/processing/integrator/ddm.py b/psyneulink/library/components/mechanisms/processing/integrator/ddm.py index 5f790fc9200..a13281ff994 100644 --- a/psyneulink/library/components/mechanisms/processing/integrator/ddm.py +++ b/psyneulink/library/components/mechanisms/processing/integrator/ddm.py @@ -1227,14 +1227,15 @@ def is_finished(self, context=None): return True return False - def _gen_llvm_is_finished_cond(self, ctx, builder, params, state): + def _gen_llvm_is_finished_cond(self, ctx, builder, m_base_params, m_state, m_in): # Setup pointers to internal function - func_state_ptr = pnlvm.helpers.get_state_ptr(builder, self, state, "function") - func_param_ptr = pnlvm.helpers.get_param_ptr(builder, self, params, "function") + f_state = pnlvm.helpers.get_state_ptr(builder, self, m_state, "function") + f_base_params = pnlvm.helpers.get_param_ptr(builder, self, m_base_params, "function") - # Find the single numeric entry in previous_value + # Find the single numeric entry in previous_value. + # This exists only if the 'function' is 'integrator' try: - prev_val_ptr = pnlvm.helpers.get_state_ptr(builder, self.function, func_state_ptr, "previous_value") + prev_val_ptr = pnlvm.helpers.get_state_ptr(builder, self.function, f_state, "previous_value") except ValueError: return ctx.bool_ty(1) @@ -1247,10 +1248,10 @@ def _gen_llvm_is_finished_cond(self, ctx, builder, params, state): prev_val = builder.call(llvm_fabs, [prev_val]) - # obtain threshold value + # Get threshold value threshold_ptr = pnlvm.helpers.get_param_ptr(builder, self.function, - func_param_ptr, + f_base_params, "threshold") threshold_ptr = builder.gep(threshold_ptr, [ctx.int32_ty(0), ctx.int32_ty(0)]) From 33344fe694c2fc16b1b6fd9373c9b25d057fbc38 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Tue, 27 Sep 2022 11:04:00 -0400 Subject: [PATCH 020/127] llvm, mechanisms/DDM: Apply function modulations before checking DDI threshold. Matches Python semantics. Add threshold modulation test for integrator based DDM. Signed-off-by: Jan Vesely --- .../mechanisms/processing/integrator/ddm.py | 7 ++++-- tests/mechanisms/test_ddm_mechanism.py | 23 ++++++++++++++++++- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/psyneulink/library/components/mechanisms/processing/integrator/ddm.py b/psyneulink/library/components/mechanisms/processing/integrator/ddm.py index a13281ff994..608b46827d3 100644 --- a/psyneulink/library/components/mechanisms/processing/integrator/ddm.py +++ b/psyneulink/library/components/mechanisms/processing/integrator/ddm.py @@ -1230,7 +1230,6 @@ def is_finished(self, context=None): def _gen_llvm_is_finished_cond(self, ctx, builder, m_base_params, m_state, m_in): # Setup pointers to internal function f_state = pnlvm.helpers.get_state_ptr(builder, self, m_state, "function") - f_base_params = pnlvm.helpers.get_param_ptr(builder, self, m_base_params, "function") # Find the single numeric entry in previous_value. # This exists only if the 'function' is 'integrator' @@ -1247,11 +1246,15 @@ def _gen_llvm_is_finished_cond(self, ctx, builder, m_base_params, m_state, m_in) llvm_fabs = ctx.get_builtin("fabs", [ctx.float_ty]) prev_val = builder.call(llvm_fabs, [prev_val]) + # Get functions params and apply modulation + f_base_params = pnlvm.helpers.get_param_ptr(builder, self, m_base_params, "function") + f_params, builder = self._gen_llvm_param_ports_for_obj( + self.function, f_base_params, ctx, builder, m_base_params, m_state, m_in) # Get threshold value threshold_ptr = pnlvm.helpers.get_param_ptr(builder, self.function, - f_base_params, + f_params, "threshold") threshold_ptr = builder.gep(threshold_ptr, [ctx.int32_ty(0), ctx.int32_ty(0)]) diff --git a/tests/mechanisms/test_ddm_mechanism.py b/tests/mechanisms/test_ddm_mechanism.py index 463761cd0b6..44a7013eefd 100644 --- a/tests/mechanisms/test_ddm_mechanism.py +++ b/tests/mechanisms/test_ddm_mechanism.py @@ -664,7 +664,7 @@ def test_DDM_in_composition(benchmark, comp_mode): @pytest.mark.composition @pytest.mark.ddm_mechanism -def test_DDM_threshold_modulation(comp_mode): +def test_DDM_threshold_modulation_analytical(comp_mode): M = pnl.DDM(name='DDM', function=pnl.DriftDiffusionAnalytical( threshold=20.0, @@ -684,6 +684,27 @@ def test_DDM_threshold_modulation(comp_mode): assert np.allclose(val[1], [60.2]) +@pytest.mark.composition +@pytest.mark.ddm_mechanism +def test_DDM_threshold_modulation_integrator(comp_mode): + M = pnl.DDM(name='DDM', + execute_until_finished=True, + function=pnl.DriftDiffusionIntegrator(threshold=20), + ) + + control = pnl.ControlMechanism( + control_signals=[(pnl.THRESHOLD, M)]) + + C = pnl.Composition() + C.add_node(M, required_roles=[pnl.NodeRole.ORIGIN, pnl.NodeRole.TERMINAL]) + C.add_node(control) + inputs = {M:[1], control:[3]} + val = C.run(inputs, num_trials=1, execution_mode=comp_mode) + + assert np.allclose(val[0], [60.0]) + assert np.allclose(val[1], [60.0]) + + @pytest.mark.composition @pytest.mark.parametrize(["noise", "threshold", "expected_results"],[ (1.0, 0.0, (0.0, 1.0)), From 0f2b15a4135f2cb12e83972e583bc1ffd23d1144 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Oct 2022 17:56:19 +0000 Subject: [PATCH 021/127] requirements: update autograd requirement from <1.5 to <1.6 (#2493) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e95cb4b5661..1407c5c58e2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -autograd<1.5 +autograd<1.6 graph-scheduler>=0.2.0, <1.1.2 dill<=0.32 elfi<0.8.5 From 3bdc151c91f479333a240ff09301a59a60beecab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Oct 2022 21:42:44 +0000 Subject: [PATCH 022/127] requirements: update pytest-cov requirement from <3.0.1 to <4.0.1 (#2492) --- dev_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index ccb1a619630..76397ebb41d 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,7 +1,7 @@ jupyter<=1.0.0 pytest<7.1.4 pytest-benchmark<3.4.2 -pytest-cov<3.0.1 +pytest-cov<4.0.1 pytest-helpers-namespace<2021.12.30 pytest-profiling<=1.7.0 pytest-pycodestyle<2.4.0 From e607c0973f74eceb514c2de9eb6e3ae7dfa29716 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Wed, 19 Oct 2022 02:22:23 -0400 Subject: [PATCH 023/127] github-actions: Restrict scipy to <1.9.2 on x86 (#2502) scipy doesn't provide 32-bit wheels for >=1.9.2 Signed-off-by: Jan Vesely --- .github/actions/install-pnl/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/install-pnl/action.yml b/.github/actions/install-pnl/action.yml index e0dc1a6f8fb..46ba907ff11 100644 --- a/.github/actions/install-pnl/action.yml +++ b/.github/actions/install-pnl/action.yml @@ -49,7 +49,7 @@ runs: sed -i /modeci_mdf/d requirements.txt # pywinpty is a transitive dependency and v1.0+ removed support for x86 wheels # terminado >= 0.10.0 pulls in pywinpty >= 1.1.0 - [[ ${{ runner.os }} = Windows* ]] && pip install "pywinpty<1" "terminado<0.10" + [[ ${{ runner.os }} = Windows* ]] && pip install "pywinpty<1" "terminado<0.10" "scipy<1.9.2" -c requirements.txt fi - name: Install updated package From 8881ad28e72782e3e599aebbabb3be024b9cbfc4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Oct 2022 18:29:38 +0000 Subject: [PATCH 024/127] requirements: update pandas requirement from <1.5.1 to <1.5.2 (#2503) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1407c5c58e2..ae63ee03534 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,5 +17,5 @@ torch>=1.8.0, <1.12.0; (platform_machine == 'AMD64' or platform_machine == 'x86_ typecheck-decorator<=1.2 leabra-psyneulink<=0.3.2 rich>=10.1, <10.13 -pandas<1.5.1 +pandas<1.5.2 fastkde==1.0.19 From d47d0f00a3662638d7a287ef17982123e750516f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Oct 2022 21:32:37 +0000 Subject: [PATCH 025/127] requirements: update matplotlib requirement from <3.6.1 to <3.6.2 (#2497) --- requirements.txt | 2 +- tutorial_requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index ae63ee03534..5dcb8c153ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ graphviz<0.21.0 grpcio<1.43.0 grpcio-tools<1.43.0 llvmlite<0.40 -matplotlib<3.6.1 +matplotlib<3.6.2 modeci_mdf<0.5, >=0.3.4; (platform_machine == 'AMD64' or platform_machine == 'x86_64') and platform_python_implementation == 'CPython' and implementation_name == 'cpython' networkx<2.9 numpy<1.21.7, >=1.17.0 diff --git a/tutorial_requirements.txt b/tutorial_requirements.txt index 003d498526e..738edadb115 100644 --- a/tutorial_requirements.txt +++ b/tutorial_requirements.txt @@ -1,3 +1,3 @@ graphviz<0.21.0 jupyter<=1.0.0 -matplotlib<3.6.1 +matplotlib<3.6.2 From ec9dfaccc482ae7157a8f92f874a19134f307c87 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Thu, 20 Oct 2022 20:36:00 -0400 Subject: [PATCH 026/127] requirements: update torch requirement from >=1.8.0,<1.12.0 to >=1.8.0,<1.13.0 (#2504) Signed-off-by: Jan Vesely --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5dcb8c153ce..42c941f0986 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ numpy<1.21.7, >=1.17.0 pillow<9.3.0 pint<0.20.0 toposort<1.8 -torch>=1.8.0, <1.12.0; (platform_machine == 'AMD64' or platform_machine == 'x86_64') and platform_python_implementation == 'CPython' and implementation_name == 'cpython' +torch>=1.8.0, <1.13.0; (platform_machine == 'AMD64' or platform_machine == 'x86_64') and platform_python_implementation == 'CPython' and implementation_name == 'cpython' typecheck-decorator<=1.2 leabra-psyneulink<=0.3.2 rich>=10.1, <10.13 From 04d726cebcbdb18142886139e46b82a444d9572d Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Tue, 25 Oct 2022 14:43:48 -0400 Subject: [PATCH 027/127] llvm, ocm/evaluate: Add basic support for multiple evaluation_types (#2505) The only supported evaluation type atm is "evaluate_type_objective", which uses the result of ocm.objective_mechanism as output (combined with costs). Signed-off-by: Jan Vesely --- .../nonstateful/optimizationfunctions.py | 10 ++-- .../control/optimizationcontrolmechanism.py | 53 ++++++++++++------- psyneulink/core/llvm/execution.py | 3 +- 3 files changed, 41 insertions(+), 25 deletions(-) diff --git a/psyneulink/core/components/functions/nonstateful/optimizationfunctions.py b/psyneulink/core/components/functions/nonstateful/optimizationfunctions.py index 6f4ce9f8588..d64947cc55f 100644 --- a/psyneulink/core/components/functions/nonstateful/optimizationfunctions.py +++ b/psyneulink/core/components/functions/nonstateful/optimizationfunctions.py @@ -1652,7 +1652,7 @@ def _gen_llvm_select_min_function(self, *, ctx:pnlvm.LLVMBuilderContext, tags:fr if ocm is not None: assert ocm.function is self sample_t = ocm._get_evaluate_alloc_struct_type(ctx) - value_t = ocm._get_evaluate_output_struct_type(ctx) + value_t = ocm._get_evaluate_output_struct_type(ctx, tags=tags) else: obj_func = ctx.import_llvm_function(self.objective_function) sample_t = obj_func.args[2].type.pointee @@ -1751,7 +1751,7 @@ def _gen_llvm_function_body(self, ctx, builder, params, state_features, arg_in, controller = self._get_optimized_controller() if controller is not None: assert controller.function is self - obj_func = ctx.import_llvm_function(controller, tags=tags.union({"evaluate"})) + obj_func = ctx.import_llvm_function(controller, tags=tags.union({"evaluate", "evaluate_type_objective"})) comp_args = builder.function.args[-3:] obj_param_ptr = comp_args[0] obj_state_ptr = comp_args[1] @@ -1845,7 +1845,8 @@ def _gen_llvm_function_body(self, ctx, builder, params, state_features, arg_in, # Check if smaller than current best. # the argument pointers are already offset, so use range <0,1) - select_min_f = ctx.import_llvm_function(self, tags=tags.union({"select_min"})) + min_tags = tags.union({"select_min", "evaluate_type_objective"}) + select_min_f = ctx.import_llvm_function(self, tags=min_tags) b.call(select_min_f, [params, state_features, min_sample_ptr, sample_ptr, min_value_ptr, value_ptr, opt_count_ptr, ctx.int32_ty(0), ctx.int32_ty(1)]) @@ -1997,7 +1998,8 @@ def _function(self, # Reduce array of values to min/max # select_min params are: # params, state, min_sample_ptr, sample_ptr, min_value_ptr, value_ptr, opt_count_ptr, count - bin_func = pnlvm.LLVMBinaryFunction.from_obj(self, tags=frozenset({"select_min"})) + min_tags = frozenset({"select_min", "evaluate_type_objective"}) + bin_func = pnlvm.LLVMBinaryFunction.from_obj(self, tags=min_tags) ct_param = bin_func.byref_arg_types[0](*self._get_param_initializer(context)) ct_state = bin_func.byref_arg_types[1](*self._get_state_initializer(context)) ct_opt_sample = bin_func.byref_arg_types[2](float("NaN")) diff --git a/psyneulink/core/components/mechanisms/modulatory/control/optimizationcontrolmechanism.py b/psyneulink/core/components/mechanisms/modulatory/control/optimizationcontrolmechanism.py index 8abba09428e..3070b74719a 100644 --- a/psyneulink/core/components/mechanisms/modulatory/control/optimizationcontrolmechanism.py +++ b/psyneulink/core/components/mechanisms/modulatory/control/optimizationcontrolmechanism.py @@ -3196,7 +3196,8 @@ def evaluate_agent_rep(self, control_allocation, context=None, return_results=Fa context=context ) - def _get_evaluate_output_struct_type(self, ctx): + def _get_evaluate_output_struct_type(self, ctx, *, tags): + assert "evaluate_type_objective" in tags, "Unknown evaluate type: {}".format(tags) # Returns a scalar that is the predicted net_outcome return ctx.float_ty @@ -3210,7 +3211,7 @@ def _gen_llvm_net_outcome_function(self, *, ctx, tags=frozenset()): ctx.get_state_struct_type(self).as_pointer(), self._get_evaluate_alloc_struct_type(ctx).as_pointer(), ctx.float_ty.as_pointer(), - ctx.float_ty.as_pointer()] + self._get_evaluate_output_struct_type(ctx, tags=tags).as_pointer()] builder = ctx.create_llvm_function(args, self, str(self) + "_net_outcome") llvm_func = builder.function @@ -3308,7 +3309,12 @@ def _gen_llvm_evaluate_alloc_range_function(self, *, ctx:pnlvm.LLVMBuilderContex allocation = builder.alloca(evaluate_f.args[2].type.pointee, name="allocation") with pnlvm.helpers.for_loop(builder, start, stop, stop.type(1), "alloc_loop") as (b, idx): - func_out = b.gep(arg_out, [idx]) + if "evaluate_type_objective" in tags: + out_idx = idx + else: + assert False, "Evaluation type not detected in tags, or unknown: {}".format(tags) + + func_out = b.gep(arg_out, [out_idx]) pnlvm.helpers.create_sample(b, allocation, search_space, idx) b.call(evaluate_f, [params, state, allocation, func_out, arg_in, data]) @@ -3321,7 +3327,7 @@ def _gen_llvm_evaluate_function(self, *, ctx:pnlvm.LLVMBuilderContext, tags=froz args = [ctx.get_param_struct_type(self.agent_rep).as_pointer(), ctx.get_state_struct_type(self.agent_rep).as_pointer(), self._get_evaluate_alloc_struct_type(ctx).as_pointer(), - self._get_evaluate_output_struct_type(ctx).as_pointer(), + self._get_evaluate_output_struct_type(ctx, tags=tags).as_pointer(), ctx.get_input_struct_type(self.agent_rep).as_pointer(), ctx.get_data_struct_type(self.agent_rep).as_pointer()] @@ -3419,25 +3425,32 @@ def _gen_llvm_evaluate_function(self, *, ctx:pnlvm.LLVMBuilderContext, tags=froz builder.store(num_inputs.type.pointee(1), num_inputs) # Simulations don't store output - comp_output = sim_f.args[4].type(None) + if "evaluate_type_objective" in tags: + comp_output = sim_f.args[4].type(None) + else: + assert False, "Evaluation type not detected in tags, or unknown: {}".format(tags) + builder.call(sim_f, [comp_state, comp_params, comp_data, comp_input, comp_output, num_trials, num_inputs]) - # Extract objective mechanism value - idx = self.agent_rep._get_node_index(self.objective_mechanism) - # Mechanisms' results are stored in the first substructure - objective_os_ptr = builder.gep(comp_data, [ctx.int32_ty(0), - ctx.int32_ty(0), - ctx.int32_ty(idx)]) - # Objective mech output shape should be 1 single element 2d array - objective_val_ptr = builder.gep(objective_os_ptr, - [ctx.int32_ty(0), ctx.int32_ty(0), - ctx.int32_ty(0)], "obj_val_ptr") - - net_outcome_f = ctx.import_llvm_function(self, tags=tags.union({"net_outcome"})) - builder.call(net_outcome_f, [controller_params, controller_state, - allocation_sample, objective_val_ptr, - arg_out]) + if "evaluate_type_objective" in tags: + # Extract objective mechanism value + idx = self.agent_rep._get_node_index(self.objective_mechanism) + # Mechanisms' results are stored in the first substructure + objective_op_ptr = builder.gep(comp_data, [ctx.int32_ty(0), + ctx.int32_ty(0), + ctx.int32_ty(idx)]) + # Objective mech output shape should be 1 single element 2d array + objective_val_ptr = builder.gep(objective_op_ptr, + [ctx.int32_ty(0), ctx.int32_ty(0), + ctx.int32_ty(0)], "obj_val_ptr") + + net_outcome_f = ctx.import_llvm_function(self, tags=tags.union({"net_outcome"})) + builder.call(net_outcome_f, [controller_params, controller_state, + allocation_sample, objective_val_ptr, + arg_out]) + else: + assert False, "Evaluation type not detected in tags, or unknown: {}".format(tags) builder.ret_void() diff --git a/psyneulink/core/llvm/execution.py b/psyneulink/core/llvm/execution.py index 2500d160997..95fc1e04f04 100644 --- a/psyneulink/core/llvm/execution.py +++ b/psyneulink/core/llvm/execution.py @@ -683,7 +683,8 @@ def _prepare_evaluate(self, inputs, num_input_sets, num_evaluations): ocm = self._composition.controller assert len(self._execution_contexts) == 1 - bin_func = pnlvm.LLVMBinaryFunction.from_obj(ocm, tags=frozenset({"evaluate", "alloc_range"})) + tags = {"evaluate", "alloc_range", "evaluate_type_objective"} + bin_func = pnlvm.LLVMBinaryFunction.from_obj(ocm, tags=frozenset(tags)) self.__bin_func = bin_func # There are 7 arguments to evaluate_alloc_range: From 81700308f89ed00b8bce109c5a5a18d044ff6c6f Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Tue, 25 Oct 2022 19:17:10 -0400 Subject: [PATCH 028/127] github-actions: Use environment files to set outputs (#2506) The old way of using workflow commands is going away[0] [0] https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/ Signed-off-by: Jan Vesely --- .github/actions/install-pnl/action.yml | 2 +- .github/actions/on-branch/action.yml | 2 +- .github/workflows/pnl-ci-docs.yml | 2 +- .github/workflows/pnl-ci.yml | 2 +- .github/workflows/test-release.yml | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/actions/install-pnl/action.yml b/.github/actions/install-pnl/action.yml index 46ba907ff11..724beaf12e2 100644 --- a/.github/actions/install-pnl/action.yml +++ b/.github/actions/install-pnl/action.yml @@ -58,7 +58,7 @@ runs: id: new_package run: | export NEW_PACKAGE=$(echo '${{ github.head_ref }}' | cut -f 4 -d/ | sed 's/-gt.*//' | sed 's/-lt.*//') - echo "::set-output name=new_package::$NEW_PACKAGE" + echo "{new_package}={$NEW_PACKAGE}" >> $GITHUB_OUTPUT # save a list of all installed packages (including pip, wheel; it's never empty) pip freeze --all > orig pip install "$(echo $NEW_PACKAGE | sed 's/[-_]/./g' | xargs grep *requirements.txt -h -e | head -n1)" diff --git a/.github/actions/on-branch/action.yml b/.github/actions/on-branch/action.yml index 770018ba85a..33c5813fd95 100644 --- a/.github/actions/on-branch/action.yml +++ b/.github/actions/on-branch/action.yml @@ -25,4 +25,4 @@ runs: git describe --always --tags export ON_BRANCH=$(git branch -a --contains ${{ github.ref }} | grep -q '^ remotes/origin/${{ inputs.branch }}$' && echo "${{ inputs.branch }}" || echo "") echo "Found out: ${ON_BRANCH}" - echo "::set-output name=on_branch::$ON_BRANCH" + echo "{on_branch}={$ON_BRANCH}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/pnl-ci-docs.yml b/.github/workflows/pnl-ci-docs.yml index 4821dfd103f..0c5ea63b011 100644 --- a/.github/workflows/pnl-ci-docs.yml +++ b/.github/workflows/pnl-ci-docs.yml @@ -76,7 +76,7 @@ jobs: run: | python -m pip install -U pip python -m pip --version - echo ::set-output name=pip_cache_dir::$(python -m pip cache dir) + echo "{pip_cache_dir}={$(python -m pip cache dir)}" >> $GITHUB_OUTPUT - name: Wheels cache uses: actions/cache@v3 diff --git a/.github/workflows/pnl-ci.yml b/.github/workflows/pnl-ci.yml index 2b1ffa648e6..bbfc1ce62d6 100644 --- a/.github/workflows/pnl-ci.yml +++ b/.github/workflows/pnl-ci.yml @@ -61,7 +61,7 @@ jobs: run: | python -m pip install -U pip python -m pip --version - echo ::set-output name=pip_cache_dir::$(python -m pip cache dir) + echo "{pip_cache_dir}={$(python -m pip cache dir)}" >> $GITHUB_OUTPUT - name: Wheels cache uses: actions/cache@v3 diff --git a/.github/workflows/test-release.yml b/.github/workflows/test-release.yml index 0b7887ea5ee..eede3fde5d2 100644 --- a/.github/workflows/test-release.yml +++ b/.github/workflows/test-release.yml @@ -34,8 +34,8 @@ jobs: python setup.py sdist python setup.py bdist_wheel cd dist - echo ::set-output name=sdist::$(ls *.tar.gz) - echo ::set-output name=wheel::$(ls *.whl) + echo "{sdist}={$(ls *.tar.gz)}" >> $GITHUB_OUTPUT + echo "{wheel}={$(ls *.whl)}" >> $GITHUB_OUTPUT - name: Upload Python dist files uses: actions/upload-artifact@v3 From 96574ded23ce332a0da29a8a6c6df7484e2f5e46 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Wed, 26 Oct 2022 18:30:12 -0400 Subject: [PATCH 029/127] github-actions: Restrict scikit-learn to <1.1.3 on x86 (#2511) scikit-learn doesn't provide 32-bit wheels for >=1.1.3, and the build process tries to pull in scipy-1.9.3 triggering the issue that was addressed in e607c0973f74eceb514c2de9eb6e3ae7dfa29716 [0] Signed-off-by: Jan Vesely [0] https://github.com/PrincetonUniversity/PsyNeuLink/actions/runs/3332092764/jobs/5512568444 --- .github/actions/install-pnl/action.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/actions/install-pnl/action.yml b/.github/actions/install-pnl/action.yml index 724beaf12e2..b48495e2e34 100644 --- a/.github/actions/install-pnl/action.yml +++ b/.github/actions/install-pnl/action.yml @@ -49,7 +49,9 @@ runs: sed -i /modeci_mdf/d requirements.txt # pywinpty is a transitive dependency and v1.0+ removed support for x86 wheels # terminado >= 0.10.0 pulls in pywinpty >= 1.1.0 - [[ ${{ runner.os }} = Windows* ]] && pip install "pywinpty<1" "terminado<0.10" "scipy<1.9.2" -c requirements.txt + # scipy >=1.9.2 doesn't provide win32 wheel and GA doesn't have working fortran on windows + # scikit-learn >= 1.1.3 doesn't provide win32 wheel + [[ ${{ runner.os }} = Windows* ]] && pip install "pywinpty<1" "terminado<0.10" "scipy<1.9.2" "scikit-learn<1.1.3" -c requirements.txt fi - name: Install updated package From a95ffa8e3a3948f93f3097680bcdc811cf0cec73 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Oct 2022 00:49:43 +0000 Subject: [PATCH 030/127] requirements: update pytest-xdist requirement from <2.6.0 to <3.1.0 (#2510) --- dev_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index 76397ebb41d..caa7f0cf335 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -6,4 +6,4 @@ pytest-helpers-namespace<2021.12.30 pytest-profiling<=1.7.0 pytest-pycodestyle<2.4.0 pytest-pydocstyle<2.4.0 -pytest-xdist<2.6.0 +pytest-xdist<3.1.0 From eef027f3d355d5820c29e97a0f2d891fbaf8149a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Oct 2022 02:57:42 +0000 Subject: [PATCH 031/127] requirements: update pytest-benchmark requirement from <3.4.2 to <4.0.1 (#2507) --- dev_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index caa7f0cf335..9a8ac773e85 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,6 +1,6 @@ jupyter<=1.0.0 pytest<7.1.4 -pytest-benchmark<3.4.2 +pytest-benchmark<4.0.1 pytest-cov<4.0.1 pytest-helpers-namespace<2021.12.30 pytest-profiling<=1.7.0 From 795d794e8fa6a3025fb7a423a331120c29c5454e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Oct 2022 13:44:55 +0000 Subject: [PATCH 032/127] requirements: update pint requirement from <0.20.0 to <0.21.0 (#2509) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 42c941f0986..8ba27238b1c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ modeci_mdf<0.5, >=0.3.4; (platform_machine == 'AMD64' or platform_machine == 'x8 networkx<2.9 numpy<1.21.7, >=1.17.0 pillow<9.3.0 -pint<0.20.0 +pint<0.21.0 toposort<1.8 torch>=1.8.0, <1.13.0; (platform_machine == 'AMD64' or platform_machine == 'x86_64') and platform_python_implementation == 'CPython' and implementation_name == 'cpython' typecheck-decorator<=1.2 From e83328a6fdbbab63311ecaa03be11098b60524d8 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Thu, 27 Oct 2022 01:44:01 -0400 Subject: [PATCH 033/127] github-actions: Fix setting of outputs The format was set incorrectly in 81700308f89ed00b8bce109c5a5a18d044ff6c6f Fixes: 81700308f89ed00b8bce109c5a5a18d044ff6c6f Signed-off-by: Jan Vesely --- .github/actions/install-pnl/action.yml | 2 +- .github/actions/on-branch/action.yml | 2 +- .github/workflows/pnl-ci-docs.yml | 2 +- .github/workflows/pnl-ci.yml | 2 +- .github/workflows/test-release.yml | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/actions/install-pnl/action.yml b/.github/actions/install-pnl/action.yml index b48495e2e34..c9eda59a6f1 100644 --- a/.github/actions/install-pnl/action.yml +++ b/.github/actions/install-pnl/action.yml @@ -60,7 +60,7 @@ runs: id: new_package run: | export NEW_PACKAGE=$(echo '${{ github.head_ref }}' | cut -f 4 -d/ | sed 's/-gt.*//' | sed 's/-lt.*//') - echo "{new_package}={$NEW_PACKAGE}" >> $GITHUB_OUTPUT + echo "new_package=$NEW_PACKAGE" >> $GITHUB_OUTPUT # save a list of all installed packages (including pip, wheel; it's never empty) pip freeze --all > orig pip install "$(echo $NEW_PACKAGE | sed 's/[-_]/./g' | xargs grep *requirements.txt -h -e | head -n1)" diff --git a/.github/actions/on-branch/action.yml b/.github/actions/on-branch/action.yml index 33c5813fd95..a4dcfd5ec3a 100644 --- a/.github/actions/on-branch/action.yml +++ b/.github/actions/on-branch/action.yml @@ -25,4 +25,4 @@ runs: git describe --always --tags export ON_BRANCH=$(git branch -a --contains ${{ github.ref }} | grep -q '^ remotes/origin/${{ inputs.branch }}$' && echo "${{ inputs.branch }}" || echo "") echo "Found out: ${ON_BRANCH}" - echo "{on_branch}={$ON_BRANCH}" >> $GITHUB_OUTPUT + echo "on_branch=$ON_BRANCH" >> $GITHUB_OUTPUT diff --git a/.github/workflows/pnl-ci-docs.yml b/.github/workflows/pnl-ci-docs.yml index 0c5ea63b011..cfe9d7b2476 100644 --- a/.github/workflows/pnl-ci-docs.yml +++ b/.github/workflows/pnl-ci-docs.yml @@ -76,7 +76,7 @@ jobs: run: | python -m pip install -U pip python -m pip --version - echo "{pip_cache_dir}={$(python -m pip cache dir)}" >> $GITHUB_OUTPUT + echo "pip_cache_dir=$(python -m pip cache dir)" | tee -a $GITHUB_OUTPUT - name: Wheels cache uses: actions/cache@v3 diff --git a/.github/workflows/pnl-ci.yml b/.github/workflows/pnl-ci.yml index bbfc1ce62d6..d72b6fc93f8 100644 --- a/.github/workflows/pnl-ci.yml +++ b/.github/workflows/pnl-ci.yml @@ -61,7 +61,7 @@ jobs: run: | python -m pip install -U pip python -m pip --version - echo "{pip_cache_dir}={$(python -m pip cache dir)}" >> $GITHUB_OUTPUT + echo "pip_cache_dir=$(python -m pip cache dir)" | tee -a $GITHUB_OUTPUT - name: Wheels cache uses: actions/cache@v3 diff --git a/.github/workflows/test-release.yml b/.github/workflows/test-release.yml index eede3fde5d2..8f1822de2e0 100644 --- a/.github/workflows/test-release.yml +++ b/.github/workflows/test-release.yml @@ -34,8 +34,8 @@ jobs: python setup.py sdist python setup.py bdist_wheel cd dist - echo "{sdist}={$(ls *.tar.gz)}" >> $GITHUB_OUTPUT - echo "{wheel}={$(ls *.whl)}" >> $GITHUB_OUTPUT + echo "sdist=$(ls *.tar.gz)" >> $GITHUB_OUTPUT + echo "wheel=$(ls *.whl)" >> $GITHUB_OUTPUT - name: Upload Python dist files uses: actions/upload-artifact@v3 From 053db3db857ad39a34102bf82d3abc2b7049df18 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Thu, 27 Oct 2022 13:18:26 -0400 Subject: [PATCH 034/127] github-actions: Consider 'doc_requirements.txt' when generating cache key in doc CI jobs Signed-off-by: Jan Vesely --- .github/workflows/pnl-ci-docs.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pnl-ci-docs.yml b/.github/workflows/pnl-ci-docs.yml index cfe9d7b2476..507befbc8d0 100644 --- a/.github/workflows/pnl-ci-docs.yml +++ b/.github/workflows/pnl-ci-docs.yml @@ -82,8 +82,8 @@ jobs: uses: actions/cache@v3 with: path: ${{ steps.pip_cache.outputs.pip_cache_dir }}/wheels - key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-${{ hashFiles('requirements.txt') }}-${{ github.sha }} - restore-keys: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-${{ hashFiles('requirements.txt') }} + key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-${{ hashFiles('requirements.txt', 'doc_requirements.txt') }}-${{ github.sha }} + restore-keys: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-${{ hashFiles('requirements.txt', 'doc_requirements.txt') }} # We need to install all PNL deps since docs config imports psyneulink module - name: Install local, editable PNL package From 923e6e7886d06264aee338f4c2780232e2f5eee4 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Thu, 27 Oct 2022 13:19:55 -0400 Subject: [PATCH 035/127] github-actions: Consider 'dev_requirements.txt' when generating cache key in CI jobs Signed-off-by: Jan Vesely --- .github/workflows/pnl-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pnl-ci.yml b/.github/workflows/pnl-ci.yml index d72b6fc93f8..95f4d611a88 100644 --- a/.github/workflows/pnl-ci.yml +++ b/.github/workflows/pnl-ci.yml @@ -67,8 +67,8 @@ jobs: uses: actions/cache@v3 with: path: ${{ steps.pip_cache.outputs.pip_cache_dir }}/wheels - key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-${{ hashFiles('requirements.txt') }}-${{ github.sha }} - restore-keys: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-${{ hashFiles('requirements.txt') }} + key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-${{ hashFiles('requirements.txt', 'dev_requirements.txt') }}-${{ github.sha }} + restore-keys: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-${{ hashFiles('requirements.txt', 'dev_requirements.txt') }} - name: Install local, editable PNL package uses: ./.github/actions/install-pnl From 912968095d543759451cde7202e8fbe50e0a4921 Mon Sep 17 00:00:00 2001 From: Katherine Mantel Date: Fri, 28 Oct 2022 02:58:31 -0400 Subject: [PATCH 036/127] Parameters: allow _validate_ methods to reference other parameters (#2512) --- psyneulink/core/globals/parameters.py | 50 ++++++++++++++++++++------- psyneulink/core/globals/utilities.py | 9 ++++- tests/misc/test_parameters.py | 38 ++++++++++++++++++++ 3 files changed, 83 insertions(+), 14 deletions(-) diff --git a/psyneulink/core/globals/parameters.py b/psyneulink/core/globals/parameters.py index 819db375349..dfe7b589824 100644 --- a/psyneulink/core/globals/parameters.py +++ b/psyneulink/core/globals/parameters.py @@ -113,8 +113,11 @@ def __init__(p=None, q=1.0): `Parameter attributes ` - default values for the parameters can be specified in the Parameters class body, or in the arguments for *B*.__init__. If both are specified and the values differ, an exception will be raised -- if you want assignments to parameter *p* to be validated, add a method _validate_p(value), +- if you want assignments to parameter *p* to be validated, add a method _validate_p(value), \ that returns None if value is a valid assignment, or an error string if value is not a valid assignment + - NOTE: a validation method for *p* may reference other parameters \ + only if they are listed in *p*'s \ + `dependencies ` - if you want all values set to *p* to be parsed beforehand, add a method _parse_p(value) that returns the parsed value - for example, convert to a numpy array or float @@ -123,6 +126,8 @@ def __init__(p=None, q=1.0): def _parse_p(value): return np.asarray(value) + - NOTE: parsers may not reference other parameters + - setters and getters (used for more advanced behavior than parsing) should both return the final value to return (getter) or set (setter) For example, `costs ` of `ControlMechanism ` has a special @@ -607,13 +612,15 @@ def _owner(self, value): except TypeError: self._owner_ref = value - @property - def _in_dependency_order(self): + def _dependency_order_key(self, names=False): """ - Returns: - list[Parameter] - a list of Parameters such that any - Parameter is placed before all of its - `dependencies ` + Args: + names (bool, optional): Whether sorting key is based on + Parameter names or Parameter objects. Defaults to False. + + Returns: + types.FunctionType: a function that may be passed in as sort + key so that any Parameter is placed before its dependencies """ parameter_function_ordering = list(toposort.toposort({ p.name: p.dependencies for p in self if p.dependencies is not None @@ -622,13 +629,30 @@ def _in_dependency_order(self): itertools.chain.from_iterable(parameter_function_ordering) ) - def ordering(p): - try: - return parameter_function_ordering.index(p.name) - except ValueError: - return -1 + if names: + def ordering(p): + try: + return parameter_function_ordering.index(p) + except ValueError: + return -1 + else: + def ordering(p): + try: + return parameter_function_ordering.index(p.name) + except ValueError: + return -1 + + return ordering - return sorted(self, key=ordering) + @property + def _in_dependency_order(self): + """ + Returns: + list[Parameter] - a list of Parameters such that any + Parameter is placed before all of its + `dependencies ` + """ + return sorted(self, key=self._dependency_order_key()) class Defaults(ParametersTemplate): diff --git a/psyneulink/core/globals/utilities.py b/psyneulink/core/globals/utilities.py index 84fd6a73f93..b75ae1965c7 100644 --- a/psyneulink/core/globals/utilities.py +++ b/psyneulink/core/globals/utilities.py @@ -780,7 +780,14 @@ def __deepcopy__(self, memo): result = cls.__new__(cls) memo[id(self)] = result - for k, v in self.__dict__.items(): + try: + # follow dependency order for Parameters to allow validation involving other parameters + ordered_dict_keys = sorted(self.__dict__, key=self._dependency_order_key(names=True)) + except AttributeError: + ordered_dict_keys = self.__dict__ + + for k in ordered_dict_keys: + v = self.__dict__[k] if k in shared_keys or isinstance(v, shared_types): res_val = v else: diff --git a/tests/misc/test_parameters.py b/tests/misc/test_parameters.py index 98af182a686..751a1f92a58 100644 --- a/tests/misc/test_parameters.py +++ b/tests/misc/test_parameters.py @@ -633,3 +633,41 @@ def set_p_default(obj, val): assert TestParent.defaults.p == 0 assert TestChild.defaults.p == 1 assert TestGrandchild.defaults.p == 20 + + +def test_dependent_parameter_validate(): + # using 3 parameters to reduce chance of random success + class NewF(pnl.Function_Base): + class Parameters(pnl.Function_Base.Parameters): + a = pnl.Parameter(1) + b = pnl.Parameter(2, dependencies='a') + c = pnl.Parameter(3, dependencies='b') + d = pnl.Parameter(4, dependencies='c') + + def _validate_b(self, b): + if b != self.a.default_value + 1: + return 'invalid' + + def _validate_c(self, c): + if c != self.b.default_value + 1: + return 'invalid' + + def _validate_d(self, d): + if d != self.c.default_value + 1: + return 'invalid' + + def __init__(self, **kwargs): + return super().__init__(0, {}, **kwargs) + + def _function(self, variable=None, context=None, params=None): + return 0 + + pnl.ProcessingMechanism(function=NewF(a=2, b=3, c=4, d=5)) + + with pytest.raises(pnl.ParameterError) as err: + # b should be first error to occur + pnl.ProcessingMechanism(function=NewF(b=3, c=5, d=7)) + assert re.match( + r"Value \(3\) assigned to parameter 'b'.*is not valid: invalid", + str(err.value) + ) From dfdee1f0f81315f4eca16d925fa33e51a759bdef Mon Sep 17 00:00:00 2001 From: jdcpni Date: Sat, 29 Oct 2022 17:03:06 -0400 Subject: [PATCH 037/127] Models/nback (#2514) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * • compositioninterfacemechanism.py: - _get_source_node_for_input_CIM: restore (modeled on _get_source_of_modulation_for_parameter_CIM) but NEEDS TESTS - _get_source_of_modulation_for_parameter_CIM: clean up comments, NEEDS TESTS * - * - * - * - * - * - * • Nback - EM uses ContentAddressableMemory (instead of DictionaryMemory) - Implements FFN for comparison of current and retrieved stimulus and context • Project: replace all instances of "RETREIVE" with "RETRIEVE" * • objectivefunctions.py - add cosine_similarity (needs compiled version) * • Project: make COSINE_SIMILARITY a synonym of COSINE • nback_CAM_FFN: - refactor to implement FFN and task input - assign termination condition for execution that is dependent on control - ContentAddressableMemory: selection_function=SoftMax(output=MAX_INDICATOR, gain=SOFT_MAX_TEMP) • DriftOnASphereIntegrator: - add dimension as dependency for initializer parameter * - * - * - * - * - * - * - * - * - * - * - * - * - * - * • test_integrator.py: Added identicalness test for DriftOnASphereIntegrator agains nback-paper implementation. * - * - * Parameters: allow _validate_ methods to reference other parameters (#2512) * • Scripts: - Updated N-back to use objective_mechanism, with commented out code for version that doesn't use it once bug is fixed - Deleted N-back_WITH_OBJECTIVE_MECH.py * • Scripts: - Updated N-back to use objective_mechanism, with commented out code for version that doesn't use it once bug is fixed - Deleted N-back_WITH_OBJECTIVE_MECH.py * • N-back.py: - added stimulus generation per nback-paper protocol * - N-back.py tstep(s) -> trial(s) * - * - * • N-back.py - comp -> nback_model - implement stim_set() method * - * • N-back.py: - added training set generation * - * - * • N-back.py - modularized script * - * - * - * - Co-authored-by: jdcpni Co-authored-by: Katherine Mantel --- .../N-back MODULARIZED.py | 231 ++++++++ Scripts/Models (Under Development)/N-back.py | 544 ++++++++++++++---- .../nonstateful/objectivefunctions.py | 10 +- .../functions/stateful/integratorfunctions.py | 8 +- .../modulatory/control/controlmechanism.py | 28 +- psyneulink/core/compositions/composition.py | 11 +- psyneulink/core/globals/keywords.py | 6 +- .../integrator/episodicmemorymechanism.py | 4 +- tests/functions/test_integrator.py | 41 +- tests/mechanisms/test_episodic_memory.py | 20 +- 10 files changed, 743 insertions(+), 160 deletions(-) create mode 100644 Scripts/Models (Under Development)/N-back MODULARIZED.py diff --git a/Scripts/Models (Under Development)/N-back MODULARIZED.py b/Scripts/Models (Under Development)/N-back MODULARIZED.py new file mode 100644 index 00000000000..87ee56c5221 --- /dev/null +++ b/Scripts/Models (Under Development)/N-back MODULARIZED.py @@ -0,0 +1,231 @@ +import numpy as np +from psyneulink import * +# from psyneulink.core.scheduling.condition import When +from graph_scheduler import * + +# TODO: +# - from nback-paper: +# - get ffn weights +# - import stimulus generation code +# - retrain on full set of 1,2,3,4,5 back +# - validate against nback-paper results +# - DriftOnASphereIntegrator: fix for noise=0 +# - write test that compares DriftOnASphereIntegrator with spherical_drift code in nback-paper + +# FROM nback-paper: +# 'smtemp':8, +# 'stim_weight':0.05, +# 'hrate':0.04 +# SDIM = 20 +# indim = 2 * (CDIM + SDIM) +# hiddim = SDIM * 4 + +# TEST: +# Structural parameters: +NUM_TASKS=3 +# Test: +STIM_SIZE=1 +# Replicate model: +# STIM_SIZE=20 +# ---------- +CONTEXT_SIZE=25 +HIDDEN_SIZE=STIM_SIZE*4 + +# Execution parameters +# Test: +CONTEXT_DRIFT_RATE=.1 +CONTEXT_DRIFT_NOISE=.00000000001 +# Replicate model: +# CONTEXT_DRIFT_RATE=.25 +# CONTEXT_DRIFT_NOISE=.075 +# ---- +NUM_TRIALS=20 +NBACK=2 +TOLERANCE=.5 +STIM_WEIGHT=.05 +HAZARD_RATE=0.04 +SOFT_MAX_TEMP=1/8 + +# # MODEL: +# STIM_SIZE=25 +# CONTEXT_SIZE=20 +# CONTEXT_DRIFT_RATE=.25 +# CONTEXT_DRIFT_NOISE=.075 +# NUM_TRIALS = 25 + +def control_function(outcome): + """Evaluate response and set ControlSignal for EM[store_prob] accordingly. + + outcome[0] = ffn output + If ffn_output signifies a MATCH: + set EM[store_prob]=1 (as prep encoding stimulus in EM on next trial) + terminate trial + If ffn_output signifies a NON-MATCH: + set EM[store_prob]=0 (as prep for another retrieval from EM without storage) + continue trial + + Notes: + - outcome is passed as 2d array with a single 1d length 2 entry, such that output[0] = ffn output + - ffn output: [1,0]=MATCH, [0,1]=NON-MATCH + - return value is used by: + - control Mechanism to set ControlSignal for EM[store_prob] (per above) + - terminate_trial(), which is used by Condition specified as termination_processing for comp.run(), + to determine whether to end or continue trial + + """ + ffn_output = outcome[0] + if ffn_output[1] > ffn_output[0]: + return 1 + else: # NON-MATCH: + return 0 + return None + + +def terminate_trial(ctl_mech): + """Determine whether to continue or terminate trial. + Determination is made in control_function (assigned as function of control Mechanism): + - terminate if match or hazard rate is realized + - continue if non-match or hazard rate is not realized + """ + if ctl_mech.value==1 or np.random.random() > HAZARD_RATE: + return 1 # terminate + else: + return 0 # continue + + +def construct_model(num_tasks, stim_size, context_size, hidden_size, display=False): + + # Mechanisms: + stim = TransferMechanism(name='STIM', size=STIM_SIZE) + context = ProcessingMechanism(name='CONTEXT', + function=DriftOnASphereIntegrator( + initializer=np.random.random(CONTEXT_SIZE-1), + noise=CONTEXT_DRIFT_NOISE, + dimension=CONTEXT_SIZE)) + task = ProcessingMechanism(name="TASK", size=NUM_TASKS) + em = EpisodicMemoryMechanism(name='EPISODIC MEMORY (dict)', + # default_variable=[[0]*STIM_SIZE, [0]*CONTEXT_SIZE], + input_ports=[{NAME:"STIMULUS_FIELD", + SIZE:STIM_SIZE}, + {NAME:"CONTEXT_FIELD", + SIZE:CONTEXT_SIZE}], + function=ContentAddressableMemory( + initializer=[[[0]*STIM_SIZE, [0]*CONTEXT_SIZE]], + distance_field_weights=[STIM_WEIGHT, 1-STIM_WEIGHT], + equidistant_entries_select=NEWEST, + selection_function=SoftMax(output=MAX_INDICATOR, + gain=SOFT_MAX_TEMP)), + ) + stim_comparator = ComparatorMechanism(name='STIM COMPARATOR', + # sample=STIM_SIZE, target=STIM_SIZE + input_ports=[{NAME:"CURRENT_STIMULUS", SIZE:STIM_SIZE}, + {NAME:"RETRIEVED_STIMULUS", SIZE:STIM_SIZE}], + ) + context_comparator = ComparatorMechanism(name='CONTEXT COMPARATOR', + # sample=np.zeros(STIM_SIZE), + # target=np.zeros(CONTEXT_SIZE) + input_ports=[{NAME:"CURRENT_CONTEXT", SIZE:CONTEXT_SIZE}, + {NAME:"RETRIEVED_CONTEXT", SIZE:CONTEXT_SIZE}], + function=Distance(metric=COSINE)) + + # QUESTION: GET INFO ABOUT INPUT FUNCTIONS FROM ANDRE: + input_current_stim = TransferMechanism(size=STIM_SIZE, function=Linear, name="CURRENT STIMULUS") # function=Logistic) + input_current_context = TransferMechanism(size=STIM_SIZE, function=Linear, name="CURRENT CONTEXT") # function=Logistic) + input_retrieved_stim = TransferMechanism(size=STIM_SIZE, function=Linear, name="RETRIEVED STIMULUS") # function=Logistic) + input_retrieved_context = TransferMechanism(size=STIM_SIZE, function=Linear, name="RETRIEVED CONTEXT") # function=Logistic) + input_task = TransferMechanism(size=NUM_TASKS, function=Linear, name="CURRENT TASK") # function=Logistic) + hidden = TransferMechanism(size=HIDDEN_SIZE, function=Logistic, name="HIDDEN LAYER") + decision = ProcessingMechanism(size=2, name="DECISION LAYER") + + control = ControlMechanism(name="READ/WRITE CONTROLLER", + monitor_for_control=decision, + function=control_function, + control=(STORAGE_PROB, em),) + + # Compositions: + ffn = Composition([{input_current_stim, + input_current_context, + input_retrieved_stim, + input_retrieved_context, + input_task}, + hidden, decision], + name="WORKING MEMORY (fnn)") + comp = Composition(nodes=[stim, context, task, em, ffn, control], + name="N-Back Model") + comp.add_projection(MappingProjection(), stim, input_current_stim) + comp.add_projection(MappingProjection(), context, input_current_context) + comp.add_projection(MappingProjection(), task, input_task) + comp.add_projection(MappingProjection(), em.output_ports["RETRIEVED_STIMULUS_FIELD"], input_retrieved_stim) + comp.add_projection(MappingProjection(), em.output_ports["RETRIEVED_CONTEXT_FIELD"], input_retrieved_context) + comp.add_projection(MappingProjection(), stim, em.input_ports["STIMULUS_FIELD"]) + comp.add_projection(MappingProjection(), context, em.input_ports["CONTEXT_FIELD"]) + comp.add_projection(MappingProjection(), decision, control) + + if display: + comp.show_graph() + # comp.show_graph(show_cim=True, + # show_node_structure=ALL, + # show_dimensions=True) + + # Execution: + + # Define a function that detects when the a Mechanism's value has converged, such that the change in all of the + # elements of its value attribute from the last execution (given by its delta attribute) falls below ``epsilon`` + # + # def converge(mech, thresh): + # return all(abs(v) <= thresh for v in mech.delta) + # + # # Add Conditions to the ``color_hidden`` and ``word_hidden`` Mechanisms that depend on the converge function: + # epsilon = 0.01 + # Stroop_model.scheduler.add_condition(color_hidden, When(converge, task, epsilon))) + # Stroop_model.scheduler.add_condition(word_hidden, When(converge, task, epsilon))) + return comp + + +def execute_model(model): + input_dict = {model.nodes['STIM']: np.array(list(range(NUM_TRIALS))).reshape(NUM_TRIALS,1)+1, + model.nodes['CONTEXT']:[[CONTEXT_DRIFT_RATE]]*NUM_TRIALS, + model.nodes['TASK']: np.array([[0,0,1]]*NUM_TRIALS)} + model.run(inputs=input_dict, + termination_processing={TimeScale.TRIAL: + Condition(terminate_trial, # termination function + model.nodes["READ/WRITE CONTROLLER"])}, # function arg + report_output=ReportOutput.ON + ) + +nback_model = construct_model(display=True) +execute_model(nback_model) + + +# TEST OF SPHERICAL DRIFT: +# stims = np.array([x[0] for x in em.memory]) +# contexts = np.array([x[1] for x in em.memory]) +# cos = Distance(metric=COSINE) +# dist = Distance(metric=EUCLIDEAN) +# diffs = [np.sum([contexts[i+1] - contexts[1]]) for i in range(NUM_TRIALS)] +# diffs_1 = [np.sum([contexts[i+1] - contexts[i]]) for i in range(NUM_TRIALS)] +# diffs_2 = [np.sum([contexts[i+2] - contexts[i]]) for i in range(NUM_TRIALS-1)] +# dots = [[contexts[i+1] @ contexts[1]] for i in range(NUM_TRIALS)] +# dot_diffs_1 = [[contexts[i+1] @ contexts[i]] for i in range(NUM_TRIALS)] +# dot_diffs_2 = [[contexts[i+2] @ contexts[i]] for i in range(NUM_TRIALS-1)] +# angle = [cos([contexts[i+1], contexts[1]]) for i in range(NUM_TRIALS)] +# angle_1 = [cos([contexts[i+1], contexts[i]]) for i in range(NUM_TRIALS)] +# angle_2 = [cos([contexts[i+2], contexts[i]]) for i in range(NUM_TRIALS-1)] +# euclidean = [dist([contexts[i+1], contexts[1]]) for i in range(NUM_TRIALS)] +# euclidean_1 = [dist([contexts[i+1], contexts[i]]) for i in range(NUM_TRIALS)] +# euclidean_2 = [dist([contexts[i+2], contexts[i]]) for i in range(NUM_TRIALS-1)] +# print("STIMS:", stims, "\n") +# print("DIFFS:", diffs, "\n") +# print("DIFFS 1:", diffs_1, "\n") +# print("DIFFS 2:", diffs_2, "\n") +# print("DOT PRODUCTS:", dots, "\n") +# print("DOT DIFFS 1:", dot_diffs_1, "\n") +# print("DOT DIFFS 2:", dot_diffs_2, "\n") +# print("ANGLE: ", angle, "\n") +# print("ANGLE_1: ", angle_1, "\n") +# print("ANGLE_2: ", angle_2, "\n") +# print("EUCILDEAN: ", euclidean, "\n") +# print("EUCILDEAN 1: ", euclidean_1, "\n") +# print("EUCILDEAN 2: ", euclidean_2, "\n") + +# n_back_model() diff --git a/Scripts/Models (Under Development)/N-back.py b/Scripts/Models (Under Development)/N-back.py index 716962ffb60..1c4b97f0ab0 100644 --- a/Scripts/Models (Under Development)/N-back.py +++ b/Scripts/Models (Under Development)/N-back.py @@ -1,116 +1,434 @@ +""" +This implements a model of the `N-back task _` +described in `Beukers et al. `_. The model uses a simple implementation of episodic +memory (i.e., content-retrieval memory) to store previous stimuli and the temporal context in which they occured, +and a feedforward neural network to evaluate whether the current stimulus is a match to the n'th preceding stimulus. + +TODO: + - get rid of objective_mechanism (see "VERSION *WITHOUT* ObjectiveMechanism" under control(...) + - from nback-paper: + - get ffn weights? + - why SDIM=20 if it is a one-hot encoding (np.eye), and NSTIM=8? (i.e., SHOULDN'T NUM_STIM == STIM_SIZE)? + - do input layers use logistic (as suggested in figure)? + - construct training set and train in ffn using Autodiff + - validate against nback-paper results + - replace get_input_sequence and get_training_inputs with generators passed to nback_model.run() and ffn.learn + - make termination processing part of the Comopsition definition? + +""" + +from graph_scheduler import * from psyneulink import * +import numpy as np +import itertools + +DISPLAY = False # show visual of model +# REPORTING_OPTIONS = ReportOutput.ON # Console output during run +REPORTING_OPTIONS = ReportOutput.OFF + + +# PARAMETERS ------------------------------------------------------------------------------------------------------- + +# FROM nback-paper: +# SDIM = 20 +# CDIM = 25 +# indim = 2 * (CDIM + SDIM) +# hiddim = SDIM * 4 +# CONTEXT_DRIFT_RATE=.25 +# CONTEXT_DRIFT_NOISE=.075 +# 'stim_weight':0.05, +# 'smtemp':8, +# HAZARD_RATE=0.04 + +# TEST: +MAX_NBACK_LEVELS = 5 +NBACK_LEVELS = [2,3] +NUM_NBACK_LEVELS = len(NBACK_LEVELS) +# NUM_TASKS=2 # number of different variants of n-back tasks (set sizes) +NUM_STIM = 8 # number of different stimuli in stimulus set - QUESTION: WHY ISN"T THIS EQUAL TO STIM_SIZE OR VICE VERSA? +NUM_TRIALS = 48 # number of stimuli presented in a sequence +STIM_SIZE=20 # length of stimulus vector +CONTEXT_SIZE=25 # length of context vector +HIDDEN_SIZE=STIM_SIZE*4 # dimension of hidden units in ff +CONTEXT_DRIFT_RATE=.1 # drift rate used for DriftOnASphereIntegrator (function of Context mech) on each trial +CONTEXT_DRIFT_NOISE=0.0 # noise used by DriftOnASphereIntegrator (function of Context mech) +STIM_WEIGHT=.05 # weighting of stimulus field in retrieval from em +CONTEXT_WEIGHT = 1-STIM_WEIGHT # weighting of context field in retrieval from em +SOFT_MAX_TEMP=1/8 # express as gain # precision of retrieval process +HAZARD_RATE=0.04 # rate of re=sampling of em following non-match determination in a pass through ffn + +# MECHANISM AND COMPOSITION NAMES: +FFN_COMPOSITION = "WORKING MEMORY (fnn)" +FFN_STIMULUS_INPUT = "CURRENT STIMULUS" +FFN_CONTEXT_INPUT = "CURRENT CONTEXT" +FFN_STIMULUS_RETRIEVED = "RETRIEVED STIMULUS" +FFN_CONTEXT_RETRIEVED = "RETRIEVED CONTEXT" +FFN_TASK = "CURRENT TASK" +FFN_HIDDEN = "HIDDEN LAYER" +FFN_OUTPUT = "DECISION LAYER" +MODEL_STIMULUS_INPUT ='STIM' +MODEL_CONTEXT_INPUT = 'CONTEXT' +MODEL_TASK_INPUT = "TASK" +EM = "EPISODIC MEMORY (dict)" +CONTROLLER = "READ/WRITE CONTROLLER" + +# ======================================== MODEL CONSTRUCTION ========================================================= + +def construct_model(stim_size = STIM_SIZE, + context_size = CONTEXT_SIZE, + hidden_size = HIDDEN_SIZE, + num_nback_levels = NUM_NBACK_LEVELS, + context_drift_noise = CONTEXT_DRIFT_NOISE, + retrievel_softmax_temp = SOFT_MAX_TEMP, + retrieval_hazard_rate = HAZARD_RATE, + retrieval_stimulus_weight = STIM_WEIGHT, + context_stimulus_weight = CONTEXT_WEIGHT): + """Construct nback_model""" + + # FEED FORWARD NETWORK ----------------------------------------- -# TODO: -# Nback:: -# - separate out stim/context external inputs from those from EM into FFN -# - figure out how to specify feedback from DDM to EM: -# - figure out how to execute EM twice: -# > first, at beginning of trial, to retrieve item based on current stimulus & context -# (with prob retrieval = 1, prob storage = 0) -# > second time, at end of trial (under influence of ControlMechanism) to encode current stimulus & context -# (with prob storage = 1; prob of retrieval = 0) -# scheduler.add_condition(A, pnl.AfterNCalls(CM, 1)) -# scheduler.add_condition(CM, pnl.Always()) -# composition.run(...termination_conds={pnl.TimeScale.TRIAL: pnl.And(pnl.AfterNCalls(CM, 2), pnl.JustRan(CM))}) -# - implement circular drift as function for an input mechanism -# - ADD PNL FEATURE: should be able to use InputPort as spec for a pathway (if there is nothing after it); -# same for OutputPort (if there is nothing before it) - - -#region N-BACK MODEL -def n_back_model(): - - # Input Mechs - stim = TransferMechanism(name='STIM', size=5) - context = TransferMechanism(name='CONTEXT', size=5) - - # Feedforward Network: - stim_input_layer = TransferMechanism(name='STIM INPUT LAYER', size=5) - context_input_layer = TransferMechanism(name='CONTEXT INPUT LAYER', size=5) - match_output_layer = TransferMechanism(name='MATCH LAYER', size=1) - # ffn = AutodiffComposition(name='FFN', pathways=[[stim_input,match_output], [context_input, match_output]]) - ffn = Composition(name='FFN', pathways=[[stim_input_layer, match_output_layer], - [context_input_layer, match_output_layer]]) - - # Episodic Memory, Decision and Control - # em = EpisodicMemoryMechanism(name='EM', content_size=5, assoc_size=5) - em = EpisodicMemoryMechanism(name='EM', size=5, - # function=DictionaryMemory(initializer=[[[0,0,0,0,0],[0,0,0,0,0]]]) + # inputs: encoding of current stimulus and context, retrieved stimulus and retrieved context, + # output: decIsion: match [1,0] or non-match [0,1] + # Must be trained to detect match for specified task (1-back, 2-back, etc.) + input_current_stim = TransferMechanism(size=STIM_SIZE, function=Linear, name=FFN_STIMULUS_INPUT) # function=Logistic) + input_current_context = TransferMechanism(size=STIM_SIZE, function=Linear, name=FFN_CONTEXT_INPUT) # function=Logistic) + input_retrieved_stim = TransferMechanism(size=STIM_SIZE, function=Linear, name=FFN_STIMULUS_RETRIEVED) # function=Logistic) + input_retrieved_context = TransferMechanism(size=STIM_SIZE, function=Linear, name=FFN_CONTEXT_RETRIEVED) # function=Logistic) + input_task = TransferMechanism(size=NUM_NBACK_LEVELS, function=Linear, name=FFN_TASK) # function=Logistic) + hidden = TransferMechanism(size=HIDDEN_SIZE, function=Logistic, name=FFN_HIDDEN) + decision = ProcessingMechanism(size=2, name=FFN_OUTPUT) + # TODO: THIS NEEDS TO BE REPLACED BY (OR AT LEAST TRAINED AS) AutodiffComposition + # TRAINING: + # - 50% matches and 50% non-matches + # - all possible stimuli + # - 2back and 3back + # - contexts of various distances + ffn = Composition([{input_current_stim, + input_current_context, + input_retrieved_stim, + input_retrieved_context, + input_task}, + hidden, decision], + name=FFN_COMPOSITION) + + # FULL MODEL (Outer Composition, including input, EM and control Mechanisms) ------------------------ + + # Stimulus Encoding: takes STIM_SIZE vector as input + stim = TransferMechanism(name=MODEL_STIMULUS_INPUT, size=STIM_SIZE) + + # Context Encoding: takes scalar as drift step for current trial + context = ProcessingMechanism(name=MODEL_CONTEXT_INPUT, + function=DriftOnASphereIntegrator( + initializer=np.random.random(CONTEXT_SIZE-1), + noise=CONTEXT_DRIFT_NOISE, + dimension=CONTEXT_SIZE)) + + # Task: task one-hot indicating n-back (1, 2, 3 etc.) - must correspond to what ffn has been trained to do + task = ProcessingMechanism(name=MODEL_TASK_INPUT, size=NUM_NBACK_LEVELS) + + # Episodic Memory: + # - entries: stimulus (field[0]) and context (field[1]); randomly initialized + # - uses Softmax to retrieve best matching input, subject to weighting of stimulus and context by STIM_WEIGHT + em = EpisodicMemoryMechanism(name=EM, + input_ports=[{NAME:"STIMULUS_FIELD", + SIZE:STIM_SIZE}, + {NAME:"CONTEXT_FIELD", + SIZE:CONTEXT_SIZE}], + function=ContentAddressableMemory( + initializer=[[[0]*STIM_SIZE, [0]*CONTEXT_SIZE]], + distance_field_weights=[STIM_WEIGHT, CONTEXT_WEIGHT], + # equidistant_entries_select=NEWEST, + selection_function=SoftMax(output=MAX_INDICATOR, + gain=SOFT_MAX_TEMP)), ) - ctl = ControlMechanism(control=(STORAGE_PROB, em)) - decision = DDM(name='DECISION') - - resp_decision = Pathway([match_output_layer, (decision, NodeRole.OUTPUT)]) - # FIX: ENHANCE add_linear_processing_pathway TO SUPPORT InputPort at end, or OutputPort at beginning: - # stimulus_encoding = [stim, em.input_ports[KEY_INPUT]] - # context_encoding = [context, em.input_ports[VALUE_INPUT]] - - # MappingProjection(sender=stim, receiver=stim_input_layer) - # MappingProjection(sender=stim, receiver=em.input_ports[KEY_INPUT]) - # MappingProjection(sender=context, receiver=context_input_layer) - # MappingProjection(sender=context, receiver=em.input_ports[VALUE_INPUT]) - # MappingProjection(sender=em.output_ports[KEY_OUTPUT], receiver=stim_input_layer) - # MappingProjection(sender=em.output_ports[VALUE_OUTPUT], receiver=context_input_layer) - # stim_processing = Pathway([stim, ffn]) - # context_processing = Pathway([context, ffn]) - # stim_encoding = Pathway([stim, em]) - # context_encoding = Pathway([context, em]) - # stim_retrieval = Pathway([em, stim_input_layer]) - # context_retrieval = Pathway([em, context_input_layer]) - # storage = Pathway([(decision, NodeRole.OUTPUT), (ctl, NodeRole.FEEDBACK_SENDER), em]) - # # FIX: show_graph NOT RECOGNIZING STIM->STIM_INPUT_LAYER AND CONTEXT->CONTEXT_INPUT_LAYER - # comp = Composition(pathways=[stim_processing, - # context_processing, - # ffn, - # context_encoding, - # stim_encoding, - # resp_decision, - # stim_retrieval, - # context_retrieval, - # storage]) - # FIX: show_graph NOT RECOGNIZING STIM->STIM_INPUT_LAYER AND CONTEXT->CONTEXT_INPUT_LAYER - # comp = Composition(pathways=[[stim, ffn], - # [stim,em], - # [context,ffn], - # [context,em], - # [em,ffn], - # [ffn, em], - # [ffn, decision, ctl, em]]) - - # comp = Composition(pathways=[ffn, - # [stim, stim_input_layer], - # [stim, MappingProjection(stim, em.input_ports[KEY_INPUT]), em], - # [context, context_input_layer], - # [context, MappingProjection(context, em.input_ports[VALUE_INPUT]), em], - # [em,stim_input_layer], - # [em,context_input_layer], - # [ffn, decision, ctl, em]]) - - comp = Composition() - comp.add_nodes([stim, context, ffn, em, (decision, NodeRole.OUTPUT), ctl]) - comp.add_projection(MappingProjection(), stim, stim_input_layer) - comp.add_projection(MappingProjection(), context, context_input_layer) - comp.add_projection(MappingProjection(), stim, em.input_ports[KEY_INPUT]) - comp.add_projection(MappingProjection(), context, em.input_ports[VALUE_INPUT]) - comp.add_projection(MappingProjection(), em.output_ports[KEY_OUTPUT], stim_input_layer) - comp.add_projection(MappingProjection(), em.output_ports[VALUE_OUTPUT], context_input_layer) - comp.add_projection(MappingProjection(), match_output_layer, decision) - comp.add_projection(MappingProjection(), decision, ctl) - # comp.add_projection(MappingProjection(), decision, stim_input_layer) - - # comp._analyze_graph() - comp.show_graph() - # comp.show_graph(show_cim=True, - # show_node_structure=ALL, - # show_projection_labels=True, - # show_dimensions=True) - # comp.show_graph(show_cim=True, - # show_node_structure=ALL, - # show_projection_labels=True, - # show_dimensions=True) - # comp.run(inputs={stim:[1,2,3,4,5], - # context:[6,7,8,9,10]}, - # report_output=ReportOutput.ON) - # comp.run(inputs={a:2.5}, report_output=ReportOutput.FULL) -#endregion -n_back_model() + + # Control Mechanism + # Ensures current stimulus and context are only encoded in EM once (at beginning of trial) + # by controlling the storage_prob parameter of em: + # - if outcome of decision signifies a match or hazard rate is realized: + # - set EM[store_prob]=1 (as prep encoding stimulus in EM on next trial) + # - this also serves to terminate trial (see nback_model.termination_processing condition) + # - if outcome of decision signifies a non-match + # - set EM[store_prob]=0 (as prep for another retrieval from EM without storage) + # - continue trial + control = ControlMechanism(name=CONTROLLER, + default_variable=[[1]], # Ensure EM[store_prob]=1 at beginning of first trial + # # VERSION *WITH* ObjectiveMechanism: + objective_mechanism=ObjectiveMechanism(name="OBJECTIVE MECHANISM", + monitor=decision, + # Outcome=1 if match, else 0 + function=lambda x: int(x[0][1]>x[0][0])), + # Set ControlSignal for EM[store_prob] + function=lambda outcome: int(bool(outcome) or (np.random.random() > HAZARD_RATE)), + # # VERSION *WITHOUT* ObjectiveMechanism: + # monitor_for_control=decision, + # # Set Evaluate outcome and set ControlSignal for EM[store_prob] + # # - outcome is received from decision as one hot in the form: [[match, no-match]] + # function=lambda outcome: int(int(outcome[0][1]>outcome[0][0]) + # or (np.random.random() > HAZARD_RATE)), + control=(STORAGE_PROB, em)) + + nback_model = Composition(nodes=[stim, context, task, em, ffn, control], + # # # Terminate trial if value of control is still 1 after first pass through execution + # # FIX: STOPS AFTER ~ NUMBER OF TRIALS (?90+); SHOULD BE: NUM_TRIALS*NUM_NBACK_LEVELS + 1 + # termination_processing={TimeScale.TRIAL: And(Condition(lambda: control.value), + # AfterPass(0, TimeScale.TRIAL))}, + name="N-Back Model") + # # Terminate trial if value of control is still 1 after first pass through execution + # # FIX: ALL OF THE FOLLOWING STOP AFTER ~ NUMBER OF TRIALS (?90+); SHOULD BE: NUM_TRIALS*NUM_NBACK_LEVELS + 1 + # nback_model.scheduler.add_condition(nback_model, And(Condition(lambda: control.value), AfterPass(0, TimeScale.TRIAL))) + # nback_model.scheduler.termination_conds = ({TimeScale.TRIAL: And(Condition(lambda: control.value), + # AfterPass(0, TimeScale.TRIAL))}) + # nback_model.scheduler.termination_conds.update({TimeScale.TRIAL: And(Condition(lambda: control.value), + # AfterPass(0, TimeScale.TRIAL))}) + nback_model.add_projection(MappingProjection(), stim, input_current_stim) + nback_model.add_projection(MappingProjection(), context, input_current_context) + nback_model.add_projection(MappingProjection(), task, input_task) + nback_model.add_projection(MappingProjection(), em.output_ports["RETRIEVED_STIMULUS_FIELD"], input_retrieved_stim) + nback_model.add_projection(MappingProjection(), em.output_ports["RETRIEVED_CONTEXT_FIELD"], input_retrieved_context) + nback_model.add_projection(MappingProjection(), stim, em.input_ports["STIMULUS_FIELD"]) + nback_model.add_projection(MappingProjection(), context, em.input_ports["CONTEXT_FIELD"]) + + if DISPLAY: + nback_model.show_graph( + # show_cim=True, + # show_node_structure=ALL, + # show_dimensions=True) + ) + + return nback_model + +# ==========================================STIMULUS GENERATION ======================================================= +# Based on nback-paper + +def get_stim_set(num_stim=STIM_SIZE): + """Construct an array of stimuli for use an experiment""" + # For now, use one-hots + return np.eye(num_stim) + +def get_task_input(nback_level): + """Construct input to task Mechanism for a given nback_level, used by run_model() and train_model()""" + task_input = list(np.zeros_like(NBACK_LEVELS)) + task_input[nback_level-NBACK_LEVELS[0]] = 1 + return task_input + +def get_run_inputs(model, nback_level, num_trials): + """Construct set of stimulus inputs for run_model()""" + + def generate_stim_sequence(nback_level, trial_num, stype=0, num_stim=NUM_STIM, num_trials=NUM_TRIALS): + + def gen_subseq_stim(): + A = np.random.randint(0,num_stim) + B = np.random.choice( + np.setdiff1d(np.arange(num_stim),[A]) + ) + C = np.random.choice( + np.setdiff1d(np.arange(num_stim),[A,B]) + ) + X = np.random.choice( + np.setdiff1d(np.arange(num_stim),[A,B]) + ) + return A,B,C,X + + def genseqCT(nback_level,trial_num): + assert nback_level in {2,3} + # ABXA / AXA + seq = np.random.randint(0,num_stim,num_trials) + A,B,C,X = gen_subseq_stim() + # + if nback_level==3: + subseq = [A,B,X,A] + elif nback_level==2: + subseq = [A,X,A] + seq[trial_num-(nback_level+1):trial_num] = subseq + return seq[:trial_num] + + def genseqCF(nback_level,trial_num): + # ABXC + seq = np.random.randint(0,num_stim,num_trials) + A,B,C,X = gen_subseq_stim() + # + if nback_level==3: + subseq = [A,B,X,C] + elif nback_level==2: + subseq = [A,X,B] + seq[trial_num-(nback_level+1):trial_num] = subseq + return seq[:trial_num] + + def genseqLT(nback_level,trial_num): + # AAXA + seq = np.random.randint(0,num_stim,num_trials) + A,B,C,X = gen_subseq_stim() + # + if nback_level==3: + subseq = [A,A,X,A] + elif nback_level==2: + subseq = [A,A,A] + seq[trial_num-(nback_level+1):trial_num] = subseq + return seq[:trial_num] + + def genseqLF(nback_level,trial_num): + # ABXB + seq = np.random.randint(0,num_stim,num_trials) + A,B,C,X = gen_subseq_stim() + # + if nback_level==3: + subseq = [A,B,X,B] + elif nback_level==2: + subseq = [X,A,A] + seq[trial_num-(nback_level+1):trial_num] = subseq + return seq[:trial_num] + + genseqL = [genseqCT,genseqLT,genseqCF,genseqLF] + stim_seq = genseqL[stype](nback_level,trial_num) + # ytarget = [1,1,0,0][stype] + # ctxt = spherical_drift(trial_num) + # return stim,ctxt,ytarget + return stim_seq + + def stim_set_generation(nback_level, num_trials): + stim_sequence = [] + # for seq_int, trial in itertools.product(range(4),np.arange(5,trials)): # This generates all length sequences + for seq_int, trial_num in itertools.product(range(4),[num_trials]): # This generates only longest seq (num_trials) + return stim_sequence.append(generate_stim_sequence(nback_level, trial_num, stype=seq_int, trials=num_trials)) + + def get_input_sequence(nback_level, num_trials=NUM_TRIALS): + """Get sequence of inputs for a run""" + input_set = get_stim_set() + # Construct sequence of stimulus indices + trial_seq = generate_stim_sequence(nback_level, num_trials) + # Return list of corresponding stimulus input vectors + return [input_set[trial_seq[i]] for i in range(num_trials)] + + return {model.nodes[MODEL_STIMULUS_INPUT]: get_input_sequence(nback_level, num_trials), + model.nodes[MODEL_CONTEXT_INPUT]: [[CONTEXT_DRIFT_RATE]]*num_trials, + model.nodes[MODEL_TASK_INPUT]: [get_task_input(nback_level)]*num_trials} + +def get_training_inputs(network, num_epochs, nback_levels): + """Construct set of training stimuli for ffn.learn(), used by train_model() + Construct one example of each condition: + match: stim_current = stim_retrieved and context_current = context_retrieved + stim_lure: stim_current = stim_retrieved and context_current != context_retrieved + context_lure: stim_current != stim_retrieved and context_current == context_retrieved + non_lure: stim_current != stim_retrieved and context_current != context_retrieved + """ + assert is_iterable(nback_levels) and all([0`. angle_function : TransferFunction - determines the function used to compute angle (reproted as result) from coordinates on sphere specified by - coordinates in `previous_value ` displace by `variable + determines the function used to compute angle (reported as result) from coordinates on sphere specified by + coordinates in `previous_value ` displaced by `variable ` and possibly `noise `. previous_time : float @@ -2876,7 +2876,7 @@ class Parameters(IntegratorFunction.Parameters): dimension see `dimension ` - :default value: 2 + :default value: 3 :type: ``int`` enable_output_type_conversion @@ -2950,7 +2950,7 @@ class Parameters(IntegratorFunction.Parameters): time_step_size = Parameter(1.0, modulable=True) previous_time = Parameter(0.0, initializer='starting_point', pnl_internal=True) dimension = Parameter(3, stateful=False, read_only=True) - initializer = Parameter([0], initalizer='variable', stateful=True) + initializer = Parameter([0], initalizer='variable', dependencies=dimension, stateful=True) angle_function = Parameter(None, stateful=False, loggable=False) random_state = Parameter(None, loggable=False, getter=_random_state_getter, dependencies='seed') seed = Parameter(DEFAULT_SEED, modulable=True, fallback_default=True, setter=_seed_setter) diff --git a/psyneulink/core/components/mechanisms/modulatory/control/controlmechanism.py b/psyneulink/core/components/mechanisms/modulatory/control/controlmechanism.py index b4d82a6662e..a100ed485a2 100644 --- a/psyneulink/core/components/mechanisms/modulatory/control/controlmechanism.py +++ b/psyneulink/core/components/mechanisms/modulatory/control/controlmechanism.py @@ -357,18 +357,18 @@ By default, a ControlMechanism has a single `input_port ` named *OUTCOME*. If it has an `objective_mechanism `, then the *OUTCOME* `input_port ` receives a single `MappingProjection` from the `objective_mechanism -`\\'s *OUTCOME* `OutputPort` (see `ControlMechanism_ObjectiveMechanism` for -additional details). If the ControlMechanism has no `objective_mechanism ` then, -when it is added to a `Composition`, MappingProjections are created from the items specified in `monitor_for_control -` directly to InputPorts on the ControlMechanism (see -`ControlMechanism_Monitor_for_Control` for additional details). The number of InputPorts created, and how the items -listed in `monitor_for_control ` project to them is deterimined by the -ControlMechanism's `outcome_input_ports_option `. All of the Inports -that receive Projections from those items, or the `objective_mechanism ` if -the ControlMechanism has one, are listed in its `outcome_input_ports ` attribute, -and their values in the `outcome ` attribute. The latter is used as the input to the -ControlMechanism's `function ` to determine its `control_allocation -`. +`\\'s *OUTCOME* `OutputPort ` (see +`ControlMechanism_ObjectiveMechanism` for additional details). If the ControlMechanism has no `objective_mechanism +` then, when it is added to a `Composition`, MappingProjections are created +from the items specified in `monitor_for_control ` directly to InputPorts on +the ControlMechanism (see `ControlMechanism_Monitor_for_Control` for additional details). The number of InputPorts +created, and how the items listed in `monitor_for_control ` project to them is +deterimined by the ControlMechanism's `outcome_input_ports_option `. +All of the Inports that receive Projections from those items, or the `objective_mechanism +` if the ControlMechanism has one, are listed in its `outcome_input_ports +` attribute, and their values in the `outcome ` +attribute. The latter is used as the input to the ControlMechanism's `function ` to +determine its `control_allocation `. .. _ControlMechanism_Function: @@ -857,7 +857,7 @@ class ControlMechanism(ModulatoryMechanism_Base): outcome_input_ports_option : , SEPARATE, COMBINE, or CONCATENATE determines how items specified in `monitor_for_control ` project to - the ControlMechanism if not `objective_mechanism ` is specified. If + the ControlMechanism if no `objective_mechanism ` is specified. If *SEPARATE* is specified (the default), the `Projection` from each item specified in `monitor_for_control ` is assigned its own `InputPort`. All of the InputPorts are assigned to a list in the ControlMechanism's `outcome_input_ports ` attribute. @@ -1300,7 +1300,7 @@ def __init__(self, ) def _validate_params(self, request_set, target_set=None, context=None): - """Validate SYSTEM, monitor_for_control, CONTROL_SIGNALS and GATING_SIGNALS + """Validate monitor_for_control, objective_mechanism, CONTROL_SIGNALS and GATING_SIGNALS """ from psyneulink.core.components.mechanisms.processing.objectivemechanism import ObjectiveMechanism diff --git a/psyneulink/core/compositions/composition.py b/psyneulink/core/compositions/composition.py index df4aba619eb..cd5da94cc78 100644 --- a/psyneulink/core/compositions/composition.py +++ b/psyneulink/core/compositions/composition.py @@ -6764,7 +6764,7 @@ def _get_node_specs_for_entry(entry, include_roles=None, exclude_roles=None): if all(_is_node_spec(entry) for entry in current_entry): receivers = _get_node_specs_for_entry(current_entry, NodeRole.INPUT, NodeRole.TARGET) # The preceding entry is a Node or set of them: - # - if it is a set, list or array, leave as is, else place in set for consistnecy of processin below + # - if it is a set, list or array, leave as is, else place in set for consistency of processing below preceding_entry = (pathway[c - 1] if isinstance(pathway[c - 1], (set, list, np.ndarray)) else {pathway[c - 1]}) if all(_is_node_spec(sender) for sender in preceding_entry): @@ -9583,12 +9583,9 @@ def run( specifies fuction to call after each `TRIAL ` is executed. termination_processing : Condition : default None - specifies - `termination Conditions ` - to be used for the current `RUN `. To change - these conditions for all future runs, use - `Composition.termination_processing` (or - `Scheduler.termination_conds`) + specifies `termination Conditions ` + to be used for the current `RUN `. To change these conditions for all future runs, + use `Composition.termination_processing` (or `Scheduler.termination_conds`) skip_analyze_graph : bool : default False setting to True suppresses call to _analyze_graph() diff --git a/psyneulink/core/globals/keywords.py b/psyneulink/core/globals/keywords.py index 713e9e16dfb..e08ab37b0ca 100644 --- a/psyneulink/core/globals/keywords.py +++ b/psyneulink/core/globals/keywords.py @@ -37,8 +37,8 @@ 'ContentAddressableMemory_FUNCTION', 'CONTEXT', 'CONTROL', 'CONTROL_MECHANISM', 'CONTROL_PATHWAY', 'CONTROL_PROJECTION', 'CONTROL_PROJECTION_PARAMS', 'CONTROL_PROJECTIONS', 'CONTROL_SIGNAL', 'CONTROL_SIGNAL_SPECS', 'CONTROL_SIGNALS', 'CONTROLLED_PARAMS', - 'CONTROLLER', 'CONTROLLER_OBJECTIVE', 'CORRELATION', 'COSINE', 'COST_FUNCTION', 'COUNT', 'CROSS_ENTROPY', - 'CURRENT_EXECUTION_TIME', 'CUSTOM_FUNCTION', 'CYCLE', + 'CONTROLLER', 'CONTROLLER_OBJECTIVE', 'CORRELATION', 'COSINE', 'COSINE_SIMILARITY', + 'COST_FUNCTION', 'COUNT', 'CROSS_ENTROPY', 'CURRENT_EXECUTION_TIME', 'CUSTOM_FUNCTION', 'CYCLE', 'DDM_MECHANISM', 'DECAY', 'DEFAULT', 'DEFAULT_CONTROL_MECHANISM', 'DEFAULT_INPUT', 'DEFAULT_MATRIX', 'DEFAULT_PREFERENCE_SET_OWNER', 'DEFAULT_PROCESSING_MECHANISM', 'DEFAULT_VARIABLE', 'DEFERRED_ASSIGNMENT', 'DEFERRED_DEFAULT_NAME', 'DEFERRED_INITIALIZATION', 'DICT', 'DictionaryMemory_FUNCTION', @@ -240,6 +240,7 @@ def __init__(self): self.CORRELATION = CORRELATION # self.PEARSON = PEARSON self.COSINE = COSINE + self.COSINE_SIMILARITY = COSINE self.ENTROPY = CROSS_ENTROPY self.CROSS_ENTROPY = CROSS_ENTROPY self.ENERGY = ENERGY @@ -270,6 +271,7 @@ def _is_metric(metric): ANGLE = 'angle' CORRELATION = 'correlation' COSINE = 'cosine' +COSINE_SIMILARITY = 'cosine' PEARSON = 'Pearson' ENTROPY = 'cross-entropy' CROSS_ENTROPY = 'cross-entropy' diff --git a/psyneulink/library/components/mechanisms/processing/integrator/episodicmemorymechanism.py b/psyneulink/library/components/mechanisms/processing/integrator/episodicmemorymechanism.py index 3286eff87c8..7cd65dc3f84 100644 --- a/psyneulink/library/components/mechanisms/processing/integrator/episodicmemorymechanism.py +++ b/psyneulink/library/components/mechanisms/processing/integrator/episodicmemorymechanism.py @@ -365,7 +365,7 @@ as `input_ports `, named correspondingly ``RETRIEVED_FIELD_n``:: >>> my_em.output_ports.names - ['RETREIVED_FIELD_0', 'RETREIVED_FIELD_1'] + ['RETRIEVED_FIELD_0', 'RETRIEVED_FIELD_1'] These are assigned the values of the fields of the entry retrieved from `memory `. @@ -427,7 +427,7 @@ VALUE_OUTPUT = 'VALUE_OUTPUT' DEFAULT_INPUT_PORT_NAME_PREFIX = 'FIELD_' DEFAULT_INPUT_PORT_NAME_SUFFIX = '_INPUT' -DEFAULT_OUTPUT_PORT_PREFIX = 'RETREIVED_' +DEFAULT_OUTPUT_PORT_PREFIX = 'RETRIEVED_' class EpisodicMemoryMechanismError(Exception): diff --git a/tests/functions/test_integrator.py b/tests/functions/test_integrator.py index 26600d7cc7c..30771433a4f 100644 --- a/tests/functions/test_integrator.py +++ b/tests/functions/test_integrator.py @@ -67,7 +67,6 @@ def AdaptiveIntFun(init, value, iterations, noise, rate, offset, **kwargs): return [3.59649986, 3.28818534, 2.45181396, 3.14321808, 1.56270704, 2.88397872, 1.62818492, 3.72575501, 2.80657186, 2.2131637] - def DriftIntFun(init, value, iterations, noise, **kwargs): assert iterations == 3 if np.isscalar(noise): @@ -108,7 +107,6 @@ def LeakyFun(init, value, iterations, noise, **kwargs): else: return [3.12748415, 2.76778478, 2.45911505, 3.06686514, 1.6311395, 2.19281309, 1.61148745, 3.23404557, 2.81418859, 2.63042344] - def AccumulatorFun(init, value, iterations, noise, **kwargs): assert iterations == 3 @@ -169,7 +167,7 @@ def test_execute(func, func_mode, variable, noise, params, benchmark): if 'DriftOnASphereIntegrator' in func[0].componentName: if func_mode != 'Python': - pytest.skip("DriftDiffusionIntegrator not yet compiled") + pytest.skip("DriftOnASphereIntegrator not yet compiled") params.update({'dimension':len(variable) + 1}) else: if 'dimension' in params: @@ -263,6 +261,43 @@ def test_integrator_function_with_default_variable_and_params_of_different_lengt "NOISE_SCALAR", "NOISE_2", "NOISE_3", "NOISE_4" ] + +def test_DriftOnASphere_identicalness_against_reference_implementation(): + """Compare against reference implementation in nback-paper model (https://github.com/andrebeu/nback-paper).""" + + # PNL DriftOnASphere + DoS = Functions.DriftOnASphereIntegrator(dimension=5, initializer=np.array([.2] * (4)), noise=0.0) + results_dos = [] + for i in range(3): + results_dos.append(DoS(.1)) + + # nback-paper implementation + def spherical_drift(n_steps=3, dim=5, var=0, mean=.1): + def convert_spherical_to_angular(dim, ros): + ct = np.zeros(dim) + ct[0] = np.cos(ros[0]) + prod = np.product([np.sin(ros[k]) for k in range(1, dim - 1)]) + n_prod = prod + for j in range(dim - 2): + n_prod /= np.sin(ros[j + 1]) + amt = n_prod * np.cos(ros[j + 1]) + ct[j + 1] = amt + ct[dim - 1] = prod + return ct + # initialize the spherical coordinates to ensure each context run begins in a new random location on the unit sphere + ros = np.array([.2] *(dim - 1)) + slen = n_steps + ctxt = np.zeros((slen, dim)) + for i in range(slen): + noise = np.random.normal(mean, var, size=(dim - 1)) # add a separately-drawn Gaussian to each spherical coord + ros += noise + ctxt[i] = convert_spherical_to_angular(dim, ros) + return ctxt + results_sd = spherical_drift() + + assert np.allclose(np.array(results_dos), np.array(results_sd)) + + # FIX: CROSS WITH INITIALIZER SIZE: @pytest.mark.parametrize("params, error_msg, error_type", test_vars, ids=names) def test_drift_on_a_sphere_errors(params, error_msg, error_type): diff --git a/tests/mechanisms/test_episodic_memory.py b/tests/mechanisms/test_episodic_memory.py index ab27e385c9a..c50b89cc364 100644 --- a/tests/mechanisms/test_episodic_memory.py +++ b/tests/mechanisms/test_episodic_memory.py @@ -76,7 +76,7 @@ def test_with_dictionary_memory(variable, func, params, expected, benchmark, mec # expected input_port names ['FIELD_0_INPUT'], # expected output_port names - ['RETREIVED_FIELD_0'], + ['RETRIEVED_FIELD_0'], # expected output [[0,0]] ), @@ -94,7 +94,7 @@ def test_with_dictionary_memory(variable, func, params, expected, benchmark, mec # expected input_port names ['FIELD_0_INPUT', 'FIELD_1_INPUT', 'FIELD_2_INPUT'], # expected output_port names - ['RETREIVED_FIELD_0', 'RETREIVED_FIELD_1', 'RETREIVED_FIELD_2'], + ['RETRIEVED_FIELD_0', 'RETRIEVED_FIELD_1', 'RETRIEVED_FIELD_2'], # expected output [[0,0],[0,0],[0,0,0]] ), @@ -105,7 +105,7 @@ def test_with_dictionary_memory(variable, func, params, expected, benchmark, mec {'default_variable': [[0],[0,0],[0,0,0]]}, [[10.],[20., 30.],[40., 50., 60.]], ['FIELD_0_INPUT', 'FIELD_1_INPUT', 'FIELD_2_INPUT'], - ['RETREIVED_FIELD_0', 'RETREIVED_FIELD_1', 'RETREIVED_FIELD_2'], + ['RETRIEVED_FIELD_0', 'RETRIEVED_FIELD_1', 'RETRIEVED_FIELD_2'], [[0],[0,0],[0,0,0]] ), ( @@ -117,7 +117,7 @@ def test_with_dictionary_memory(variable, func, params, expected, benchmark, mec {'size':[1,2,3]}, [[10.],[20., 30.],[40., 50., 60.]], ['FIELD_0_INPUT', 'FIELD_1_INPUT', 'FIELD_2_INPUT'], - ['RETREIVED_FIELD_0', 'RETREIVED_FIELD_1', 'RETREIVED_FIELD_2'], + ['RETRIEVED_FIELD_0', 'RETRIEVED_FIELD_1', 'RETRIEVED_FIELD_2'], # [[10.],[20., 30.],[40., 50., 60.]] [[1], [2,3], [4,5,6]] # <- distance = 0 to [[10.],[20., 30.],[40., 50., 60.]] ), @@ -130,7 +130,7 @@ def test_with_dictionary_memory(variable, func, params, expected, benchmark, mec {'default_variable': [[0],[0,0],[0,0,0]], 'input_ports':['hello','world','goodbye']}, [[10.],[20., 30.],[40., 50., 60.]], ['hello', 'world', 'goodbye'], - ['RETREIVED_hello', 'RETREIVED_world', 'RETREIVED_goodbye'], + ['RETRIEVED_hello', 'RETRIEVED_world', 'RETRIEVED_goodbye'], [[1.],[2., 3.],[4., 5., 6.]] ), ( @@ -142,7 +142,7 @@ def test_with_dictionary_memory(variable, func, params, expected, benchmark, mec {'size':[2,2,2]}, [[11,13], [22,23], [34, 35]], ['FIELD_0_INPUT', 'FIELD_1_INPUT', 'FIELD_2_INPUT'], - ['RETREIVED_FIELD_0', 'RETREIVED_FIELD_1', 'RETREIVED_FIELD_2'], + ['RETRIEVED_FIELD_0', 'RETRIEVED_FIELD_1', 'RETRIEVED_FIELD_2'], [[11,12], [22,23], [34, 35]], ), ( @@ -157,7 +157,7 @@ def test_with_dictionary_memory(variable, func, params, expected, benchmark, mec {'default_variable':[[0,0],[0,0],[0,0]]}, [[10,20], [30,40], [50, 60]], ['FIELD_0_INPUT', 'FIELD_1_INPUT', 'FIELD_2_INPUT'], - ['RETREIVED_FIELD_0', 'RETREIVED_FIELD_1', 'RETREIVED_FIELD_2'], + ['RETRIEVED_FIELD_0', 'RETRIEVED_FIELD_1', 'RETRIEVED_FIELD_2'], [[10,20], [30,40], [50, 60]], ), ( @@ -168,7 +168,7 @@ def test_with_dictionary_memory(variable, func, params, expected, benchmark, mec 'input_ports':['FIRST','SECOND']}, [[10,20], [30,40]], ['FIRST', 'SECOND'], - ['RETREIVED_FIRST', 'RETREIVED_SECOND'], + ['RETRIEVED_FIRST', 'RETRIEVED_SECOND'], [[0,0], [0,0]], ), ( @@ -180,7 +180,7 @@ def test_with_dictionary_memory(variable, func, params, expected, benchmark, mec 'input_ports':['FIRST','SECOND']}, [[10,20], [30,40]], ['FIRST', 'SECOND'], - ['RETREIVED_FIRST', 'RETREIVED_SECOND'], + ['RETRIEVED_FIRST', 'RETRIEVED_SECOND'], [[10,20], [30,40]], ), ( @@ -191,7 +191,7 @@ def test_with_dictionary_memory(variable, func, params, expected, benchmark, mec 'input_ports':['FIRST','SECOND']}, [[10,20], [30,40]], ['FIRST', 'SECOND'], - ['RETREIVED_FIRST', 'RETREIVED_SECOND'], + ['RETRIEVED_FIRST', 'RETRIEVED_SECOND'], [[11,12],[22, 23]], ) ] From 255236b35e90782084523e5048d90766a9371919 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Nov 2022 02:33:03 +0000 Subject: [PATCH 038/127] requirements: update pillow requirement from <9.3.0 to <9.4.0 (#2515) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8ba27238b1c..c0f11406a12 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ matplotlib<3.6.2 modeci_mdf<0.5, >=0.3.4; (platform_machine == 'AMD64' or platform_machine == 'x86_64') and platform_python_implementation == 'CPython' and implementation_name == 'cpython' networkx<2.9 numpy<1.21.7, >=1.17.0 -pillow<9.3.0 +pillow<9.4.0 pint<0.21.0 toposort<1.8 torch>=1.8.0, <1.13.0; (platform_machine == 'AMD64' or platform_machine == 'x86_64') and platform_python_implementation == 'CPython' and implementation_name == 'cpython' From 3b204630358c99fbe417118411cc104a91dbbce0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Nov 2022 18:28:52 +0000 Subject: [PATCH 039/127] requirements: update pytest requirement from <7.1.4 to <7.2.1 (#2508) --- dev_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index 9a8ac773e85..223bc004f4a 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,5 +1,5 @@ jupyter<=1.0.0 -pytest<7.1.4 +pytest<7.2.1 pytest-benchmark<4.0.1 pytest-cov<4.0.1 pytest-helpers-namespace<2021.12.30 From 05eb311cb0a7c5f077d708cf64f083e8896da97e Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Tue, 1 Nov 2022 21:20:42 -0400 Subject: [PATCH 040/127] github-actions: Restrict statsmodels to <0.13.3 on x86 (#2516) statsmodels doesn't provide 32-bit wheels for >=0.13.3, and the build process tries to pull in scipy-1.9.3 triggering the issue that was addressed in e607c0973f74eceb514c2de9eb6e3ae7dfa29716 [0] Signed-off-by: Jan Vesely [0] https://github.com/PrincetonUniversity/PsyNeuLink/actions/runs/3371876413/jobs/5594642498 --- .github/actions/install-pnl/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/install-pnl/action.yml b/.github/actions/install-pnl/action.yml index c9eda59a6f1..06ed2b4ef9b 100644 --- a/.github/actions/install-pnl/action.yml +++ b/.github/actions/install-pnl/action.yml @@ -51,7 +51,7 @@ runs: # terminado >= 0.10.0 pulls in pywinpty >= 1.1.0 # scipy >=1.9.2 doesn't provide win32 wheel and GA doesn't have working fortran on windows # scikit-learn >= 1.1.3 doesn't provide win32 wheel - [[ ${{ runner.os }} = Windows* ]] && pip install "pywinpty<1" "terminado<0.10" "scipy<1.9.2" "scikit-learn<1.1.3" -c requirements.txt + [[ ${{ runner.os }} = Windows* ]] && pip install "pywinpty<1" "terminado<0.10" "scipy<1.9.2" "scikit-learn<1.1.3" "statsmodels<0.13.3" -c requirements.txt fi - name: Install updated package From f6a987a407835ab81a575b1833e6be3c87137c1d Mon Sep 17 00:00:00 2001 From: jdcpni Date: Thu, 3 Nov 2022 18:03:49 -0400 Subject: [PATCH 041/127] Feat/add pathway default matrix (#2518) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * • compositioninterfacemechanism.py: - _get_source_node_for_input_CIM: restore (modeled on _get_source_of_modulation_for_parameter_CIM) but NEEDS TESTS - _get_source_of_modulation_for_parameter_CIM: clean up comments, NEEDS TESTS * - * - * - * - * - * - * • Nback - EM uses ContentAddressableMemory (instead of DictionaryMemory) - Implements FFN for comparison of current and retrieved stimulus and context • Project: replace all instances of "RETREIVE" with "RETRIEVE" * • objectivefunctions.py - add cosine_similarity (needs compiled version) * • Project: make COSINE_SIMILARITY a synonym of COSINE • nback_CAM_FFN: - refactor to implement FFN and task input - assign termination condition for execution that is dependent on control - ContentAddressableMemory: selection_function=SoftMax(output=MAX_INDICATOR, gain=SOFT_MAX_TEMP) • DriftOnASphereIntegrator: - add dimension as dependency for initializer parameter * - * - * - * - * - * - * - * - * - * - * - * - * - * - * • test_integrator.py: Added identicalness test for DriftOnASphereIntegrator agains nback-paper implementation. * - * - * Parameters: allow _validate_ methods to reference other parameters (#2512) * • Scripts: - Updated N-back to use objective_mechanism, with commented out code for version that doesn't use it once bug is fixed - Deleted N-back_WITH_OBJECTIVE_MECH.py * • Scripts: - Updated N-back to use objective_mechanism, with commented out code for version that doesn't use it once bug is fixed - Deleted N-back_WITH_OBJECTIVE_MECH.py * • N-back.py: - added stimulus generation per nback-paper protocol * - N-back.py tstep(s) -> trial(s) * - * - * • N-back.py - comp -> nback_model - implement stim_set() method * - * • N-back.py: - added training set generation * - * - * • N-back.py - modularized script * - * - * - * - * • showgraph.py: - _assign_processing_components(): fix bug in which nested graphs not highlighted in animation. * • showgraph.py * composition.py - add further description of animation, including note that animation of nested Compostions is limited. * • showgraph.py * composition.py - add animation to N-back doc * • autodiffcomposition.py - __init__(): move pathways arg to beginning, to capture positional assignment (i.e. w/o kw) * - * • N-back.py - ffn: implement as autodiff; still needs small random initial weight assignment * • pathway.py - implement default_projection attribute * • pathway.py - implement default_projection attribute * • utilities.py: random_matrxi: refactored to allow negative values and use keyword ZERO_CENTER * • projection.py RandomMatrix: added class that can be used to pass a function as matrix spec * • utilities.py - RandomMatrix moved here from projection.py • function.py - get_matrix(): added support for RandomMatrix spec * • port.py - _parse_port_spec(): added support for RandomMatrix * • port.py - _parse_port_spec(): added support for RandomMatrix * • utilities.py - is_matrix(): modified to support random_matrix and RandomMatrix * • composition.py - add_linear_processing_pathway: add support for default_matrix argument (replaces default for MappingProjection for any otherwise unspecified projections) though still not used. * - * - RandomMatrix: moved from Utilities to Function * - * [skip ci] * [skip ci] * [skip ci] • N-back.py - clean up script * [skip ci] • N-back.py - further script clean-up * [skip ci] * [skip ci] * [skip ci] * [skip ci] • BeukersNBackModel.rst: - Overview written - Needs other sections completed * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] • N-back.py: - replace functions of TransferMechanisms with ReLU - replace function of Decision Mechanisms with SoftMax - more doc cleanup * [skip ci] • N-back.py: - replace functions of TransferMechanisms with ReLU - replace function of Decision Mechanisms with SoftMax - more doc cleanup * [skip ci] * - * - * [skip ci] * [skip ci] • composition.py: implement default_projection_matrix in add_XXX_pathway() methods * [skip ci] • composition.py: implement default_projection_matrix in add_XXX_pathway() methods * [skip ci] • test_composition.py: - add test_pathway_tuple_specs() * - * - * [skip ci] * [skip ci] * [skip ci] * - Co-authored-by: jdcpni Co-authored-by: Katherine Mantel --- Scripts/Models (Under Development)/N-back.py | 342 +++++++++++------- docs/source/BeukersNBackModel.rst | 81 +++++ docs/source/Function.rst | 2 +- docs/source/Functions.rst | 2 +- docs/source/Models.rst | 3 + docs/source/_static/N-Back-Model_fig.svg | 1 + docs/source/_static/N-Back_Model_movie.gif | Bin 0 -> 256380 bytes .../core/components/functions/function.py | 52 ++- psyneulink/core/components/ports/port.py | 9 +- .../projections/pathway/mappingprojection.py | 22 +- .../core/components/projections/projection.py | 17 +- psyneulink/core/compositions/composition.py | 184 +++++++--- psyneulink/core/compositions/pathway.py | 85 +++-- psyneulink/core/compositions/showgraph.py | 58 ++- psyneulink/core/globals/keywords.py | 15 +- psyneulink/core/globals/utilities.py | 40 +- .../compositions/autodiffcomposition.py | 16 +- tests/composition/test_composition.py | 62 +++- tests/mdf/model_basic.yml | 313 ++++++++++++++++ 19 files changed, 1051 insertions(+), 253 deletions(-) create mode 100644 docs/source/BeukersNBackModel.rst create mode 100644 docs/source/_static/N-Back-Model_fig.svg create mode 100644 docs/source/_static/N-Back_Model_movie.gif create mode 100644 tests/mdf/model_basic.yml diff --git a/Scripts/Models (Under Development)/N-back.py b/Scripts/Models (Under Development)/N-back.py index 1c4b97f0ab0..903a82b862a 100644 --- a/Scripts/Models (Under Development)/N-back.py +++ b/Scripts/Models (Under Development)/N-back.py @@ -1,63 +1,100 @@ """ -This implements a model of the `N-back task _` -described in `Beukers et al. `_. The model uses a simple implementation of episodic -memory (i.e., content-retrieval memory) to store previous stimuli and the temporal context in which they occured, -and a feedforward neural network to evaluate whether the current stimulus is a match to the n'th preceding stimulus. +This implements a model of the `N-back task `_ +described in `Beukers et al. (2022) `_. The model uses a simple implementation of episodic +(content-addressable) memory to store previous stimuli and the temporal context in which they occured, +and a feedforward neural network to evaluate whether the current stimulus is a match to the n'th preceding stimulus +(n-back level). This model is an example of proposed interactions between working memory (e.g., in neocortex) and +episodic memory e.g., in hippocampus and/or cerebellum) in the performance of tasks demanding of sequential processing +and control, and along the lines of models emerging machine learning that augment the use of recurrent neural networks +(e.g., long short-term memory mechanisms; LSTMs) for active memory and control with an external memory capable of +rapid storage and content-based retrieval, such as the Neural Turing Machine (NTN; `Graves et al., 2016 +`_), Episodic Planning Networks (EPN; `Ritter et al., 2020 +`_), and Emergent Symbols through Binding Networks (ESBN; `Webb et al., 2021 +`_). + +There are three primary methods in the script: + +* construct_model(args): + takes as arguments parameters used to construct the model; for convenience, defaults are defined below, + (under "Construction parameters") + +* train_network(args) + takes as arguments the feedforward neural network Composition (FFN_COMPOSITION) and number of epochs to train. + Note: learning_rate is set at construction (can specify using LEARNING_RATE under "Training parameters" below). + +* run_model() + takes the context drift rate to be applied on each trial and the number of trials to execute as args, as well as + reporting and animation specifications (see "Execution parameters" below). + +See "Settings for running the script" to specify whether the model is trained and/or executed when the script is run, +and whether a graphic display of the network is generated when it is constructed. TODO: + - from Andre + - network architecture; in particular, size of hidden layer and projection patterns to and from it + - softmax temp on output/decision layer? + - confirm that ReLUs all use 0 thresholds and unit slope + - training: + - confirm learning rate: ?? 0.001 + - epoch: 1 trial per epoch of training + - get empirical stimulus sequences + - put N-back script (with pointer to latest version on PNL) in nback-paper repo - get rid of objective_mechanism (see "VERSION *WITHOUT* ObjectiveMechanism" under control(...) - - from nback-paper: - - get ffn weights? - - why SDIM=20 if it is a one-hot encoding (np.eye), and NSTIM=8? (i.e., SHOULDN'T NUM_STIM == STIM_SIZE)? - - do input layers use logistic (as suggested in figure)? - - construct training set and train in ffn using Autodiff + - pass learning_rate as parameter to train_network() - validate against nback-paper results - - replace get_input_sequence and get_training_inputs with generators passed to nback_model.run() and ffn.learn - - make termination processing part of the Comopsition definition? + - after validation: + - try with STIM_SIZE = NUM_STIMS rather than 20 (as in nback-paper) + - refactor generate_stim_sequence() to use actual empirical stimulus sequences + - replace get_input_sequence and get_training_inputs with generators passed to nback_model.run() and ffn.learn + - make termination processing part of the Composition definition (fix bug) + - fix warnings on run """ from graph_scheduler import * + from psyneulink import * import numpy as np -import itertools - -DISPLAY = False # show visual of model -# REPORTING_OPTIONS = ReportOutput.ON # Console output during run -REPORTING_OPTIONS = ReportOutput.OFF +# Settings for running script: +TRAIN = False +RUN = True +DISPLAY = False # show visual graphic of model # PARAMETERS ------------------------------------------------------------------------------------------------------- -# FROM nback-paper: -# SDIM = 20 -# CDIM = 25 -# indim = 2 * (CDIM + SDIM) -# hiddim = SDIM * 4 -# CONTEXT_DRIFT_RATE=.25 -# CONTEXT_DRIFT_NOISE=.075 -# 'stim_weight':0.05, -# 'smtemp':8, -# HAZARD_RATE=0.04 - -# TEST: -MAX_NBACK_LEVELS = 5 -NBACK_LEVELS = [2,3] -NUM_NBACK_LEVELS = len(NBACK_LEVELS) -# NUM_TASKS=2 # number of different variants of n-back tasks (set sizes) +# Fixed (structural) parameters: +MAX_NBACK_LEVELS = 3 NUM_STIM = 8 # number of different stimuli in stimulus set - QUESTION: WHY ISN"T THIS EQUAL TO STIM_SIZE OR VICE VERSA? -NUM_TRIALS = 48 # number of stimuli presented in a sequence +FFN_TRANSFER_FUNCTION = ReLU + +# Constructor parameters: (values are from nback-paper) STIM_SIZE=20 # length of stimulus vector CONTEXT_SIZE=25 # length of context vector HIDDEN_SIZE=STIM_SIZE*4 # dimension of hidden units in ff -CONTEXT_DRIFT_RATE=.1 # drift rate used for DriftOnASphereIntegrator (function of Context mech) on each trial +NBACK_LEVELS = [2,3] # Currently restricted to these +NUM_NBACK_LEVELS = len(NBACK_LEVELS) CONTEXT_DRIFT_NOISE=0.0 # noise used by DriftOnASphereIntegrator (function of Context mech) -STIM_WEIGHT=.05 # weighting of stimulus field in retrieval from em -CONTEXT_WEIGHT = 1-STIM_WEIGHT # weighting of context field in retrieval from em -SOFT_MAX_TEMP=1/8 # express as gain # precision of retrieval process -HAZARD_RATE=0.04 # rate of re=sampling of em following non-match determination in a pass through ffn +RANDOM_WEIGHTS_INITIALIZATION=RandomMatrix(center=0.0, range=0.1) # Matrix spec used to initialize all Projections +RETRIEVAL_SOFTMAX_TEMP=1/8 # express as gain # precision of retrieval process +RETRIEVAL_HAZARD_RATE=0.04 # rate of re=sampling of em following non-match determination in a pass through ffn +RETRIEVAL_STIM_WEIGHT=.05 # weighting of stimulus field in retrieval from em +RETRIEVAL_CONTEXT_WEIGHT = 1-RETRIEVAL_STIM_WEIGHT # weighting of context field in retrieval from em +DECISION_SOFTMAX_TEMP=1/8 # express as gain # binarity of decision process + +# Training parameters: +NUM_EPOCHS=1000 # nback-paper: 400,000, one trial per epoch +LEARNING_RATE=0.1 # nback-paper: .001 + +# Execution parameters: +CONTEXT_DRIFT_RATE=.1 # drift rate used for DriftOnASphereIntegrator (function of Context mech) on each trial +NUM_TRIALS = 48 # number of stimuli presented in a trial sequence +REPORT_OUTPUT = ReportOutput.OFF # Sets console output during run +REPORT_PROGRESS = ReportProgress.ON # Sets console progress bar during run +ANIMATE = True # {UNIT:EXECUTION_SET} # Specifies whether to generate animation of execution -# MECHANISM AND COMPOSITION NAMES: +# Names of Compositions and Mechanisms: +NBACK_MODEL = "N-Back Model" FFN_COMPOSITION = "WORKING MEMORY (fnn)" FFN_STIMULUS_INPUT = "CURRENT STIMULUS" FFN_CONTEXT_INPUT = "CURRENT CONTEXT" @@ -79,10 +116,11 @@ def construct_model(stim_size = STIM_SIZE, hidden_size = HIDDEN_SIZE, num_nback_levels = NUM_NBACK_LEVELS, context_drift_noise = CONTEXT_DRIFT_NOISE, - retrievel_softmax_temp = SOFT_MAX_TEMP, - retrieval_hazard_rate = HAZARD_RATE, - retrieval_stimulus_weight = STIM_WEIGHT, - context_stimulus_weight = CONTEXT_WEIGHT): + retrievel_softmax_temp = RETRIEVAL_SOFTMAX_TEMP, + retrieval_hazard_rate = RETRIEVAL_HAZARD_RATE, + retrieval_stimulus_weight = RETRIEVAL_STIM_WEIGHT, + retrieval_context_weight = RETRIEVAL_CONTEXT_WEIGHT, + decision_softmax_temp = DECISION_SOFTMAX_TEMP): """Construct nback_model""" # FEED FORWARD NETWORK ----------------------------------------- @@ -90,26 +128,38 @@ def construct_model(stim_size = STIM_SIZE, # inputs: encoding of current stimulus and context, retrieved stimulus and retrieved context, # output: decIsion: match [1,0] or non-match [0,1] # Must be trained to detect match for specified task (1-back, 2-back, etc.) - input_current_stim = TransferMechanism(size=STIM_SIZE, function=Linear, name=FFN_STIMULUS_INPUT) # function=Logistic) - input_current_context = TransferMechanism(size=STIM_SIZE, function=Linear, name=FFN_CONTEXT_INPUT) # function=Logistic) - input_retrieved_stim = TransferMechanism(size=STIM_SIZE, function=Linear, name=FFN_STIMULUS_RETRIEVED) # function=Logistic) - input_retrieved_context = TransferMechanism(size=STIM_SIZE, function=Linear, name=FFN_CONTEXT_RETRIEVED) # function=Logistic) - input_task = TransferMechanism(size=NUM_NBACK_LEVELS, function=Linear, name=FFN_TASK) # function=Logistic) - hidden = TransferMechanism(size=HIDDEN_SIZE, function=Logistic, name=FFN_HIDDEN) - decision = ProcessingMechanism(size=2, name=FFN_OUTPUT) - # TODO: THIS NEEDS TO BE REPLACED BY (OR AT LEAST TRAINED AS) AutodiffComposition - # TRAINING: - # - 50% matches and 50% non-matches - # - all possible stimuli - # - 2back and 3back - # - contexts of various distances - ffn = Composition([{input_current_stim, - input_current_context, - input_retrieved_stim, - input_retrieved_context, - input_task}, - hidden, decision], - name=FFN_COMPOSITION) + input_current_stim = TransferMechanism(name=FFN_STIMULUS_INPUT, + size=stim_size, + function=FFN_TRANSFER_FUNCTION) + input_current_context = TransferMechanism(name=FFN_CONTEXT_INPUT, + size=context_size, + function=FFN_TRANSFER_FUNCTION) + input_retrieved_stim = TransferMechanism(name=FFN_STIMULUS_RETRIEVED, + size=stim_size, + function=FFN_TRANSFER_FUNCTION) + input_retrieved_context = TransferMechanism(name=FFN_CONTEXT_RETRIEVED, + size=context_size, + function=FFN_TRANSFER_FUNCTION) + input_task = TransferMechanism(name=FFN_TASK, + size=num_nback_levels, + function=FFN_TRANSFER_FUNCTION) + hidden = TransferMechanism(name=FFN_HIDDEN, + size=hidden_size, + function=FFN_TRANSFER_FUNCTION) + decision = ProcessingMechanism(name=FFN_OUTPUT, + size=2, function=SoftMax(output=MAX_INDICATOR, + gain=decision_softmax_temp)) + ffn = AutodiffComposition(([{input_current_stim, + input_current_context, + input_retrieved_stim, + input_retrieved_context, + input_task}, + hidden, decision], + RANDOM_WEIGHTS_INITIALIZATION, + ), + name=FFN_COMPOSITION, + learning_rate=LEARNING_RATE + ) # FULL MODEL (Outer Composition, including input, EM and control Mechanisms) ------------------------ @@ -120,11 +170,12 @@ def construct_model(stim_size = STIM_SIZE, context = ProcessingMechanism(name=MODEL_CONTEXT_INPUT, function=DriftOnASphereIntegrator( initializer=np.random.random(CONTEXT_SIZE-1), - noise=CONTEXT_DRIFT_NOISE, + noise=context_drift_noise, dimension=CONTEXT_SIZE)) # Task: task one-hot indicating n-back (1, 2, 3 etc.) - must correspond to what ffn has been trained to do - task = ProcessingMechanism(name=MODEL_TASK_INPUT, size=NUM_NBACK_LEVELS) + task = ProcessingMechanism(name=MODEL_TASK_INPUT, + size=NUM_NBACK_LEVELS) # Episodic Memory: # - entries: stimulus (field[0]) and context (field[1]); randomly initialized @@ -136,10 +187,11 @@ def construct_model(stim_size = STIM_SIZE, SIZE:CONTEXT_SIZE}], function=ContentAddressableMemory( initializer=[[[0]*STIM_SIZE, [0]*CONTEXT_SIZE]], - distance_field_weights=[STIM_WEIGHT, CONTEXT_WEIGHT], + distance_field_weights=[retrieval_stimulus_weight, + retrieval_context_weight], # equidistant_entries_select=NEWEST, selection_function=SoftMax(output=MAX_INDICATOR, - gain=SOFT_MAX_TEMP)), + gain=retrievel_softmax_temp)), ) # Control Mechanism @@ -159,7 +211,8 @@ def construct_model(stim_size = STIM_SIZE, # Outcome=1 if match, else 0 function=lambda x: int(x[0][1]>x[0][0])), # Set ControlSignal for EM[store_prob] - function=lambda outcome: int(bool(outcome) or (np.random.random() > HAZARD_RATE)), + function=lambda outcome: int(bool(outcome) + or (np.random.random() > retrieval_hazard_rate)), # # VERSION *WITHOUT* ObjectiveMechanism: # monitor_for_control=decision, # # Set Evaluate outcome and set ControlSignal for EM[store_prob] @@ -168,12 +221,13 @@ def construct_model(stim_size = STIM_SIZE, # or (np.random.random() > HAZARD_RATE)), control=(STORAGE_PROB, em)) - nback_model = Composition(nodes=[stim, context, task, em, ffn, control], + nback_model = Composition(name=NBACK_MODEL, + nodes=[stim, context, task, em, ffn, control], # # # Terminate trial if value of control is still 1 after first pass through execution # # FIX: STOPS AFTER ~ NUMBER OF TRIALS (?90+); SHOULD BE: NUM_TRIALS*NUM_NBACK_LEVELS + 1 # termination_processing={TimeScale.TRIAL: And(Condition(lambda: control.value), # AfterPass(0, TimeScale.TRIAL))}, - name="N-Back Model") + ) # # Terminate trial if value of control is still 1 after first pass through execution # # FIX: ALL OF THE FOLLOWING STOP AFTER ~ NUMBER OF TRIALS (?90+); SHOULD BE: NUM_TRIALS*NUM_NBACK_LEVELS + 1 # nback_model.scheduler.add_condition(nback_model, And(Condition(lambda: control.value), AfterPass(0, TimeScale.TRIAL))) @@ -193,7 +247,7 @@ def construct_model(stim_size = STIM_SIZE, nback_model.show_graph( # show_cim=True, # show_node_structure=ALL, - # show_dimensions=True) + # show_dimensions=True ) return nback_model @@ -207,15 +261,16 @@ def get_stim_set(num_stim=STIM_SIZE): return np.eye(num_stim) def get_task_input(nback_level): - """Construct input to task Mechanism for a given nback_level, used by run_model() and train_model()""" + """Construct input to task Mechanism for a given nback_level, used by run_model() and train_network()""" task_input = list(np.zeros_like(NBACK_LEVELS)) task_input[nback_level-NBACK_LEVELS[0]] = 1 return task_input -def get_run_inputs(model, nback_level, num_trials): +def get_run_inputs(model, nback_level, context_drift_rate, num_trials): """Construct set of stimulus inputs for run_model()""" - def generate_stim_sequence(nback_level, trial_num, stype=0, num_stim=NUM_STIM, num_trials=NUM_TRIALS): + def generate_stim_sequence(nback_level, trial_num, trial_type=0, num_stim=NUM_STIM, num_trials=NUM_TRIALS): + assert nback_level in {2,3} # At present, only 2- and 3-back levels are supported def gen_subseq_stim(): A = np.random.randint(0,num_stim) @@ -230,67 +285,70 @@ def gen_subseq_stim(): ) return A,B,C,X - def genseqCT(nback_level,trial_num): - assert nback_level in {2,3} - # ABXA / AXA + def generate_match_no_foils_sequence(nback_level,trial_num): + # AXA (2-back) or ABXA (3-back) seq = np.random.randint(0,num_stim,num_trials) A,B,C,X = gen_subseq_stim() # - if nback_level==3: - subseq = [A,B,X,A] - elif nback_level==2: + if nback_level==2: subseq = [A,X,A] + elif nback_level==3: + subseq = [A,B,X,A] seq[trial_num-(nback_level+1):trial_num] = subseq return seq[:trial_num] - def genseqCF(nback_level,trial_num): - # ABXC + def generate_non_match_no_foils_sequence(nback_level,trial_num): + # AXB (2-back) or ABXC (3-back) seq = np.random.randint(0,num_stim,num_trials) A,B,C,X = gen_subseq_stim() # - if nback_level==3: - subseq = [A,B,X,C] - elif nback_level==2: + if nback_level==2: subseq = [A,X,B] + elif nback_level==3: + subseq = [A,B,X,C] seq[trial_num-(nback_level+1):trial_num] = subseq return seq[:trial_num] - def genseqLT(nback_level,trial_num): - # AAXA + def generate_match_with_foil_sequence(nback_level,trial_num): + # AAA (2-back) or AAXA (3-back) seq = np.random.randint(0,num_stim,num_trials) A,B,C,X = gen_subseq_stim() # - if nback_level==3: - subseq = [A,A,X,A] - elif nback_level==2: + if nback_level==2: subseq = [A,A,A] + elif nback_level==3: + subseq = [A,A,X,A] seq[trial_num-(nback_level+1):trial_num] = subseq return seq[:trial_num] - def genseqLF(nback_level,trial_num): - # ABXB + def generate_non_match_with_foil_sequence(nback_level,trial_num): + # XAA (2-back) or ABXB (3-back) seq = np.random.randint(0,num_stim,num_trials) A,B,C,X = gen_subseq_stim() # - if nback_level==3: - subseq = [A,B,X,B] - elif nback_level==2: + if nback_level==2: subseq = [X,A,A] + elif nback_level==3: + subseq = [A,B,X,B] seq[trial_num-(nback_level+1):trial_num] = subseq return seq[:trial_num] - genseqL = [genseqCT,genseqLT,genseqCF,genseqLF] - stim_seq = genseqL[stype](nback_level,trial_num) - # ytarget = [1,1,0,0][stype] + trial_types = [generate_match_no_foils_sequence, + generate_match_with_foil_sequence, + generate_non_match_no_foils_sequence, + generate_non_match_with_foil_sequence] + stim_seq = trial_types[trial_type](nback_level,trial_num) + # ytarget = [1,1,0,0][trial_type] # ctxt = spherical_drift(trial_num) # return stim,ctxt,ytarget return stim_seq - def stim_set_generation(nback_level, num_trials): - stim_sequence = [] - # for seq_int, trial in itertools.product(range(4),np.arange(5,trials)): # This generates all length sequences - for seq_int, trial_num in itertools.product(range(4),[num_trials]): # This generates only longest seq (num_trials) - return stim_sequence.append(generate_stim_sequence(nback_level, trial_num, stype=seq_int, trials=num_trials)) + # def stim_set_generation(nback_level, num_trials): + # stim_sequence = [] + # # for seq_int, trial in itertools.product(range(4),np.arange(5,trials)): # This generates all length sequences + # for trial_type, trial_num in itertools.product(range(4),[num_trials]): # This generates only longest seq ( + # # num_trials) + # return stim_sequence.append(generate_stim_sequence(nback_level, trial_num, trial_type=trial_type, trials=num_trials)) def get_input_sequence(nback_level, num_trials=NUM_TRIALS): """Get sequence of inputs for a run""" @@ -301,11 +359,11 @@ def get_input_sequence(nback_level, num_trials=NUM_TRIALS): return [input_set[trial_seq[i]] for i in range(num_trials)] return {model.nodes[MODEL_STIMULUS_INPUT]: get_input_sequence(nback_level, num_trials), - model.nodes[MODEL_CONTEXT_INPUT]: [[CONTEXT_DRIFT_RATE]]*num_trials, + model.nodes[MODEL_CONTEXT_INPUT]: [[context_drift_rate]]*num_trials, model.nodes[MODEL_TASK_INPUT]: [get_task_input(nback_level)]*num_trials} def get_training_inputs(network, num_epochs, nback_levels): - """Construct set of training stimuli for ffn.learn(), used by train_model() + """Construct set of training stimuli used by ffn.learn() in train_network() Construct one example of each condition: match: stim_current = stim_retrieved and context_current = context_retrieved stim_lure: stim_current = stim_retrieved and context_current != context_retrieved @@ -348,55 +406,85 @@ def get_training_inputs(network, num_epochs, nback_levels): contexts.append(context_fct(CONTEXT_DRIFT_RATE)) # Get current context as one that is next to last from list (leaving last one as potential lure) current_context = contexts.pop(num_nback_levels-1) - context_nback = contexts.pop(0) - context_distractor = contexts[np.random.randint(0,len(contexts))] + # + nback_context = contexts.pop(0) + distractor_context = contexts[np.random.randint(0,len(contexts))] # Assign retrieved stimulus and context accordingly to trial_type for trial_type in trial_types: stim_current.append(current_stim) context_current.append(current_context) + # Assign retrieved stimulus if trial_type in {'match','stim_lure'}: - stim_retrieved.append(stim_current) - else: + stim_retrieved.append(current_stim) + else: # context_lure or non_lure stim_retrieved.append(distractor_stim) + # Assign retrieved context if trial_type in {'match','context_lure'}: - context_retrieved.append(context_nback) - else: - context_retrieved.append(context_distractor) + context_retrieved.append(nback_context) + else: # stimulus_lure or non_lure + context_retrieved.append(distractor_context) + # Assign target if trial_type == 'match': target.append([1,0]) else: target.append([0,1]) current_task.append([task_input]) - training_set = {network.nodes[FFN_STIMULUS_INPUT]: stim_current, - network.nodes[FFN_CONTEXT_INPUT]: context_current, - network.nodes[FFN_STIMULUS_RETRIEVED]: stim_retrieved, - network.nodes[FFN_CONTEXT_RETRIEVED]: context_retrieved, - network.nodes[FFN_TASK]: current_task, - network.nodes[FFN_OUTPUT]: target - } + training_set = {INPUTS: {network.nodes[FFN_STIMULUS_INPUT]: stim_current, + network.nodes[FFN_CONTEXT_INPUT]: context_current, + network.nodes[FFN_STIMULUS_RETRIEVED]: stim_retrieved, + network.nodes[FFN_CONTEXT_RETRIEVED]: context_retrieved, + network.nodes[FFN_TASK]: current_task}, + TARGETS: {network.nodes[FFN_OUTPUT]: target}, + EPOCHS: num_epochs} + return training_set # ======================================== MODEL EXECUTION ============================================================ -def train_model(): - get_training_inputs(num_epochs=1, nback_levels=NBACK_LEVELS) - -def run_model(model, num_trials=NUM_TRIALS, reporting_options=REPORTING_OPTIONS): +def train_network(network, + learning_rate=LEARNING_RATE, + num_epochs=NUM_EPOCHS): + training_set = get_training_inputs(network=network, num_epochs=num_epochs, nback_levels=NBACK_LEVELS) + network.learn(inputs=training_set, + minibatch_size=NUM_TRIALS, + execution_mode=ExecutionMode.LLVMRun) + +def run_model(model, + context_drift_rate=CONTEXT_DRIFT_RATE, + num_trials=NUM_TRIALS, + report_output=REPORT_OUTPUT, + report_progress=REPORT_PROGRESS, + animate=ANIMATE + ): for nback_level in NBACK_LEVELS: - model.run(inputs=get_run_inputs(model, nback_level, num_trials), + model.run(inputs=get_run_inputs(model, nback_level, context_drift_rate, num_trials), # FIX: MOVE THIS TO MODEL CONSTRUCTION ONCE THAT WORKS # Terminate trial if value of control is still 1 after first pass through execution termination_processing={TimeScale.TRIAL: And(Condition(lambda: model.nodes[CONTROLLER].value), AfterPass(0, TimeScale.TRIAL))}, # function arg - report_output=reporting_options) + report_output=report_output, + report_progress=report_progress, + animate=animate + ) # FIX: RESET MEMORY HERE? - print("Number of entries in EM: ", len(model.nodes[EM].memory)) + # print("Number of entries in EM: ", len(model.nodes[EM].memory)) assert len(model.nodes[EM].memory) == NUM_TRIALS*NUM_NBACK_LEVELS + 1 + nback_model = construct_model() -run_model(nback_model) +print('nback_model constructed') +if TRAIN: + print('nback_model training...') + train_network(nback_model.nodes[FFN_COMPOSITION]) + print('nback_model trained') +if RUN: + print('nback_model executing...') + run_model(nback_model) + if REPORT_PROGRESS == ReportProgress.ON: + print('\n') +print(f'nback_model done: {len(nback_model.results)} trials executed') # =========================================================================== diff --git a/docs/source/BeukersNBackModel.rst b/docs/source/BeukersNBackModel.rst new file mode 100644 index 00000000000..ea2a3adf26e --- /dev/null +++ b/docs/source/BeukersNBackModel.rst @@ -0,0 +1,81 @@ + +N-Back Model (Beukers et al., 2022) +================================================================== +`"When Working Memory is Just Working, Not Memory" `_ + +Overview +-------- +This implements a model of the `N-back task `_ +described in `Beukers et al. (2022) `_. The model uses a simple implementation of episodic +memory (EM, as a form of content-retrieval memory) to store previous stimuli along with the temporal context in which +they occured, and a feedforward neural network (FFN)to evaluate whether the current stimulus is a match to the n'th +preceding stimulus (nback-level)retrieved from episodic memory. The temporal context is provided by a randomly +drifting high dimensional vector that maintains a constant norm (i.e., drifts on a sphere). The FFN is +trained, given an n-back level of *n*, to identify when the current stimulus matches one stored in EM +with a temporal context vector that differs by an amount corresponding to *n* time steps of drift. During n-back +performance, the model encodes the current stimulus and temporal context, retrieves an item from EM that matches the +current stimulus, weighted by the similarity of its temporal context vector (i.e., most recent), and then uses the +FFN to evaluate whether it is an n-back match. The model responds "match" if the FFN detects a match; otherwise, it +either responds "non-match" or, with a fixed probability (hazard rate), it uses the current stimulus and temporal +context to retrieve another sample from EM and repeat the evaluation. + +This model is an example of proposed interactions between working memory (e.g., in neocortex) and episodic memory +e.g., in hippocampus and/or cerebellum) in the performance of tasks demanding of sequential processing and control, +and along the lines of models emerging machine learning that augment the use of recurrent neural networks (e.g., long +short-term memory mechanisms; LSTMs) for active memory and control with an external memory capable of rapid storage +and content-based retrieval, such as the Neural Turing Machine (NTN; +`Graves et al., 2016 `_), Episodic Planning Networks (EPN; +`Ritter et al., 2020 `_), and Emergent Symbols through Binding Networks (ESBN; +`Webb et al., 2021 `_). + +The script respectively, to construct, train and run the model: + +* construct_model(args): + takes as arguments parameters used to construct the model; for convenience, defaults are defined toward the top + of the script (see "Construction parameters"). +.. +* train_network(args) + takes as arguments the feedforward neural network Composition (FFN_COMPOSITION) and number of epochs to train. + Note: learning_rate is set at construction (which can be specified using LEARNING_RATE under "Training parameters"). +.. +* run_model() + takes as arguments the drift rate in the temporal context vector to be applied on each trial, + and the number of trials to execute, as well as reporting and animation specifications + (see "Execution parameters"). + +The default parameters are ones that have been fit to empirical data concerning human performance +(taken from `Kane et al., 2007 `_). + + +The Model +--------- + +The models is composed of two `Compositions `: an outer one that contains the full model (nback_model), +and an `AutodiffComposition` (ffn), nested within nback_model (see red box in Figure), that implements the +feedforward neural network (ffn). + +nback_model +~~~~~~~~~~~ + +This contains three input Mechanisms ( + +Both of these are constructed in the construct_model function. +The ffn Composition is trained use + +.. _nback_Fig: + +.. figure:: _static/N-Back_Model_movie.gif + :align: left + :alt: N-Back Model Animation + + +Training +-------- + + +Execution +--------- + + +Script: :download:`N-back.py <../../Scripts/Models (Under Development)/Beukers_N-Back_2022.py>` +.. Script: :download:`N-back.py <../../psyneulink/library/models/Beukers -Back.py>` diff --git a/docs/source/Function.rst b/docs/source/Function.rst index eb23c588103..2dc5b4ca902 100644 --- a/docs/source/Function.rst +++ b/docs/source/Function.rst @@ -12,6 +12,6 @@ Function :maxdepth: 3 .. automodule:: psyneulink.core.components.functions.function - :members: Function_Base, ArgumentTherapy + :members: Function_Base, ArgumentTherapy, RandomMatrix :private-members: :exclude-members: Parameters diff --git a/docs/source/Functions.rst b/docs/source/Functions.rst index 4148855aa37..7388dfd3541 100644 --- a/docs/source/Functions.rst +++ b/docs/source/Functions.rst @@ -9,6 +9,6 @@ Functions UserDefinedFunction .. automodule:: psyneulink.core.components.functions.function - :members: Function_Base, ArgumentTherapy, + :members: Function_Base, ArgumentTherapy, RandomMatrix :private-members: :exclude-members: Parameters \ No newline at end of file diff --git a/docs/source/Models.rst b/docs/source/Models.rst index 8e8eed5db7d..24e4e3889fc 100644 --- a/docs/source/Models.rst +++ b/docs/source/Models.rst @@ -17,3 +17,6 @@ illustrate principles of neural and/or psychological function. • `BotvinickConflictMonitoringModel` • `BustamanteStroopXORLVOCModel` + +• `BeukersNBackModel` + diff --git a/docs/source/_static/N-Back-Model_fig.svg b/docs/source/_static/N-Back-Model_fig.svg new file mode 100644 index 00000000000..836e7b87bc9 --- /dev/null +++ b/docs/source/_static/N-Back-Model_fig.svg @@ -0,0 +1 @@ +N-Back ModelWORKING MEMORY (fnn)TASKCURRENT TASKCONTEXTEPISODIC MEMORY (dict)CURRENT CONTEXTSTIMCURRENT STIMULUSRETRIEVED CONTEXTRETRIEVED STIMULUSREAD/WRITE CONTROLLERHIDDEN LAYEROBJECTIVE MECHANISMDECISION LAYER \ No newline at end of file diff --git a/docs/source/_static/N-Back_Model_movie.gif b/docs/source/_static/N-Back_Model_movie.gif new file mode 100644 index 0000000000000000000000000000000000000000..3a11c1f8eeb93530c4c851deb7fd52ca2fed3756 GIT binary patch literal 256380 zcmX7vbyU<%7sr3F3%m5vjY~Ju(h>{O-67o|AdP^;!qOp~(o)hOE!`^JAT24NpdiS; zKIi?MGk?vTiTj;9bI+VR&s3De#I1?3keDdo|G)r%0RZ>_Kp6lk0H6T?Issr106qc0 z5&(P!fMWo-0su4)0B{0;AONTUfIR^C13)AIBmzJi0PFz3+5Z~>02L-CD;P`zg=#>d z1~@o)|A7YZ0ZM#)R(yOkK7IrtAtnhX>3^VW{}obFc5-r6Dk^{)*r2A~r=g*xrKO|u zrKh)~r$?jd>BSfrn3&vUn3$NEnbFM57%UhpENE8NMK(4^E_U{RbK>Nb=i<`k;@SqL&_>vw!CK4n9gaG0HK%<3)xkb1EaX?($8!ag*^&e<285vA@%u#uH zh5taK6%-bpJyZJ+G+Ip!qz+h>RKz#Sm*B>Cj&33vl&Z*LzTw6Cu( z@CtbK3LP9A{IAeP&_@0T8XXx4Ac3DzQEn)=n3#j0xFE}T%lLRSMgm4cLIOG=fj*g@ zHJQ~V*+t-ufcG13APq?S4>US04af!Z{sWE9&&MgmDf?X{e{txP8;bfPp`={qV-3ZFC>XP9zH(#9a6FZe%lcSj z>1Z-sA&yR^scbxrL$BI=ys3OLi{Ex4U!}R?Q=XXj@%ni4yDx8LBXQ|fTPkNtmD9K^ zCR(cI-f5Sr7O1vXFVq+}yKGFf)-2Uq4aU){wbiaPJIz&FOt!sW`{21dQK0srZlf#U z=kdnmhx)G{!$5cp>g^5NgE6Gs&!^fOcSlp0)e6-+n)WBNg(N0 z+0%J_vfS+Y^>a_x@AIv}c=RK!-tOChubK4G{;U1OXe~>#O$zS zCyuA>a3`LBgkv{Bc+qJ$QS6{{H%aQ(;cl`lHs@Xng2H((Rhg}7?~S_X(O#Oi66b!p zzM=DehOxsDGZXDJYQ)1Tn)4vr&eUEti?{gFKF6)?=pfH?#HS+5Yti{zLBPS$biQ+4 z)wj1{*j$w!!A)fcMKNsEhb8aGJ7${(nSv{RZ|SBk1F#0B$moK?T?R?HiY#O zctQ;Sq^M^μc~X#BsNg&8a^r6+Jm^ zepTXJ`F5IZ{*ypkcGGx`liTGGAy4+$0HKy!xKcx2w#}r#l;O$uuK0+!H4Xy1=jck~xe<5$BfZ;f zP6K423A$4Y!N1WPCW|cYiH=Vi;6D^Mz)crmn$WTuH6c}z()hN}qD$}&sObWWsiEcM z_qQT5uCvm4Lc+lb#;%{|bR$<930ePd!O^VA88`7g} z4YeVQ!VPYMl9zvT8lqVA&tLq#ew&o!eDvaY2Qqj6@IB& z$#N>^2sSbMdN@n^w;X3>rm!0U45lIFQHeri0-Zpqs}F|Ey?Q@+h1W3Il`Z(!%4JH- zI+Wo_$+N{^`KM+LQEEjPiP5s9G=#-_hHi~me}kD)i}1@46jiB8)gwf=0+>8pf>5BH z1YqQlXKN8iVe0;o=}rX|J}d$mRrJ9?{UCtrDA}cj5nPwv56Aw23)tDf7x+Pf)+!KF z+_J1LIFuXk#EmQq!^KU-0E`)73IIH76)a0}Z#`+&DVR0n=?Jw8L&f)%B>%dM~ z)?DhM#GC?ScoToF$n09MR++>NM1=sm;}BaTmz7$ zWDH5eppkaJRV%P(P>qY6_+`2`MFz5-uX0JIj=Q4Zj2<Gd(CTZ`lfNiGW19$d1NJv zMIN`;DVT*50An%GXH2YX_WG&`uK>;_RB)86?l}h!17t;gWwC~u z+S=2kU&u*ZT2goaemB4yq3wEW6{(+E0sSdqFy;MU&-$7T-TX4;%413i5+uN;JD zgAG_U6?~tn;BQv3Je3EHXv7SKz7@BP9Lg#M`|-Ud4%Q>6|IqW`v~Q6&uA8(tTTnOU zAf=~`&(BAqYZ<;( zQdW05V1CQ3eoeSH`U}g^v{-Jhk7ngj!$P#E-=$$hfPN=oh){kiS0s-%Mvg_F-LrP5 zx1U09=b-pM5hM3G;_Hs2PD;FTEK(bhKrVbWyZPyHjdM60RIyRU9ocNIIgg#w&cxdy z3pPtJlsEaf=JS%*!Yl=kSs}mvVvOy(5JcPTE~F$AS|x2eOpw5|5Sqm9O0TxzO~FC} zR64q!By?ze9*LUY zZ7)swA=);m^$$qbK{HAeduzv$eLlSp)7WaxcQblrd2oC*1GmNBMz6kHz54KIHZ^pf zRitop;WRkR71ZnEK)N@8iNkOH#O9LVkaS0xs9=au-s^MzxC{?`$CLBh4Kw3E*%fYkF3lYzd@YIf6fBNYv}6WY z&S4WC6;r=B57mXoVNUo9Z(1L4^(EimfA>$gCGdJcJ75RCEuE`4q|J`_z4Vs#T&5c3WK(j_m%`DED{e3tFe+HRHuQ>QVmJY+)|-jW|QLAfEZx_+y;tcQ6d9*D!=> zSPM;Em=^5?_--kTPMMP^8%A8IID&elwH1&k7=9xF8CweH8RK{q17mEbuH%2b->Ouy zt>o$RNK`wVcR7-wQo49Rx^spA=_xPQqFR7~r(E`kO%}?17^%&{@i+h`uuRQ5^hmTp zCVN|2IN*^mS(FtOr=Anc;E?*$7KAMYgHjfQq`|;{3s=jDYGVu6e>zHV4?;^79dyHP zYYej=qaK6d`T*cMFa~K6hI*yEg(21Og}e&71w*L;$EzKc?!+FV40|0w-37zJ+gH^G zptafrot}`^v`0K)*m=_iuZ!{W!m!I@9`SfWs*Ph?pd49ZupAERyk#t&3`}NLSL|(E zHDf6=S!i7f%3}t1}mKXfGIkiBiR5_g5_%Q=W4-nl`e7$oIy6RaPloxA{YkgMOJvAfPa`2 zLu__CH9PO^8{bRZFe^3644kFkq!`o!{ufwE4Y}lzaMI~O#%Zt*45z-xv%{nymO434 zhafptz2P@*Ml2ad;A3TURr*#_{$Bw;y^A!g-Q=Z(zz^w|U!4nn9I zku5N^=JE~93S8mUyWp+9jlr_P%va&lv&wX;${Z`fxnRQ8Na&f^~4%Xc*%@DuFxp=CwY5Akl;_$)Za14SRK#!6fcjY%t z%)aIh5@PLA?xaeq8b>t^!(k-HbmuSmVEwXw9{ezkBWDj~$*l;Upm9~Da>v!#fv$!rtTMHG0i7SYr@G_UT%9dYqHs2FBIWylT3btH-;lH|xh;Dp#jE z;ftH7hUlNDI0S;)3MX;FKsS|7gFE+^2AC?-gq)+cpU81l5R z(C$d;>JY~0ki#GdgJBYpH)~$uGWs^==XHv2cgTd}J5%DYaVxRVVn)Kcs$X?+^mj^b zcS#9RI-(d~|2;Fj;%i_3=`&S69DR z1cP9p477&l^%Ae^KCchP_>o74pqLvpmFE-64Z^DJqfm$8=aUF^d9>kq#jSx8UiCBO z_p=Lh1+9i>aqH%C4{WIouvm8jvJj`W*Zc{cJZ}AbbR?Yl9oAx9t*-`g@&^Sws~G8` zQXAe6;tlbhezXVoafH8u(IpbrdjGB&s^T7^3CC-@8vMT86XQB;G&K~sH^gjR&$Qn= z)86Y5HDX2AZtO;!n)1qkC$ZK_&0%%KI-x*7r?F+Ne|8xsJ*qEYz1@S3cwtM&qHjb8 zE#6NqHKuJgMnsIu2?u!xXFoC@BGVyQ{D9Yff&0!Aw-|<7z(0|bf|IH}k;6Yxd_P_r zhSw#FpS3rdbQ+m51siMd3V=`Q<_wr(*LWyAjwjp;QE);Nui-4ghtR#iF<_RFuWCn8r5E z!YTH|t>2m$^Hd=%CRhl=A8Np>@56mJjgtyMeXOuew83(|YO`+G0anoBFue8kPq5#g zAYQJ3Rk4#Q{=1a7y1AGd(^x)M(3NhyvKo>FV5%5~6JUj{4-ab42RraXi_r~uo$53C z_A}pp<$vXVi*$gx1GX8Wv+rc_pM;Q(UEm(AW65t|g~4#S6Xx#A=Gw6JKKT|eT;N7e zW0kOjG%eQ3ek3F{B3FQy~H8fhEI+HO*-K4=oa$IhAynOJeJ3HZd{em7^ckpH&Ac-z%=^-|p0B|Q z1CK{nr8&kWJijLOw{#?IoxUHXC?;6S80tUspy{EmO=*rsFtI93D_#NT>ANxHgl5|Vth@oMJ?w2Ay5GC8_9km!%}eyqvXY4Ydn81)bzw|@A8$tW zP2K)^N!LT`YW)3Yd`+06Cd~Oekr2b4A4PY@kP}T}6}=HGsLthqe-CR$J&e$mu}^=L z+)6CVP{B6T%UJ@VFTW5>YT$TpzV`}Hd9);+Rx8VzhHJy@qIgz>}1V92Tss;hM@3WyW>jx|HA<1hP0_B~y|tYJOsl5B?nVO5LS?Z-ws< z;bYiB!+Y9q-^e~*-c#VT3Fkj#hS0yY!$f2MNL(aNNW5eIep7T^|3H5l<9spnPdBZ5 zA+&c%yvq2wrDyqB?Pp6&YkxVS@0D7xf%JM{1_yw@###d^4IAK%;kfjFXu(!l^6nBq zeo(f~u2K}Ipcj~0cTf2Zt41LrM|WQ}lh>jbNv(IFo-5+MIh3RKO*6kj^kO{*n_yfU zI0O`P^^bHbGzyaD<@g;)sfn~Opa%R{84(h?04DMs9sm^Je#ex>a?4`_?BLwSEMVi# zF#KDEJDqJ~fZ7scOqWu-l}v)c1us>8yB~>5ElP|ZLym|gL{Cgpkn%RZE0BJTO>cVT zG+VCmR*84^onA3BmZ}E5*)O!)sw#82)BT%jgId#$Kh4UE?pfoGQ6Ok>B;k7dXq3^X z6PRgCFA-W|6xT|7B$Dc{fZrHQ5|DbpH}DH+kWv@YaQTROWcd*M=lbmHQhTTKNebqN zT?oG{hUA=5bSX9>h1n1YV3mVBLh{Swu$RHu{$}qam#PEpHS$DH(=Q(P$hHvWkS$8` z2$2}sH7ZhCBE8D-vL>`k2_zE)p3y&Bo35p|Wmbyjp_8v5m5Q^vc*dG#*QCO(lBZKe z^N0jG%3eS8qnrRe+0<;Ldx)89gg+k9xto0=s3EdsM>@%omaAJ_((cx*DQSYkh$nnD zr9vwKa4@xBR_``zE1-!K=X;-|#?8I0u~HM(WlJq)LMpRl92LnG=;7;WJ;f54FAQDv z;5Zp06)~U`!YPKH>$Boe3jwaKdQCokSTx4|`>jSglvpmRHuC&~Mz+kzgD-5)rTvN{ zEBnfej4B(h#ps)g22-?Oy0@8me2gMCE_6XCDSLkDA-)Z zV`pjO7c=9@VN8d|iDDn^Wk}B`idjcFlKQ{HubC;v@u#d16SV_y7{L-HQRFlVJjW$+ zacp&zUBr(j{4E>zf;6onOc)(@;v6eF9CB$EITg6<#y&Z`RhX+tYxAK(Hfz-!4Q-OG zj4`_C zd%mfpmfh!P_2ZjzLKJ*l7Ax}ymwDNiPgZd)-c}+XOJNd?Sju$*ulh7;JajDW0?DG z@(^l4!lyRJU^w(yCQLY-upoO(rMkFk7gNL*Kr|97Bp9T97qjKC7vne96 zJL;b!qZB&pl^t$!@sC9rr1*(h76KI64rQ^v;*7t*Vpfu`hSIpFlTf#BtxinMkBCz3 zkcA+|xpALmGB$d`>~jmK1&N>ixt|GE>CRuF5m&cKVo4R~#{$x=s%>2AXx{ZjD#}ks zDN~^dX!1!4)LI%vLT@L<1&bo6FRWtJ^LNO*WTRPaO9-4`pp1eTFuW+bY&Xo?s$k1d zqXh%HZSKO{9)rY7ox@V_oRR50wV3?w+~6SEk~$i!B4Rfkr6lg-P)~(;p9_d3LaPa2 z2`VD;0^Ch7eIAVqTt8iSB$IGBt_8mW?S*WX9R2_-ItHQXVLB%lhDGGVuRz%}jg&11 zg5B&m+g!Nv*$|kSA}d&L@H>fZ>;~{dC+FbLi#2b>_%bZ4s7+2)8iGydYtJJq)u2X+ zafR3BLr@N1pD>hOm_B1bHhr@lW}>>AmYCwQvPeoJo*|NnG?gD=yb7D=(Jsb8C-ce| z`M^Q+`l3zA<(TA6Fg*!ipd#mRh|^OpK__=q9IVV3 zYiP33xzr)486muAWnMi#(aFmjQsp+LMZag> zqf(e|$yy=P0m)*MqS$FEKcX6{SM%PCaZZ}OB0K2ai_L?-=L#6ps?3}XLk84k8*taTqp%~EN0V}?Jnd-ij>UX zRzD{fH<><0aqAO=q*6v&P?Z%!oKl zmKWH0lWnd3t4lZ6Uf7`}S)2DWQi5;Xp&m%%a({ZoGJ@C1VoCDz5tKF%B+2R8x|Lhx)OD;cWb!d8X&%`{oosn{KfI&-F8 z@~Y^@7*QZSoL4n%2Cr@QXJ+H8rx)XJDUQ7BNcmQiF7z*|SW0z;L_q9r;4xKy6z_Tm zTgL#uE$OyU0~eT51`iAGm}OxNe(_2n>b6}hUbb9vC~KOZ@Mi;^crxEuKV2+?YAtE( zBrAnpwz~vYs9cH8B*ia*Atk%VkYJo8CY&codAJa$lDUb#3i}mtvq+T!A5S{a{ZUei zl8d{4&$V1zOj1vsD!2w4xacd&ma)OF)hp_9Hf?52El2OBxnyVX)RMxcsvAnF8Cm2T zoFl?#1RYK(f@zUt(?W)By-LjS&vu8YVq@0i(h0^RSky@$xwZG-<-~((eiG-&Q9XN^ z&adx=L%sFXo=mwqR-|K~NB0WmbSSGb-bc3FhgP^8Seuk-kz*dmsljHhQB@z~_G+tX zc;<_cbRBG4Sz&$^OpNCKC_fV&DK>`KR-ku!@@ZvEKuU@ke=Nm~)bSN(8vl3_NX&U6 zB3zF&^iQw$xMXGvsYaRD?D%-n{b=FUIJ=5Sbfq#x5jvbL#9{-|rPmNe28Q&_lXkwgj}{D|^ygi5Us3poW5Xu*;j zoVaRy=;+2Elk}L3vz5c{Fdll|p=^BuN84FN)s07(kdRzkV@;cDtn#-o%i}C3@bd`L zGfqe&iaWksQMd<)A*;soPT`h&W%C#ReJWEtI~FAHFf9v6fHWTK0)_ zr&)M*d_`OiYfl~HQ|x0~P0;m|gjo({5c!wdFZwG}xt!|6JY_>eAP*Am1xW=Wti;E7 z7+30FKxcyOL7mj6led4;`VrY5?X;u%pDp8Re`Ac+T4(us`6an!ro6B5V}RCmu+FYL z-@dAjsDN4vBD)bk{IDix6FjIin~Rp6nihCj&Rn32$2gWIMv~DtOB$zOnmhR1S+(V| zmry$M>`K8VVrCnxCCichJ6JENPoSh&k757)GyFE?tBmYWLHz zsWm&@Qv@OOk+Gw!<=GSjlg%!(n%8Yw27MNvlErvqi~blqlV9(+xxJW3zcaqOSqW}O1yPv> z#z0CHP5rb@D_Tt}6IZJWOuKzeD>NX!#*p{EDF4IN`f|vtK2#&GSx}<_(S!os5$f$Q zsx4%#U3%58$jsMxwIal{d%>)Ff35egSs%Ii!1Zci5vpMr60(IF{?rgQ(a~B<6uOKG z&NlzRYC7qQ>UeL4?mAoV{<}U+Zt;m={k@3A;Csl>C&;3&#m3!GI6X79gdyYz6#|$} ze6r{mUf=97tNd#|Ee)x;Hk%dM98k2_XI&dZ7^fcc7hbOfH=qK;P=4AdZw{0f6y-7f z_3!@IhwHCstSw;C(hCmpmPPq`q5_IguclE4*9krP3-AnBXqaVC49Z{G)Q2B+M*#7J zZ4qH_!yc?WVJI&SFaVEq-7*b?u3>NH9}}zwwr`bbpuDreKzoD}Y@7aIn}KcHiv#7Y zjPi9t`NyCFW!J9^NF6lRGH+JeGA#YAAimR9gx|NhM|Ow^A(*0DwHDih^kx{;`V-dW zqP{6;_BvEa#tsc1d`^D@Z^QB-qocpna-y5}BZpC|7~HeQ)`4%uNJu+M^Yhlr09)91ECy(V&VjuWo3#O(jf%sT>I1Pit497ZBv^UZ-wNf! z57GX<@095PU_;=~?XegR{|k$zG0}KX-G&@w_mOH-61Nt65^!^G5hrm9YbjC zau_AXfby2L3T*f%HF-z_B#?j4d)ZOxAUZDdFb&)K-%=AoA&GoWthY-?*C;tYJ3m>} zOQ?PN;!%vDbAlg);>EW*L77|)dggnFFAa{Sp@;dKM-|WQGGw-Vdm%}g1Yb#3WF6u{ zD*x$>^4hW|;j{9AZ}gv;&TDM!wL)ICqgtPzwA7(K)R_ltZB&Rr?*AUYG)4uqL;Trr z4(|t!T8N^fQNH|;m!jWz2~R%qK5vq-2oiH`F?4NnIB98vwELZ&_N-6Y+desPbY_6~ zIH3ZTokX`$uSE9-lAMP4ARxoj*|XJ=P`8OBxADxg4}NPC3maz^t7tAi+X_BMFX*AK zpFMs5atg zJS}rRbHOWB)RJE07_|j?6#((MaA(lmH*m0S46(1M+{C5$rd4-A&j9g_fdpIeD!XWf zB_;>=Iflr5^WsM_KG+P|Kc73iC~CFBuh_XG@ZiCQbTc?rKM=oXaPx*<@?g6%I-sa6 zEm$=ZkFiXP4R#$UjxZ_h!zd1eU@s}h=VQkpK>-k7{vXflJkSQeZ2p}|gBBZ$W_pw4{zS!q|_uQn-%3o|xVsyt-4CR-N(tr?~FZn*VwA;e= zv$nM84NWmNbVzyOFrT^m=*vk@bgH>0^50t%K!w4t3{LCax$JfD)AS{Z-8cE==V%GqJ*DwnIp2c4} zCb0p3uL=!fLn~1_U#-2I{AQAFFo@j-dT$;=uKly4aNWc_qKHCLP`>bM?%qS+l;d<% zYLup}zc%Fdm$h*{D)3QCXv#(1XPYfUh#|vw9?r99iXhLWTb}@kNqtJB2Sj1%t5qLZ zBmhHt+UCa#i1n{P_TsIgr+bE@HZQ4uCq5yNnJc5i%;uv)8U@9K6rt)oZh?nWB)_AJP2NLfGrG3S*&<> z+~6<>?uQF+D^vj7Nk-+E|Jgl^0p**GYO%aT=lu>&c;Y(~{Y&}M$x8{9da%7}84TV= z1$m;p+wVQLqW?;Je$9F%Q+!lie+BSC`~rU60-6fbPedak#2Ub+MbY$zM?OxFKzOhM zr#nC8&y(L@jgqgA9AjJL;}b+wLSrD_fc5YbFE)&r&|jP7c)!oK8Nu)B3O~k> zu{uIxmXPZPO}n%s_luuf&)oerBhelH!%-(8&9rTodR zop8HDKIJT&-{E_4e1vy=#W`Q8bG;DhI9H^Tx;;BQ$$ot@8S-;Ebuw3PUEJ^T^bBTyhs?x{ouqE`n zC|lZ)Iakji6+7I!;i#yRclZ|Xkoq$z|Eqm0OUCKc;JfM$r4cRZsn$S4NvCQfqP(_Z zP;x^j*GYQvi8<6(>TT~HyShU*Hwk zn-b+zKK7v?ZGEg04r$}L9Q=%KKQoSNL=F|QesR5TAH00BeNo0gDQ+}0)321OaN&B_ zqVV&H!(xrltlY|OQa2=OF6l!ujz)`^luDlP;^U27!uR6kKQCQx++6h+uD9|<7H_)e ze3bFqTsQR&tYX_ge(Yj4fCVcK3IsV@BFNpGQ2{JV!=nCh(j6>OJHfS1zTPdt;vq$FGq zVPyyhK_r{9LVLI#pNw1zhhSt|5 z0J?3$$i_!~2QY++WFrYB#ME&QD2NpRD@rnqzSesk85A6cbP^Ql?cRjgy_#fsh_yG82JfVzw}V$TBEjuIwn2j{p-VG#%kd90A*6Vq@1^ zB5%!n=h?@c!HwqJrN&56ifx!lBn`Jk04Af7bpk2DSGIi2;$!j*FPUPmMwyt8#^R{t z(@+9~xXii)LdY$C#-d0N^GhW&B_uf@7A49Qm~F8)ll2gQ1?2J*yOS}3hhUDhk(lDC z*@H3s7@M(~hw1*u0$@S72|x8pv}c?*<$YP{Q;g$eRX!N?jvW>p(>RrkPB$xvZcpy_ z%@-#_aHf;KEk^+kK!3%@G=wN)8a`5vpY=+~pc_KqszlidzRZyoFU0s*OlZ`Jj9`BQ z(GnTZmHabZoN9-q0qU5QB{5)ni|+(q3&T@0L=pmUCGoo|4e1FuUO~$ksx~JIi$wU% zi{NVXQ#&AxPaFC6MeROqy~u*H{$2rpK@7DLX{!2*A2oVp)}La2*KIKl(N(`v6*_`` zdj3*~sXd)pFYqWi!9~BAhn$6xAXe9|x`ioN8lRmRb6i=IKHq43QNonBq%$HQ(t&4zlNM7=u^O>;S5jZo0p*ZSsPId%t}2Kg3T+HivB}Ps_c};yHiwS8SgP zgh^w~;GI)-OjN^nQJ`Tg2v5-&Q&Szv^H%s7JBJ5wWq%-GXd6T3-I)XF^N&(f_Tlp3 zkk(B7o>Sk)!E*SvvZcnfjy&a0s~gRw9Zh?~tj>uElnME|z>fl5^7U&a9nL0Zqx?pM zsw`@c^d?gWQ;S6WrEyd>^Olk)KUL^0-iwp!goed{S?sq4&?1il zKNjh81NX1Cz5aj(J?yw8j#do;DHEo^L}(b6x>c+a>myqbj#SYsQCq4)gIuEvse}2h zmt%WpFLvAeDtt$wEUGUNhK&gR_pW-S5}P2thNs7_n|B9GYs45I+9x$$cr1)YHy*o+QlTWRcgpjz0={v^f0 zP2toxr0@GkeJ@|plSrN?Sx=1YtuHD4&DO^7BSoG!O9B>R1bO#4>8#~*SE)s0nqb16 zg_H5H2NlX)>WEJ11-|vol^!*$adWjpg}ln=`c90B4f2$$8y(vgR5Wm=&l^RYC5opx zGz0JjIg@q^q%WgcWscl=(Hw;m7;w=e%}?D@VfbONiCBL$pGYQfGq>=m(Ch+qpC74L zSs;F4s9w3n{Fi~s0nDR>%78)zoDl`11Vx{h4VI2d6mmXsjz|bAR7DSuGYFv?m5(i`5(;4d9u_}4)e+^-M zEVyBO@W!OhhOMmp$%NIxC9`7s?k@rpsUv_~ot#4hy@}vqp*`C+{7wvnv2hJx{hf4q3mpz`jb*gpL8-jg=#h}C_?c2)%kxz`SJuVq#dnRNFA_7kkO z4m0JkGB9Q36h*Op#elhp-XVR>AxSt`9D!D3#acqZDj3P-OjKvv_buJRGhbU6O7XI9 z_)eRe+UWlQ#buTPf(79wXll)(nL`A63H?{318Uk@S52{T4?M^?CpGrZ1e-wS=@ zoO#Im({l|R><2R9IP7d@!MezNKAUqZhuuJWM~W>B2XcD)UFs-K#Y1-E7R}TLutOQh zyqi=}z*(d`EL6wYcBX+YJ2)ciYoG;uSyR==1jsX{`PV0x~I6wQ~biY)P zPBTPL&)UzBoXrT9JmKVRP_<^9c~{)Kh2YUG!N%Q?Qz%tJIu7Wp?_lg}7`)$Smm5$t z`1f1M6oHiZ6nU@{QtU3`iwLb-S_PsjAx4Jb}m(x7a)flTD5kN{B%k4!RhD14atlb>m|9Z zg7vGKU{kLbC%W#??3&Nd3#sW|jNsad<+whZF%}sILNhDcLh}>L`&zr_xZT{C{oLjl z_oa3WSPjn>Ho2ayCUG_%neDl+urb*cyZ_u=-jDun>dtpUbM}_){de5QN{5fzH>!u5 zPC|PgQ;t72!ety&MOx{;UoiAEPeD!D%#ijXo!V<-{^dCK(W~A3yLv&DJ?{{1GdU~z zOq^unyt=-j4IZY?nS9X$v)CuGE*mYPb!<%oXC2QuRhFV;u%zH%$$P{_zNZ!2ro_U9 z(}4q_7Cn(x77;V8I>{c4(u-D?dcI@&gTtu1w*7ZLR`zXB-(ZisbI>PQp~X!EJxCBdW#w}KEq z#Y^TjTlGLe;;-m4u+dLpwIM9hpZWC7wG-!psNDQOeh$@lS-ReDO@5k%of+s3O(^kb z;2PoSArtlMRJ-TcWoX8OYYcnSXrv1ch^=N6^}l5_?i&o|DDB3yl`ksJwu^*Bb-LcK}Oc6f8HY$ggRDt`gd1*fN@&#ckx`(bH`gaOQZ+>RHHs zrYJ1jJo9TqcsANpC%HvCudKe!#&}fF+?WyaFtkJEQM;nA^w!%{%~Oa?G^lX<+8&d3 zBcIPTO|}U=^Zxi9;ne|uv&USc{1IC-t&Z^7r$nKT`%!OdRPi{Bg@x7CzyB2D*45e# zIr4FS(x%1z>-ul&Gc~>EAW_~mH``Cmj-{kdX$kjlL@{fKpWY1yv2g-E(=K|gdf+DY zQcv~T2FlyP<{NAEt8Bp^S+wKx2z?Ta2YTbzR8O240_A>LcrRKXiYT?0<10G~cj`R9 zC1(+zcJ0xFuCMEH>_55SR)+?Vf5Zy-QV`&{sG0jgy5jeJ(+A^H#>1KP;9cRTY_n;K z%=?Wx*)k%^kFfVOuKz_V9ZZ89fiDa{4u=Q$D@}pq(<6@S_n^Ds$6~Oz z-FcNhqMl27b4wR{ozbT6BB47)@N}R-r?5+Cw^O`qM4V=8Mg=qo9Vg-ee<@TVHGv)G zmDa^xB@Gha-52bc*pki7d;7y^U3k+5*)cBCI8RG#`^)dU2 z-8-QHf}!0PfJ+3rtGhYxAEbqXA)HaA;d^|?a>vh2InH^H9U#-X1&pWcPaHjbw5d}V zJ5}A9$@_fX#d|)g{CTlg9>wm$pV$I98?=ph{T8{w`vs9Tk+iZ&&-`@1(C zo0*pbc~c6IBOr`YDPnsYXuCPah30T-OdJS-y9KfL`@gsSu#>ut$9qd|ya4@tle?r* zdsa2=|2>cOy*z(9J?q`t%kNCOz2c_;ew|n;DHj|`_8h63-t)aLHw;L~+(FMM3q$DZ zOT{j<0k~Zk;eneBw87@)-NxerK)N!Og2=No-{c=LB?i|r;vO;d7_}0g2lp7N{e)mE zD^MKBzXmYwqY+YGZSVj746Bv!3n=loYUKL^@=N< zIN?H%ZC^Bk9zeVTkjY;`g9i~NREY2%xNi^HEtGgL-NSF-?oHGy!Hypfh8#%;05pFcQJ%zeFcJVW2utQn>2DXx5IX1}lenvm43jA$A$(;f z{{?^$Swa~s+OiwokLqbhT)`7VTD~X*(n=JB5>Oh2) z3(Ck+Fl6FP0Cp*MnzGGWfkM}Y3VM~bODZR2zyyp}oyBkVmKS{IoL+Rf+5j~<*~X@B zlQMcUc3cp$VUHk1k_4@C>p`zz4=z?n?D%)(218Un2-}}*+5l|B%{$)KnoOvckjdgJ z0Gcr5Ao`HI&$@;78OI`W7BXdn=uUxe!h+C1L!xpl634F(6N08Yg;Ye*#TWk~|7JxP zCz{7Ocs{hzqH^}3F~u1XYSG6bHPYs-g`g>r$R$6l$DxP9>CrlUp1f!tCb7(tNg)?H zQp+#rnx{%|PPC35=fr^y%r)7>OB-T3(~?U!iQx#yA^ADSxx1{>%ei>6G*Y5D+k8>R zFA1FsPKgd>GM~LHQZpe9IrQ+u5GBILxxNbR)KgD&d)*tvBk+7_^GUHn#}b1e$?;*+LXjn!4$UjOrkNj-p=>$noWW_w-L~-_(jJ@*mK#QW zG#0m=GV?CHO?}$Q+9GoZhY=ro^Bvsq$v^EU9>|>o2jRb0-n41TL4R@YoWt2#Te3ek z+d8x*qUYLg%0W8xcl&uqao8J&QRSS!K}Yu4fltVC%r|!0$9Qgk|LB}~j1S)Vx%!!B zYH>Fw@}1WoGDjY(pYQ%Jd(^@Hn{@8>5gzQpVFw@Y*)ND6cr4609qtwK_%D!AWL+Y_ zb0&xF_HD%vcO22ZzM>^=yyaooZAM}7A@9IWB zJQa+0z1!XfQRokQtOJ4HIEUYsw2yU^kR0zQUkck6J-+3v955VPrMx8~ydW=U;&8_e zahRohl8)!V;R}cUlr)3&DX7c_7h>T-%2{&>_BWgu@*o zWYRtcSi*0Z10D=4W9axXj}MkG9X`a8K4w_La=gQicXZo4|I#SPE^fqrc){PLgoqb@ zkT7S<)1Vp#R&tN-SQ$OTEJcF^L9+$+VvCf!CKKk)XpzIv)$k7ganDUhV>W4kr zA<8DogO*tWW#D)x4t30fjJ34bE!*+RBDy1YV1i={zcCI{_R^Te!pA#+DL+*!Z z)*+8(5|mv0s7E^xT26H=5>(UUojKewONtIFAMr>BG^1(5 z33XF0-&|cd0rN~bo}-QmB_(Zr6W=E|Q~H5EGEQ4gU`wNw4LM?BQwP~L?@9T^qEq`4q{RiAMj8m2-iW+RPh6i@w|{a;&BgZ z-gP4u)hk~UxDJjE3m@)aCkV~K4td~Xvz(n!Kky+Bc9_GmJEG#d)On8DA{HI)pvOdA zt1ElJLmggKmMqD&k9hbrnCS?sU4x4kv&Q49a#YrBVFXx>2zEJyjgBVO`l@kYR;o$N z>t3zvLSB*!i}_6NWuF_}iR5D*?m)*==)-4bEUUZd19q_0Ja0@IKe%wR0=fJNVAvzBS6RU&nm9RL@(T;hrHp3IqM?De@ zVA7>yj6~AzM7RqW?@p++3kHWdR7_y!O}D!5J;w?Ar4M(NoQHJ>t=hbd=-KORKPE%}5W7mV3f< zj>m;*-M)R?qrU3M^|il2j;OAQ(hIRPUN8+Kdqnul;2=jj+%aY;^GnqAX!bcwEiQD# zL&$Hw2XQ05)^ZSg9`?{jyoJV(eAGi8?pO!J*`2v^sJ7x=+XswGjqgpFLmlma2e&^C zz<%h1-tcILy)$0IaX7ag$cCDp+i{LR%TXNVNC!LKG3I=XICMMj`OnQUj+B?P9P5b3 zv?&)`R&U)@PLFlRbN;+}(<9vW{|>gRD-3ov^;;qTo=d=ebl#;KjOI4)_|&DPY?8mj z*?yM!cqtA#BGXppnHz^Wp8k$_(BmHYF#3a)ZI5}t!{q2F$Jak5a=p_t} zacq1YHkU^|?x7E=$BQ34C%Mnp;d!k$-e4(SEyxXbX6l+_@97N(InMEPcf_OP_>ldt z_mK~K&_f>Y__^m%pLz&K^K9m{Y2K7e1WenkHsY$v=r&OwgimoOZu zMRfWnjk^1(Zm=+AQ&E#{{(`9-l8+tS0UqRm9_#@h@9T#10MJ?ug3xjQECd*Gv!vRqz0rPXX@%2KPY*eaQy%0SBjV2RW}D zG7uetP#nlk!s<^QwvEM%iOSre&p1$yxPcoU0vwtU9n_)qSTG9hVG0{C24_$%YES~R zumW{Z9^k?H*g+l8;T+0g9NO@`3@*&R?((`v)as4gmT;|BFZtBL03~l8q>u`&5U;Qh zAMn8*bdU$#!3Q-k49C#bitq!$gzgf;?&JdRoXO$(tQEg39K=B!#^D#qfgH-A9EM>S zgYg%8F&uR9%1}(T|BSGV7)=Br?h+dh9C*#c|kT&Di?TiW)B5#tpQ%AsAKx6Z$|6m;eQ? z01l~<9oz98w~-smK^xt%7sWvt@o^o8P20Fl67OikZY}FF^1T=`4zCd#$3YtCLH^LS><;+ShAV;!iX6?=pV&~9|@X{gIUQ!(~0TZP1+EfqnpibfXajJN39PT$GV-KuEeUGgHjgQH4VK;&EIpeSiRS*H|!TDcY7ar@fe_un z3riFT8`MFik{v759L|9b*8xU7aX{6xMs+0%w{S!cF&&;k3`}ATjC3TekwS&>LhY~x zVbAievs~D-Aly?T;PZ+iPy+Q~0`nmmOyUR_a1ZyP3eQwZndA@iG*0C-3opSWfFU0Y z6;Sh3Px-WP{1i?D6(1JCBo^UL_s{|JG!z##R}KR-ia<$Vq)`FYPz{w7NtIF&^-}jV zQ*9*^76brH;~+paPDQm;Nl{c!6;rJ!P5og_7ots}<}*yfOH}n@>LW3Rb%TPU2#ytA z|8A61J#|?*YZopf1r}9Tc~v2N^=VS$JfO8=s^KK4wOiR|Q|m-rjinlBLMS@bTsaFn z5Tjc4&yo1AyZjGyHg!DMHC}GxBwjUNfoBjLI<98V9|*cb|WVw zgHNq>UUjTqDF+a8A~zWJUFyRmx?p2N=V6BeWIqKNXhH@u;Y||uPDmD$prH$tGczPM zUcHH8FU?{nM`Qs25L$LxzBMNnwr7Q=C^A-P*Mu0_KqNZhAYe8nSk`EhC=^;%Cua8cwH{Y z_fvj$Za3Freq(PxmqWCJ3ix(((J2&!A}T1CW?RZ;0S<5RCL};31W>n4R2MXO_ICXx zU4=q(d3Pf^p(MH>VuQDudIEE6LRBj_e`L39l2$qU^>~@2X}6Yo<7H}%0(%oe8zR9e z+JJ7k7n1}c65`e*nqXL)7m|QS8?+ZC+F*Q%L>q9_RpobDIs``-cYLd15K78z>|~IE-C4u*$d=pSN(LSdMujjs194*!Xs<;d=ofd;>X!2-tx0 zRgC+Oj1&1ozL$FOppx+*6Ntf*@c<@p!IJwG7XTp(nt%?L)*uE!lkvb1SYbzz0hN)V zU@h4}B0-Y};gm1=L3BU|Dw%GHArjgk3)(=H=>SM}!IDdPZ7n$y|2i3!lew5H`IryF znQz%3Kv|SY`5np0!z9*dKnw z3_>9kIw2B@pfY6IDr(v!ZrUG8nxToop&?qLDcYhX0~*p5o&BL2LO=$L7a`h!DW-$qE|waGgu8mJJuN)e)zHx zBC`SDNc8$aQo^qRJ3R^lX8YP8SRn+w+8?rDcMIaNV>&vtK?k6DNJ61>_t&i(=rgXm zk?(lB@;E2=IB)tmkvqY5sn>lYqac3Pc zzEwL~yRnO4cLgE@?&P8sLc9SW1&|mJnzlfWW1amWV&$8@55m4pVlA#ayASz4x?8iS z+9`mSFm~G@ntMm-*I2Csu5CCvs=*MP!VHERxY>t}|3P-GZ?>&Dd?Qqsdl#BKzGA^) zyC80qwza|}I)l5Zxx5Vmy#1jmgtZ`Oe5&2MD)hq)iU1~#L#kb3VK=!dxWavZ1HS=) zi>tv{kK8p-e92jSXA^>GWBjjYe0l%Fe(_*76oN3&ITFyex~m}+%m5lPAq$4NAa-F8 z4565@;x$|xMKU=p;(5e3Xpu=4#A8>)^}N!KyKl8N1Ue&k4c#Av!yufZjyFR+Z0R5i+IAs!oT_o!Bz6MGVd5)TcP{WC5(3*GGJy&Xo5?$Yq@f{{YuwaT+aFSW z)nh}--NPhuBhn`w0Eql#UpvRu6aao389Jm3|8{{G4B;z^zzmM#VcAd<5PI^#A*-5)Sk*yo~ABoM8vL+XLnd;!!N)cWBznXB0pJzGDiHo7 z4qLxvyTB8o)ct`b(w7xRLgM`)zfm2`|NQ~&)&8#Ap5vvydrhM1MdIrHZN4YsM!lpX z(3vJQ_HVl)vI|0M@8dIiUM`4%4&E6LV%z7RN99}A&+pFM+s5eCSRj%fbu;^y3*Oh| z{_O!^;Z1vaW&CTS`yg!mZ&LuKy?(Q;LI+@CGz69=YChIudhe-#Alx4A$wT$gKC@pT z_>CIxk3;ocg2r1vG!#N>WmX|zc18&Q>kl73HsfRWn_CN_CmMXav^`}lAB8-;XEk4s z0sZ_9LLe%9bL&HG{}uqC8DLMJn^il~X(9z?#Kr+aTN3~>koh}!5C8yZ2djlZ=nt9< zg@X>#0Z;_uK|BB=Y24`1pBVr;|59l4r%hosgj4{8(x~l~KRh8vmOPmfz)F9E0A%^t ziR3|67%c)w!B8VEf*&CQ0zeERF#vQBUfgvs3xI>r0CYLHv?9WuGMj#^W@uuUvS!=5 zeG50P+_`k?+P#Z6uim|U`}#!|qN?A)gbN$)Yih&YQ(O6>L322=WQ2?-TfU4rvu3kP zlWzVDI`rL4079Jf$5*Dt zjdTowEsjzpOP3lA;qlk}IRLRacd`cH8X;`iMQa1NO*$EB-V&U6fNIgrbTEvA7*VFzQI7lL)aFqghfehni0}w&xFQ z3~@-sLF@_kByDC@lSvT(P!S4lARdZnqKdj@22zVYI@yDEStuc7kQzqmrJ8Ok7l#0H z3hHKAu!UVGO+jNu2u)-WLJ@4?K_f<5`a_hgI4YC{G}^SGP-%b|A}B$4zcwb*K_kPOSAsw%6lB1EjL zLn%~5RJ+F7gmY%m;U~IOArncnMeK5sPrG~z2rjUNV5&b-zSI=KAp{m)s1{$0abH;e zm~qE$Nm^H?c}?i?q!E^kGKf<}m2%5?{idv;9}%;Q%%7QI2FzUH@+pQi+gw;RDa<+; zAbG;n$v{P90<_CcKfNeXOG4c-$aIN}7s*tch4t26Z?^K+Vrz=?*jE>!b6%Yku@hKy zOxy;?X3tG`Xn^rq_oY_P)wNw(&)s+4g5xD+#e*MCVA+T_6$;vYozg^7)kGp_6PdKV zc;=eZg`k3)|0BA$T!GW|x7nkAj(S&yqt5zainESP({-`TUuk%(-g@qzcOgUCx-S@d zT&K%b`q;z&K6qDhBhP$ZuQz`eGJ{M+sO->R|MkY$GZ;Kv$HP^;*xzsebv1O7kG@&Y zr~hXrEy?g7`|?vQ_51>M|JL|!h5xnw_NSZ@QvHv7>-!%oMiPQH2ylQ5ycjV8(?HPR z&n^Bl3mS|di41CRgBdnT08qjds&IuYY@rKZ2*Vi4kcEl>032qB z!yM{xhdk_|4|AA=B>ZrQMBE`CtUv+)hg3V1zeWMUvGxScC+iLst650@B9<>q|JOHENHn8<@A@&MJ61TK)6%tRDX z8rDn-0W)*cj3%NOX3f?avvt+zgjN#R%4~}BLl4B;I0;lu#HI6`7)k*&*@?JqZcdip zG{Y&Z2hV(hMjJXfnm#LZGV-nSp9V_7)CgKPc_I#A$!xSTchQQ5 z|3+OJotr}Qwu?asDg|vIT1QW68C{}OGa0R0OILas9Y_nNPW!0YUI7T$$N&$^q^VE$ z!kBmd^e`_qTR%zx0Dkmis2b9NzL?rFn{G`Tr)UF>iZBS=L3OKhDJNHn8c^2tBOmi{ zhdRzdj&OWqt)u|JTGuL$a-d@!@TiAAzA755;_*<0S~7Pno%SsQ4Jhu99n5Ru44Cqi)wR^Vb6|HObs zHSpOWoG#1^p}?GIzk8$Jfmbh#z1Th4F^p9?Y*YWnH3A>RmUbJD_SDc~~ zkr>2m&5(sA6hQ{z>Xm^@lOi1?s0mca0&b1lh6vM^4#OH&Y%4rl+D16Sx9!4in;>8v z$RGs;KCnVUAcF*D!3-20L?k+KidR$@yN7{BCsKtGK=jnUitUM3d7Qtn<}DxXP-}ZJ z8xC=V0~|-bY&qi5Ss>em3fGbHYLJ18Q)I$A03NW3kD`NBu^138&O|6!@d{iJ0~xxa z5Pp$C3|zc|6`^P?Z0P{tCY0pMDPA!i43UXb@QlWD!Kt7^(1lD8*2;+KJZZwc&y_bMrN-%^6}`nWTDF7#d0Vj0SGftK?fxcL72PH z20*-6&@=uTe&f7fSht}IdaiXp&7kX0@N6!~xN);|fCp9(I@L^lAgiez1q-nGQo=2jfoX_SXiPLo$kU`qn{D?UZlNNk8)2I9P6M*xC^6?c${Os>4;3z4FexL5E0r{}Z(jbf^~{9>>f+ z;KA+5x+Cajz*a}Kv5QUsq6^z12o;)hid_&Lp4HgJDGb4Ng6p6QfapZcioRDL!<}mv zf7(2lELo1D8Z`AVhm-f7?!Gn?8mAZp*LzijHVh&ZZ)YmjLotXp1dAkH7{nECJ&z!Qib;+SkZ(k<|F?jjRrpm!1IkO^E&+q2rhMJD)_E;D4}$HI^1 z#1mWoIDJhY&3`L6;z1#PgkG}gz;?^6xr<12p;cc?q(=Z9!mAp3xK-%t)U2qO4Ui08m)z%oFcKx=X1hkk(z zZD4CcHHp8ofPqMS7&Cw05Duhg8P!09bs-OuRS#BX{|Gu@Kbar{>cW5pqHupQi+lkM zN~js62!ONW7o=zylkf_MUy7QK=M;V~89vQwS` zj|c$`C#ZW*bs`EOR6W6p{xEq4HX=&_00k)&2vLu?5*}D$V6DBGmv-| z0N@<#u}c~e2`lkHrFa*ez;h-wjK4HYocJ{D@K)q77g`rv1QA<+Fe@|?0+|6e&7TL#e+bZ}v|VH6A@B`JV?bU-6S z(-KYaVyTh>`hh30)dsY*6Bi*Iw3cWh)sP>-kS1~vI&}~_XO*$lN_VLddif;hah4&F zmIu)jG~rtN$4Wf$9`GTIC?Nyk1QF&_5v$ z=?_0inM>g$q=7d2eU#2@%w} z5^{$SL1H8PL?=P=6CbgPO<|dF!I=j!{}soW7sw!B!#SE%GjaEsiKiJL_wZKXfETQ( z5vjr&eo`kU;g34z54QP}+(;Mo(4bP*3BS2L$RGpLco@ZL7RU(^Klu{O*%KzkTnCW? z777tq(n{B<5n90z%qb-%$_jK~IMDf$gJ5suSp?=u93ufSJW-@FN}~|jnD99!gYq0Y zpbh*K0)*ud#z_pRvNUCanY6QiW~mJ-i4h@D1PCe_ZBP&edN`Xnj0!3m@9o}{WKK8LE9N*om-rT*{~Q^OIj1ptvrD*(VT z6BhyjR~#Z~of9K)AUQ5nF)KYGnssq3%j%{bQ;YywesUTbpV*A6nOLNuB^-h`1w#<- z2qYMx5LSv8ez#Wi0A%d3J=5c?UBzdJxgD9XWjI0`+j$D%f@o!NA21;&n@SnEiX9!o z5yv>HZ*vk;F)l{>Wmr+Lrr{3=i>mv&8ggQgMR26_(l)HoENZ%m60(OMSx*{2CoN}{{@}dA!Bk1bU<&&A+ZqAuVn%eL18M)`4l7B6r&m>twf|- zVx*P&v$Ms|vFAYr!7@`sQL_6IA-tm_Wg@Hzl&2?=vVlsOpDCg*D>l0j z0>;^rNV^%w$CNmW5czmuOJXS6&=VsPBZu*V9zxZ9tLhIVRVP(}s^!TU04uycr?$2G5mF+n@!=2b zL9QBcjcrl9b&`d^A+o&M6^?rlg*z2Xdl*ZBl!}Wo2l}j#s~N|KZj^fvPoZm(Aiw7k zsKO{>ff^DV;wW_?|GM7*4?so~?MgQRgc+-=5s>;3%n2_CM|!--;lb?@!Vk>A>Il8GS|cA(z3LVrSzw+ivAx>&m=U3U zg&QQgY8U}*zFq^T1xgf7@?B(vOe50 z>+n{j>%K(c|Gj&{!aD_UV=2W5F%UEL7RPt9_7x=Mi#mx)pm=y5eW{a<38?Vlw$0K8 zuGJEJvW<}t3LMd<5=;ebIS|Xdoe~iOLP#td%)v`SBUKi{joX*bF{&~O0NSjo$DGUr zp&zqKorZ$QAEB)>k^)7LT2=DMKSCODgC)BW6m-Bb9Ffb>gH4?LG%@?KS?n_L;9d`U zG`wOsP3*9cIFI29Cl3_HaKR6fC5rad8QYU{w!9aXdtldrz|2v##6T?s5n7QDSS4zQ z90D#r2O!&NoouRmu)Guv!z*PmYkT>wHGL2_{Uk_=(&^F$c8Dd0!zNd;=%)gQ8vR9v8*;xti5 z%9nC{QwL;_y*(_F0+?)HP;8EPVFuYtR6(fNi|rvY%+YsPI8LL6aD|A8;-PPKsNfYg z=#!{f@XPY0hkp^(C(0_HZQ2;)$v=#`PQwp{_Z2eX`Xl2UtKg94hPUYP%dnL3+2heTb7Sq_K?Nm?`+iOJ*XYC>OP>ADz z;r3PB12j2;LIs(S;F%~(El~s|`QR;s+NRyiR8!tF+2I$W;Y;~w=F^AhLkxg05&7ca z!$=H<@XUbl<1214%u?iTDtMu?n`<==>rEl2C|Os`Xeq&x4HQy>Qlg?fk7g=fw*J_aEH?2_h!#G} zq0tZTaO~eO4wIX1Az;|!R1K_Pc=RR%S$)DB>9T5>@CwyEPRQ^InV^9$*95dq?&a<=?7`Wl%~58< z?3Wb|>;UbPK@UwXS>s?2MQ2beb$f=;mIbzUG!b?OuS)mca(nh=ua59@gWpEJZgjH` zpEz0L&<+`282m5~>QHK?9_J?~13Y|E)qo3-*Omm=Dl%sXq0sM_t~u`Z{|cdib2^vu zDNph%-!Y+XS^l%UOkPrHR z`T2kk_An3d&<^Ra=CC03FBgm1qRXml^wPD6^jpkG@C2m&{2 ztpCn&uWw=JJKEr_DR0e&zcuIX=a@Y>72jmd-~15|<#SgA;=5PuG5Eb=c>sq8c}8F+ zhGJpnU~Q0G%;j9sm1fk%X4jP&qy-IdwhMBWTCC+-vNdc0CSt5#{~9tNW_a-XGFJ7k z01z?&3?xV}ODJgm6fR`g(BVUf5hYHfSkdA|j2Sg<UN01>ib^>rp& z5~8+}E@jT_M{k|DZ#i}D!gVWk?bNqA zno#Lth7O=Xkvg4vwF_A_m)hv8Z8A^?o>Q2gjwW5&^l8+)04zn_+VyLhMF7Emt#2N? za^QaM-YM>!yL$X+4=3KV$?W3El`GtQxJW{k+NwcAMr;?Z|5vO|oeI?{Q>I9f42>V; z!@+s<=^d0AG-$k`@ZURy+Ro|~u3f~CL93<>_A7x3D#9OaUKyk_0Ez%jwB;If@IeSA zsz!z`BDC?9LC2hO$QkEEa>_X;9d_QC=bnBzlySy|s6dWJ99hEU zFf;sXO$+scY$Zp9w85hwJQ~u56+jSFkQD}rlyXWcC!#`!D67k$RtZk6NHYo9+*8(7+s+Wg$^C4wHDiS1D+8Y3I#TJq!0i; zc;Rf#FoR*!D77>NJfL;x6aczV*tHt$bwC+!_2qY1&upW@s8@m}l75Sbgyjo; zwijrDk?CY;wz7!is;bitDOR-)21JLp<0g_0|ElHo=#(xl#RF|3Mex`fh{Yy5gf7mu z`|yZ%>=tovcSMG9$Q`6Z+{i831&_3H*3x4KwuzaTOw33tg~FMxJZ*yOK0VtMB3b?Q z*EEFvp=u&YRqJ`Hl*5fPywE}g6_Dtm2H=|pUGz)M$i(#7TfTDR=3AwsiRi0$3Xlz} z$D1uW_&)OtH|*Xek}jl5L3!q_Pj_kcX?+EN$J>|Rr6!xdJgtME4^xgmARoVc0LY*X zorvH3yu~&Hx~g>vWZ+2#F`(5|BNM68fgy5dC@sigb+|A@7NjCT0ydCViW8xv=m5bI zrZ6LhKq2H2^kztf}N>g0cV8-LCmi}QuNG%yf`;Pz41(j zNP!%$n29jP?S*VX!`3~-7HG5WXVn(Wg-Mz|AS}cFxE0{bYhCId}q2Ga86K)tX}}A);jB%5=1KU zpg7V-3P!om)v>}$5HqH*6uK}6q#~FCC7H<}`Vmonlt_a}K{iENHHn%uSK$HI z3hMDjS2~;~G_Eqj>R{ETntr}YVwXay+JxZ%e6`iD$y?q+LuyzI84IYGg&Hy7r>mP~ zOlOY$7-#=yKJ?Lar<}!5{}QXp+H!d|sXb(u*1C!`UN``3usv-62bfqnYJ;kst<_H?mu z0N}{WkQ+)x2UUEdVBh;7ZOEWk_dS@g{CK9T-7XU2JzOg~$6tp;af$$#?+%A}Sp+u- zW9N;qx2gevWddq@6r(WYQsICKKV*#y{!D06sMCjqxIt7nrHa1=lMIU~z4@!2C_wCD z5gUk*CN`jOv58~ls0OY;))q`Is-@ad(1Pvrgb0X?CnkCFwvgklcb|M3%uV^qhKk9T zSdq67(BZ$)(}W7P|16PPN;8+Rv+tQ{li&Ti`Bhz}5)zg8+Q@0526D#AsiMgzJR|LW z$mMfuFc#xNHwplbB+?ieykrqRq6dklNID7nFT|mk(zG#y4g$StB86bcIND>F$85ku zOd8Y_;pn9enl>MQdetqd2E}G=s1%HvLzE;aIe#b!B;48}t-fj0sx0i*bP3VO{!$7y zTii^%GL~BWgANUHFz#Kl;HB88m_){P6d3SlVk@|H&H6V0gnF{&0v#T;da_c*WgNg^Fif z;~VF==d5~G|F#sa!)E&TJ_w!>P4l}n?_;#ei?a`W)B{B8NXI$KF^+KPW+yyIL{GpW z4sw_S9qMS&JnWGV;W#Cq`<%9}wjqpEJVFjt@EYf=es!#8UF%!tI%IcWxE<-Q}uL-A+_I>y19p9EqR zi1!IN%8?FtyeLL8sc15nEh%QiLKAuLLcO~FdC-Sm^rPpT2#|0FL5PAE?iH!oQ;Q>g z$YULcXFPAhfev`wV5JZYNHgoO9rBl(y|JI)b~+9zN6yBSA1-cb*|pXA_pm6SDV zAqqj9|6y6A=YIFS|NYOapbBz0L@I>g@rCyGe3i8jKHNc$Tra&g+Cf(grH3K zp9wzTfez%0zn=gHa0mx+5QlKcIk^)+o*0L9m z07zJeq2NEKz?haZ!jssCctD3AM7(eyhjUN|cZdgiu!nodhkcm5iQtEQ$cKBVhk1Yp zc6hrTOg?c~hkD>biok_q0;?kn6J^l6JkWxU>AD(#K}~3d8HAb(JcNteK&?AHLGXlH z|6o0BiZ9;^A=t==bU-|EaK0DO2R38~n5zePSO;_XL27^mpqK|GWC}nCfsqTvi*Ubk zs6EG12N%JIq>~8`X}fed!`%P}bclynbO^g@mv<2p>+^&^(7y1i0yikX$f}%Q%swRW z8?I9UCn!eqL%&{9ME!EG(!d9F*o|{|2YaZ*jK~LhP=^5QiE#i$Q!I)sz>;&^h#Z7J z-T;SlXoq>ohtpWQcvuH_$U z$%?3lY}}1-|44^; zILYDY2YYabb9hH~xP`9@t0_^2R(OOuyhE;g!5AbvB7(XMq`J>T0x$RjLqtSF8nHIw zF`|%%anuQNz{Q!Uhezy*bpT1042fOPzp;#neuxKJ+zE4d2by${ey9g_SW8*p0RUKn zN*K4|V1`-%MlaaDW*omqFu!UNNbJ+TgXF+xJOtH~#y1+Iflv^uR0)3gO17*Aq|gVB zY{0iX%aFjBbMs7ph=;7)iF1<z~^aEBcX20iG7oG=G_h!DdBM1$-)5pV*9C+=+7F zN-DvJb|}r9I0t$p4xp5}Dj>?O`?^06JEX)cVF*eMbit?egt7CYGy}Acu!jW%2Y4`z zd!R*kxXzO}rc%7Wd@u)}BSJBu2L|nlb`VRZ_|4#q$gQgZ!8}eCGfY7!$i&RL+;oG~ zKe9Vb-Scz?b(yW_EJe8@_1h|X7W2c9?w3gw8Ux;kfsMiYhAUkpyb+i25QX+WN6fmaMo`C2T9dejCh59{|&r0{RU`VAp`-d+`)yK z)d5}W7w?fM?pO=B$xf3 z0O;A0=n}OE+NlVEA{eb|P`7~yGe0>JD}vjHU|WJf7tbn)5RifWI}P?akB*fieAo$G zZNr)^f@P9hs+bMO^@k$?fGkOs8<~iH7)LN=S&eX6ujRLSz*mmgj3yX_Ovsy+?6wpb zgiydL02sBbVF;hyg-ij2RKbHxxs~i}gXYZ?);N+I|Je+x-p5Xx2T#hxQevrG+orrGJnH}L{g&^gIpkmb> z2-PWJMZSpG)rn^I;)-B}mxZ@`%?Xv|2=&F5#D%LPxhT)tH^t}|XrX21pyP$G2YNN0vv=WBN77FLLO{|110rn4!wXN93zh#i6M6bNdfrsKB0W}sqina10>ZHRse zV};ObkI>bG_3QHhY}2?Mzz7iJsNAW@D#22X{bFo}007DnwV9^lB)02>2(!#ikoz8p z8~N$a_6Pkg2;EkQ?v9A>eu%umjPX8_@`mZwW{47oJoMafXDr90#=Kw z@T?R7jHJDkyLF1-wj;}2)r&XRL-Nk}?+Peh7ZpiR#vfU4UD?ys~}J z32^8L)yUi#ZlQ&MAr9Y={1Pwoo@=ne12QM`(b^IB&1(SoV~wx}oiGRab^!V=ja113 zDi8={SqRTkkTS=EQBvFf|G?v825uqA831z-KkpJ=jA**IYi1r5zvdQ?T>a~FJSgk{vU8k?a+%0+IYQWP zhKf_*;eiNEAzyCR4Ho~HFC@&8+9v|5bx$MB6eqgU(gx;?leC&LD*`I#adLeQt zl>klVoSZ@Y=kx65aEib$RGZuxf}Uh1vwl_;x#p{cHTp^;_BiXME8Y7&&(JcHi)M4V z;Iw5pz0J?!>{RCU_9yg~nPP_t$Xu}rgOzBv#jk-)Y%P%Y`+m>BA%u-HNW`x*bos*& z#NF}r4`(G?soAfd4}OQ>`qJlrx;m-8ovJ|zje%%5Qboj83{+O49XfTN{|Ze4SDJ&X zw9l*U4j;OOGX#bkXNEt}5wGz>SMMGclsV9U_U?V3#ag)YNiIy3>6;&eaq5jn5u3|y z#&4s;8yRSWmyYPBeRQCREuHbHWu-<>f2<-lTSdUl&6P|NV7ScVHdG2J~d;^vF zH>Dsdu|Dcq_dm0}sCA0SuWmWt=r8NB;g)T=a}x%3zLEAb!H#>UL$4xZ{Go=a?@LM_qq{tYKt$`5z{juTX#=(cb{Wy?S|p=`)c zQgq&6^W(SC!hHG7mYtp??aDVYXYd|ELY!q6hM2Uhm7cD{Tz--m5dzEk%VwDb(#`I_ zv_vo#3~aiR%rgfk=Egax4NARO7K2N5GwaAWAW10#+n`N`nF5API=!^<*}6ji(G+=0 zuXD|*$j6$LqdBuJDw7|!(Tpbxwl8lv6GrD`JX`_h@vKAf&1MB%s{u8zWqhDv5MdUt zd4e!&(C2i$El@=ke%AAX4D`QY3OO@*fS@HC%Aq|#**z+Hg{;zJTy2k)F6%bA8$P>IsG63UtBn&63KZfx#B|M}gX7lp>8p zvdm=58+?R`c@JXXNG<03)~cNf^rsW0_z8J|fx+3%1-XAJ zq)V^mxeIpxqkXa_KKkS9aoZT>qr5Do!{~yXF3yho4jOOA%7@;FEEJ2VEN@wTe|Q_u zd1<((RogMEo(YzilYX0nrItVBDVdwsR%ewqoi(HcJu`NCP}wa%UhrD|Hn(_gf{F0- z^%pliXQ>s6<2auW3tj1#D^C{QL3DpCmB<<~aKANK|8-q#qNIUy8bj^N!a*#$daxck zJ+-(s&ThL1lyBn?fiqMVul}XW(b%!|EXEZ&)E(5x4pYc4XT(vgU)cY7;1sWJrsyo~ z4jbic0mt>6&-f0d@+Ea|#)G>)vt6A}TNRqfmrJg8j!&ALONfY5n?(Kin@g=Y;C8$a zrq!{K(fb?oV`GH9ba*iB=ec%`zF_8>9Hi&sj2MOKP(6lUgeA5W~X<1lNzhA7p= zR2&m3ILcaiT&U`fY{9M`Km_1LfQMoIXnW zSX(KrAS>v`NA0SrjP7hybEig>qR9Ol%XAl+Lp3V;k>LPr`gz}vrNOB5pgEN^iwx#2 zcQ5&hQ69apN?e}VT%+S8zBbK_P&{Mj-}Z#~ZQSI>FZn_npkL_(Efp!D196r0n&3x= zZB(8d6c4UzPO<*alh*l%3MN6!U4!D8w1-s@%L;{3Z{?vU1+flan+sL!85I5N)iWtp zwUs!6wf%dy)7sm$cJBNMxWqbwjdF)z#;`Q8j0MYU8cn(^%s^!0hW|fnYqb}zGneux zb2*EkUHug^iH_HNuQ0KsD%o>In^mzPe+d+hVGnc1)gx(rw1t|u-#gPul)YI%aX3*< zXVM*Hy$u@TaybI*OL1~Wh8sE?m?VdolRh>R(g6;?!+!7&^}@ zdRou{MujDovu$jp-5ymeD0&o7&Bl=rW=p4pI7|gHdMX&w8`ep*7Ji!(VICpve&7r z%_-^0T_M3uH#YKmVZ4g7|`Iz=PlwU{p z<)bLyrD_2}d2{tW$*k0!i)hwQ6rpO}l?S0o%8)5c*FnEdJS{nf;NMM zz-|hwbyKn}rme8NNgg^D$M&}2{dP=Sr6@8pe2SCp+yg_xFQJS{IGKj;Hd$)(0)*)=ZO=i;+F|68N!6|S+nC4wxcoc zmZtR4ux^HxcbWuYDmAC5)2e&V0bxOl&)IU;y~UtgyQ|5K&o0vg7_pUkckFW>p?ZnU z=Dd^nbkvp+SjNn9fYC1e{qCE=y~mSHI<)ULiz2+&iFJ>G@b$;^HGd55eRIep<)Bk_ z?twpA#z@INJ>RPw^LBzXUwm6ea+R=V^~FxQ``YGdV6=J_C%PwW=q4-Ez^icbP%cS+ z!iYRD&531;flz+Ti2TX_DyjC`ieD)y!q?Q#g7g8k)| z;GpA=4FvK)O<<%l6}mxh~oh^no0>O ze7>LMQdZm2;psBYAgg{N89Gft<8s-QZ8-#)ASQ|28AO`RslKx%_HtWs&}rmvWf6do z8mtnZuv0qPR%zFl)<8u&;}8yVN~dtBGVQ=s(&Ph^V67EcDO8m*hqHz*S2q+FdW~kR zFPG@r)(CXgmeJ=^Lz8PD)L(L_G*)R>I_t`ih;P6%gPeq@s&yxJ^tbf*Fod3JJM1c4 z53ZaanvqD5qM{Aqyv5GUzSR#oT#OYaGj5Q~^;;}O)yBHJrXqUM&Y<{CZ7xp-ePb8% z@S0|T+lpoY>AGnUy=z&?ow@Y+VTjm%beL;^?tTymUnKskMJJpOz0eRe*6FTp2f-ET1ql`-_|Qplq#)=9 z4%;l8xnLk77*6jDr!#~{t?<%H4Mcs~k937c_PH^4;;?SS+LEGR8VH^s1Ya>ouy;@d zf-7N)>ra61Y3!C@RsRsG&V|utzjB3}4iy}z`59!<$g@EslJJksP$z&!FTLHW_gNVf_mT`6f>WtuP9jxl98jW-dEk*EZfS{dlddN}gE%>oU zLv*|veYZ;u@o}pz-p;@a;cx-*Rs;{)lbWH4R4TTT3BlVsC^GE2?oVPwE`H!K-40sjWRRZ$!EA`LQ=`@iLU73)y{4M2D@rKtB(gqG zquYS3==x+yz3Maaa$IMGhJm;J703gV4^{j;~0{f&pkT3N7G0 zj@nC<5z29mg{7U0WSB^AAb0@;J)SpZC!DKyu&L-|rnV-%hxdg-8F@i0j78}EfEV9X zbM3_azeDrn)yYEmj)*gg(cP1J7S3zoeea4CWB@G=)}xx^2Hvn_%a8=EYpNF)rQVM_|?%I+f?=6^h zIk~M?O^$omwV7BVer@Sq@Qfk;NjvO$v-03ZC;R8DL-Ry~kd7-fKDuDkuRkF;W-U8j zExY#xBObx8U^jTs@Sn3$_zB7!8egcNoQur+{$nDnAAcxF6HAOQQ9#6c(0pa(Pjm)h zQ+kDfsAtBHZhv^C+N$kY?DX@jAPO(ZEmF@t&4IUH16&_Jb%Y-hnFxM?7wlhN7^D|D zm89m`*lRypxTa|vN|0HhL0#{OoFc+<3}cyw9deU^A!WsIcrfwV#-H(zy>zR`DKUP03~TrSK|*UhpXaNn_y9O*E9s)9d%XX5cuJZ7E5^`d zzy72qx%gTI;-INlsumLn(#z_4d}{W1YWm@mHjNBYG3x_!T6QhRzik+Fe5*zq94~;C zLbLOtfUzo&&a|y7>>8c*8!UbV{NEA;juH;R?eLpqRnzp6-jOE< zI_hXI;`j^Kc)Wk5$p{)aae=|L%2gCYJXc*ZBRavKOpKT*p2uX|wZC1p|1?2qi&tG6 ziMcY|0C`qlDL&pP_Z3qoO;kI>1h08Xo8n6KqR6X>5gYB&oEk{AVbS*i@E{NtU}-n* z)uR0S1V(*QM57_8l`1J&C!@rn?(fHZg3u0;3S-bfNaAZV4g-?MHxDEL7BLQ7!MCmV zS(cy-{!TU75fLavXPi=j*Zl}h4R00W9XOI&9qh2(8o}>$o>|gyi zu13d_XgzysFZbkkBK`w*ykCjM@Wo>d!dBISp!?tdo0yJVwV?(noZm!x($?o1xCG|x znvJgm4Ypq&y9~%F;_JC{8)mkKCu$J;+p5L+WQqs`DM=as689@e)+5zo`9SFSinm|? zH5m8GwHIV2c*SWfV?ugS&WHEQ7|v7B#tx_!&rBc zU(f!Ol6L0kL@gGGm3*a-wN1;;m2Ylac<{!JIJQLx+oKH@QGoidp8zg-9C&ESZx7p_2~fbPcS;~Yur$oNb}d{!3ea_T?3I4IJKYYlJRI#S;s zBICYt#7q5}#AEnxvvrVl!fwvhwd{EDEbTu1tBy(G@c0kqX!+h#+lsZZnUc_^RE-nurjOHfp7hMlDz-X8g;mmlsq+HVY|l(~x$p z$j-=!fyaNhPG@6YI9z96jzlPJw}Co2l&FId#Ln>cXOR;W=F+Vi^l6dgffc0Crilfm zAssOXB^UBrr);J{rO$sq($h(7z;nn!6ED6Arim!rD$PnNJ3M-s-ue2xsO;&H_KD2D zB~dnUQjnEYEUdTpCo3;2*rnAm>dUv1Pa&Gb&UK4*Y#S3kVVHMcCZjf4%Y5g3_!yKw z;)vXUV)S>m*KqH?F%7PY`f#=S9Knb90mPEOc4%wVhPujootDqxBh!0A+pr;iQqZ?2 zTFWZYJ8^xJ?@spwedqxMpR-Ssk8p=X^e*$=d{Q*_U4U}Ur`zZcgm z{GCC>GdIN1Wc1~8cJVQd(!)*X>fUFR-aH3=mtI)VRmW5jPPais z=T>ar*5FxEhD@;sF9WJLMl~d@30AIh?T7vQzVx~b!6nu7gAPt#?OA2pFoYYg8jDLr zClv~DKqQbb$_RmdJ~tU=0k6j^Iq`sWz}e6PmycS|afKycDvZiSH&4*AE9f_stA3%R z$EVIPYBz%t#ju~(zp32~E7X(eq{A00WSqNs%%+UFMzqrTQw+t`-3+S@>ij9=w5a%J zJjXanC!E2M5)o4|-Q8s_l24eo4#^?;?Kx{PBn0-+myDh+Zbrf=9#b!ICjQ z$56`$h(x37e<^MlSWAV$yTR9~3=-~3{YQ(nElg6*!yN4jv;rnvouMm1EOTT{W8fMrco?b&QUSwLQmI@y`U*q|l4iD--u-jBIgnA7r)ycGY*BwH z$Dj!;*<&2(N>t-qdDhwEJSCA-6MQw;;_L~5mR_n!q1S8KlOpf_Qca1Cu;ffhOe;}O zOD$RFOv|iCQqRck7U#?;oc26?t+2xKbl2DP1d|c;|I#LOQw%(@|MhOqG&%^xRm65vhsOH``P;G zdfsQ-sDHG}_6e-{%Z};FbYGnFZ1TUjmPFC5xYv~AuXwh+qg(ZUy`I17`|cmzn*Rvv z^R>WfW%~7?C7b8#!Rt}e8a(9H?+jW%wpr=k&QT%gcZ)^L)jQ`Ji126UyF@b`)jgebh9>iy@$JVN;HV`%kNIgK zfofM@as(4r_A5fKGb&q^e}+x=~XaF>@Y2oX`nS1 zx7nUOfj*VWX8Ol8ocLF3h+{^%Zdn%YvoXuWbel|f9$u^2 z{Jl=5DXz$YfGJFi7vdc`mHsnGoTIxlFTi$~Sv4NRCA7eOVMyOc)B7+RBz&Onk<#FE z8&^XtjGS4qPpk+0p+fJFlhE(uHX1eHAYgdeHOc6iX`D+!yqhLIhq0scFliRrW(#jn z5!k>cR_lG&{%ssX*peds)Xs;4#cRdv4dSSFX>3eXmM;%N1M>m*vTN7#h*J|g(_4Zc zQWh~uJ%e%zBx<6yi8$sn?;dhWV66WP%YvVkKZT5K7rC(W(Khzcc2+!~wPcvjD>KEV z50d_BNA1XJoI=;tmUrsy@%ZuTSGqU#CN7p?-d*+|~*m|3d{=e}@XMu&Q z*87N51%YJW2eql$M{L@!7pFyXyH*>|R1O6{PA|P1V@?S%6B1Auu?m0S*c#XN$oWJ$ zS!QRF-Y54>?|bheM>kZejDD@|ALa1@<5~3wMBTc-h6J@ojs?hj5?+%f>&9O$C!QND zb7A=(7fMs09CAYoM3KE8zc%teDl}Dhw4OT7;BaurlbRkTGb;QG#ZQ+3f}U7+rltp^ z6_^CYx*qwoHoDUE(a#+8<}C7G@_FmQ1T|{!>N9j}Pw0I*>ou9g@$wIQNC@6kGhw|z zXOY7@d7+lvvM!4vLB$ZMq=ad49&#!4uX)&lfWWs$$%P(NF8I4&c=>Y&yuya}%j(07 z0_f5z^Elg_zFL~au%Ts$d3W_YE2wJ*@RQ`?A?g&)3&kqdyb{|S6oCwj_p5eQ+X_zu z!5Q|7vuguEvUEMS9_SME2h>`_wFJo&+gK|mu3*Hfz(>v)gC0(|&H4}JO$t27#|HB1 zqtom*G0$|Ts=tg6>uSqt@pe^QhW@psWNSEk#CfhS1GV^>MfIC`AF%lEZf)zl zvu}+ZuIiVvUO8T^x7+J|hM1OhbTnHg1d7*_r!+|yXIv4oZ`DfQXY$hSlI=uWiH?1` zm^Dnd_?bj2T((!R_A!b|BUkXvS=ieaCrpa&nPa)yk zEqy<*aUYhsoJL}$I;&N_6vf{6_)3blCV7}PC+kd zWmM@_vnc&h6Qc)>M2C(odDNy8AdD>=~T?bVmU)kSc&s0P+9KAZ-;z1OY{DmLJ4#XxY^5yp{ti42BUbfQ-&%MYQU zbL>ayVqH9V*LtHewd}wBp_Lr?q_MLYWB$zTO(YfX3m(p=X`Bhk84`G0%!5ts_M5H- zGtF0=M*Gi@uYl9aC=>;oktW;Q7${pd^k-Jt6(wxLSb*_@J=1`lupN3fT0#yIsd*`k z@8{+G5W04i2leh3xZ>e%^U!y9vaS`-}B+lbzA&!tBbpl%h zS$u@VdZkDZ#kn;RKMTLGxr;Ea$=$|0&PRxdiTlV(9YxiYLV#{$&#@@c91QU-D)})YWQ7+LS1d$ zu%PR^2)=e@5EX(sNiIW@DopZIhLMT~Mg`8f9je7CMBS>E0o}_{;df;xlM>?Eqo{gf zTHB%2WFk73(XYIMBfU6zzu?~vW51_p9S z%Z{!^Or$JNb>2_{ayd|?9I-RPZ*(RwaBlGw5jq+ zQ^`SsC!F9foe}h-U5dlr2UVPXz{Mp)#n)xgK;yRB|0_Plg`=V#%)wcF(2Lpk@Z#{C z{KVlBLw*tC66hWv5$vRB26ap#4ke?24t+cLNeGAPXBcmG}*w|1*U2or?&AKVgQ;h+?UXuV7Wu=&dt0xL~gu*00P1y7W^y?jd zjIzlhN_@qYG=l6nk8Pd_kd)&vDSC0LkhFIP{HR7iJ)XPZF2t5U+pwKR;&g$co2dGqO5M$AviqcP-_ZC)%P-#fz9j2S9f0T->G6Uav z_lzW!{YYg(Cw(L?J$W>p74$lijHgsC6?^l%^!zP3Ba9}@PSYvn)leFvJce_~iIB)P zp}V{r1%0V2UKqzv4~8iG=2O6%G_x!;EzEmdZJW{$VjRx1vt&kZ=Cz%qy2zt9>BT$? z^ZewWKX!iZRwx=1mKPDm-Pgg5VVsN)D^Pc(CxGV~e#s#u4lU|$K7evoK9z5Dko&Jq zJ{N8KJSDzZJ`dLUS$^iXc0m|lcmw|{#?D}zeT};0k~oIu|K$8Dz_W}xH($!=w~jPE z%L7cvBe#?|#NVB;t`7wj>`axMR+YR(m4YjkqD)niR#mb^Rf;QBs!Y}DR@K@?)p{${ z51DFAtZK}QY96oD*fG^QS=G7~)q1Ve`Z3i#v8oFxs*6~uvr~Wq0PHs&jvjyoU|13Z zO;ZzfxS^(mh!`#o002UG06*{pPal+Y{{;ZAab7)0f1TnXa#^C02l&*X#n^P z0N()M2mr1D080ei?*>Q!fDQn-06+);!~yr!6`%_MwgBMt|2F~v1`voF50429egFnP zA|fLBA6Ohx9C}hxZcfa&mTdzQ0)f?)+FUFYo_>1$+Q1f2wJJf8Ys#eextI z2>bNuQy>C}h`>fhM&1u*6m#7Fz+&U#fOz10LV{1C4;sB^mu#1siUrbuw6rvAS{iF6 zD|aThA;wTVTl`7(6Cel3`5#zpP7Y86y!anjY;iGBDN*(Rz+$Vbfkwcw(GlC&*woY% z-W=ZCj8%E50(1Z!9UYyWSZrtK+qZAAZ{LD?LA|}$7w=zC_froIVX;F)!^6WkBRDTd zUt-6`CMU7L6reVxHZ?V6Gh*4ozAu$!CM z`vGhNirb1iJ3FKYq~b^7*rOxh7&tyYK0U?SpV_N^S53T5y#60p;0J*H@gw7B#{J|a z!IIEmX;`oX~3otg+TkSSN3+_Y>@s2y8?gHZB#Lnv2aX z#uis&tDCXS9oUZh^BKYpO<|`NunVi$)otwdG4}X7_B$4fy`TDEEbxzr1=(5-kMo#k zu^wx!7)pY0>J}s0?lmCgGVW_*yPrlf@hjkQ;1;Xchr3>Q;Q>J)$OdGt3q<}=udPuJX8~vWk~+f28n3`5YV}5A1GC`{WsEhSGRjz zESlg5M{M5iwJhxA44)K)4tRjuRFeE+$-q3;^%qa z%T11{Hss>!4<{_NDnWs|J)MUO^(IZW(>YaoMm(9^&ny&Q}B-F)ld!`FzqzqL{#gdXQ1;?BJtSDWgsynnc+W zL%-OtsPE3q=(JZ~o$qasa9$_k11gDsOO~X z`0VY)hbEJzt52>|dTevVKqYPoPK-{z4bQR_>fZ{#6pkV+xh<5^8}e#SS@E!Kq22sU z?G3T>qx9ee`V#Z!HrxfS{7{ELYyQRVd129HtMf^*QK9csQrbwr=|+coN(QMV1xOu` zR6aVZkvS|RLW&~@5m~X>)l3_GIdZNBA$+l%JZt)Qd*p7hAzkKuCvrCEIk|Sw+`s8^ zP1=Ak2jd6!-nrZ}L7#not%vTH%uk0)Vjqoa6<_!I!Pgzb4VUmd;yP9P`-CA9yNo>B zuLTrkx`&f(>t3s=(K$aF8=?ByX8weZOGF}dz>%7ld1>y~aqFnapOdDm2X761y){xc zOuXoLa|#h=7%m{Rwr>6(oWIHywMWQMb$5Hrxo>@r8GMF)hZ9p>No}hD-@0{jCLWp4 zn9%ZFdwF+z^5*=`v+MmUYE!0knVgR+?sH9>gn9c-sw?1!rS)+sTa-Iiu;0l9~(>_ew830KcZzZ zoBjuIWUY8=$*jBDR`YXnBm^9yzM8a+%Q@0b>&ejehz9 z5#%>yAXE>oj{UnumWtE=g8(Gt13na(3VQ*{sUn<&!Xc)EYibOk1sD$wyl<1%&NAS3 z%%QGiP?kQQ>XmHztbb-}T8)rfp@8x{6^FtC+P4^$our1V$`!!UK+N^#G1K>gF@0B* zcf(Yi;U+h)_nx|LH;w@Ed(rY#txUwAzK;y9&7swT4ADHPg~txT7E7JN=kb|fy6P7z zN(6GU_Ne}b9BQ7iS@E@T?8oR^lfsK@9-2zWcyssfwB;*cn)%-!6BVv2gh}gV-p)-t zS98diRvJ;OGEQPbTvtKK4o|syrp>1$&2kEsgzLnK*fd;lWyR`|Zn_Ia^Xb*ACyB5t zMq29bSb?^*1m@6NDr$zoDwzk3#?L22iZePGyTkbPaoQ>r&Oi{}btD_Z#Nx~FA5Hdw zjS4Stz|_tgAR3HX;?)Ilb>A!IS7YjC(vBcCf7<5I=_Xs=YD^J_I!!Qm@DWrDhpcxI z&we9*a?y>{ASSk6R2sy)*7JG7>Sud1<+1cL7}17x4Kd1HQ#v>+Nlteu&tpU5(Vs37 zgX)3KIu|&bS?ZS+?8?tBojD;1`QwC(YCthna!fEWd?mdgh9og`#Kz%MnmGfC1e4eJ z&?RN{$m&6v+N|$3a zq4d3iUq-mTl5Ezm#SlL=#N+pETO*L(rmHs8V67K3zbXo$>U`Il5fZsc+SJNa&!=bB z2~8F|+hM348`9GIJ5OB)qeuZp1cT?&@SfK)PcM%Ms#Tzss$)Os=L^tWsij0+-AuW^ zY|Z>ODs|l7fQ2{9{T5OqPsyBMf)P&nuo! zv+%E+k^Uf=W9c7f`#pwG3oJvENS0vFy!rWE-$2RJfAhrFyEv2uYIUknxJFMn=!CUi zcfES?Go~XAGAit<~H0?Vt6jQUIpDfv1QB(e2*Hi+;73`yID1 zwPZ|kJBWMX3(L`2|CFE&o%Ib>WEm9Nz@aT=*|lJ(VlL+3vM@IEkB<1eE&b!odJ3Ga z&V%Ed4c0SJoQ?HR$GnKdw7;`m`kycLh1EyXjgB%L;e*EvoU7OTnIB$d>tdMy9YATr zrmZTHMyC47>kmTT_OuPu~A-eySbQmFJXChKsI)I~mgyYjCgsK7&= zCITmhn%H5-6=byi)$;AvUoWS-MoBgpKM#&CS(5g7vkXT*?A>xZ{1psWr`~nQ;EQiC z;IwfS4=?|_CHpaqK_>dqEGGUb;v%sg+Yw_CfKpX*VAnvt``()Q?pEblbNe};io{Kik%>2t7bZ*#WEiUQ>2YpX{`0r@wSo8@0r0)Kz-Etu2M?}p({e0+$#%@p`&HJHFUScU(v*Y;JtJsqk% z?5BAQK|S{kJ#54foEnRlE%xkn4H17UlF%ZS4h$n&hOTT7eVWrmo7=1yg3GT}=`Bz^ zU}na8Xe&TpP47laK(?}tcP~DbvWTn6LcPWzL)+6C^wYYVk(yn{%E-lQ98k-Fnd{3{ z`JrG$y;JK2Z7>k4DVJa>N3A`^G;0Xn+{9m~@~NB+$C@WN$|X_-Bs9zs7fa9^qdni# zM>^j(QmUs}$rDiMX0Vz~U^ge58;uHBPI7clG>9d+Ymfg{7raTxyR)B^f=q}hpsB># zvl^(Ok_n$-jL@l}$ui4m&8&pYO9IKUFq|7W9y+<&BDRQ+CaD@3jfkQ`#@5`V)XF8< z%@gl#;@z1?!{kt{-KjrAQj|A|>^AYsETWxZAtGa`W5~$90_qhg5okXClR0Ve7XEBO z+Nb)ENq6d5C>W;3;)Q(HYR_sEn!eK=`bCaV1H?>Cuj=9%n9q$?e`zmf6bU|^mQL#lIs3xtkrh$5lMj-lvB7&KX7mhw+b z7iAW4a+326$!-8aeti~Cm>r0ba;#h#31kN_q~L~TOC)Er{AQYMrBlSJWfd%^9LwdX zSlaN9Lzeo}CH*lq1*wwdIjaBVhSKJygCajuXBJqb8ThB^hvnJq=33$<0{rQpjnXX` z^C2~PcaRKOsgzy!{B6E`Z+R1g!@NbgoHKBykL7bG%N&C&4AMT2m!F9GATyY;;FJqf zW}53DpUb$Djry-(Iw+(2f;MP8pH4n4&9ZQ;Gfga%m76i2;|s?NkHT^yy*@so^6CP| zFT{3PVHN)swO1!UyvZ=;&o6~?D~}g-S!%~vkS*L__x}oz4Y_X^i%*>rD={ER`=UIZ zyvQ)hr3=0HF(j=q#Pv|(%KM4gKvV=JT5cfvq*F5gg}D2IWCUwS)y7}EXUVLMCcUj9 z+#N5lZNQ(cCmR7sDmPfdF9>Yy`KY9VEC(__6be%>rzeI4OK#v<`4a>ig0nY>S}sW3 z!%89HOma6mNVUTKhWuG*SxY$)>Vn`=D+_ZH-lGkCzYX;wC~-YNvYAZE?LfI=NH%6j z`W_%@i6Jh95~1t~-OBNi_U!gX?|mqtWZX{ZwzK=8}8+6E+3YES69=TKzdSP|ZY3y)ap=ZAI{ zt@H=E?&FPS)l^r?Wgb07Xf$^>)|L^HNVekAF%@LisRr_b@pvWIYA`7Krpc7*^id%{ z!v;yl@Ty+~IU%weu5={>E$g18K1rko5RlfFe80N*;MQW!B(Pl@F1oz=w5Ldym-zZ; zgGDka3c}{rpA=?7iYiyXxqa2eADC-g_hbaxP$LpWr`38K|5Mk22A%z-tpO1ZS3BYu$~6Nk=9yq&zQ2?lf;|Hj2H%0c`7tB|O9f3E6rvB}xM|bEbD-67rRU70%DikHZ0goJhIqBr z$NnZzg#1U*){8sW`_W5y6!8wfLX~q4PCzLmE(clDgM57}m;jSr1p&woy05W)xD|cL zxV(~W&u*FDGbWoL1>5sH=q(f}4KSehm_h!(435maioiP%4jBPgKVFtl<3be)rA$#P zUGUrf4_EK-D)3FHWZrqw`ztWwIt(x^2;mI~>HeWy0le6z^0^+7@mddB9OAolQ2EkH z2$M312|axGH-|k3*#qAnUr`A9ew1ABdzEZmw7B6>t9Iq z?Wl=TCe@?YifXME4P>vdg4CBiCCl=xb}T{;&11Rn51;_yxjA7sNvQ=rapc+um;ZVQ zFBy#L$fmATf8YSF)r`p2aYaX314I0LoMFL=u>r-gc%}Et%t+KpP7&p6$iRF1KRs9y z6n)dv#lnStCg%{Pg8u9V088`no3+3?~%5?YQG_A;{UzZl#1n5dsh6ZDFq?Ol346*RZl02j7FleossX7_td*1d3_-Whcopc z(~L>eq|y~%Gyl+q9_8`cj6JE!x;s^Vv5<&k|8j&7s6w-0^r=+9OyK?6T4xhj4g{ZKOJ4bNY8ghPl zL91v_xEuT-aBd0@hb*9?$9Afkph9`9Z5|Zc1>0uKGzwXH0v8M*Xq5e=(xCkrG*e;8 zF5jXbFFuzcoGF?s^waA_&b@j;jEnG6x{wTWIrN3HNa#DU-blI3hOCBN?3Wswl&&Yj zCBnty9A8?8yKMoo41wY2Su-8!$&o{XJ;KEzP56T)h!ru`NTr~ml1V}%H6_*x3^l0J zT6<-vEQV&i2>TC1N7G0nu)rr6ZKD)4d({1vHL1lo^faR0ROf4}I>x_Ut`!=zRsy#& ztkG!vdi)^X1*PyfvPrNIu7XA|ALIGPlPoexpa8ONC^6ce&}#P6$18$_4Wic8HEdKY zjd8Gn{Y$*FkMX{^`pO6T01$SVhv=agm*(_QWwh+gC6thX%SjSqLyh2;+K~}-+ z&Z=k#b`+Kb>f-%kN)j#zXxZq0-;gkd(4vp&-p3Hz64(D>M7lx2MYF^TecJ>`=KKC? z`@ac@1c)kuQvJ^{3I1E6oav4e>o16Q&Eb0+k4c$tnR!W(2Ccg`cf0lmQF@nsY^xiv zg>(yS+rnLozzX2I)m8S_V~Lb9i$i54bfzM$WH)MNm+W>scE*RuNbO--({Ro=#vs~O z9^zCLo^)FgC*?+iL3rr|5|xsiO-$N(9CTl4_saMftIiu-%xnFd)Z;QP< zAu2P?%)-a^IJ^wzay3AzKc-Bd(0F6K{ikHJ)sU2D?n3cWdM$&*Ol3O~?&lW+uAjXk z+-$aQ8)+e5>(srfzq~4(*_X(?kj6#>t$5@F{Dx-Au9mc$i}toN`+E*m~820clkByqmX&ly554_<^T(NW3L70XLso@ zxxP)8O!FmZgY$pr)>)~*&wGUMF2C|qG9xnoBwE`Ta8Xkk%&8a&nY8sM@GB=AF#;#V zfYC9^{pWuYHkiZ9tE^7&^f|a+h~pv+ugf??vh9D+Uy|6xj*OHOmC}N&uz=frkm>68 zb&O}IOvZLP_O}dv!;;vo|Crqi60iD;a<44V)u|rl*EtDJ)%#iEAN)jU%zwb;UK+jd zqhb~fl=El?BS{FCl0{8QzNFs?SRfvm!)w}|0{>2L?2Oa^^o!Pv4n8CEg>CvGlfgXt z&ZQ?OVuu=rO`eibcX&*ud{dt94Rz+X@paHSik-`FE@JwzibpNBBSnvn&0gBPi^gX* zJF#eYnJ(3QL08Z7e*m*UOuqoQ{G~~wR_Vc=Xu)YR#11cdB%%6sIQ0K)+O=)p z#+_UDZra5{6bcEK_i*CHjnCHBYO_f#h*m;ICdlvPlah=Vqkb~P8Oyt?wG!k611WV1~--y8`$>j3-_PCM_Utr}G9@M}v=9viMg>NL3zne6{2?5`Lj`_!V4 z2JsZB8q!QdO**WBFX|YTMmHEK1(*tR*+JPDU`Hu4YppT#W!Dl^@An^nMkd3UxC|%#*;TxOo2Sm z3Uwj`sTlKQzkW{=1eu0W!STCQuO-i1RM9hSy_3j=Pm>r2Mmc4b@j7D%mr7kZ<^q$` z1E&zQkR(L?L~&2Xk^s=6KTmQJsarj^wNXP3Jv;+MeMvMCMHT-|n&C`)WyU&dt$%6- z01R7#d26wW!=(^$b$YZvZR`L*50*MySicUVL?r|yJjU_LEZgJqReg&(>a^6_doR9| z$3{GHe0MnGrLYx;JTFDPK++CW?$xgtBvQbbB|Qg(rn`|WcDPMN&tTM1r1vG!lM{)Z z7pkixr@i(yB?%#JXl>^mt!hN!v%RMA7CM9~bQe+(0PI+O$Xrv?i6@x>r*v@l;DeF) z-nZwTxN3fb>GAHzuS!+Ps^~K5HvkaC&Q?-D;@LOR<;*kWRfhU?d9S7SHTuX0U;y)? zhZ!_4H~|x20$cJ6L})=l*)kx3im`(fhz?HBs6q%L1}XoIO=$-mLtIfxV+WC$Bn8Hj z%Ka3$LKX_eP#bg{0$WJFnw&vJGk62LfTRiIsjy9j0Kgj}=q9CE>q9ylo4YhYCQD&T zKH8gM6jyi#B=|&pQly=pYQ{H7$S+6QI0FExk{*y&vOWN1gkZD>kPa1@(C0aN14STJ!wUfT*RpUigBkd3S@e6L4rvIi6P5o-L9KbBFSU6p6ai(E+iaI|B-0!; zwrQEo#FsGvFsFQVa6IKnprk}0tSU?m09C8k|G>G=H5n?8Gx;Vz(}*AiF^G;|Dv1|N ziK+@IbYMkfn$uXQ4WopNJuH&lFAgeDj^5`cK?!I_-BhF_-BU>mWm7dA$j*J8)0qNO zBNtz)z%T_)3z2M5RAM7io|>l`07#@J_t?`l=_z=A@*<~T(U5YsGmOdXYSm0T1;jIqm04`j!Yx^66X@~klKE4Lc4*1f zvi4OjGZ>>#{hA|cSqod*Qr01XsC4Ji6SI3w(`a@3lPY*&C?VY~Z)j#SoHZ3y z$`fMZJki6*K~$I$1!rw9s~X&HHn<|i&SM38UGtRfYzEaA1~quc&G8a)G2K%7I9l6y zQFp65<*rwyQx)LG7c!HpoG1nMzWCK{ZS3RTBB!c4p8m>2vK(1XVdGZ&N)fR19cqK~ z^1GuBPi4wuUhjBTuJNrB0MWB9E}@o`!C`cc61HU5hzYg`qgXEHW0iXG*Dz6Z4;f4O zXZIqzuF!%L8Sm1=O;^;vR;<{?+f)CKRqA`>urSbpl0~e{>c-*mX~Sp=jO_7n>BC$3W5 zhU?rr3^O>uQoR|0uWR5SmzZAJB=x?`e3dlkI!>RwmLBw*xc|0Kiu_G#6g>>$28$_V z_Vlx6eVr^=fR@*zp0-GU=@-CkS#g@HpdG~vLrd;>vJw_yA*TD6D#$L`*>=-ru+nQa z(`=m1>D_XVcS#^7HK;l|ZRsHMQT&mW%e7tkh zF3#PWB4Sv#on)t7o$x}nh2aS&Y2}tQzQ$_zl8QYuI4dm)wW654gJX>zOonih^M;36 zp?7eN+9xmkDS&Y0`3l(@Vmk^r))H5mZq;eK6Q4>vB+`14R$sT z(5?Hmv;n%tIdVc!xXMu?}^lBOT~~{yENJ4(d~1 z`p`#*^VPwQcDMr`@sP(n>T!>J?4uv~@$Rmm?xx`9;~w=WzC7ZAeC4x)`Oe3_^rv6_ z>gyl+&QAw*SO@NNhj;kC@S_KNz=!fXzma&ovQnpH>WARNhxd!W`Afcbus?NZzUPZR z{o_CD^S}QKKmjB`0|dW$poj6p2Yuj&2E>|5ld=5ZhkeM03WPu7!$5bazx%_#58S^H z96=J~zya(&6hy#!P{4d}K^VjlspGT4lY~}Cx*O?-eGoni)Is{wKp)IM=?g&-JO}Ip zKodN|1608!G{65gSrFf|4}Qo8dzc4!Xa{vThjJK)aQKEjM zqpFcBi{N{|IdlhhPzO!q#9qY4Z2ZJZv#xQBUohjloIagaxPWJr1p2Xa6M zc7O+Zz=!`w^c~>o6AQBuf}{sJv_o>x!-f>eh8zR{$b>-@hjKtfMXU#W=tltR2YtAQ zj?Bbzh(}QDg&rt^k>tsSBt?}Z#))i1i~N+cYbCST2Ygt^cVLHfFb9zA$*7ddKm-S1 zT**ZoO8&sKwHush>ooh|$attpahS@o9La|~hwXa@WXuPm zL2ViVc1%=WEtim@K6i{;jRc-uEVPuC`Z9!!mjw=*C;$z3A z49!9nNxuZldH_~CiM^G`2YPr1bs*NBtW%ZzMtG2i1Z=?I zhccCiibT_SVmot`1Y7coM~w$|2v?D;N^;Fd@gu)|$kg0$R$DzjT71f4#m4_$Rfl(= z2iLTVr`QV9s|tRY2X-I_UxY{yrO1Uf584FH?&L*qI0tvIheu2_E<#7*=!biVhjl22 zT@}TYOb2#g&6dno8;Qx8j0bl>$a&4ia5#r{md9Mh|0OOySQp{0k<Mlx*&q14@E zfy3TKNR`b-asXS*)eT9}1QL=L)+hv2FqeIxhwM#VLG)0GoQLT&U!`nJ;O#~Bm5048 z3vB=Z^QybB;0Jj?hyDEq%MDvN$|JqLD(2f<7k zy7boHl~_O=hjo~T^2LvQnB5Gn+>s4Yr_Gwq6iajvO=@CIK91wOz6vjqTWph{m=VcOvbn`MV{Amu$Y2Y0v!ulOvk;MkF%2Xy$va%hL) z)fd@S2gLQma9{^pz72a|2OJK>aYzSvum?-79etn&Bqn8ZfCq5a4UbiaWF`l7KxY~T zpnb3hcu90zys=+MvycnIn|#?L+WC49J9fEL7ZfQL`6jeS7mP*hrXxLAIA)Q~1;KqLou zU~1dwhj~bc{RIbeXa{^?jP*%m0{ZHMRP!*sBRyqJf@rpnl!?XMbPnhr#BuxZGkhjXarYhG@C z3h&e%ht9?emUihqjPCE=r+tuzdLBe|Xl9_O0=M~Xq2LF0NbYvnYshJBKokdgkPCir zhhhzfb)bj!wxdHv??5Dn@`mhokl3)!?)b6?cd+hn2#0nk?yuMf4i`jmknr8+tb1^W z&<4bG=x@2e2jLDx7Ow{i=Olc12O0Ona?o&?*a4_IxuBSbNfmMditI@>hXe15esG6f z1&4L`@WFcE3;#oMpog^3hjkD{bBG73j<5ei=I%dS2Qn86eZcD~eh2$TG5n_HDlTNS zz=t{i!_=;E-8yO@=fiOr@|GB{0QZT0XmjYk@&xLKctBimh=;O}hloCh3BRs=fQPR( z2RP>nerSg(j)xNuF01zQJ!}UT$BKObbFh8~HCMB;#`7TG^RPg5K&rfc@qiZA~|bLjU%KlDEr_^rr$?U(@o_Xm?V z2_dqOUnqRlIQ*iqHbbEf@aT1xfS<(AdVhHQ5IB4lVtNxY@|Ta#&DV7ulkVadDd@? zH-O8V9|^WejJQvG#;6*X6N!^njkxj$x2KMkXnw!&tvE4>mDqTe=y-pC%htn=*9s$% zAbkbV{;OE{Znkx;F!a8!bFL8klK_Y_(ERyB1mG9JgaFPEL;~R9!xUPO5iADrVTc)e z5<0y2@QuG;0FWdk_y&MKQ63@gC`k~P6NpG)5%jY1p&)-Lst}w3z>7uOzafSH~tylV+^Ga9qEg+eelxv>hS5Xe#8; z6Tw1a3I1%DLK5IYF;o9k^QTu2U_@R*06^6+grLZh9g75jsN~6zk{#-ad^w>f$pB_Z zZiN&;h{6fOrm&;QP6_~~b@F@}A_T6tW9^Asi%z$0dTYOdd)E&*T)A}Z0>Eo{ufM+j zko*lixbWe`iyJ?dJUQgU&n8j++63H^5&5-oa1830v8_QrKp#Wh!5cO@hV z04WqAiXJ{WSDF8MnUxnAXP&Xxqhs3emzzJ;$n~Rn=&3hiN%M>|9621q=@LF_twjz! zcUDE+iGo#G*p5B23FdiX0&3l$NYz#(nfryHjY}ywK@@`CEx2C_k?FCWV@gqpgdmdG zG(%x9R+yAOJ^`RwUJoIp7eYn>1q%R@C}q)u2?Hf)eHrjeyy>BBU;YXBlSXwJ9c8RSFxMX`w|nT4}WP^boAi z4MnR<(y{6i5=7&kY;M|jtNE9t=)>Zw0$dxXUe zluPV6CNlp)xH2@Iz%Eq=KrsnrChe@}+ISC3AAG!qn$8pyH z@SKoAR+nTZ05qwzazeP?{WL-K^v2aV{4C4QoLqsUEaALXIw{L!30{)qC8a=lj^H|I zRgskX3cx^FgROH(dT9%A)E$335Hum|Fc3j%5*xP6DAAm+=~V%}1En_6?GJlpKMfB! zYb72nKFMfwcJ(ZO2F zh*!M;CS-Gz@*cz>A~t`_jC#f+NC97RK!V(?QuccY{@y05s`X1-|H@tl2}Z!y0S#~o zyT`@00uFqr@Pr+rm{r=bmIy8@EJ2c+9rmX;>3B~iJmZ=+k~BS+-KageBb&`|Mvya1 zY6j!-Q(Q_a{L-!R9r_Hnqaq(gD0tRg3Ssm+>kB!hw!8VCsjjTH*0FyyO>`CQ@& zPP$|VjZ6qM1VNL)@w0>Wt3nD^XAzr7v4(Z~ALckXM&0QpjfjA#^x`p`KKY4ZTqy@E z2}d1ArU{aRyqYit*-M+0E`uRFNi<2QyJ=RXqGrRWKU!G7o*-1AKiL=mY$(QdxVx=Kr~g|#7MJ>6Mz zCC#-~C5t|E2{wlp*P-m73P7ukCJ8z?qP#Jyr~yD}>o!_bYK1Ezx|2q^q7H82a(+4; ztK|w>+ra6Rml7?>TUWQwP~rA;n@R{a2bz$=S|vYKnL$g;iq^?$kav&0ibG_F*Xq(X zAs*cd6YqqqJ9bxliq+9?*U3Asdeu>my^1kK_tupeB(-llqdd=}l=$rc5hHZTErm7% zE$D%AhWQInr+6#L0x*~U7$9Bkx?NM|>3QZHkv)8K9O+rsutDA8d3XPtvfEyli;+$1 zOeuT1y57|)IyGod2yEa&Ah@A_ydhDWBEJZ?xV!#{?N-*|kN-^gh-azFJG%0;26-|e zhlLsvqU@>%>BW2jn~G{W;t_RZ`$mfAqS`ANz8O!wj{RSgkT~rm2s|8xH?J&6z8B`nr zq{Pym&Pe{u!R`q=CBNL~Y&PwY2j#P0@>%CYW?%>QWqP%&8ZfRxwFB2A*rER0)o%wF zJ~hwf+NR8h9Ww@}IBumJ8;%GS!%PO28Y*nW&)9?jo2WA7sKeVKpMR6KNZ0runeOx| zMZG;#e~8uVnsuyios<8O&)3<>#f?~{LH{L)r1!&8^*BZ;_yR@i+x9^JF}WG_Qxk6WbsA zbPoJbdlgn!rJh_ryrLJ$5y20h-|<@^w!6!ZQ$q_*?%>KQeXRGV7tJENAN-rZj_#(D zf2YBhJuvU>bi6G@^8&|j$E@Wf_%LP${|)fJ$wvw>ghjvtHqjhP9bALCgu-=9+PMno zWfbDT2|v&oZHWZ%nO2{z-`+vpeAHg+{RHe$pVooLrFmVnsh?74!vj9VYPnnh>I4Dy z%>ka&1=$?HxYI8=pxQ;?(*%(`IEBz1nNpmZoh<)LE9>W6CKu$ z6ZXzD?v^Qr;x=;19~g58%8aKUIU98B;@6#f!yHshmh6;TXUC zOH3eN!pH+xaN>xJ7e}d{6HbYK!DK|uq`=gq(h&qsR-uX2PEQJ=5sg&4kQGbbTu3hB zgDnL3JR)?Y3(o}O=MkenKqZ{eoziq-uK>;Z2$NFK-dSK#7TCjMYsm=pH&EgjIgvWwhfuURJ#sqVrMD`}G{N_f~;Bb=LXKW-q0su+r1x1W! zcWq)xP7D#cMB{wJ93@PC#E>?`#-WK(gW!+$@FjML1>X=?LQbZ;J;YD89L~&uxfv5e z2uPgS1f~3oMVK7nEY(&>+=zhaH>@FT{bw8sD0mR)FCA#3pv;2$B|$i-NV2~h0Mc8`IszqWi|e#ZcS;>5aw9G zB7@23u$2ThouNOVsMM$^GAJASRD;?mmLj2#in< zO;m=$r;l|Talpx)2~j>u50&KT=V*)K}3X#YDs8lNt9=++}loI z>QKOHLKG+trcT_*5U(bLuhQ0cy~BML%Sk4LG0xW0K*PsCk{-kc@WIqY2?)-3PlJF2 z*Wk(we2|ey#$`Atv)TV>ol2JsPAOa$Un`&r-L)eEV7}M&g<%15Xrrk)7 zKm(F)COXn7rPin8+==Mnsb21lPnbrzc!zobCe9LzZ}CWiq)|uk&bo?;rY=cL%z$hr z*0y>=w>qP1R>_qh0)K4D#lq__j!A-C1~yG-q#tWqYQM$`qujAEtU8+!m-aE?dAN`km%T8u~rrTj+< zT$3Q&hS=Qe8|43Mesm)@wNk3V$vBB9H;F69daU9)?6}5l+e(zj5X;W8E1SM+#<~RN zYVN*9e(&cjQJ~ECg&i#Q)T;zH%-C$NwpG$>1~CRF3T^x zfvfyQ$+*f4yaDWGQ0wACHp;h$NoP^$J!);JXG2}C zf){#32CAR;gfBdmTVkOGNqz7|(VGjL{M)8@_cYVWX&fQu>r73BH1Vh9d@LK@`vM`133-1vQquUQR@%ke2`7ScB zTIwDXL?34|ti;9_0!utTM8lGD{dU(r>}RPx@0~<1LEPy#h=Ug|3pDVXOAIqz9&?8j z12GdL+J3{gjYu<>!x5U2F(dNVqwvy&_{ zSg8NAJ14|E*bz7cGja$ASBS$Szq39gvo;h%kx9=rBS$}vg)$Q(KU^ZJ`N zRRr`t!%0EYB}4;cx{Cw>+f)|!>abQQ6Du-&%>#aT2WibKCXp2`_knyieSG8CZcf!9(FM>st5 zTi-QalLb61^*6X-v5@Fihyya_HDMQaRosI&k(yL@2*Ey&UmG@L8+8H=hf{9#h;;uo zL6migHg#lowm;;9FB3=gD)dhiM>{OGXQwtck$WVMbB;akvMV4pK|yEQ*( zb~uc-Z%;Qkv;$jjw|OsjYX>%X7mGe5HvhK6S)aFjCyYL{17Ckre8UMo#P#XH_k92N zG513}@HcS`_Oj?hXm7`tj7fU?cH(SiZikG;I6GT7E!*TPkffLR!P=Ck#IT_HdwskJ~wy+rv7zI9seY z;p~Gk(#bmiVt#_3p2WBfpi%+*WxC5j|x+&d5BY%{e6Ju^8IytyQse?LR z_d`7#x`=-hiPM&sBRV(;`lqvcWbcDKphJ=KuRHX)^z6fz$ND$H!$^u=_lWERpgT@Fn7hc6Gdx`UFEjW% zi~6s-(mSAh%zHCG&;vWHyKH-UN#jF1?Dlk7Oa4h4J?Qfv_jb3fceJg5WK)3rL#c}(vEJcxb8wF5m!{OTtR!M_7K z#QiT%eTUTjTipFv)VSD-!#YHK)8hj?;QOAh1JA27KlFOecZE8{!##w2?~8jq0Kf46 z@;a3L=(_{WH;g&3gFN7a_kTZf^aDP~13Q>Q_J5SBPqjY4!#Ui4V50*-xTB}fUqOQh z5hhf)kYPiI4Qo&yBBZWMwTDRxx-fy6#$!Q(%jVdZrsm~EyLFS@dX9s`%VVb_&j%Zp6T`8`Q~Dy3?P8;qRF2sCgatlT*VocTB^ zEWiOJ?C>Uj;Cn}%a=5~<9Cqf3(8GrIktZ+={|o={9u-%t@u2q})9Av^!a)ZfzHDNy zCZycbCmwWmMC-HAMzis%x!g&oL8ru7haUW#Tu7gJ*eM4?j{ajuw0yA4QZ^9hd1oDS z%&pz$5tAN#Xq*$feej_N9(K^_Z!mGtdB>hI{e&o=)x5m(n{m!r z=bd?GY*a!J>xrkGbjnE*#?bCzR8xiap{E^lL_M#ZblQn$G*eGqY@d7RQishuAC%`h zs)VG8$hb7)Y(8)bb4S4SYAvXreCiqWoGH8NQL1Je(kUZq2Med1JmHzg9(`!V)*@)- zac7-rTXpk9U(KaxpW&pG^eJ=HX(!x!$c_IOr+)Oo6`po<<@U9gV)B?~ zm7RA4%m+M$Zvt3gg3TGH-DAsHC!R$yjtCz}KkioIaK|mTWEStC$5UO~{1Pg7NQNue znTAaaE3F>JP#k>q71v@30j)=#cL*g>QmC$dIoqAqO!?Sx%t>eBc<8w&pFXqf2T*(F zfk$0*%BfjWM6Hv$r1unT&EAi|8Ry$|^p&R`elL!fpMaGfcpbDop4%hb0_4qYnh@m4 zoS+R4hn#b?b|)Tj@zJO6c*V}g9(tB07;UxfUGklQ7B9)Mx(vQL>vb&0d>@yc>^$ap zm{#W;av+y99CO%_r&PvvHmPUKwru~n_d6q|oE>-Ik%u08@X1G?*=^!yAARzqd|gU$5*!PB+%cjHN{SoY@J#&Xrw;qU!-Mt!q4oF|L6ekldnfcB0p$UYceH~Y z>OjXi$}x@|zK&K5IDXU z#?W35ya)P-XOtm1=Qs1pNE!dlSS@jgV;qen2RX`74q`;|k&RqrIQVEr+yrlUVRTQR z*f*_2N(vm|SY#u|@yJI;l9j4N)I(!&jeGLsaz#G%i&5(va_8$dew$-37q1UY+`ZI-hw4VcVex;q%ohZQjxfm2Is9QoA`dPW1Jlh9fbwa)2p11whI z19x~-haST#%g+ChSFXoGFC?d%jy1ZYjqUXs_TbyBGl2H9PS6HI{w6>uu5caqRbY6< zTTpVQce>~hA{&dCx!)$ZWP7C}J&tSNbUb4WNo7NZ=^Bpzs@KEp&EJ7pYo_2*WVmi) z;`a2xy?iJmsYj?D689m(9Fy3`+Dmefg?vIxBsCcMuridBtYju{`F~l4vXxKRh@>{+ z$Vj|il1Ip8X%>UQN<~67#^PikNBM;Ku(O@PjAb-0S33mL+mfhM#eZ2;^58UuHDM4;h?HVgdW29fte=$$@*D^(B`Pv5OS zi*Aoo`WK+Z_sFwUj5jRY69H>BroOFC#_LG^fP0koeMdxBE{^s}=E z7G(c-0xB4y2br(^;a~r(*|jc6VG(kpX$P$x=zc1YgEs4dzbY)+9CpK>1{LM5g5noX zby0v;5;NaAL%RO5RG1N}`KkifwXLKOcx zu;QvuA+FDDPS67vqY$Q#1=HsY&Hx50;ultK1|;FZXmH@pAO&b|=~fUSQjq-cZwIlX z5b(|jOGXBj@F7~k?Vzp+^DWXKPway5AcpW|u22f80uiW<3l)Y5!4M%-4*;~F3dgY8 zHV*>NZVL+{3rofg({Lsv;S1+b!DcWGt0DOSVEOLw+v0Ej=uQsd5M$)<4=W=1{!kAX zQ4chs7#dLzybTu`k@s*x5VXJv?4acgq7Wic4-lahbkHA=;S-S|+!_%P{6P{Tu@F4b z5f7pmc3=k>vG9sP65c=y-XIk1;0J%95jhbdevuJ1ff7Hl5j9~L^&l7zLK*+r@Dej| z6A$7QQ&AKP!V@Bq6SW}{f6yQhq3{Z!3TEK+CPEXUFfoup5JDgX1fd%dQP)6F1Otx{ z^>3pN@f{^X6M}9I+06_e>=)8b3r3OX1mVH7LD=j77mDG}kl^H?VZlN`6A(e-cA)kA z;TPI%2TqR7LckN?OdGBZ0181H4l=3E(I2!R1ZIHuB5enX5fxtU0!gvmLZAdijRXAw z@c`fqL_rihK@yOl3T6@jYBC{ilGsWzAr~?P8nPfBk|-sTBHs-fN{<}Gv@3_ql{bA*n?J#G6`MwVTvJT`z;N<+_40eDVSI!I6lG`NnAat)RNm0@a zqTEgr8D6k65yG>QYSIeAFJlqr1QP%VbKwj^;sVnkS|J3&5+y~iAS$yURFfjKVF#G8 zDn!BY&TlT6Ez-1d5Q{La>T#ar5;+;d6N>K+vkw4T59BP;4n(ggW3SsPY-6`Hw|L;3gXtvtu`kU2}BR^LZBNfQX$kc_S8}kUJl(Z?Iiu7_4YFb`tv^v z!aKoJJPU$64Z=L%v)xWl8)jfQ4T3sH@%s|9BvnJo}UFGcfeMG@8FniC9v zE(9J7^eDCF@UGq1&?%D&-*^-n%FQa1&JMI->J-8~ud?%WGSkig3A_!^Do-NxKoSbm zAT*&08q-QWfh3^;6AfZkYqcP5HCGElQxgIjI@KROHB|o*!r*RmNFnt&k)bHO;1`Mk z5daMd&fwDetr#wC29V$l(rz=opgL#b2^S9G9@W_55j^KHQhDq-rO8p(m0m|r)>=*r z>JR|(HCFv0sgCm?Oj22i@(kh=5!BM|)^s6I6)T0+BwLl%kQD#|78M}P79l7!sW$Z>l(RK06K4H^ z{d^NPAMD!v)gO`#S|wr+WlakPa9wq+Em4nAugYE#b@SYH47sfekQEvtO>2v^Akxl4 z8#C-SZwElnLX%DlAd*D?^=lorAcQRhE|%6BQ$YVwvJdsM@`P>rT7lX^)*k}YR^62T z^cDd4)^AxBZ22`I#+G2sHmNK$L1V8W7K}aFj2wmS{~ z(`i|4XQLKg9ZgU zXzy|jB6AC3b8i;_I``+w680juAy9$rTyl0%m(@-=dK07Pk{k zmJDT+V9ie0eDENGR2$-RNRRg*paI~#P1ygEkp2K5dI=XycfdgX0egMcAhaRuxOXxa z4uwOO+XnX_i0%t{l!Ci4eH$X#n9qIT763F?kdU>0Ki7fLkRd>{-Mk=)|5w+LAq1Kf zMOpXNT-TEdSUP{sEy3{GF4Z5(PvmZPGyOr<6pu$S*l2Y zjMaiiSRw2{W9c|X!8d#b6cI9Qk9#z!ye(wM7$MBKLCrLU%S|576y_RY@~RMspI3GTYFOeFPu3c)ZV&E zr*|3$4dNjw+M+R9qdD5sd`+bN&I!i1S@BM%lgbW|K@zg{7fMeHe&MGD0T*PgBZJm% z$CPxXn2Z&=*i_e`6MEjRIE?>Rc`n)Pbiwckt3jZP;THtDFJ#aip#c{fIy?IMuhTCD znwBO6AxRHH8$6-hkU(AATH68`mE#)Y=GunpdM)p|va>@B)eIGoz?CUN^+XOA)cOtB z7PO&QipTGup)<2xyPYw)wtYji)l3vhJ4l>B=Jv4DoIn#07`IvNlw+H=xo)#1sI&1A zS=rjS7elwz3>gp`BC;-&iBb=uTe@@23rw~9WP4tfTe^|8wY@tosN3O?p%B^)pszc< zHSMj%yS%NBxzoqF-|^O*Sia$6z2U8&zf}+l+rL5WvXi?x;?llx`}_Laz{LWNxZ~O{KQc_#Z|n-;{X6w{Ka8B#$|lQX}rd5 z{Kj!y#u)(sCT(G|VWK>!36{m~(v&|lyLIsgDV-~?VE(lve4Ila?8 z{nJ4m(LaC#U?9{@{nQs-zf(um5s_jsJTWAk)$p|3Qh>W(UER11y)j$cSp5%I_}1Mb z);CSSiPFM>UETjSFMsQs*BKnQ&HC7bJ)KLf8a#pQLf{Po7TT*#!>9QDdYuTh{UB06 z*~8r=gni5K)Y{JgWXt{Ag43}*@Y}Vp*=sumpt;@6UD~T#5=Iu=;~U>W4ZKae`Q$wg z>D{%F8sKf>+&SwPkTCY%V6hdx*m8Xb3%*(QDuv|16aXL|yLW};GG7azCdV09U_Y( zgkcfBBqaatp|+jCc@10}ssS4M8W&mt6?{??Bmof$K@bE%5A47W-arPuVDYs;3mEY6 zt6&x(p8*?x@x4F>-{A1|fbjcN5;VaRL_q*?!CIkVud^iO%bF*Po9v5C;F~b**#jTg zVWwtj9N?knS=0#z^?$WN?_mKIe$o@|;0!V`0kx0#Y#9M#x&ap-^WWh458nPX*i1eJ%f@B7O zktF{Cq}bs^%NaX@3Q5u=sue6;#gIYMW-;eOZP1Vv!v(9AC{2*0E&Enmvm)t=hG0+p0wcz?0j#bnC8#Yd3FBdE>tM z{R=p-;K76o3r2@euj0jw)oK~oII`r%8AVAFbf(IV5G6yP^zufKBu`a~-JHyr=TB6g zF2C`jlru@sRnCYOWi=u)PiHvq(9;Sl>fywT8$XVGVX+X9lRJNo5HIw)_Q)9~EbiPo zck$-cvv=>Fy?XWL!E2|k9JuR))$N-OKfXjQcI4ZijIos_%vD5^OyPtVb`(ODRl@(> zHylP9X;O$CurcLSZGvdh%2fnu(~2HDcm9F_L zIO@cMPd^pyW6wM2gahL@;K0LTkf?zsWRVgTgNh_}XyKn%NO&O?ZXOyrT>z3mZNFt5 zd4Ys%0L-u<06W+L3zcGyN#=B3%ur34YCg2$nhDjzBYWt?qYqm5xKj>{=Dg=-pE{lK zhMz2I^NS~fc+usSRoHCe#_St5n z?67B!JT|tak2`%C22MSv-fGYpXW%*=G^iAUC`m}XQHUs=@`@s%q7?E*S0nAWYM=d}L+d*E=tYk?*8#b0q_3qrnXiRt0pSj3G&&`_LLyV?8C|BJ zg|yV#3vj@CE!UjDBX+wcKh_n-4nK|EV=lu6m+9cdbp4{q8U7(*hbFjqYH^X;;G&5g z&dzX#CZ>kWa?25wc%sW6B3vdvc@ltOB39ORNfm-UMzUT}u8Bp(mG=E#z0 z%%LPJGfnlxOhQ`qbTp6IPsDkzgla`wb2n5~0PCy3gQ8w8CcRhV>!Cv#8k=5A;IFja*l z|8h>ck=d4BT;WE}L)bdc?Xz%T;qZxiae_EedNrC{zJnmR)GjJ6f^1lYCVc~se0&5a zKbh)6^0S^`KSO8pVCXQ9S|Lbq4$B!{7E%oMrxsJlhG&E+e)`41tUhDT2YI;k@%0ls zVD_}X7$Vyb?#X6v{Eoj<;o=RR(N+KtXqT%Y&@S&wQ9i8amTGJ+C;DLAU&yh*x*%c) zf78SaW*~{<6~{sfks!z{0*%p4#)NGNA6X=rmTE}igPO>o4bvis5KQfcY-u2h)bTZk zHN*=|A&DReq6Yv3>L4>5NeHy?FI)gX5(WvJ6Ha&#dJM*KJXFgJ>DG$>9n8RhW~ zN74#^kg6mzu*1KS7)2)t3w;GaM3b~IwPJK;26`AB918*syp1FUh)9SO<1#8Q5yV+d z3DhCUs76KB@s3kNL{mb51XTD#k7zPTHD(Y36%wR>2&o1D3MmQz08k+z+++wQNET0U zDUDsMB}L#O3S0IN0Bh<;^Z+xDhZsr_ftvv_1wl)ugrIMd000t1L&Q$PVu}SJl@mm$ zkQW6e9lAt|2hXMr0v1v&dUV1uk0};Xu|yDhkR$H=VFv(E%Zw@Lm>rlxOMhI>38U#v z3Ow`+091jPZ;)dOAQjJlL1Qyu(HkP|@JB10;KGD7(?fPx;#v?9f;V8Xn?zk6QPbi> z5wR{X7@SB5BtCQ|Kin!D1*hR`=nEhJ=v38Cpjc1jRE?l3LLZ~_w8N>(10 zNVT~=OHxAw#K6ddB2PU?Z(zDVf-FTi&=5qJwt80pTyY}*;0g>Eds`7toRwgvHVLsSQ~=ldDjf%Nu5ZTkIlG zfhs1;Z$VT)zl=2^#Vv>_0~NTb1#Sn3kgGqo+RW)rL>_{1M~kx?0;-vouPp??TM;r| zkE$}U)CwqbU~wR5C=nr2tZ94=;@2?QHI)Qrh!oAr2PL{dq$Ld*cuZ9OArNkt%z z5OIHYjjC+Eg1#;zq`|8FazWryL%SxC)0iz;2ymiDNCWtcraH47uDwYBXxq+(H3+jK zxdS9vb|gr^nqvXLTZA+dQlvs3u>Fx}eLhy7?S(BtQt4y0z_i@94j`Eeu9HvSXV!sH z$EYRego8?WGg;Qe(Y6%6X-YSVb$-Y%{bCM;vx%2XBP3n)hy*>TT;=0jV@b5YQ^j5g z6?X7X4V8@$c>BZIqeP@#1Gh>uzj>M}J?onPE6bdftDIVACVGMOSy7xHMCaD5g0r}4 zkU;MnA@MB8nF?Y?L{B-WHpYp+f2`oSAY!{BSKp$a-1dnm+&=n{A92+-*`~PkK-wTx zpr$$9uH-c$;s}hWRQXG?hD%Gh80Vs6DgP%Bc9^r}6BE_uOR6=rO zZmyx;{P88IFY>ab_s8i|zh=!Ng!6gEx0gWn#DyFk_3tEz=M$}ExD({;dK{-)q=-$# z=WYk_R(?bjAD_Ahh8+*bs*_D>Qt6_ds>-b>bUU8{Q97jXfH}PL4-#lz1FXSxY_jkT z8*o)CHU&`4GZGG($WwdXAabOZUG;eXU(Hl7b{1_qByOr#S*!WfCmNuknw;Qg;9mHhOXC)1hIQoBni7Xj3Y&d1R+=ll7|Rk zY6O7{DS!m6fC^0Eft~d~mUKi_Xa@k4L@7oV$PfaM1c`ogaQWB}({zL)ax;KIJ2VzE zdpKJ-2yXo|N8TukvBeoqh!F4~80gZ7QY3yYgC(h07J1hc0}*9hCv(McIjlx-cQs#I z@&-8-P_ywRQ&Ma@cVdHtb!zxfdx$=ou~-V1!=PXP={rW$;B4c@Fe{BIB&<54S{@WvMPaLHglm3#qf_15o6zQ z5BV5S7s)c(V0H9Z9Hy}l7-1GEL>zdz78;Y3n1&3;^DhBaW@?dERgjinIWeCEgo=3( z0Y#T1QV)V54j^I=U&D`(SvA`5If@90JBW`taR$A27b$QCiRqXEvw<3^m}W$6Mq)vJ zQI_|ymadbJdIXUzvx-`fo99j#r6K4xB7p>j5TJJ|FsbC3 z8JaeN5DsgCP#V zBpLibHiZ%YHjNsnwet&zP@I?}30Nkg+5ihrqfAJ9>WAP^)jyCGw&&yLvX7L9OelnUSKVgh39jVi)?* z4oM0a;n0@K83N6UKh@9*z~Udhx+%xve1u{sOu}l>(hA?otrqj8VTzt_3M%xVu7Cj! z>VT$e(GT))4(w45ZF-ys+C|nX2?Zq^z1oSeK&du_46vXHY11T5Agur!uq*?p+zLXW zvJYNk7~!A}^B|vNkq__?u`ps}{h0zLs7v^Q2pUTg&{9&4g0i)Pu}iWPXEYMnPOFb?LB4(!kl?(h!q@DA?K4(osp zlfiLTp8N3ZFiRX!5>ZHBw7#}*$4}h*SDtv4aIP`t^pESqA0dOA!fC=L+iJX z z64JsT9+AADR74@bB`Eo4Lwl|zW@~I=m|uzy?XE( z$ty<9>lHJQ5@(SAWy_;` z%7Qq^zbT-r*PFeqVH8Nwy-KmX{5u=$^+8;+13ORyG(ZD6kOM$K1Vk_dLm&h~pl($x zMjympBFwZYP{38-1uM}KqcOdx+ZN@^v~uJndVmTm9K*cAb19pI+p-VxfDZA?!+s$S z>rfB&TcGAdgywrjNT8_`T){qp!9jrvs2~a(OcW=D2!8eh4gkubEXodG28B@n2%U@w zCZ!3Q;0dB|$qekkHW9(d0Shg>J+xW~cARo~yvG*vNr-j3g3B)bkRxGR$bcaZGeVua zI#5J-NN04K#<2yXOwF!fB-)TfBr$*(ay#q>9D3Z#Zj!s{i)$035A^^K>u?U_kgR_p z4&{Ij><|z4P_qFm5GjZZ8-yt(QIf-91=Kvsvmhq8;2)3b58Q0W<`f*^Y%S{>!@&tO z{jd-EkPrBP5BZ=E`%uq&Jhh1Do%59f#8gC1I~h*^(4uS$V}i{9K&{n)3VPry3N675 z%_1`#&NGm|G`%(g5_h=pZ2)ivw4`PKF`>he1ScKJyig_tOAsy%(<4#;1%)81I^8FI zoW5`1t5nT2L{e}Yx*<+rJT~Y|k<1uAfYhKo4E6#Iche32Ktw5^2f7L(B2d`YP&K(N5?{gF!hIZ4r_)mOvDD2gLeovZfQ*N= z*&Yq2g}$i(WKTILTvO%9&xEFua? z_BgOWKtVOx0;2+-&~{E8)5y-+ww3HD%jeGyD&-w%HE?70x&KW^{HV&W) z{Aln}$=ESm>rP$>?d9-(f`UMU?o%4pXR{3sV9koT*1ApFDGLpzIpsTuwjnPjsOU1@ z`R{_@>;Pj84j|CV3EiVT-FFJ^W3C5nX7gndmLY@x*+6a3Rl^JpkkSI0+sd@wXr7w& zp5|H>noD0K7CA9qEnO!+Im_SxN-d%ap5E)-^UA02NU!u@PbKZkE_@x*Zyz$l-~efD zrEJgOdT(nTAL&|F^K_pgdSvfv^7NXWJbZuHn;PPrG2()cT`^yQeJ%(fkN8G{o7+P6 zvd!o~!wU{@+W|ZIDB0vhs0~M->(vkf4d3}Aa-Ybe4X6Op$*n%NVEQrKs$h=ei?2&> zBmAZfRCqr7(4l2ouO=@J-T^cV4&dD3{N=u32VyRAai8ljm1@b4A`+^)`8{IDKRK>I z-dauPc7FZj@8zdt^KilZ-v1m<8Y&u&-0@%kJF4ISl%43$;^~(U5Gl_1BUsSjL4*kv zE@ary;X{ZKB~GMR(c(pn3aeDf*wN!hkRe5mBw5mANo@dNon(3Om!1>=w20Ey(&kN^ z21hx_+0*AwpdPisD&zzJ03>=)RSVkGX+~{Mkkouy)#_EO41WRel-283uwlg_OxbE| z$ZC?10N8;D+F7_QKZ%H2*REAu$0#4l6@#CuiP(g0k z$_s!Hc5qSVyl|6>&Y?vw%)9rf6G5%0j!vuzfLYeDWlz*L!_I8nxo1m`T})8SnX7d> z-ehC=@x;|o=~)W^wQA&HVF5rC-MVc5R4T7;=MENtDACn48EyNv`(a2v&aVfnocYlz zgIRi7z0<0?$^%QilCul^mV+Z);^Y1~Zq`DA8 z4t=U71wA?xaYTPoAkZ=2W&ofHPc|g+DMFHXamJ7$obV_u5ZR1Ii3l@{M<7cQMY|x2 z+%5%88WYc`84CH)$f^Q)0!k_+itr5;L&$N*DhZN=$S%X2=!_W{6Z6cw_}M3)d+MQQ zo_XYnM+QsciANqd>8a)_ELhG6A%r(pLNlm6tT=R0gOKqAOU$&81hFDBLP z9(mX~N1R{d1*aT!;Hifngzg)N+*PLok6(a8+NYj()-flXcoW98heZ^2c${G6Ek?9^+&RZqhm*dQ9CqmWGpea}E>({ZIF|bAhWMe!9dr;@I_W`J`C935>ZMpD zSl$@4Pym^kWs^O=@L^{Ey6d+4ZoKoBS%f6eAY>F?tik%Lo__{FAWepDte$ne9{cfc z&}pNv!54MH31=$@c%OOLAqR5MLwDF5ckHRSB4t%{LK8E&v&NQD2tgwWy=%AqcHDDk zSp_TTNJN!k#?}0(UnKD&gu~BB{H}f6fmig$BLP4bb?zBnOk_d`@p`H8ap!aB!x!II zbmG}ZqKh-0K@i2S^H`NcL?+p85p1yke*E*-e}De__y2zY22g+l3?SZou!Wn=Vj0{6 zO7V`Tu{U_aYF^1lI>t9PSpc9{OQQz^sg#|6dC*k$SjY1&c)}Bw;~n~F2rkqHfHS}b zIm>w49=2cuyx9Q%ftzr}0vYwbL~v_mox$rov12ZzKZkLY0z@d-;}$HLc0EJcrk zy(@g+SVumzAqp)Vj(xJ(1~1?ThbLH)enlJw5_{M=~UkJnzlUP#p5v83wPzo&XAyG` zcI@_7?g$G#M&y7c$W8W4y@WP@t%mwdAy$4azE8{yIgigurH7>GoDQ{EZiL zMXGz6^BndXBp(IFRXF6~6nvB;Vep8sb+j;p3AT7N{iv_Pn28VE%%h(P1ILMRVhvub zb+q)$q=q>xW3RX^tPHLj?_leSNTSul6`D^&_90z#1lB3jNlHr@^V=AL1J(S``(Yi1)&yR7f$Eu%@ol_j0=Jco7P1iX7&}gJ0kD@lAW{E z(W!>Tvr=(}VEiZ0fah?4_yv>o`@8q}Mlw3l30(a%BFG5wNNDb>i9Z}NuyDB0#Zr%W z6+9w{;0p}liHoN1;6(nQ(0={V3ds!VC|f_u4*X$Ut{)`?{+NyvXp4jp6eH|hw|YY? z=qqt3+#(ZWPmlW4t8NgiqeAOz_d3|O9s(pxrXbZ2l-TXCn!mTa zL>xf~<+rXXwR1fLGuYl$W)XA#M*JJ(fC$vLjs!z#e(i$rNA9bCl-8~Ph58PWwXVRC zX`>5<=z$NvSH6uak4MA~&MAZ@F4Jp&d?^Km5QQF8bRr5N0riskg&_RD7CrF4>M*W1 z@IM1A1cLax?emBA>k0!r5G^o3-D8O0@)e!S2)e@xBuD{C=)k*50j5wp)G$E*Kh`mbzJJ19>hzYa*2txlGLjN<1^?QrI zlfr@sz6E56en2RtD?h}Tj@!V*tC)vh@iB=YK!a#NK?;E#Af;-c5ZDN`>L7^J8$|#x z6XR;Tg3tyES&1Rpmx7p%#i|J9%9VusIFqS6t3U#;gT*)mfJsD%?C8Q?q&od8h=xl6 z^H95Az&Zkf1^~DWL&3($J4VFgjomwl5a2yARK|x8#c=EjXfQ=oR7HYd#e#Una5R%H zw7TiomyGc{0N6w|Gzd042xC+TwOhs%utDQPzVDI;Uoi)W_>vdI1sEbnb0mmHyuNKT zh%Q9OhlmIA;Y7!1g`xvVrzo~>IJSw{K;fXhuK2spQ@gD}nh=3%Bax{phbO@C+2$sCQmjt|cbb^_5MRBxAd(27NS`o5?%Y#_F1Ox!x zE6aiiON2 zf4IzZ{7kZN&4kF9*qqJU)XS@@%iO$6t8=_0ag7&jJiPqJ%R^6sK!O<{NRAXv^~6Jk zM6UDu2Jl0Ot7tKx><%qwVQTwb>jbsZV z4KyTRioc`9A!$j#Lm-N+VTu^7jSzqa8r4mLFs?Ju z16;+`{*=hWIV2vn2<6kVipW5_lZh%&krdz(wM&Xz)q_^_hsdi2ZLmdzgvNtNiGi99 z^O%lS*b4BFNKueT*;@-OIM&1IO)#ykCW)ojmPf*XPujaz!355Y?YR)bH8{Us(s*K!rLyif2qu^qkaKv;&0P z0Xxt}7Db2?%@r0K)xao*89;)>L|Ksk=spj%2q`s)Uto&kNr>tIP;4)0GdE2x2WpaD0Pz^#|xgOTJqyD3#jVD1@j?i9+}iz;HvZ9nhwI+Htf@ zb&WM388fVmD-UV%+PHVjrltJJl&Wy$JK?~INdMD+ZUK@!@nfh zy*1DUlt^mC3W!Cn!qt@vbO^z`N3vbqJ*CtVd4o&6&h0gbp943}y^Csyyx>gWj@VR! zvxqM_O{b;4T|33u+sAk`#}7G**3H*hREQzzwd{#igZNcn^cA_Ah~9k)=9J$MG~0rh z0Wu|sXrh#=6~jTPhH5yjf+&PRSwKk$ffXFgq=aF;?cYe0;e?oAgQ#GCxL{P&V0xXs zm@wZThFyOEUfGML{xmLs+|xBISmxATgGgWcUC`N5O=F9Q1A)}b5DrQG-GQyLW0C94p6iGi-?8;JV`LaepFn{ygsJDI_;2Lh&b9@Nk2h8i@)>Nezu5E zMp}!|2Gcl-H-M)=Il-9_JoNNpOU6mQ!-b47-Dv8?PnHOME@V^;an*pBxwMh7j)Xg*q620Vo}94)Ec)U{zOu2n7W z&Ub}|_dp6kz`eEOh%4s0B=J4MJ6#0aKfYB=HJ&Yw!<7{#$ad10t5)K0%*?KpJUoVo zbMO@b3+S-;5*>AGis(>WaY&3nxrNx=z{rR9>eOL9%I!dgmi-F6RtUW|321Ow+^ApV zIwf7<9UxBv(;fJo?W77^&+|~`%CN2^K#k1H4KOF9{fQBjvfnjFu zg2;z>q1XWb(TCB#6tRkU*m)e2sYXM4hz>f0}{RL^Tri$fb8zz2j#;RzrKk3 zP8l6~vM4)nuL!LQV;KNPEyu>+u^?nzDQyl13tW&*fZcE1zO}C6j(AYCTY+#OrJ3?N zuN2P-J;Sb<330~a2Swv;MEkEx-0`b$?g2^he)z>-K?in=j(KR%T(R+t*oF@$GSdni zB=-mfYprtH0!U~vdWe_&y^4DHFGKEfuyDNFOYsU!m~;5U+%PCAckl`aiT+-&MtZOf zXY+*r05T!7u$jTIxN5d9i*w^F@1J;YxMOrcZ;DnB)D}V-`S*5eVw(cq)UP;RX}u-H$doX%qvIlrjhnQEG zaBv5;M~q2gtA5v{;6W^Um$oLe0=)O`c|iAD0f%&$hq-SEduWF{*OhfB_^!taXox2E zKJexl`mYIxa!7|_sRw*;d5Y)2Q*>_HFAe{fCqU0c!zgz z2X|j3{v;#fla$VszMVUOI5!K7tG>awN%; zCQqV7IgXvZj4ofoq_~aOC_-qSh;ZOwK>-5?5F}8rfI$PDMvaPi0w>WZUd*0CjVg62 z)v8vnV$G^`E7z>cY<&v!#mCXIX3eUwLI;wnVKLvrjVpI9-MSb1!3&piFJF`E^w|w8 zcrf9@h7Y@x6avXF#Eu_Bj{MRdyK#LlW6mtn+`D{|KgVP?%MpOmrcVR^2oONP00Iln z!j3I_HtpKBZ{yCb8+KAOgqm&xEqplf;tc!6v#fbJNO0@`8;>r1I(5j%v;YvKtvdJa z-VN@Xw@x|p@|n%Kv)AuCdnc&hmmctXH~sqd@8iGk<4d4TvzW0LpnwBDWKTNgT~k-(XgAWepNeDZ@0-=Q$#>G!P@1#Q>f**cU&N}hj!ytynAp#$2Q{0qOelNlpql~fH zpcW*qEOVlcI}*pwJm`1?qDbX*#-ou(BKZ(BX8`a9G)Y1UB|rDb(@r|%1S#Z6;h3Y& zJM?%K<#5HQSXv6L9K;M*VkKdNn{UDyr<`-rS*M+M;+dzOd*-SC6jQteC#Il-%GHlO z?wFG$bL6Zu&pw1g8fk;A>=1$`l41(sKKRrVk2~w2V-7juD7sfT|@##0)P{ZIa_Ua{Oq%j zKKbN>kGJ{gqt8D4m}u=|un4jP04WIc9wO$#8?U@_VY{uj;>J3!zWcgVj6ix+!>_;t z52_6+Z`g`NA-MR13u-AaOt8cgQ(Upd5{r^T3YT8ovB%Ou2dJL0go_r~D517FeKB%2((@y8?oH7aKNp1ku1MLxOE z&r@H$^%jewt{u5vkGtrkXWhN{#o0CxO7f8WCg zUcCPQ12BL)I$x9WC%~{p4`|anAOjodK<;D@0B3;z-2;~hb~%zj&e<`p=ANT;l^< zft?*_!gOr>4iJ?%$2;Ote#Nju2tptTEas6oP0U~*3u(ypRYMZ?(?a78`I#-A(2`($3!0GS`Zns*qarnD)euEOrG*G~ zHl&|9MV(2LYSpX4OdGB6CjfRJh)5nJS~CDE>SBu3v!bA@Ox%}1TCCF4~WnNDsX`eIaZaw zGM+KTeo+V~*bhEV=;TARfsEk)B0>*m!12Qzduxce$SG=Oa6Os^waYvjo z(@dxe?zLYCybw?tgE`C+vsIaT=n!obqZOJEvJ=Q$ni)7D#fb1kD}F)Lefby{sL(_q z?BE3?T=~sfE`%pw!DBHaI?-;u6vSB&l4$XrW#(fb+d&KJ zGR+ROpa)H~;?Rm-HLFh~iAI;E%Ua0s=(J#mP5&%L&8@>?s9vtRHSFn#K{kc3q%Mc z1#fui%X^+^ZFu69dN5@Tmzq#-65Z!Yk1a0Vpk6_Ejfe(I=nOEq^sA#)6z(bI7sI&q zsAE0sGtFKK3X$-PK;!Ijd9j6jEEx<#<`+zQo!IMs(qFV$5*3F?HJU0pJD|pFuH z$wUxjsHlvs@B*1{RuI6wXRhq>$9YW$i-GtfuBdrKLhR=SgfO}>DPr(J5aFKEoFP|; zpX))Ggb@3I^A|S%LkQ6da(Rwb!|87(NI~>angyDT^>9H3(d^)DyCeUdHPgf1AyN!m zy}=VW2cbcl%iTA!4)bJkLT9_5On+G7?cWI&M2c{pK^##J8DQ%Hz(F94?{QB90suQj z-~gcKF@`7DWElU|hVK|BYZ9Hd8e?*zv){nbaZpnVmoQ9+RNW4vb71#sw}QlN;(HF^O37NrWZZa&%`^t4 zRYNsE11y+V_7TJ%D#XY@gdtX+9(n=*DB@ z&1HoD@0}nc_K@I}#tHf%e8i)3MWY^0qbPP6ZrzVTXx?d@;0$QNLnIw4_}qh8!kjQyP*M zv|?TO-myWXKUm*jjZ6xFT}2w4cBxoFEaO8+V;-hrzyRa>0Gd}4mMoE^RSF@Qz(Pu1 zq6DdBQ0WKxm; zKgcB}E`(ts#4iL)6&eHr<{xT231J?lC+s9{9RxB6MC15gzy%{*QYK+k0bXk6bso|T z0Hs~9VnP5KSsvmJ45B@jA|+O4AR>f&h!AUvCgo`X;yqyIi6ZU>< z!6q)}==cM076bw&mn0adBoNKJ80S!?06R)xVjhwD{NZz2i6r3eeQrVUcx4md$UFk<64BkP2~Q5r<}&8I=Ir$&Bbb^w6-%w|KhWorUJh2*9s zD#WW5B?Nw=fck^$U7mG%q<9*qKQLlx*j{4(Ba`|=H7X&6`qK{JC58q@ZV4m*hl=SJ zIZV>cg*GscU%5{8sZ(d^9-0tMyxGsmY~TC|0u24*e`x~X5M7P_CvSqJAI)EU7DSI8 zQ3@19kiLPP<_eQk0U{uR*&xAhCS{UdA<=-{$edPU8U)}>>7{6c3nkeR;b2e(O!)n% zKX|EK*ujjB>8pN`Aj|+mqMaVZ70U(QX|T&RLPNWxKqE2QdoZcUTv_KNgr6RSKwKk8 zT4d*aX!u!YkitjA(9G#IiRx9VEKW#5gdaL`YC%9l%6zIQwvFVhOa`*di-K#V&P6qn z6|BN*5$Rhd&IR02gk4=kF)*cfxLsu6>u|i<|&3qkU!b%03V>=z)3_0>!Qn zEI1t;sVvLltOEg^d;Fx@snA;457*`F&^}Qy6oR@q0U~Vd25FTYNYj@QZPXT#(VC6U zdE5x4oW_9=)JkpFE>Xxd6~~E!%yv)8F$q+IG<|AcEFP9L32GG@ybYB;x2q zpU<-G-WCzQ5uCPh+rH)wGN3{soUODRoZj;7;zH5I8ScAzfgp&&zu`{6t=ZiIuG}*2 z<~q{FQJUt_;;F%c;@VA`NrD|%nweCmCQvTtx~?@%8n9j2VniY?AtQ1kEAk>U zaw9wPBSUf|OY$UBawS{xC1Y|XYw{*@awmK8Cxdb*i}EOwaw(hgDWh^KtMV$dax1&? zE5mXu%knJKaxL5PE#q=7>+&x1axeSxF9UNh3-d4&b1@t9F(Y#_EAuil^F#D8LIf9U z^|2ox8%E$FiBiNfBZP-daDPHXt!PXNd~7q5^U|b@zzRe)BLq33Mirp1LpV&4N(54- zszDGTO_P|6u3r}faGn5*1K8IpNyxPG6 z^hL|emF-PB7xYB`qpMnTIo@lm9Yk%FEF?}eL{v0Jvvkf}nSc5NN1vIL{v}9D;Bbv_ zZ&I4%QVT;HM3kMhLC|wBdSpb8CZ!cZ$>PFrA%eIb1iJLwkot2=Q?<+V5q1ceOal{5 z`@^fXVyHgML6qTX1dLAyrB*t`P!}aIrVI?#%xRo~+(4db2sH;+f$6MtRqJ)f^pV4a zK)`4b!nVTo(hdV=jV)uUp(&NRJV3@3(RXXTCdKwC|H+u*&{ojffv|M3V057 zfRk=tDWD>R!%Wxc)SPfL;9i`o2+b7vcI9>i)Bc#2z# zfxo1>P_{h-xY<+);^AR(FLsJpsbWeHlK^*ce_OI;JCU zp}2}ed9N6FP?pAvi(-uHcWwc|;JC;CVmpjCnvf&Z#^mj2#F9c9gY9SA)8IXhS6W zK`?qTq)fb(VIu*fv;?1X-)IUfuO0wrrS(UCI6lvy=FI>&9nrcFM6pK`sj*>KpT>{J;}@!5jSl!6SUaEBwMUe8W5Z!$W+;OZ-gLRoIX%MfB^-T)h5l z{Jhv#icj;BY3_quCa4QWF--G({SLw`Ex(5R%a84J%kwQd9heWuIfXYObi5jhv_yn+ z3fQz@po^&qY_$yi!5IdTTFlW4ER)nz3I{WTh3SCU%Dc*if|rICXwAs9e0KyZd@Sp$ z%{p9^VAfkSd-PWQoOXYf6HR0vDZ8#|H`3dH3JrKgDgV0o&>QjZ{9E30`bR7jI2h$~kX0w74C z696i`0DuT9=FEt&Bo&h6Nt8m28aa9#II$v6ty{VBOaTCyLTxEMDIHkM6SQJ>P)QUd zN!v$3c2p|F1pt8mS%uKbN~9>$A78z{Itpz05-H$_zu0zltWXaCJp+XTsPg!6QN05N z`+Wm=Y|70an?8-IFhq#0jN4*%Tn#TYfvxZYfaO{%vcAwHz0tIiZy`iEt+EU`vgAOJ zB1x`fjk_ys0J8nj#(Nt&&tCwvtPG8EGg`Hck-d$pd~RLWQ#TeKxV`&UkaQKI@)S35 z@qvg0z0A&2I_j*mEh1>XlH|Vx6MR=w39u@| zYal^t!|RL={fX>E0Q~a@krq$$r<$)$d_y6hS`5pdGfsR_AkB)Z4ZGPwXt9vTd;*Fx z0);#=Ic-MCk?YHSh5{bbZ|ps#oosF)KcQ%Dj3G&$0!8SwfoFg0tdqC+2(D>eS=T+L8>jwlO1; zvc*ORN^Pq_U3~B^Bm~lCh7bm=G{H*;(UuDpf zkO0_W(n-VO6faK&D)UUdesPmcXxzjLPRy>nQJ`esShiU~zSQ=_Zv6=nnnUplp-6!; zWwJqH{bAKrTqCM;2QK~n7vO*e9+=>Q4L+Dl4hI?+5fGORL62Vs4HtkEB%KFP6W!Ow zm%1dh(7V#5N=KxI9*WYdbO8ZD=^dnpPUr}Tq4$nR5v2DLs)(R;X(AvhD!L!<|9d8x zoy=^q*}LcL-nnzm@3EZQ%S9-B=O;3*f#!(mVwBkyLQ)I+%&Bz*3S|uj9&ot!7lLAXua~3VOoJLD}-1_{)Ifz#G6Z2tQT-HgDa z-2h;h?6Iy~;uxQm<7FZsvTEEZFlhN-+d18LlJeW=5?i&CDDj3EzDI$|7-Ys?6V>lQ z)0qzwb5~yVFE3%hE$;O9^h?FKU#ih`w1rBd`;ATYJBCzu3#RIbSXlXMY+Ee~knIG( z_LA09uj-ty8;+I6R^?U|*=NtbGcB{UM3b@(IYE#HQytjn*QK{QR zKH)#+K=t&BBmvDYMcBgxptwv!`kEah3aYcqV@|zIO2j1W<8hSkJ%=2heD;^~7o%)ajzd7!M*2O z)x}J`ceG{rmwKp#n$5TSKGE>5L$BIghNz*Y+Wzu2U(xN=R|Yv5%f|7Z54c(Ug&BeL z7))4gqRn--H$qIyr*U`JHu1C@ax1s68VqrQEv~WBsE+l-X9o9^^Cmj{nHX$-Ketel z`q?Se#wni?x0zk&&t9FfMsHiOz+g6`D&Xvu6h@Mh_v7{J4*c7c(`}AvYwB;htrNV7 zN~HHness|aOr&eL2%7gUzFphZHYF0@>50g#WOCbDy(;DXUU=V9Q5o04SKDs8?mI{ zFIAgHB_N;PbjKgo2HXV#u$NKEb_?ZcxQeL-1`N`pRK zE||1dW%DlNXV!Wpvfe;7kw(#EwFI(1KFhsjLR#k6%eo_~SEb9HI>p1VQ@Sj)SYg>d z-%7+u`(os-seWN zZi<*DfzH9J-myLZmrY0Oi#Hj_X69XfvtJVu$8J6YmK^A7!=#ySbgTYVi}O*$Y9l2j zhA%7VhN~^9qkE3+8*}@{JZV~jO59_C&Q||pf2^6`&Dbj|_Pd{S8by4+Bs>4PO*?9^ zWT;=b!wTW?$;9`IpOv@q1L65A(Gtk`n52_aVI;hNAiIR@L)h56-Bs9kXG| zyqsML5G@&3&o*zZo5YjnhNl^}Sl9c?P`q3%m+y?A<_VQyXo9#fLQVdL?EC>H5wNXD zcv^7m{7iSYks^Yr`s3HQ>bzr?{g>^emO>nnZd*#TfknQU-Ep&j=g+??<<2tiDG9o2 zXnY)KWbdbllF538GqF*lEw^vd>PK-KWHFUZiObAI=Z70+Li z>s28gaUves{jwv%x#oCDJEW#PLZec4RumV4?Hc#0Qg#8AqNyd)QE*c z!-O1!Oubv=`pW)W!$f+g9nO6e7k2uHl+9FbB41vO7H;txkiy2#4=9oY~??JitcZ3+-C`O^aw9qbXH+8JyY{% zx6wUQGi)Bek4SWMwcOL6cr%gM?gSQAE|Tz{%8Y5$089=0BfrxY-)LOq8+ge>RA*iI zbQ)qK?ub-3RfVXtSMgL|{SVc!HH-2jDv9>m1-Xd}y!rS#?woStqM6i5z9M92ZNlmBA<*_+Y=epa1&=AjY zBWO3}Bwl!;eAO%sWZXg`a&OPwdPdC`E&koJ_!F^;OB=ppqOzA;93NhcTez}0BOyZ- zm-w=Q3MTT1Hi|zI@)bfT&~Gb zxvgLLR*a}8>22`*X^k?hvQo%(hd;KQ8J>CTG~+gnqsS*}>efk_WONrLj3ZT25Ya6oBm$K=h++3~ zHlhijk=XEB@?8+5ihC;AFO#+)gZ5FDKC=riM(7gV2{Mwc%#^MDrlLCpyN#nrb^Kaq zq*BxW>dBWyEZ)(ZOnDE=b4G)unM|%{8F$G?G@Y^4Z!#5+78JjJ6AM>n%+w&QqgG-m zV=QTH^u&X$X3D}Ak=-S9?9RIgW2stpLNy~h_v;K>R1sN}`$DaHZ3ZH@sH&7Dd%eKV z@rw{o2Y>($i1fa?v5~5>zPk8rQ35;w0E7que&9Qy4nF$&0|1Z#04)IA0D#*7pbh{| z0N@D#;Q){f0M~8w2oxF`di`dNW{v(&5ELB^!~ox7W8Km2@$q}XiNUr>wn<44UNT;C zax#>h%$CN+lg1;KA?B0e17rf3{|SOJGlAzo!GD6F!a`UPtn5EQP+1vJ1LW7_Lp3$E zwY9-@!F6?zVuK>k3beMiwzWY}TU$m~Mt65N)ZP8&4OH;9fToWI=m!P|A!u-LXlSTm zv;i6$o1BDzDF8WzoSK@lp0Tc&sQ?y$g#}1xQD|{-5xRb@u73FNp?R%&Z4ClG0_B_K zo0|}{xe06oa@%q{J3D-Pd}RA%VuxbT;o&s_{P_6z%NNM*#7^m(Qrtz{#eaf;?*R1u zd+LwW>)<7YNNFKjR*01w;ueGiudkmRB&PdK4!Qe6z9CRZ3>0%c zG+9tqAyilfmDNFYtx)UrDGfq{lhEV>wD1A?unlb=K}X-9Zx94shdv1e{DrZiny|{z zE;v@Jv8LidG=fXJ5Y=2VoK&RdvNkrnJe)@KQX%@X&QdL%-Kd7pHED7rrO0%mP^-0K zxaQ2{O*N_xbE0w?=SOUr>?X6Ko+X; zdDMg{j(87Gbl&!rG+)CZqvRbeO96$ox z)>pxmS_ibXMlqM3wz)>hZ!hhxwu6N#qgtEku2)CznjX?v33U4}$Qe+Tw`qe2c8R3v zK=sXT8TP?{MSqc-L<`u)Lc)1_!BZF@l1^-q=FZUg)?S?52NE~l{f%rzadX{@&nve6 zhW}QYzUm$bf0&)XqVzA!yA1LbqGv+ItNEclNO-F@qR6E8H?D2|#@IcmP4hCiMb0ou z+NR{0xqjH8p9HG5^=3EWT=(u|hyf%4HvntDTlzuYU~Tkl5!Tr`$e8*&`XQvRI0h{} z*6|}Kp56P*M$L7jjES9SJYv9>eRp6`C&RpIH^<8VU^mw;j-MAro66*TTZE~9N}G+^ zm>ftjSuRr+q8Q_1i>h9E`J$FgJR{lvy#*UQLcu-<{qG{sypY&6I*Li-WPjGOh)L^t zJgI^OhGDUKNY{mh>fv5hS&rLbbyY=;fSOYSbFmIH9we5c>p^WCdtHptgx~{WIxt25 z@^*=Cc`W(mD&^5SBiI3wAF6a3Bh@Zh$@D`BEwcOCpNdgypR?PET`d{lOBIsS+00&> zhh4|*MDP|O>mNs-OOok_j6JE96RZkvm82*YxfCVcFv&L83!~YhmQ>1&^P&nUQ3Qga+ zzXw=#Iv642Y!ArKOHOq@r-KZU%Yv$yzUUJzeMO7-KNynD~u7_ zW#Zj_B_Jo=F_d6a`3i}ocQhFrrvA}v=5t+%CYIFWkjcw3KKJveY4rA=x!R*hqN~m+}V>d*`i1GDBSJP2}-D(+@F%ThKY8i1bj4Wx0BQT~v&XEl(|S^-`NfCKC4d z%V`kd<`Uui(&)cyDj`IFm;mG4&beVtr4cQiOQ0ifBC?D{as|b2hZ>b#Jxgs0;kY4; zKf3%LAtexJd8a)(Wgl( zM`@B4%z?IqZ@Mm@()pbfoM&27a=Fl>=K>1z)SR*l@lOh)GSKn_O59$o)o||D517B@ zjp;f^xmQm`>u>Vha^F+c=EE02eal~&s*nyD(De`}us*O{kcJJAnY+e-Tdb9`@5X2R z>B|aMg@|R;aZ!EMMl`}>`eJM2&}{f`qvxj=ytE|_F{Uo9bl6oA+WB9zak3Z1!emv_ z-E$MUNc+@jpQdfbXc_ILY!{rG z$)6n3d)WTA%JP0T)1>#S(CUwNtvMm_yGOC7V}M+e1WGV2cr~Rhf;7fu*xG(1*%T2) zipj3gb4vVhX!)~i?f9`$P?ETWor4=iWUN)vA8O`^NJP1LU3a_zTqOdpeP_3yEh2IK z_v+7Hfw?a?&{QxH-=E#=uE!6@b@V0KIeW6`jP9%?U9VJU^4^tN#-C^?;_xU#u|tv@1K0IK+mZ#p7niMDe;4(`Bw{{;LFPo+ zh1UV-QZgeoB9ZCZ7i~j;N1{O@O&doZYsa$ru4k|Rh2LL83aep!#_Jk5rD+0#$N2QZ z8n4=4hgq0Rp3p^H!hJ;Xbl(Ye&}DQGFeW#-J9qzCpIQbu3{DAU)wbw%x7&Mv)|)*& zs%NfGo#e3GD`IsrN0hP@a&ntH>U)NVTy0Bzf4v^3 ze52By^OH5?KrD8H?Lf;cpVYXotwrkZ3H_<%tno(cOFUth2MDeY7yM}>jTzb)mcRS# zbV1Xm>FDA82#SjR!0xVQ7iZBf#-Xv`+wDb3bu^`naqY4t9){$tE}_~ez3$b&)xLSI z1hgS|eKf@OJI?q9+aE2uKmKf(ZXYGxV0t$&zI2zY*PV4JRPW8!ql2IR3IsH}_7r?E z)%UrqoyCH%S6g>xgBYd5O=ja`{Pb~g*H(=Y=0QCB;PItBe#dzw<(zz0 z0sF{lE<}I$wm@fM4RV+-r2?d%*(vxt&R@hhjV9Rsr@n+Vmq9oW0B?j{- zUJ@3jD)vTzp^V`X-4C+WZ-o7RQIh7-<>^tc&dB2I==J*OT%^fNFCwL7V$}9&WJp*l zuu9?piAvSO##1_9AW~f>)>wu{bBuXblw|XIE>r4(#RWrVu?{kE)LyaG&G15T2E%x_ zw+x|_0r2krCK2`c}g12@)qZ3iF5qY#& z$c{}98I=G&!x+RT1tv(Z#H**rZq5-)j0NFcDig*hl$l57^UYBD zE-!hxDqzxuW)>jfL$W?bJ!`aMGYCxC=?Hu;Lox8nTt$75 zr#xU5JspoPjtTg7nRdf0?WWlE@Dcn%#RQW8^B{&y>-rJMKDZ%*Voz`gj5oo=lRr;a z_yq}Ii*=65ig3;`!`7`Bx z;l{i)&a%Io#k7+V)tNW#oBI0t3XkW|-%U=ocs|yaEE>qh!<57Mp0mL9ITohV%Lv1k zh-&Fnr^l1D(GFe#z_HhXnE=Z}z*Z7q%UUnyJK?W!q=S$?bu)kAo&}4h3)$~X zaCbbMLu{6$8N9&;fQVQ_^2g)E&9_&o(t2gq{IdI%bpTi+7bJTlBD;+YH_44 zK}CpQW|>PZ6!Q7!>YQ2jm$ebF2pq9V6Dtdv@EY6Yv7wp=z$;GSo33O$_W!}eMnuVa zPT{q7@InME0tq@}iBNVo?F_Ka7%&cizdprM1(&I$M`Et6tI%bi?BP6*0wZ^beCh$YKN6Fkn2I7j(* z5CHE9s=^*3Rz_i#RF%oU$guia-uA>loy)B8r3;bZlRf)9yPD$QS_0*er9OW4w*1vT zU+15Mqv_>kB{FG;4^-6ZI%+Cj{)Ee(63{c}rBy0Izv!=DcPsLEM5Y&kukk+TX=RLWQz6$Q65BI^IBeaQu6%6}e(|H) z+?6Z>al^R}9V9^(fmOZy-Pq3Won`pEXG26O0@eVRbE(4g!J3N?JWdnw$Hud4EQsJC zazFDjZwMCFw8IIy^SbUb%?T*o;SV^5x>`Isohi5*m@EpFJX_9v;*S$rZ+@ttDJCz( z&%3BmxYwmwds-K(-KJY@J90`OE81$7AS&zGN_IxL!HFQ+5kxxpdlwUVRWO~$0(nqwj`;6Ev4djd-~H#YH8>GgTlI4bE6T(7UeKY-n9P>k)*w{}ex zo|tsTe2%(5kvC1#LFlZ?OSyx>p02l`L%iBzoVuZxzeHXKk*R~oKB9cc@>*vwjf+b9 z-eKsK5u!p)24Czokv&7*14JrB&7M3U#Nf+$*{nfmPp9F`z?DUvcGfgd+o5rUc-&kS z`JPw_(Mj3-hG6c^>|^0kl^&vEC9XMTVk&7d8N`|n;$yd9EQizwd;rx-|1q+cptv`I z;Fd)5v)?RlnG%dpf-Tvu4CZoFdh7%}m;wGPMhBK#Nn!FJbq&e$)v2vHqmQ~2UZY)cA1m8<|Zcg}-gye#b7;0_Asj~|_ zV>GBFpE_6ijt4lGF`=5q4HcV z?+|w1drWudjds%!TPW$oQ(YO3*5OqyUF~B^#$(Di=LF;32o$cZLq7BL0zNf2&C4Vm zFGMwPO!@C`d&%5sL>dvQM(>PdfciAq}qS zzRc#EzpfNgqXa+v5gBps9+_i&e>~DGM`KSgB^8#^o;BZpklXVHp~yDw7{YC%&-K25 zP6-L_`j2?eO%a}vc@=lrOqCHAD~vVI<43l0Y%`@91g!cf3)&Oc+*wwv)_jLQQ*3eb zZoO`d*j%b`nn;$=kH-a>F}fq7+~Aqib8^ht;Crfkp>J@VL9Fz~9W~p?_vO}!?LNXU zgbT+x-!~0)*qoB33Jm3@&$Ol_gboUJ2^Ze3B^r34vMS0JD(^e^ViGh!%Ci+?qVe0T zG*<^-M$oS3Lw_;!v^BH>3w(m%*7Cl%!;X(^=z7DzFCkUNS|6KKF`g|ljqJW_FO)6y z%hhT=9^H#^ijsX0S}XWmSutLPWOm~Cf*9s|90A2hMeUBa7243}t3FQos?2`Z9j-2s;86fqNYlkvFG5OLp z5yO)dc~cZgRPK1@%41_LK57*g=kfbZ%6*)*ii~*nd0Y;n;_7wa|b$*(m)| z)w49}fq>OE9l}avREMO^&In)Rmsd2MYv zxbPrzcd`nA#FAY(-u|i04UXPvSOidwF@9Ru_bRA)J?a5@E%vxdQJwXKw!PtV{>f(> z=-)ByrEyw15!7Yh;Lq)RAEo+$DrG|Lwc+-k7n_ZuWV~~ya%WO&;iRUD+o8%&J?uy- zaOdD=on7;AJ;hp^_IcIW`SY32;%TQ+P&m*;Na0b0%mJPIS<@r<#NT+ekW<+al+%7( z)-)tib|cBdl=RqGD~0AS+L#gXN}v~tH=A% zQS?)$chf1&Y>BFalN-i*E@5k0L+O zp8vXta0`@9-A;jiNfT8siT?H+vn?QfTycth%t3JeMF;cqD=A)SPZRt*6D$tX3CxWo zGYFd$v8qMwj8d|ZKI4`*FHfYG@Yn~K9Z9asXI{rCd}ACo?^nvNW7JmT9uPwjTe2(21pgv_JLs zdc}kaq9b)96cEDEyxzWg=~Z5+2BnYkxpiJ;2Qk1eN!`udz?-x;2%= z-JNM8cE7-kcrJ>+Pc*2ZuO>N&hz2NwSC67APSiy_usL z@?N3OM7PVE=+N>a>OIH|3Zs*yr;OZ=eR(_zU}!~mx{!azJ@sk3MT4@RJn5M<2`?CG zYPl+Ven@dCb3AeJ_{>)ChTLrJsW_u&jur4_phf`u=3!%2MhgkIp5s;edj8yqPQU#4 z`g2i-TX^)r*~w_5w#j(SzW9MO01VBTI3l;+-NKXzcJG%#*6RC@a4gXOE0KBVdZBxIbHk59cK2hf?kxfNZK(2HL7%`g1knk&KDI zv>ht+@L_!xUsn7FsNVcMWp!B>g6@bI*miZ(y=VZ*!nw~|ZgHe4%TmxSs?4inudTRR zU{7o2Ld<=s7HU}bqR4IJHEK*mb5)c8sM)DD8`}U;$5|iS7VmsW;WamupkceZqb9U0 z=Dl)Yb?NA$IppD@t>;Nw5ZsSxj67tb;5Kf#pWD^;J~g<~eirs2f_Lg{EhX*f_(8M~tks=(XU1-&4j~^Ad)-pG6Ek1L(q`@|Hr!e#Sj<=a7R* zTYLg_c9TI;;PPF|lVSq4OAm{S5G;i&NgYi#q@`RRR3h>!NPrg4Df5GSt~d7rt7NyMeI-mtj$UB{P63U8vH~KFe-q zW$m=$rpuyUs`X)tA(K8VZEVmoF>+hdl_OG*c9`+YDAC_#e5vJ! z$!%H2QvJXATy7Sm4^|DAIU;LbDNkTAkY#j5*KzKoZD(rYdDqA0E?M)Le?znY=IBIL z2OGt9ECNq5Wt!!a_NWtSa`3q|x^XZ$VM@~)a&N)K?@y*cpgkYQy6{sMXNf{7I+F<+ zhb@`8!7*jyJZhpx9wN2OYfq#*IEQB*5`|cQ%lhI@yu?5(6*xM5M(r)OZuncq@HVV6OG{?c$uQetX|k>UPI>@vwH1QuEt;53_^M-ihk_&NAx! zdi0aCy8#!q!1#4y@SfS?Ta&t#;SWYon#p0vPLn7uGDE#?>TqQ9%yrNOp8N%T^k6w( zBP!;HW6Eaml;d&qs+;H#-eAwovw6?`y>}`!Tk^B){g-b{Z(I~~K<#>oSNGg#709-T zllUEfT(mley{I#b$az;<)Q*ZcVCZwV@XD{nvJI1}>YN0&P}6U+SNzEp7`y%cz5jx9 z)BIzp1^2?4gLUt;04;XjJGmCbo9U3Nuw#}TTq8E_PNbpuQ}>&{kz1DfCV!7st!kD# zLf`l-+@4v_XLc}u{lc-*`o*crtJqE0E!W)~$=(sSpBae?5;B5puvaB{J<$*E-O!Em z%=i|EwkQtpOj*4pa>XCvx%&(Lp4mYTF0)K+*4nCcVsrm(_vc|I5*$nTdTR@O3BPW{ z=x*2I(huMF{@-7*;6-jk5k%2+_Q-lU_8F37b~a&SHsIC*hS?3Rb$dA#jdF^9{*%pP zUT?|$r73Xre_pkbsCtmHO!(^XW}QRCX&^z-X=LlIRVsyOeJz-Dj;0w2zva0)C z-+aFoPRsaeH9Ydu*(QD9WAh>ac8I?cuN(((Mz4`0xqyc+UvGIBP#&^D6`K`#%-PY(| zGo!s1m9~Emk8b{+(F6U=!4n^Jz-8}gDHzg+K9_r&9@x`jSm}6(-WWvxOnUoEr`Nwp z$#V^0hOYuQt$OV`zAc()J)Spz7_P z!MBuaZxO7@)H{86$BMLSeZl&Dp`OZ&LsZqQ)juz+CZ~1|m-edNK~_5f1BLsd*yMUVMZA zU-aWs#UMr{&9w?a*506wpiVA4j&+q=G?zdm?xXUJw~#Qkc>lW$0M?V2>ycTSTYS-||0TT%I4fJz70QIN-Q*BaM2RbMaeSK@bN)=TG%C@O`;|Aks$K%KdfsPnuTD13d(@-54x~~ZwUUOH_sZ&GV z@tXQ;vFz+o={w_b-0IofBMI(mNmp5|k9n^7LZRlP0e0x?;lLb^S7l8g2|+unadlSh ze|>GQSPET^B>!xT1y*T!oqM;v@bh4R&ugu zZP-&3?H7R##-W=d!~o>W{K>TNF?29u(zi*gH)cw|EY3P_)Jq1fmIa2afJ1X&2nHQ3 zkACBhdV-xAlf0)-2zqx+1&mGgOL9+Fz3l6TM@gctxiQ`(;|q)T#&oAAeL=|7eew#c zAmEm$^k<@T2+a4LT3egma~xY#8+!EyJ-tXipE=pRf({Nu`!a$>-0G@oAdOJpSEGJ9 zW0jC5bcnv*ceUAI&sowK^o>wOpE1?1LTJBubcp99*0TjnZ24!d8)y#tuc z=>PgqAAF%tl7eIO)4S~MV>=$$Yt;yJoN1v&zqQiIJ*JKL)}F&|5Y39i-<_njHjvm+ z<`&XpRG%fu#{H->Ak7AUZ4Z#Cqvg2K@;kU{vj2EGpJ)xP5Sh@b9877u&E!KOSC7ux znvN$4Xy;C4g5tZ{#PvMA7zynndm@>nn?!N5;B! z8uDZd?u6(+WV(k}S;w^cK7fhG)qQ(Mw7G|}6KkpV_h(F(F@6*{fAgj2zQsa!%& z_81r(2>P)v7u6YMUugtl(1h;&4DzYWzHfsu>bh27Wz0koH(ER!ov5}DW`3Av*h&)$&b7<#XI1ENE z)*D`}`v#(O*{BCR(2?t^b{&LL9eDC98sZ*c@#4B;e2d{O_AjqVX9^#FixHQh}cU=R}h_b>4z`2(5p znJ>n+NaN^hk5l%hXrm7K)jlu~W2~(JUZ+F9E>qXV@~3QbBrDo44}CtI7MizC*07dN zZcsU$KSQhIi@0Z#yh%_&b$#*yG9c_9@so`~)ph%y_n20bKQd4ZdE22YmeOuY#N-&H z@Wn@K=MlNB3@_@U{V^Z}*f3Fh*ng>C%XwwfC8TaUro=eC_B0DUj!19AgEzC7c=>kM_lGDYh;5?bv^uLkBZ1 z-`gAXM4l1&H$py3v_2sP`pax<5M!VC zfxa7i5K_`92={1JbU*$HvB}0xPB|EKaw9wr+)&8I8 z_`?9c!#t`-xfIt=$Aji;_*$U7k%p6%xJ^LcwlX&uxbNDza*gmhR3Zd@t3LL%m%jdJ z>m7)0k2?-T9V)ju2>lb_(%AY-hxTWDJXVEny)op81Ov@ItT@;FE)Nxj(1C%+P`YUJ z$a||=9NM?aA@KA#uM->~9(p_C-mnh_p7= zKkCGPRmpZ@^Uzv! z-gt?}T2g3&uZh=3u_i+MU7q#_Z;(Q&-dJ=S4(*3f^N9y*QX3)Fy%QxXP@I7@tnWx5 z9Q>mpc2%(_aNDu14}BLJTx{@Kmvv+RJm|%Ep2OG({7eXU@!9V^Cp^3kY%9-rbPEO@ zuZV`dZ3v4%H%fytpW)#yXn|BT4dw0HAKw(*L59)9Fu3#0?Enqt73mb37#3TpvUGlPq|WTr|d`MTtOv+=Wv$@!baUhcf;-8)zL+ z%}>;?_|#vC8y&dem0sn~+#GNv6WB23k8}j{zB5C%=!8(a8Be>@x6Xl?sj#zW$C>2=_Pz{e3 zIVH>Mxqss_y#sxN91P_9ouo(3gbM^V0@I$7F_8piF#gJ@q{uQ0%5nLXTbY{o2&NiO zCt;q~G+0MQ3M<4tZxACQnxP;rp`blQyQyYjaB<~*sjuH9ia@G9`#)8bu8RpS>&HUA z;P8N-VCEedM+?S=FLX)fs-#(3wV2{`-!CWP<foB!l!V?)!-Zu z$VCTDza-20n$zc%> zWA=lVpC(I3OcvaPwQD7J=!J|_G3U}GWSXXQoP)0`Q?p&d-=5aXh$UpN1m^4qK8u)K z9!z})=SpV}BbxhHYkv8p-LGEsqPNkf0r4jjAd#58`Xxb#W$2FqrVwiQ8=I<3mdRAC z#Ah6EwjgTRHbwr^__)C>yGamU9mFXW})8MZCTwog|Qy zLbwyNaX***L5?I5usE4V`ADGN?NwQ?-=}tOp326u<`T0}UbSfFx?WGKw=X_u<{Y=) znQu?)5(Lc6M+aZCo=UP9{53cvHv1pMLp*AKrWJV?6#`bqHPw~-<5C}pfVeZS!M zJq~Pq+xri5MK0fQ`ai4Y9QnIDgR{H_KUT@G66|NN10YbGB=epy3_bj0a9p46;9q#M zP-kMdttm7=>Vq8E3nTq}mU`GuR5=bOt7dd8Kt|vQ~vYiua`b9N0?wfC5wz?%i z4`Wk|lw@;+$x3rXrYjh2CGg`nafT3BT&P7wFmlZ&!u|#@Q40raG65!P%Oid!!%}9} z0X(!YK%5L;`-b9Ov*%Fo^K1sDM@7vmr4Lt6a|@J~#=e&Z?N{G%(E64v!Frc<;DB1u ztzMl%A;469Jt4}(VkK3f1-2LDfbX`BMJ<_fKmSvVP0U(QPaFEB}F`}r41!V8H0)j-gh{V&gV224TGT&j~n`V3tnpPW^Q_((iZ3| z?c!W#TX>p3-v}ibv8+`=-ZQi2j-uY9Oo@2k!C7`Y>JgLk&MSUia}zjAfork4w54m? z_X|YbEV+8E&V2JBHMK6!4y|#2FY(hh!h_No&yBTA42o9x@v_O^h_P(BIa%5orPck2 z0p!(cSC-E4U0!iWxkFs?Mc-``;yzKB6YrN=$71~teZYRKmX@)1=%m4qIA50h;q!+x zmS5B!%zAq8tRVsjgUHT(R?f>k`gnZMElL8JR!fs&X_bDxJZz{|pI_4>?0_#~=w=t; z-%+o3z-b{y*)-RY7m%!O^H%Bj_*YW-`^Ct0+6qn17FtD?3T?hImF1swgxO^5@Qfpa zQa5B>OwzM{!~2jH(e*xB!LFp9i)_#2aT0u2q6lR-YL@mXzPZB20~5Tz1m)6mUnoPI zl*lP%9`oD~Rvh=COO=m#)^j7^xvfd4R*x#9RPAcPdY97uv%=RVN|-F1Fy9z5L`$zM z2f?nF?b`|Mz3POUsCdr8t!+CFnHN86p9?;_BnwmKVpYVn=L87VT0F+nlrIaAKHQ9b zJlAhI8E@7D@H+Zro!=N@=ahanl^i_6g~GG-MnmD6$$EOIbI*nl~$un7P&LO!iD5RXh!uF%`&Ox9akNkGNbcUot?s}C%Cs)C= zrv^eoR~!7Cw6qF8e_M9p;G_O{ggsM5h;(1Xaeg~I&|6Sed>u39Utx_!-RS1OD) zUIxQxyBuyD>HA88z{c~tT2#kZ550}Fl|5;t?l8TlJXGV9ap~zTb9%x5HbB{B?^Gv# zPr*ItGV!9$jAou`UZDKKs?qr_W@fzQ(Rt>oRr{sZVzwG-oBKQNU-1~?@;M)N<(+a` zR~6P&p75YPwGUZQ3Pu8T*`Gv5TYh+T)R(DxRn&3b3aOsX_u`I@qGcw5ohQ@5t3-Ed z6MtC^!0vbLfGN6T5`c1M-&t$)kJ>0Q(??VVsOC7oKR5SeWHi~LNHA>%?emX^qYK9qW3xlS+}D(eAL}h<(N3 zsIYF~!!!NLlqX5i7bcBncvpOdg(GWIcww?KC2z!PtMHJnN79x?Zn1)#U6~r4B@|D# zyu)SLYIot5ANmfma;b&KADF$K)!+Q#TcAxM4d!D8EP0en=9G1&5|_u_Fac>jiK>q1 zcZc)R95>eCG+$%t5wj&%7$y1G6t3CK#h=w0)%l-WCqpS|HQDByfc^17q@U;G!-<;^rm_cH!b^@Lp4 zc^h$%I0rhGnY!e-20R!J#Vbg3lii|<$@6ogH4qFKJ$HC!w;;*s^`Vb3npuR;jQB}- zy!Xqvv{yUZ_kWq|T;Pw)sfxdcKpR_)n1V61lK4Bru%u`x9S*2<<9&r zRhB39nDBj!$(G4eav^Sd(#f>)6f>w7{98k|>oFCRS|`dRKP5-R;njnNuBGJke)rJs zjH6XGDN78VCK-9rHnm#l?MPn}#|^`07hRJ#|9B@3lD*0{uNU{Ks7@PYp@5X8p9=== zRH#}i>?W?24b2zLp#DwupTOjJ-mZOim;RCt?zY;VTYOgaMS?$^f?w4!i#DiyCwRxb zQvC13(o>L8QEGN~wPFg*CP7>&^Q4@WCC8@p*Rj5D5(}>Ap&1;g|CO$it_%f{Vl^5Ej7_OME zJa=&(Bj8I|sy;!m+^UFZrFd^?_i@i;Wl)zmC!gPaXm$k)6jX`tRt9g{H9Mih~Ap_S1B1(3uR8$Tvs% zB*~ZP0h$jt|M*f|>4(snnw1w>DZROFW~J(vS>PAlN>{{US;qQ3+I_RRsF)CJibz_`;dIiT7_;L`+=JUE5W9hp*`nw>2FOXTeh zn$#C?!IdGM1n`NBN&rue9TGpC1ScdOTjdna2!%%s4oR)T<{-sh>BVJ53p{WIJKV{4 z8J+YbVM1ub5-MLpycM|!3=}$9QB0vD#ghq+M;4}_;@R6F6$ZR1g+as&rFESbSl%t& zQ)S5odz@i_aTCRT!(fdFkO`Tp1&}fLgmU?zXhqb6d}7~F4|+M8=Ew(?V2dA$MDgjx zwZWm(pvhwVNpxLdb7&zoT?MXGK}=wkiuss7wAMGsorR3rYn2xs>KD1GqV!Eq9>Q2H zsaIGW7bNx*{#nGQ#o^xQ#S;Ob{m2I$ObU&W*-$WH9m16Ywo3;3gBGCwfHw-s82VU0 zti{D4$2}-ncj@0J*%KYsjuZCIH13uuhT^jQp-crL%B`S()MG-x;5ZVGIRaY^s-r-z z2#!&bxs{pJJfuAK99Oi15Jt<;m4qFoUa?rjBvHvm>=Ra5#Aoyl-{23*#1G)SB0p7w zn;BC>R>h0OPpO|6R>QN2QeQ^@qmL=kRxr@SaYgqr zV;u4a{-~2>W~FITqbT|V&|rc=NDNUl*?!=kL9AAQ{XmI{V~o|!D(D_MB7{|b;AOPq zI$l=28Rc6_PfB7K_=(>?t;T2=-wt{XR+i>3Sr1>*j!xbUPfCPXT7`02jYRe)uKeak z)ZlQE+h=SfJOTho=>4^32t!l#dQ8*#wNoe5DsN)MIf=;vtaqqS(uAgG9K*2;9} zazR9difTz{Xi1c3tK8d8VCqo7YC;q!4yI1r$Plk4gs;-pcfG@X7RyN{gfY(6(>%k+ zKvEvW2JpeuMF|Mbcu#|X1lQn-4t$W2NXBJ2D6`rBXq`%^-29eGxJ~-pNF?A#HVF+u z7(`4p(5YqOh%k*1ksje_D++L{x8}|hgli#HW4WFbx~glBw5x8}gqt>Bk{E&^gqvz; zgG1N{CK%J|sO5tWsixgXk30jCZYDa?DW%q@CY6e9tJRNO%d{1+HPo{XsK=_lVV60LmpGMRL!i-|2-Wz)W zTX2p?!$yL*W?GC$2BrLm3|tc++=kfPYa8VMYkqViH?>l#!O1v@C^v~K$9k;dI_$W{ zZQDwe#}LcTvMZauYsR_+=4$S~M&wY%pavms4a(HwHZD?-hIiyfTr31^I>i6fuD)_E z0?E;|`bo7Cgz0UClP*gyw1KPqMaj5|4zvO6Wl-zhLN?0OM#LIKkd?c}&g=qZZJdPO zXv1w#$}r?dz8x2i<;R=ciD!Ppw)x38S=uxhiHzdb`9bOfn5lF7a&%8r)2dBiQ`$M6`_;8Y^v+=CsND4dL!!_byR6yJZe1Orbe z;gTB+13*jY+Dc?#tFq3noeM?`T+_P$i~v^&PQ8K|u$>mfQ2H{4WtLmZ<_nT~=+<%* z3UkHh!bv}j(>NHcNP(|U+=3T+LK19Qka{YGKKI~_yJ@1`FFG1Yt zH;985FUvFVn@bEcT^@6X3`A{gOviLdd|xCli#=npO3JiO?=(-J(s*r!BCPO+47qH%Oc=!wD}F1d#EsOINj66XX682T5B9LVtrg zjPqExwOdPX5P{cH4o5gV^IP9FUXukpEcG|IVX=_tR)_;K<~3m#c2(SiH<6lDcL>2g zk6#-$WE*t?4Tn>1^@w!;H9?ehi8gg)ceX#|gD(?D^(ypF6GuBNwr8g{Hsiynam8c% z2{;djIKTsEtF~>MbU(lYM-xX_FH0I~NIcZ`Z4dWJ+e2D^13I8~hu8yO^9gZ3H##57 zaAdWP4dGkRU0|Oxbh|Y_Xm&V^wr@{2IJ5&>Z?}0bcWVbWco&O4BsTxH!&#rVd?$=P zv;$v%Q+&e-KE(Cu!S{Us_c8ZFJn%Ph4ED0uYnWJ zJ6O0Y$%AswIFP&lggvMOkE2?7qalxj13M5=ke4_=z{7s7ML8+Oghy3KUx<5?gLXH$ zJM)7(+&7ayfC=Lpl_>IWW18b$5}2LpsnyjH5Yp_X9nwxrmSVk0%U2 z0QPX8gOA%em)pZSxHwy^H{tAqG1AF9966zTc8#a^N3D02BL`+HMT<|jIJg6(N4hE9 zLnD8bn-gPhBRV;_L#cy0UiU*i9J+{q6N%H7mm@kj2>Pe9dSvf|JfK67^RGMfx%BMA zm&f`y!NZyBx_R@%Jm`6>r-Pz%Q&t#zIlM!%D?4E`yRXMOhL6Ri14pJS#h=IeIlwr! zOOHOhLzLhD^J|;BJfL`BvqL@9`MIAouW$RaN4q`BLxR&uJJ@8gFUnZyjPq& z1iUHrgFLu5Q>nu|3JB#|SyV5(Le9U_@KhOg^th;P`dP(C$JM8v!n}a*dd(Tg` ztGk2A-*Y>xx<2QFI|#jotb;sgeACA?Kh#4zM7psTjT^fwxdXs=Jxu3AJfuU% zd;FsV$G7W7xL*aw8~ZsRJKHNo!5@4%Y`Hr#{Ea`*IV`=f_kFbJ!#uFV!Jpnbh}iPko2f{af7qSk$=Ki^DoZeAD9tJmCADuLIAkGe7it z&Ub}6#KS#=eD8~UJpjM(|MEJN{ph;`%{PoWu!B6{gZFh|+OHnFrd>OdQwLL~ShsKCHq1BA9XfGnJDPhpG_F;C^VXfk z`;FX7dQK6>xc`5ZiM-8_DbuiA5GZZJo1<d1t|p}1(kC8tc0}v5&_=WIs=3@rr$ML0S%)6{oLoqsdDtlj zLyrDqN3?vf%u+TG=XqxxbIj`uoO9f{&&&z4vg^wX(Gim#d}y2#yM6GX2Of6N>2EM` z(0Rw6GW~=opVhp)^P6$bS?8U3W^7bK5bKGjopj1c62{Q(VN_Fv_MxX8b3{F_oOIfW zXEak!U2LCw=u(HxJ0FziI;w=EiO9G#<7_@~3Uf!m^=d7spM2^W^qeWX>QSm@8`3Ew zY6lCaoIK%~#~yuX#nvKd<#A`7X37nr$l?jd6k`a1k49Kg>M2_V1msVr`==AStp)FF^&iyNI&jY;&8_; zw`3OYp~q8Q+x!wLcu0mT*qMe+3@fc3#!wu5^%d7*2?4D~o_7c(QBtU`eL35m)=c@> zam-0);&|w}C!apE><3VL=7C3Dbjqn&Qbetjx}^6MY|Y+}z!~S;b@Y{|9)2&5m!E)@ z9(WzJJ)YYm+ydmyZJH3|$(*1K4~Lv{w00*Raq-co?|8+|#~yl?CKzqC?OpPnffg^x zvAPVtI_q^T$9x}`o$Ng3c$ik_9C9F+G#qo-k*8F~b~dSJ%(iU*xc56Frkovj;E{(O zd+^CepV@8VXCHm?xrc1r+gT@_-Iuo+;muQj2tjz(CEajv#39EVbkf25o#^S=yZY<- zq5W;`xrdK?)MFm;FdI9>cMfy#&Tv18k?{PNK8d_1O}Z*y1K}q>`r!?Fmz&-__P3Az z!KEJn%!dF!S3t9cFCFMupg61+IecA8aS|L0eB3dj3rdO`-0)2N=BEz(!NY_00HO8x z7eSMhaC;~89s%V6k9V|#9qK^GIm$7P9lnno02&$CY*>+J}gzXRkn{7ZkF8W!ziHI%!uV_r~UHKLP^q42~jg$gG<{lF9CpN%u=da zV_8D4iWz=*u59XxDFbOb1xS=MgU)MX*UXyN8fgr4Q?z0hC7MVkFafG){8qjwhFGgA z7C6RO>Sveg1Z+SytfkH1>+)8Q%x#vlEe)8=V7faP&PNqDNP$yJYaIF24|+xeq?6EE z5w*_gZv!k~?pzp9E%K2G02pjn;Fn5sP|ki_qh4ZLyI4ROH;Q;npgELFP`oq)07P(; zDN|XFAP!M_M*LcK(`UQS$pd$IREHkJE6dLRkXNq9LN6qzn~pWQqmAwL8usAZEHi-i zvro_lLjEQ|D6ViF_f=qc#amEvrgysN5F#6kn7Q93xMX{!BR!65-*h-*3`t!>hUprP z|Ekx+?9JbSS!<@?Qe?PpW8(Jo!M%JOBdJHI9uoH$J$GBkcE6gOC&hO(7UxQL`K;>bw6UXn-1WoZtB!AeC!HOAs(AV>Lx_^`8`!Hi`zFImls z#RLkJx(bB+c+OzH^PTe?!kjG*__41QUzUYZ4Y9pwAQxNo{$lwW|{QkSZoD&C;H~@6eaFoy7AD{qqo4xJv zi#sIai@ODAS55PhaRRiM5W1T4pzNb}s22-H0w@wv@|Z7u)N36 zd*R)vUwR|};P<4?UFUk|8Q-P3@-}DN^&Wb;+I#QGn}fFV!551*BGK+P6yEqKM|>o# zp42DM{PGLQJjS#!e9*R``3BNPo0H1>?n|vG;D`J8Yf_Ct$N&X$M?cDqAp`CgBv78<4=k4gwVaIWcZ|PPLAySb1@NWmPV-WDp2unr=mGB`}!R?@~3G*$|AW!Uq@F0fpWv);PslpJb zjSCfq3BeE{R1W~OpbE#Z+BOdY&u$9~A`45#4byNYBH;_?P{C#}4XYvf0ATs<@Y~{V z{^(8);Sgiw@DD2@`Tj5u7%>kpVHg@Q54`Ob8j<&I0T8s{3Fx5Z3}O%>F%Jx36?D)a zjNucFA>10#5d0w$B5@Er(Gd?~7=GXf7_sn%Arjg^3)&zQ>EH)@p%FO|A%2k&F@X|4 zu@Nz081o<)4?-FL+3*rGaT5>X6;n|Z3xX3OkrTBc5`WMj459D_p$c^1^dy5cO}P4)Gl&0uzF64%y8NAFLPBP76kn=m6otv_aVD;1-7A z&ye8co?*dI022&B;(nm@{Gk`x?FUYd%}{_7;7l8?4FCo~8xAt5&e0#V00ne__af~F ziqRBa?gB}%-B6$eMU4af0r3F93_t-CI3W^{pbBOZ0BSNJZj#taG9ecd1sbv-9+D^} zk|N*j8A^{F{h=CE(jpO}4InTQ98B0`(jl}#3v%u)ZehW2a1`;;*m}?%sg52w>mJwg zAvj?l=kVJ9>=5>nklhSo<;s%Hbb#0<5AV3m4*g-}mhCWgfcd@;0J09`P~hbJ!3=(Y z99PZ@)RNmI^B{DuEJ;z)45Hjl(imQ_GZDhGl4{Zl!Y^YH<^&S}2y@{KLgE6`AXWhd z!V)D#uOKS3AXJkgwBZMsu_{2p@y>59nJv<^auAC!t?F@}) z*|R-=^gZu0?SyhZz3tMh(jbPxLnSfoATaxM5(-73s?$&f6P!%hM)g)V$ z){qqd0`?iS00}Tw6%k@qC-(e)z!ZqJATCWM70?WSH4-|OH$gV3hz(#*^-2jA1q>GE z5EdaQG^sZAAe6H;EfZ$_f&F|FHXrQT{M8?l4O%5)4`odY25?<?*XLK`#eHtz>O&q9+<3m}q2|MhDfwjhKJ1umA>8dE_3QL+#9 zv+{&(`c{G3Le?Jw)K=Y;{`3|A_||V(7Hs)7A;y+q%{HklG(lspAr_21*^C^8?c~Ug zQ~PZ|MN(%Mwjzds4%RUba?@#9ZD*qvUL8$dbBStM6!ESW3{f)|{b6EB^&l=yV_UOG z9hB&>b7RBwZe>6w8`fOQ4F!I{+hol%{Xy7@_GCX30A8UAjP7p(H%57PR;RamuQzc4 z7XWzIN!{~zlZtJBj^Y+V;%M)34I*<3Vsmd706O>Q$`bY>w;@b{>|AnoQJ2+D)b*sc zr(m~45zj-%aNnYk>=w5ZO_mI0lVHtG*nIFHf>aygb4ZW(Af5r>yiM5ul92uYAbJTG zOn<;Y{Q-M@)*!SY?6`L_7Y>C(mfHsRAc*b@d6a^?F?|~%*_h9L;T8ZiSCEjkem~cN z(U2iPwB5WQiT_vEi~$9j6h&G0)m+z;3RpUS&Mm?4+Ah@}%1`8Ob~F7!))bFNG1zcN zF-Wgc))-D;zZO{$!biQW8jRI~M_3`~Kx64RM!`3H1=J8SZI632sk|*@#~2~ZxIxV{ zh09GI&lKhwV)CjGh@V%8USYum^^>2t*5c9l2AHL?7&%Lhi@6Y4 z1@UJ)_aJUT26W)$v|+(=j_9g0{&sRUH`zAZ6i?O~ zuH=$*@~pB7LKq>c0TVi{3P8bsOS#sdHuzH69;V_SASICi z^76p%kRaRCbfbSy3!Xt6bdM+j!4OhS2N*H!WT0p%vn;a|(4OE47LbH*Vc)b?S?3QE zWDVvb;TFnGskE&{q70Iw^{K{r;~~fj3E-T zwHHcH3wq(F0pS*8ts{fhZpV~#rI?Hry4Y0Lpc8uDt~iYUR(USj>~z8K2dhD#hoKh) zx-VqV9iQPA8ag}r`mfVZ2AY;803k^aLK`@t+>k(B+gjTK7?tB1Uu5jy0WuF z4AsmOkieBGLiI%M7Sy^8*cP;*Sc=E*prJFfUAvtzxwd^nwABn0N;^oNK<4(a)1Ckm z5E!>v?UZAiwz+PzC8)FU5n0*VxEDjW)yx{DoLIi$V!h$5pT89l3fsRy?Xr`* zIpWg3ar^uF+rY&Fz`t!3AhHb}9Kua)7+{+K3DF4uB;mtBJj6wO#7VrwP5i`BJjGSK z#NPk_R{X_bJjP{w#%a99ZT!Y@T*egv0N^0Uef-CPJjjK7$cg;Nl|TuKJjs2yJJk8a7&Dp%o-Tckr zJkHyE1^__L?flO1JkRxf&-uL1{ru1M8~|nj& z1xNq@NWcYfAksB`(>cA`UI7qf4W%!<<8!Sj$Q}JI-e3BG6aXO2wEYLF+T`DqB($9x{Jjt6)Q=aIj5*6#>G`6CD5w8a#+Fp~8g>8x}+X5rvG2 zIx1S6C?g^Yha5Y4RH);LnLLsIOx=3sFQv+rEd9L#gaiNzFPMzAyoocX&Ye7a`uqtr zsL-KAiyA$OG^x_1I!yuK#5AhZsZ^Odt%^0PKYHc9dHo9Z+c?EMQku*zz;KpCQ=<;Tax2vO)j@C~%Rzm6U6Fcf~Xd;dQ4DtN*2#?AeD?w!4UNA=BPNAA2=boa*Fzdw~Y z{{7Uo^#V1BK)?mJ5DGv4JTX}wgJ1&6Dz^;tjCA}BrOh+L+_DNNm<*De7f<{!Ab|-P z_=5~G(PYzvDz3<4i}qFXLlrL0D42LOGF4A}bAhuCJ?u%PPdw+ymySL+9*N&%NG|!4 zf1CiK1wkSjBm)v!v;jyYpuExxN=)AMOfRp15(yw}Xn}-@Qx>Fy7JzWli-cd!NoSob zdZ9ydcJ4`(jePdB4?5?eKA?)=FpR%W!(=#kH?Vu%@vi710eBpqjroRI>i zjWLBql7~QQ68M7|hOFY{r>?#VD^P;~&`Yeg`e~~_^MDf;k9Pf4k2!Od^J}ig>STr) z$Bw7XD~8}1912MPm;neTs|@4mve#)d%qo}wVg?c+dZdFFhP;xkw(73SqEk?@`zNlj z_M^^R>+D0_Jza$*54_sy=C5GSfRaZ{i~ZomnBXfVA(00ttul;p#(!*_xHeq~Ov)zKqPSnYWHDji<1j8!$D&>=BL(+*@S>2~d+lX*Nb?m|xt zatqpWGaY5ob5H0rqx&T46*)$Zw<}oc+`VKTOB3GHD|txS5E692iTIwqa6$*r4C#=^ zE0J%$E+Dl3a=u^QeD=flH|YFmA3jlg#SZ8#0&=+PDh&e&<8A}8iruo;SQnkW$>aBX@!D_#{6a|Je)jZqYU8`*2|d9{_uik6AIIop zw|t%OIIrS9tAq3R`q&MEgzVKAa%y86h@Iu=PA_GI1Pk;;*jM5rNo%YRX zAH4EPIZ!ttd{poo=YU{!43R(1X~Pf)M1dJhu|N;jNgJkUjzUnNKo!amFm;>ZUnKaY zt9>IJ_{vH(QnVC#fTbQb{EH!ez&TUc&LCO#mfSN%bigBOBGFAGT=AfEZHfBoN|kdSNiJTIo={K_JAXK3R$WyI+luBna`rQj z{upLJgCZ!etOKu6!GsJ5_YB*?5TZUgB^_@vtF+3oRyXy>J;UmcKPt-~n84^t`iab! zmi3=NEiEnqfZDI3Ab_(3fP_v7fT1G6nLahhZ`qWR)@oHHuuaaaO6d{+C`%jviUDOm zwQ!Ro-ejz^-N{;0a!TkfWrQKp!CWbOP#9qsy;9LDjqdT4a1aHsENRYHLzt2hN$44X z(A;883sBDbqaN{PuTnU1*|}-Ma#hIYQap>3(5~d3g<|> zgw~ezti*eT0iAk1f*^shR;r3IQfd$+2-v_4tRa_|;@%m(<11Mk3VjE2M}K&(l%D&6 zA<`P%NZgZi6ZLZ~_H)yJSH+HWdhi7655YY1~TtQtqH@CC^Czb8*Z1;gz&j zbETCgjdk)z%dXZRLaTyshuV@%%V5);R%oM3{A052SdaiHtUv5%n^xas71K-!j8HHL zkE&Xlaq9&{Tn5coq?8WwoYj_pVn>5?H_wPXqj;HJlrXzUhj9FB>{?sNmoB%X2FawC z`f0Er6Gd2Fp$D{wB25Jk2+)BBNsGyfiQ339gtj#WaApF^KQ22ZJ;rXNRpOMcme!8X zmSwByfP&MUQrN;(NUbuBaD@Ya;i*+MOZt7$A0T141wlg5bPIs>suciwniVK#RaRc3 zbhjy~f(*3r-IPcZ&QeLXOpbl-;#oT9NRf7n$o&l>nmKk)sHu?u6V7N%7%;dx>nA|% zi0YqEo996B`MPM0+^n3EVtMF#9%6Y>glK`SA5{qp=^&-9>iZ@k_s73e!UWiYs-JX@32K3oPOrSf^gXey%o{$n6;rSk8?aNvQ-Ph{g zo!=7ZKUd}e5I0Nsn3KKd_X#W6^)wq@dX>c7l+e_Q&0Wp^djh0h0eE_;_Z9WQdMV*K zSt4h17F__yX$tjU(I*4vr4zaL4e&sKIzekVr!H})OTq^L`L_}h7I|eS1A(=EX`(w9 z7Jd5`HX>Gbr-W+v#BSS%a4)4f@;6*KfrDr?ao>kaHC0OhP)X~jfYQ|x@P`v&q!Ni? zcJ_A*J+^HQ19Je-d9s!jjZ_fVw}I#5WncIcwDu{6^bO{gfc{VsAVw?NAT6!(3^NgD zDltqVCv{)ZFJjmd{p5k#(p&YUbt4#IgoYEPWd=bwdA{WjaaUr#_5;cBcCEEqwv|#e zrVS>NVzNeSF4uA}=7X9DQwb)2jJOhx=v<$}Z;EmMd;Y+LH_=Eb;yJ`KSpJ}LI^axg zuv`1rTB|4(MP?A$r-#PlY{58&ir6UY@D;n4fV4(}(xg@Ilm{DPQgdin{sa^wBXq== z68$7lNW(Z)(1$xAQAUM37e#a226!9M230j|Dlte}Vn;gQQZOY`VIz;|m=c*~LY$BX zCc5mlrp! z)dsgPOmq}?bU;mZad1#zHwxXcZLL5;AgOEF(YHO=72qUTxCuFNy zBTcIbriNoHX7~-@fQI0~4q;Jk;%S8E^F=P?5I@icm@tJ#stvb*32jg_1!4vp+N(@c zq@JosvCU%emtY8ritV$jHkg!s7tOy#J4Mad@`4MeWHpmfxgu#zdaAPxc{j=LYKaUmH}8@Xl% zyzv|48yxw0ymv$YvYWeS{xY?zdmiCn4(s3!@jwsvfDieg4-K3T_+St701xeu4(uu) z-%t+mfVElbW~OHhufPd|&<37{zaJq38W$N}@F=@`8Ih0~ps*R8aSNXT8lf=^H*5?! zY(+WD8aLb;H9W(xAse%S2~{-+w~-rWup7NmA|*^NRp14ax)6qdrTg18O1rry>o0wK z!Cmad@{zZwil<0WrUZz%PK?G2;Q$WM01U7I3ZMW9kN^mP00(dY22cQLtj7jY1z1uN z3uAVk^AR%<3HocrI}>-n+N(XL$lB5m^so-A^Tm_=73JU#duyvmux|kff*w%?izmf; zybu*Y%B5`o%BIW!qP!SYQUxQiASPk9J<$s}%MgTY$cM}@Z@0PI12ffI4lql}>j}Nm zySWpkhdtI2NRUlGBqqRX2!PO5Ua)vN5XvHQ%GX@V3n0oe@EB0s1{sAJrobh+85MpO z9HWK_p^D4eqD(+##mjp%`(O|9;12764(3n}%_q8i$YA02Nb=aPYT_BLvxyw-7w|C)qCc%uB|Pj znrE+oiwogqtpb^e2_Cu-+N694jPj~Hf!bQ2+CpP7v`r&9&C`tr*vzdpw#{C~FbIDA z5GL6YoxIoH0o=j;2rMI#$Za!uV5ibOAqZ#FNXyIUEjIu|S(nWaO>{PH_>PD7z2W}CHL<&&1=0r5O(U$J-=UyjuaK)9F;USG;(L)vyG>8Kz~aPGq9sZw}5a~ItU={BB&4pDiGSJ;5y** zI~_5rN-pI{q2A2hPg;KFxl%ng^bAL2;wa81*3bk+AOuAa47yMYwGa!|U_5$SFl0Um zbl&DYLEy}dNip20{%z0T=#6<697BP~5K~*rY53WHd0b z-VnCD>n-Pg?)@CizAod6KBgc?Zi5K{&M2U5?BJB_*zG@s?CEKa;QS-)zh|4q(+mA* zrNa|2++G1iKv&B#)0=7r=`7_Kx#5jgv*!*gmX$o!5D7Ue>>2aj70?9eMepE#5H-Ew ze{|RGZ3a8;@2~Q|yJPJP@dVlKG5Jl(R`6wQz1X7`*UWwEIo?m}UGa1R3Va4S>n_EQ zr1Hli;iS9;d=1r$4cDqJvb{0sNE_`dA1gbJI0B!uUeL%4qvE9e1)`V#*irn}Alup^ zi*P(I*O&l^LjNiUdv#%b5L2e@>oVk|dlqUYrR)ZJc-+aY_DBoyjP7DT5BK2FfsZTH41x57(+Q=l)Y>@Ss!cKHZL7~PYMvg4 zV_o=k!i`7+*7X_*Q@=Br;P}Gn-{npDc_(q~iaom~3NMk`W=DMs zz*zF>QJeCt-u-*D)d0N0ev6LWxjF_xhTrv!_5K1RX*)Vb^9}&;#1rr@fe2dgLAR=? z0zwKaG)bHJj(f%s+2%r_4f-rR>OY`R5%EN|IBCN`?*_u8B^0Z=DytZ4EUFg(II(d@ z9u2~fx8Jk@MT^;>ldi{juiG-vQW@bvtGI^Ob0-!MAo7r_zGrP5qrt?%-MdR1C(D2 zozYlqwX8;lJjzvfD;buR?3qZID$EQ&*QH1ZzL2rkLT$C+wlHtOB^1kj!vrP6f)jp8 z223O~#f#Zog;imRcHnCniYvUu5P$7NA`eq0-ZBz|Gj`Gp9Yi*H<&1P_IIw1SbxMY4 zS(Ye3yQ6DA=}l>=f0$Gu{WEdgO6u zopjDAXPj`r`R4J*gGdB&$iWeZ9COf7hn;uku_qsXih6i9BqFi9ziom+<&iq9z?m=D zW0!q)+H1G{cHDEZf^Kp%>*pSM+DXTpaU7={a&gK* zXB~L#*+-#qi?=Zk6!a9|)_v@WryX?0x&D3l15pKj`YSId9e3urr=OFC;ixcljVD0M zh(#v!@CAJJPJs(#U;`caz}cyw2xkyPD0E?s^NBsI9N|b|4}JKDAk%xS zQHxLr;tbV-U=^)+MJ#3!C^4`?9S(ttV6e@G7ujG3lLwU$0tO%MAO{fL7!GwfqGV&V zkTFn@E;{}RAMij258YTlaDXEm;t)qT8fU#f!fzbwn1?Jqan;FXzRg)BE^$wRoobq$=LAUaVC3@(!) zwX{=hlv9;_q$7po7^434(N2U6t{(7MhdH204M;$)9P?1R+8zvU#iJkbv6dTAzH+=D0_8O`bi%_-&#-IsF1)~^M zS`gSMYBmm?3O>+LzjM699vH1iKIEYeKfZ4qB)uwpwD5z|wKYJxY$FQ6agKJ(BOhIP zUOd)OR}`LO9{qTQ8U9t1x=`{n+t9@&-OvQ6ZnB`EKxoqFX}~QcaDn;^UO(GPAgz)n zJSlYvKh{CNbkKvWAh}2XJH!FMau`#t=`orXX8WLee6@bzNJl(mo0fjq!yV^9ArnYr zn${qL6_2ndJ{RapSQ0Ofs>9?!8CZlb{NbPpT_%Cx>7mi4L>_5{965OUl6pw=ed}25 zZHv>3JG%Eh{fI|SDQAv%{B|+@s0Taj+XtuMq=tD}MlFc?1u90hi;n=Kd<(+ZC{8i5 zy;H$d4`D&8(h*84vJ;g?!VmS{w;n{%$7!!O4*fcqIP7_uil-9~@>1>`^605%^5G6% z0;48DD1#?4L;hz!zs*x7LYr~Uelsn)>*;13jEz4geP4yUnCgI z&15UH+lk@Xuu?Lck{L8b5qkg=9Pr@EJ#y-fK>H>xq|NkZ@?nnmfuoqqp$Dkn*N*lY zCCN+HtO9qi;3#+6BoG$FV<*hJ3cI0#y2xs&GVS(#F7 zLGA%eWtW8>z-))PKXVnh`a3fGATH&=QSJBSL${QBhf7qKXHD|?(6s=xz`KIyI^5IH z(QTWyyl`0mrv&U*`2c1-(1MRNe}mE%$ILT+Ktq=QB_BCehY|Z@55PVy9kW(9?>Gx( zf5tp7jCE|dH(7_k^$X-p2^cge7v{9&gS;sgI$(?;1KU>pzyOj7_r9g zq)Js@YyC4rU2=kPw}lMyZ0R;>#^YA*7Lr459Aj7eS%x74FZ7)(Ql}g`ti7Onr2HHt zDLd^ZsNKp6zBtOo?4Dr<_c8LUFhO{nIsiit%3)gaf4RknM!yw)q+IkC(Z;h2VvM*h zh99PWT)sonc%XDP=-H`;>%Y?ahQMAs^)?Pbl>`*b=ka)Q1Hc@D$ajSpW56jv_T#`| z^V6sQMr4XSV8pQ?vo>HvN*siOfWS7yz#Q2KMvNpaphV(hh<<3P#|ekSy9fhB zKpBVxRvZK*op(?Z&-=%Z5Fmumi*!Qo(veOGy@x7Ynu36cG-)D`0HKE}(xgl8A_5}4 zHvyF@y(=O`DSpV0@BH2~_s8w*&CTxY?d zbkU|R0b#>}DQjrT5d`_&R8fd13$KCu7<%3qjn4qSkZ7B_Y;8px70bwMLc@62*2S@Pa;V|=`Ji%Z|Yf@3i z0UZGxqpMo7@u^iO){2a&kkFtIt*}n+?yB*)RO9<}Uq=MunF#h~tUne@LSLmW z4~|fi7Re~tv=ThAeksKG99yoINKGoTAEE`4_pkk7|R(&~5yik~eeny4f z%V226zCiHE{ll7{L3wbhZ(uqA)df@n|13Hq|aU>ceVar3juKE}& z(wYncm%Lsf^U81pkC;CsjT(817>&Z|W7dYH!9bV@Hn3(4yBFIsYLNLa-eRrms}kvS zv&xTL$is|e51$rkOiyNE->YfFV@tU>c(Pw9qQEOlz=(P@`8mvhDcYms^m-cYOB)WStDncI(7^0^qW7{Us z!-i~;i0?|n=b@lA5y<*ne~DT*8Wz-siupigHeo+E;>#vgldO&o}wXr5P@$}mZVG%`L^K5@amMnTvL zM>#4$r)xt5ypxRJh*%^xFe8B{!00x&H$(u@C5Nc=A9p)0>?oYZ;3!GY5CjK$m_3-% z2}l7v9c4xRvI10J1FFhEy=W!-f`d-aJl=KzJ0CmR z17S_=B_T3KV@t#Gacd2*2ckw&!-PfjHX1T1ijBE01CY=NRzIt+(88I6f89ciBWog7HTqM4=+vF7d^)P~Bki{PH=2)&HD4An z71l8kR?*Ye_H24=#|z;yrnfEON15d?ewNP2ijtexe1Xzc{-)TzB$BDsmPsZSWL?rQ zpH{5=H4I`QG5lKtV*Et|RU31C40*brXk!o2s4~p1jdz+R;Axwle}WAYs88Ip#Oy_j zUA7IfYIE^UyiqnSVjblWNMd_3`T3*dO7y6pCZg2~?O!vMu8e(-FN=G%V0p-vtafY! zV*;l|($fK$-^$B%(8eh`yFV93)$pwxO@C30BFR zqKECDj6wcMW8e9hORCRVof*Fn#p=ZKXz(saLG)q@zsLNBKNP~2euU875c~SrRBS^@ zdG$<^)=ke5r@T7bmqq~l3V*|b&@@Q&*hs7bB-X8Zc22#3;%gsp_-o&rEQe*dQDZzZ zT%WK$VOf44W4L*SAaQcR%&U^{t_{mLMi-(^A^I3E$7np)z|*x>Y_?f1&*$mRHM`76 zS%HOnqMI=~QdF|*RTae42B!KB*o^4BL^#t_se@E!`^SEvelM4r#Y~puO%A$I^oW9D z6U^nw_r&ap7-{GI%==bzGwA8aAW;a{W3r5WGKf`*O&_N3}*Ndwp6{>xX8}bSgTP zqU1@&jVE28_x?rRBgq0yn^LQnKJM78y{JNWRD-*^h{*-n?n{|zqt65>)2bo;U@k#J zNGYMs95qw6mR<(*W#?ey^vv4DgSs`lXy@S$G18me8wPSYrjUzw@RPGhf$MH5hU@9#2-GVV5Z%) z_*SVCx4s%;6~pqfM7*ltubnfWhL*6Z8+ok?uMfMEP!2r&D^9$`OSa|$xlxI8o3`6L zK5}7p_Q6f6u=~7%G=6FRRT*R`it?c?|M+n;l_EN?&1iq`ozI-(v5Q`FB021JblyL{ zXC-^l9JisA1F^kJ)Gu_ZpHY!!20j}X|81>j$(0O)i#fBuy(NQ!=nw~(KTn6%yy!m1 zW_UiMzVnmo*E2Osfft2&R6_|OB)^5~p~}m;*9_e;tnLIKNKPc>*<#tqq?*Kx6F`dM zyH#^8&N@C7uANkfXA#Huv`JUQ3Gd^ht;2`q>6&r+(X1JPiwn;r1w!Ae@I&Fv?-q+r ztxvpCH%@DTlczerwZ-CXFH8mv-NWHqLzTzCXA((#jK=-5<$WoQ>NX=hBym>id(oLS z=-1O@rOt+J+vM{~Ip2qk$Ke-U@Z#Mcd$*|Ca#51FCpiCSo>5#K@CUJ23zw#_MNkp0 zTCDkF^V+#j#L8n1C}k-c@xa>fC95CK_XjREiXxyD|QL61Z z<3)+1Vu>?6(5%Xrww*t|;Vm+hOr7ZExAqsqAL6wyi*y3fry*4w@%N9niTK>>i#h7t z-cHPu1nZoSK8{UzPB!XU$G7y>DoQqjrUKjdZfL1af%VU!yj;^A(vV@nXAx^%s_`C< zN4^gh!cZg#1(Kld(@*ZA9dB8m1;q}!C62zYBN7sr_*6Ue?P4PB-%5g^-Fp?xVO((b ziv+R%ngAlK@^{0XSbHSrGyI?Jb#Ve^d+;aM!^In*PEhb);_2q(i}xH#L#{nVp>9*h!(qSY z9-nt4mmKX(zEQ7Ntbs?m z9eTbq5cLoG(cs{zFTtxf(}1uXr*y z`#tnz-DOJYxFvlOmlEAa;iu~Sx-6?gGYUlW(Tk}(+kJ(QQlX0=yFAD9rYu?MesZvl zZX$qsoBtd{6#)=Zq3{T>42Zd+5KUuKQ4~v@vu?`fP5ZVar7Ivs;d`U?zUI!jI%{dJ z@8Ujk(47E)F9zMZOHqE=q)@>wOC#M3B)4eRrL9h zZS4>ji>dNq!@+@N_lTKe)2wMV8^6H6W4(7v9t*}QXPwQ-@|&mwk%)C1nnLmSJR$%4 z0H!qI0#gyIjE27(xlg+k*L2bif8FLvtDMwGJ*?g+DYucRP9eXY?Lwh&&@elx_2zshW9Pj%{qi?Px)+U=38hu z8z2Sd_lb5|X&-d{9y+ZhhXe2;(UU1Riodl2**zo2(03+oJkx%wAOEg$dnO(|x}fN~ z^kPYFgZbAWtJKT?QC(b~&OZ3`U;D=>jew)oRn?~am&w7!WO1?=yPpWT#3P||qlpLxnu}@7UCT8R^vu6viAlz)kEOLf$v2OijctIme>*ZMpz{qi5w>0V z*6}#1lY-A502FS)-^(S=M70&e+DOT?j1MmTG=jR{q8me8I(hgN_eUyAsewmS8~if& z`_2a%Gpg)(g*AIMfuQx2+GSe~b^bBy4arv@>;`J98l>!7lTsPLd=d!@qqciTBzfYG z2Jn(`$VW)2oIF-yHX1jqq28(0+kPk6j(tN-6&ET=gc`fZrJ?p0nz&s%e#PAias&7z z*(a~vEX=-6$cEG#S3Av$@Q66p16qa2{znKh4-}UvEs+3#k*BnI{iJ-Rh$dfvvYMJy z-GdWDx1G0}95j{`C0#1G^_HBxNITVXpnzOcP~Wy4;Q}DNA7P91pG!{XY)HSQ1``E@ zQ^lnp%yCi?xuX(1<}S&(DSgv-O=#05Q)FeTOwZW`ki3L3%ob&4D?;~s+!ECoo1l4c zA)Tj-Uz&?EL+c|Lp^M^ENfa`*zWKr@Rgt6zdjM6b$e!8$!M9J$zpAn7y~d}Nc#Cc> z*~WJrL~L8-(QYBLHSzf(53SvxQos~hA>8xsrp>}slx)Bv5PekP2AvFVsxPt^^_00! zZiK0-eQUq%Wb3)Kq; zk6h1{!=qZ}!tNS7SmU=CafS~pHKKuIp5HqjyU-^QbyETaU(o*yNq&8S`%HPRUbp^a zh}E}aK|~t<_l1;7d1g>5Q6kZ{)0w^;AmSIl?Nu0vDtMTXZP77?EVPHrW*SMa@MB#_ zX}yo_djo@H*t%>>BtN4x;Sm_Br}JcYWC`QV3}U5>g9b#5 zPG3*iJ%V|*^%*02DgAK|^NI?`j0K*toQPv}e0u5X83&~T$8iTB`Ooouwj2Qs8+I~B zLSGn(Gs+A|q|6D%+D4^ls@9@a2sZ{I9R@_b|H0UHW4ej8j)jU@*Y!QuNpl7>DR6~E zQd>P#+2RADi3F&Vy9)EYb=G3UzrpG|2c*L1SlL8^hw58BupwFdw-bn1HAfI#>EyF_ zkT6FxA1<1Ril&-KH-0aRqNx~-$UAVwAJ6-8oqhWjBo&L3oN7RL+Rx#=ork4p{<`0l3lzJn&76it1x6_C+m$Q=U$4jDO4W~b=uod-ef3b)L^-I z!q1cKL{)BeGFk0?zC^XKD=;0!Sss3NX|c(W2Vm|vPj<>`Y|w*0Dl9G4F-X|T&F|=p_5W$F+AXh&eyd?i2ivft|=$@ZTntzw+Oncuho* zE|8os#1tmO7T0W7xH*Ow*EEbz5oVSicbcEP2tTQQx?#WAtr8&bWKsiuKh4q|8toez^K7KH^R41NnsQ% z;Y!I?V-J`XnTr~;Gp=dF(910Z{4fp?Ht?%jYrLwU2KcIPLWv-ulGF0CPiP*uA;SPQ z7@u#(Jr*f7L^`ELDOnEOv6;zlggZ-c;u=d78M9mORvM`0L^m?AY_5MWnAY_lbOfRLH7 zv%W&h4MfW#Gi9-+>IRN1XGr_64=Y$TtYUB!d1Y!BgWT{S)G}$j3Mf+xFH_UtmgIVU*T|)A zu9(6-65>I3AWCZ#$poeOcJUims z9LjkiMvSPQ@*tJXiB9Abu3~`&H^`+U`|v{|r`|~RiW>vxemWeb*6xE=PC#el8CSn3 zqiuc=ff5KIGp%M_%pL%JXyK0j;qDg4Ko?i1X{E6Ba)wBZS!3UyU&kia@-D{s7W2t7 z6B_^aar~O(2s6^lReQ@V_9_W>fqt3HqpnkdJOk}uHr1?(IX6c75qpl0c%rZR7?}tI zFgU+azL!z%cw2LV$Q<|AGwBrKZZ>PBNF~-Vqb*9MQpYYHofvm@C=W24mfXHa;?V;R z)yHw9^(qztgTveCOq{C&4iH8r8c^*|#R;^JgMXEWvLO#1Q6?glyV_1j})b=VL z*OgTO5$r^j>RY?XpomuZ<6=GyJB_#ecehHe&=m3T*X&H8m+F~Ugj!$Ys2FyfJ*rhP z+cGg8F8=|MIaN0B*Jf27oswqzs5chw>UU%qsiK`68nda74E|i?b6!#~gSg=TW}Fq6 z`B2mNQ*z-FUpwSKPWa8fd#i^G(|(ah2^YwD1z|vci#FY(q>+0ZaNzi-=FQQ(l@mLd zDDH64U0uw{R)*fi&)Xd%>se{JYcZ|XFzPuaNekZt9!)-A2j+iDtmO+#TB-DH3`b}+ z!t2~UkFr@{f*O5X;DE$!wT!j=mR)#j6-(NF^A7YtU<#&{NS^jX(K!Lx#%W(wqozc7 z+#=#ZJUvBjTydpd-}_|JQqyYk)fC8En2lZ6Vc;f7BXYZ6+|ey?i{l~u=t;lkSK_0s zgPD`kN;S?kVbi0XgBhWJEbFbb{O(oLzTW#PrZNIWcOvTUm8DV`l<)W66;><4)VbgK z-(}=8MVSsB9UY}gY^fEHgZ8Utgk9gL?%(~z^^{F?wN}LKzs$67rkP9LUuQcr;xN#L zQY2^15pUTYHcV!R$Cv{%H>n-B#21 z@au-bOpGz=DujLS-Djz65Zt_!JfP_G{*Mi1{1&|K%0%Y&HW_;^#;K27`i#T7iUKbdN}bhia3|DOB2L~vzts5((KHV`V9A0T)0OyyFij(7y={C(QlJl9A^76A;>&lizbtLejCOLPy z>=?A|SQvXD{1Wul&s0Pk1vPrRWC~&M$UAmRb>unVd*f0{OafS2mTuPm1ynmhFZJ8s z%q%BboU9xvxeV|~x6xrF|*NwetIGQiCw)f(}SDtd!bA;VGFc4Klucy{1;LWPPXL; zBqx1d*M{W0d&gKvY9r>eB(>>xNIx%r2C62I;+JMD3;zM9^2gANB+$3u6FxE~4loLg zTO}x!r2FgUp6QnO>s6oWHT&y#oay)Z z8;qS9%=#NHof&TW8y%h*o%$PJpBWPdpvcZqv;ih8=O)|%ro!i@QUPX)=Vs~w=KAO6 zW&swq=N2vjmY(O9e(xU0`y&wx$?|9FX7)x@yq?MZ_xUxdQ+>H#YAp9B!#ch1&&nJE zDO;x6*6Q4K<^=4|HTDfWbx^w#O+3eQceQ6+wQ-gUM33wh0}uVbe<|l~{xTv|gXF28 zHV6I~!enh;dOqC5aK>ra*&W^H+HYkq5pK&oMUxv>eiR!%O z92P|~CU1!Ld;bj#R!l}l>y?teSMPr4Jf!)&t7adrq4b?}r(bp=XZPp4uD@iNlPYgP zb|k@*!hF_#sTN=QEvn$$zfr%EkeL)v$H|a;8lfy2uPzQoC8y@%)&>|nXAkCuMd03A zZwC$nB4b^xEqVHX7+4*Q5ACP4TFz2N^hS=5d#F&Ct&H{y$)Qi%Rs=Fs!4G|m|M+Z< zI1CWc92;*)^A+6ff8=I79S#nV&9hQ)k@*Ru%#i~wD0&@I@a`NQ4ed_fuKJSy2EBeY zkgUoGpb z#f5oASo2Vr@fnA!OYUW9`b3JYAO+=zj78mQ(~GCRZOe>Q1;S3)tmJ&1=SZDk=|xWv!x4$f=HZv+2K{M)s;7wQCtaM=~#QK0sP6Yp96#XA1o^b&gGAF zKV-6-7N;*#rj;1B-osb43oMq}f8Bn*eN~6PZnaXF_)6Mvt^W4?j->V9Tbr|itmJHt z{zre)!KxiB-)k~$|1#co*_)G5^=^KH)}7mAb1^&98BBAZ2x+L`$N~PP*vT^C0$4iMji2c7^kH z^0r0BGe;Y|)e zDO-)Osw7|L_ka9K*0m(}S{o&qe!ht0vDap06r{Y_R1e?Mzu7Vi-?qKkb_w6{yxH*! z-wnOlMThSt&_!}m{>PBI-%NfmmU@tDSY&u*<(Z3zJ{`e!{j-7fS&`Y$y_o#tG34HJ z^XmbwrI_f?_CnUYMIH<2&^Y=yLBnn=;#qz4`gGpb*Tgfk@3!Kf{3>NKB#jCFT$J}5 zrVLN4nnsuV9xI52HIAmVHHm_^5TGH|0p^%f4e{|T?-YW0`^qZ&y_ z${-y?<)mf+0ELb^{7xEiw1So`K_;B)u>joJ;G!m}2$Kr9{|=hr(n%GtZapK>BkqeT z@06R97=LJ#uNeI2?2^_URiqxrAQNKFuWnW@{@AGvS~=||mr4P0d_-TNCQ%JJ0~Ow% zD_RuKw1;%#tX;sNkFTxCNhA2325xFV-THW$m%+fWPxa}cXx0;xVg&X17qgdr&UH1{ z7tYQAim7B}q1siW7f0zVJHoUn^%h3J8urJ6*kZcc`hDPk%z<7D&191Q4u5&FEOz{7 zO(abv74h!D{)q~cpXiJ%;IAZQMBV%#vX9Ysk?Epwz2X5k`M;{oOP8bXuOCdZIuyE{ z-N+GKJQIG@+oALI+xwRsFT?+(nEK(0ufa}^Q- z0uHgXCR)s-A-rj7T|Oew4@-Wag2kQDR!7j3h|(=(r3gN?x|#iSYhU0<+2|leq7&LfjO;x;5VR?Lj@o ze|q<$EEljXY62%FC;&+ga~=TWXC67kxH48SN9jsgEk%?x`N9u$$THw*;tLgb*cg$S zM!CAx8?DizD#LWZ39#*ufIrlMS6gjee77gLiY$t;^&R~hy}|r}SK@XZZGr%L7Ty%b z-VJFCb^0T-)g`11S(HsRA8zmm!-fXib9FowT>=Ai-3@d)9;pnuPSHe!AtIB*?MJN30J(?J&em@8{N!uy126DViaasI^b_{ zIM(!Ti^~;^NoB0boFd!Z?2qf_@9N5k6A-z!`UkYizr=xwYzhhe{2j%x5isD1I~-?O zOdg96@;XW-<7aS|PNvd$Owqhbux&~OF9Ogaar#yTksu~U!eD;u^mT{~Zgrv|#&5H5 zn&O0>=hlndQ-7Eeoqewpf)R_kc%xr{#aPUf0j)M0@Q}|5+Y#O`ekwBLZ|tdX$d)GX zljhv%#%4(vCB-qP7e`evHpgIb7`{9focRTm!4e}zZ8r`kl4S1t35`w)EBz564KeHI zjr`CA`=ysog@4j85>#ovEo8kg8hNJ85+=Wa*htfg?AL_L>T{@z$Bv@=c?=QGsb;KZ zIGI#R&Sym&>Z98!G1D}_gcl{i9o3k#Kk4*2?4h7;6lSTSHmyV&A}&>417n|rI}1Y~ zOG7vZFTHzY1P*oGn<;Bs67Igw^ryYBExn>iuj^fNbH!6dv#j=o4SgZ&s9u$~HBm2? z{kYT%8jJyyS^i<-(mf6VD*e~xg-X^~Z*g^ww8HTOmVkLu7p{ih^&i>xiH|i3sAuvM z=L()(J3{S?lL9?Bi%rM&5c7hspP!f)+z?D_V1udN@DW!~v+0=b2ag>nc?J@V=&{?+ zbhsk%lbKHRhgKt3tnuxJB(S8hqj_!`p1?T{a~H}MXjH#+9jIP~cG$Xwsyeb?gE(d{ zLoLG^SDzgTUpHM*;Qtjx~18q$qc zjNZPlI(M|=M_adrw+Q@<)UgAX3}=DKi7?!8x?eLsp(8Ee5%fRguA3Z6-0ugjeB`X( z%dw?Q+qzem&tm-WdSTJjwK>1C9&x2_8)BaV7t@Hp#la$5v*>%NZ};NSy*Lh~9x{9? z&c?}hPCY6S#S<1RuIB3|S4!^8t>FGBWQTVo$|qZiuIcrYUiLO&UDGUyEs@tp_h&I+L*_e_Ws}b- z+$2b(*7!Zm#XS!wGHb`{dx#r`iT3F?K>K-ZcFZ2c6jW z?oig@D(`KXL(y3 zn^YV0GuFi4+R943)&B04c_iXCz@5n2pu2fLhD=ca|5dHK=SQ01wMnJ5#DFx}Cii6A z#1B)@4|`7oS!77w*o+r6{|2D3BRM&3$ZJ}*h;R4j6x~ibopVTRm7k6b{H64zZOqZT zT{T>UF0H>kOyJ2#uh&EG)fJWlOl~{FofWxt>o(dq-| zjj5&OhkjFS?eB6n(j=>4^F>!q2@yLrLF99=}P&z`MlYyVqOkqZ5r%vEdZ1niP1$uU|G#;Wy6zcw#PQxcUC!^;jstOM%5_ z;ZIgADM`~&q>o*8Vt(F29XZD%)c;MAY{pOkJ%a7q0YAp6x{y4wn#6iMG+mow7sXIt z9!f7h-wo%M$wxoK6TO*xVW>h3?g>|33jBLR#y`BhhXYY((6|D~9Z2!}TZy zsg~z`onE5K8Xs3UrCjOH?y+mDV;5&6S(Z9UR>^;f^>F`35bIut-dNg=D1xH9)m-X+ zTs?v}AX*Dg>ZR_-7t@pk5y~iVQIo2E)k8PHqXOICrWVTfk*{@_juzIsO9-bziu7E_(J(4 zSOk$&+)l99B?jZvY*YF)f{hs)y<`A^bN|uPHW+WjlNKYlmPHMguO6fvG+f+iQaKGI zZyK_)`|3l~$OMQ<|2Yc8uBRAM3$$LaU!#Cm@%eiGp`9;I6z>!f0`GW#P`HGxw z_mNXJp~VqWyv2#e}E(Z&y>|8qNO9yYc6AdRCm3(ugF-k zz(|T?%NkRm5CS!$3h~?SOyiGPzk6ybqKsq_(IvJMikZWwv2lDBEIM(knOKBu@#}I6 z)Pp5?PX(1v-R`EzPf`-Ud(3|`=))rwd~CwyVkYZmCR?@6&}nR_QHzM-#IEhiQWi5a z@8ySWFepTe{b}M)>=7XCz4y%T-iHH~m?AmyUX}^OGd|=uE9C7t7VEZ{@bGV8?_DnD z3Pok4{JV6;$PJ@x5hYPB_q{^qL=zV8bp;s>IeLD%2TpS2&WdBSykDg_7f*S-{N(MC z3SvpluvniQDNYC-Ck5SZ;yTm8>9><60jfddV>9(zGZ2?ax>hjseVw{7QsNp!B`F<* z1F+6!nnklmc9oiM&@tmtS`s~)_8aF`C3lE>o}k?PHD1!`pffUmY?Jb(Qm}4-qy4$6 z2nMp^g9>_xIcf9=x@AUYewaSitET(n>o3EgMY61~+h=YtByAtU$xlftKO)T;)1&=) zn*oQ=Pd_%?l;-3cgeSP&?lePD5lBw%@Hul*LJ+`t2`117@Bk;C%t+JROkEA7DS20% zkN^Mx5imdw07Pz}*xMTbpaKAP01yHIiCfYg0F(nj0|0yifZOAbZvb$M<~sm@GXTId z0stQXNCSW(0NnmVSpa}50C)nx?Wu7%0H6UN836JDpc(+)0l+W-d;x$J0N4V6qyK*% z0LTak$o?lh00Zza7>EXRpN59^f5PKwXuj1k1<^9UX=O)C@|DK;85CjATkwQYILPAb>VPS3&Zjt{9|LyKwaxroh zadAKjpq8eV{-5wC($c~*!uWgl0C_-O-Vd*+sPsSKK}e7V5~=<_;qmI~5A^i(|0g_N zU!Tx`kk5dR+K8IPgvHdf#QcB4<1H)zD}c;~3~y^|_dns~UF2O{@OT#&S6A0tl%Sxy zAl}o{>wm%nPXHPp8o&?0`}z6%;{yT$UIhjMp+IOTJ}fNk_9}V#k|l}-{XgOHXf%KU zs-mO4V!g1~osjsDhWPjg2@ety@C1njiHV8$#6-3fHr^Coj}(s&si_ia5`JlZKsu2A zKjHD|=|B#U`#<6Fd3mG-q^1889$#7tQ~^0vIryro>gwu{nvj|ryh@!4&>eMH&llo9v&OR1LFW<95FsV zZa-;XK3NXT0<*JtkvWmMxjFpYoWg zg5A!K2NH*ehxo%o;0N&YC*JYMQT0?c<}Bvyf5HRj0RH?u`A_og;3dP8(c$S>@vJ;} z9$~z&G+tT}uc(37Fu)s_;mz#ub}o1qFT9sO-aix{ios)UJ17I6k%!MK#h2FLYnt)R zw@vE9_l@DlX7RI2_@zz!=Ix8m@Mm~D{&wg?@W6khtU3*aJyAFsA-my*qP|!vZoNF6 z#^QklMk$Zw;l`4o6u5dkt8P>2NVK*&y+4Ng$7Mk5>tL(>GYnR^J z=wIg9Qq`?=1f3qNjJ>a4{S-k+#%|Eou-=Cy8Y>%Q zc78bBoi5R@PvYKln^ul?UY-2d@%PtSUjm-PxU2K>bZ@r$!Bp4B>%YgFV{eQpLj0j-j)3Uym*%g3JZ1Q)Aa-;V}Xm zSe`8RjW~hwij8;?p$bk?)~H>l1nIBtn@Msz6`RRQfA%(0etcxyd@cogyp^iMSGkpD zAiKYnE&`QzM4>DnZ)aLPs@%@93w&IXu|>bXovji;XeiOzVj9n%gD{1_{W^;VncTvh zb{T9@MY{z-iu|W)=&+v z-^B2O?k2PPX)WQOEpt&7p0VbZ9UBt02+1F?d}DV)YAGBchgK ztfr@Sqca`-aFXS9G9oY)Qk(d@ywg#lcuOsM$n*oQhB+9v9DjERHOnD0v^N7C2(D18 zsWdA?XtEd6ZJx}cv_)lb!n6w~b5{X``kW8oKQqWo@={}~+Yy9Z((Pe6Lrf+?D;Z=D z=s*cIR^OidT?taPdiT6@2R~8pRr00b01MY4@_E8asr@jEd0Xw=jNi5BHY4(fH}j<(@uN zIdH~!HHqKE5g#M)Xcn3zu*OXv%^&;r*X?)eKXZkcRoPKE~jmR+E8EkmTRTf;Rw~az7>+IJF89!v@xByqMuIUE|Gi zgKztBKB{6G+VL(5s=*m`(mFgS+is#{KR!tluuEF()x&TZJSfF@n)o~J`vw`6# zRfYW#y{I(CFVbqTJvh4+mRcZKr&MleWKfg6G&MN44^_v=uGi~C87iPIIo7R9+kILP z3NKeJc!npW^(UZxxyQo~29a1mOG$yfUhbQSX}6h~tQ$BnS>8yTnmK0?A!iwNVWn{t zV>5qCe+5F}gF&P_H!&Y#Q2J-wRG5}@P6}3#HSKczn9fTsgz!?p z_-l8Rc^%Ci0IMcO1~uW9K8MdQtE=5Q13AQK-&un~d4lxjd#|S62E5+f@#m!Df;~&q z<{g#>e8wMt{{pwHt+ZYT463@!o2xJS*gxp$Pq5E{ySzYxsRt>|49!i%pJPZzfp1Ot zEAGr?Ottju(ezG+QZiyTO6jgkWFPb2wq|t^wFsN{Y9JyK6FN#>Yek$;gJY@08mR!# zU|taPm80|rh~H9v%;t)awoiKab_F%gVop$(8SvA=$(G8f^d9ko%)Unw&eUk<8@wyO zjC4K~O z@xl>eCDitG^jitI?kM0twUPQ6-(tdTR~cba1FxoT2qK)9tV@sG7E;y*Gipk+-&uPj zaBFEvGm1J{X5^Xp zU#j&J_r5##d3DsD+fIr4U6x4C$-g$klpz5i@8F_Yj5fJ~wc1+R09TC8`sVnmT^J zRGU^USiL0H@P~)8!i>o*C#iQU3($)jremFOnfftt!XDyo@6JZ6$pi;c@#ZeB#V)89nI zMGbbRFn-hG|JF@v0MFZ7;~NfCKl;qrLU-rz<`!XhcxbsAQYtdAD`{_m%v~s+N z^Pc?Mr7F0~pf#U zwr#c3B}|nlvo((oagUIs3V(C)mg>5L>smCBREv|hi<2Zv(cSeS&Ow={;fu*~e2C)&=BMCyZ8Kl^etQN~T?MPPK zAZOkktwV4R=M(pH9{2ttSwM=sTwL!}22o03;L$T!u{a4Wg#O>KQ1lw9;;aD7nGd;1 z8ZA!Z=7Te%Ch~(r|Lo!dj#5_&FiAb~+Cn@O0Nd;$7j>OFu`h;JI~a}25XdHPyD z_wff#v)(K|De|Y6dij5$0dbr=5D~!-X=OBw6-XlZg2*PsE`C>6WA*p4e4@F^r(%rG8dn?>kNEZqO|y2wc< zKfRc|5T_4JgI`9ZK2P_bAT?pmDOMORN-54Ae@(RTSiZkdfAVpPP)Nz0;bQv}ShkPJ z4H!D$ChucmIHEzK7GI(h|571`cj_1dc?j~PDjb)3mNf^y=_VDBfq-&K?b^fiZFmhs zOhf>Z=kJyFEOg(LlcNg_pMERz&Ix<4$fI;n;)yDsRM&NUu3za24!kN4KY02qho@cl z?F$bTe|Qc!62_QC#Dl+jn{4A1>A_PT;nd7qW{n}c^Nx7&vLdg_HFc4@4IrV-B+6?j z({_cHwvcEqR@Q{ry~*M3KPDM*1*bGrKEaTRM0nS>R}Y_8$rO|4MW7zJmBf~kzjO>C zAF3YJGwK@Us$bJXwWnjwVTWJAWo#^l*u8?`+Uo;@Z#ge2gq~LCG;1Zu+|7k2-G5sAFruxhTASrv zKza*t6ZFm(a9K#?V0?|~(mRy;yPF=;{5c}E(8yxC;+cbY9%FYMZ*hCa5b5(ehz_j# z3z2Wf?JObuB$Q*SfrQV|nA|g}J_2=`^FBsVG*q8s*fHRChg2TYz;{fN9NLzV;HQ`? zuPn@&5bFD+D)y--NF<^-Q?Wfgt?ic@>@bipKR1qr3<%R}Z}9xkN-72kd#Oe#j1PHN>@^ zU4IgG{^N#Y!8eB$DY)ul#a{86d%Ok+ekG2wZ(U>UUHsY~C6@b`<@(soJN}C# z%ctvI80~m?*rzwnL2gGimxCJXs2L8mFzM{KG~h5)qDpZLQA(_>p4Qb08F2REux71K zt^PDWN*ZR-<+41e2m>=t#0-lE43q04hR7XfM+EucLQW zg1i^MOV)M_EqA;n{Xz4@N3tVGQ&1z>;v=d5hN~mUx)87zn87fScl!x2h&D+vYa>Y> zk#9A5|M6{olz#)UeW4AU|II6KJ38^fhpYF}Aw?3?U~3ey&oN0#H)-`68I95?`*mlT z8(4Sw9o;`zT_-fV;r{?{K#;!&YfuIK`F#FbG}ncrl>UAwAO+f()xL2xUM`14aj+ zaK}EaqF~Djn3U2H`wXD4d9YLluo=)Sa%Y?X2RKQOn84LnU7txi4Zt;TEUOLtX`xM! zPZYf)o@UgdxO92l)@SLmuMh(_5Y6_=qX-wGd*GW{!i%oE2W$=4G8wY#wFXShsZIK% zYXGE=(wU^RO_43x)QGRh&;%ynzUJzo>$gl?! zRR$d!Da5b`L)Y50U7Ij+n}L@exFxZ^fMGX)31nR<3hTOhT--DX$4Ugq;8EHpP{_Og zdcali+qSZ~;cMN~_^PXb+XPF;(PY@u^1UG%+U9+PpUQEU{icF^1L8d`D2vMTecgWC ziQ2s#+5lWHoZ9Pl&?mDBgx%BsZGv%%3^x!1?#-(r76bC#6HV}60x@yjrB)USOIfg! z*knoCL z40OJKgOKRR}CNNX5wO%C`RVqby1VKp{NlP#%9?eAD zfR5zY?L~6jVnS`NYj6WkP8W$j5R2aE^<`d?UJwYkNAPA4cvh_gq3Hy{>Do0+X{KKQ z@MWn4QKmi+sNQLmWgeoM>#u%)99rPrEwQvz;oH#!EZ__1j_%F$;gxj*jxG?-KykrN z5a{J=_{Y0A#o1W$e5%16cbZ&Z9Z&G#e zikDxMD>sEXSPh;#Q5ZiSF zht3m5px}9rfDri#{uKIuif2iW9fj`?z25F3-%Pf2_5OfR#QymCE)dg@s|bhY|Bh!C znQomVnwg*YaxH6}?{ayD9(P^(5We^4)U)Zm>K+^KuFf5=ANwKy&vJk3M^}$tSXJzn zZeE^mNx*LqsdTOOCD+G~^T~he%kOZcg!gn2y3n72*0^0AhqE^j{{Z1n;6Q=}4GL@$ zqo64Opain0BJc)_J$vYM0RVuLKwki=R9w>pKn!F8jhLvIa?c=t9WfEa2*BVg09jP- zna1KqL2V{j3|s?X8bM+Jn%vy8%HJOVz6jEcq3z&QsslR`tD05oR<2#Wegzv=>{zm8 z&7MV@R_$80ZQZ_w8&~dJx&+UVz*`C}Uc6d40oYSl@LL-vS?Aq#?#;e-w&&J(dk7hi-iMj5%Pu)4ku`fI}(cjPfS{j%%PxmQ@| z;+aknG%zQwn1cqx&e~~k6i_$CNv`mmn_oU;~88)83c|j(Dby5 zHU8k_l`x})H(q%QTvn_`r@5htYv<*cJ!JMU!2~#Tb@iD+D(eEhfwbAz+tvm&NL+F+ zI*BxO*riH>^ghh{3jxPu@-&PDHxl53DYohLdkuSOXJG{#fJO)Iuc4 zmzXl*cB^T8G3618S+I5Tz zI$|*M4}uFNrVF%v&FEdLV%mBvZHQR~lRbt*IwPf-c3R`E2PgbvtG~LhH-!slysmH4 z4A`<8!gW)dSN2%ZC%6G1lqzD%fEum4^WM9OzBymI=~a9&xpCKDPxNrF_~HNnh+)Tl ztw%No0L>fd043BYE&roeGxt!K6D0+0Rk_ji=G!t#yP)F{Q&`VEd+no?y(_#vC_%jK z$0rN7PH%9^@;x|>p^Qgx3B5&PQsnfr;3NJyH#k~vP8(!i;RMb|COsoi&w7F*Ujh~Q zpzd*n1R{Wd1Qs~KuADA6cH+sAy3jZOBdp;Os}P=qw1JBh@hKxs0A2{GVhRhk#t11K z&;J6rh5`C5b%Bc?1$DT?<}`3CB$&VhLcl{Jas)|&iox2df(%paAq#zj2pm3GH~1Kd z6MJx5MoiEIIB>!#UztXrIAelotYUqhS_3EK#+~Pt?sR`s9RiCuM>^KUhg!)106c)7 zIwlZBWk6aCzLpz0l#!yt~Jcc0y7e%9(lD4>pL6~BVV0p#c zGGd~Iz2Xl^;#?Up@kREuF?u!x+|&9vOInWQj#a@005ou;TE1=?rl3P*Owff#yzCi+ zTAW$5L9}w6Ycw&aLK)~V2uzs&;uXF?h9T3Ujb{+!lAL(N9&)xK&OmcroG4aXw6O=v z_-QJ*c*F)f(}h7O(LJs-oyE@Su3q-jp8~n%RHhMw3C!uATr(x7nxM9Wplc2MOUpDs zp{b0}^CC@{NNZRS10yt{3SB6}8t9OcKm1`3gLp(rPg>GHRD=$2C_@*r(1bIwV0l{P zXs}9ZnI=5qlWp9!Xj&kX%?!`1il*KOgD(Z4Fu~~ zzE+_MSqLc{_Fz&>aKaS-xF7~q1=5>e{ehKn(Ggcr1P3%7Wff$N&(NF-)!qebSn2F) zY8!{ug6Ibqs#2}Io&kzApp6A(P$O4XQ3lYCg&2b%XKSeFoZxbX1#k%3a-IQ9a=!3> z5RzCk(d8KfUaz;Y#V)*9I}l=+05-7AuCcUXHgaZR5SKN~zeJfEcWTFcxnWgums7?w z_AZS-Wl(f!D9SCyQjXyTaILnB5aRd)ya65+E(T#dBOq_7bs1=<3e26|bt;+AAs#Ak zyGj@R4|@5G>OS#{U=qJ+z<)47XbxOrLm{K6sR<7#u6vk{2x`R4;Z$ybG#+@$bC~sv zQB#ff)a3z7Sj3wDF^iSlqwN-@y(?aFn|=w0qjmuk!(~o|A4VQzq-nwsty^~|hOolxA8BL1)v)RrjCCMJLBF;cO z5~-rL&W~mwUo7O6-PMiGkmivu9)Zf=t)k7!F{=dytsb`8@W$#{u8K0^T*>ZebVDXB zu&QTWOxGGdSVc`{FKugBabgWfqZub!y&YsU!34zO4A0Vo8q4gV*d{cUwDHEyGRTCi=m+`DcSQEqm)%QpI%RfQ?p@q zOUc7-B#mkRX#=c2op$_+)*EoM5){vRd#$l%B#Y~8^lgH;Cs-V;4H*HYbepC zS&OM_m)Eh*@H793o~Dk3)@M%7_r2z`6&Ca%cq}n z%xS;>oIeI~4bOGZq{6-%%g*l`5H+zF>Zd?qL>F zgqqs%jE00hCw+XLe=isb|M1GvO3@UP&%SCKSQMY^N`)if(LAo-ry}QAvCoE zK}f+HOt^@XD+6|Uh1s(}1?rxTAU*rzqZpKn*{ZEnT0Ie&28_t8C&7gnNFp&&yj5d7 zcv-sYTb=@<1E+~T94wsf84j(eKq>5sb1Sd&ijg5K!jZ#>_7aoSv#%bMmooze0cxcG z$UC6p!$Oadoi3|Cn7h9>EDHr=u!xh9-spl8oRQt>!#7Dje)E`RGKF%fAx0CH>v2On zgcjN1tRK=nN8Ab*D}$SpF&>$QZxKT_@j{XF#CQTZVgki^5wZn@Jx&6Jr=!G7^23v9(3Bv&69Ys#rJ9-)T*0Jif^|VMTGTE{tBOBhfd{)n z98@1X(=#Vw3eYhW{m23sT$2(s3PbCfe|swFS(-msMrL%C!-;|LyS_=JL`~zg&uc^p z5x`hP6#ruZ04$Uv^gQ6}8Y|ixsOm3v!3955$1SlMhVaH|+`VV;HDF64Z;X-u@$0ug z3A`*B6*9bmYHJ*UWWu*=!E{W>JF%LN@*a4Ez;hEOb-TzXVJG7o6|kd7JQ+k)ddYwU zpn=q+b0o=<)RK@X3Vu8YhlDzZGoGU$#x;2c(=(NwV}TkJ74r)@?GZ>Kdpl)hMWAFd zXFP~KP>7VA#q6WG3=9trY?Hz}yjPjH!K)QvM5?8<%0$8(gBD~s zi8(YDe`GZZLWWf^nk*2d5~-e5P|V8oj(VXX#QDo|=?!r$66wWLtLcONl$}pQ4uu@RG|s&)Xb{VLEFMD3S|?hl*)c-N)y06 zsWHtYtfiqz!vT$=PLfd{RSQ&65s|4*)zZRI;zDDg$*w8B15FnHj`YZX;!rwZ&E?#K zcF|5Pl@6~Qh?enE6vZw)Ts1uuub^ zvP4Ym(Ld2gqJ+``%%1*ypV+!ebh!cQX-+z=8awreNwJqNouw(OvRsT8Zk$t&$;M0V z7z{h+pMMG^I8eO~a5|=CL0#RKv zbh6bqToe}QM6qnpev-!(Kc0QlDJiBf>fE<{yS0+k}zbW}Sn3TVYUR$U;7 z{58&*$d3WUZS@#F6xgUy)8hoNHht4o8%bYiRK1H6MF*Z$2X<%&cYud@kcWAwhkM9}edq`EKuUm~cWBzDr3ZV!hpDaFZXrTt#T|{^kbZbseCXP* z4cnV-2eMU%o$cA5MO(EcTDEQ5w}o4|rCX<+2YQHFe9#Ae*xN(tPtQybe%Ob6AY8Ad z*~4{*u`OG(b=#f{sgylLlkiaE%Ur$On6v2Y6@)bvTD|7>989hV^CN z_I2O)h2Qv<-}My-axe#USO<5A2YR>%edt}|D9f8e%Lf5pnT>~c$XRqS2lNdGaG2i* zhTsT(U~mwJaS#Lm7=vqIT6u_C|8joEoP;G9i|bBF{0xPoYq;1y=!7N+2FD2H=M zhjnO&cZi31z}xIy6I{(&#CX}R9bb1~2X#{9u7{cx;u0NZNP6;e4oN zMe)j90?#2$i+;d|d2okxAcsA^=5}7;a4?5;fQNd}2X*2*WEEZa=!bl$hiOiSQ+DTo zj^JwGg)kn6bI4zLu!nsR6@9P=dAQ^Mb6Dp-=7xe6=!=G6a3BYSeg}oF=OAg<;S~rv zwqtfchjNf)j8^G@)@X!w2M+dTrfNNsm7HhJQo%6iQvTT+0z zc({je=8D`z!g6g7eujs2Fo$Zs<|U5mx2;{O#$0;1YOL03uIB2jwg<5G>*PgT#pPlz zX6IoZ2XttMcv$E*;fHg+d?+=!5(a{HtoZf2e4IM#YTtoC5QBF zY+??Fa!7}lp6v9vQJQAhPFiHHxb1XC>2?N(aDd_7-fQ6YYtt5O;=TvMKJKty?y_}m zRK8}Qj_m6mQLm%}Kg&}&ehYrEhj%~+*Jfc&=41|*T2MBO;tgMVm}W>O@D&~hb%+Oi z&@4^>Bv<;X9#QJKHirmL zVyKp3b|7)i)#2zUUYM0xE%s%Jrte9HXL+uN8Gn&N^@n_*hj&ni8+T>$-ef0+2YI+# z&h^~xAYHDl2N6c#b@=7~V0LA4sBU>EWb97EYt70>ctNxP@OWScDxYLyzVb7MTB+R! z1#gZZPw|>1VPMAbBZhHxcn5m;XoK_0U`Z8<1q*(d2X-I_7Y5|WCS*U4<~ojOr)FVr zI0tvIhyR_=dGfyX=!biVhjl0i7nfj7P6u}AXisMH0ug0Wjt6(ZV>3tMa5#r{n1@C0 zkW^pwR_9~bj`dmRT(%}&nB`?(#%wW$@w8s+7|8`j2m=)W02BZNL$K&0o^E9BW=)@v zTc-zzhIZW!V{w4&dB}(KSf(plScsqlbZRqwn1^;a2X#MU1DACU#)n#OkOCgx@-1su zkKk!nYi4ITuk_acM{n7%=!bfs^c5C|Uzcow{|~vQYlL@SP=AM6_ydKCQ1Z|Rd02-^ zKjMR42cg#Z7}@76b_aC`Zx;^rdLZ}z=!XDr>LiA3b?D^T-jd4h=b4Z8l~0Fv7aE{pf3Eo!4tsfcceUJHJZ2Xuh@Z&3JZcM+v#>aT}iG(Z6W z$N@Px1zaW!e1L}mFYt35R z=yYfYd8qq8sq849^5cJBP#5*#w2H;nR zdgu;)hzG-eb;t$?dHDMID`@Z_!h{MJGHmGZA;gFhCGN8qZ=E@C-!^jW*m2xCdi*Am zELk#Nyma9_vg{bn+&Xym^qH(_^Cr%mB=yyUhjC@gmf_OH%ct|G&3*IOkqa7B+&Omf z?CnEp^(xk^2l?Tf=Pq4Sr!0pS)oK=@K5xA~*}0+u0Dxve!Ig_S^DN%HXZhWW$L<_Z zu^i3+xx1J342i_0{FwrPGjZaa^w0?|*RI}3y_z={)F)4!II-c_8RbmU9=maZH;zjO z&t7KJvS*vrN6#JL)!)p)D@}VMzj^8c3nr)Tb=BX>mz%5zPhHs6;ndBG%^YICdQ5eH z180t%yn6(%lP~Wu-)(fEbq|%7-2B4q?ACj1rpo0xcl3P2?{C_9I;yP(PCD)(g_vUg zJQBlWkR{~~eJp`Q&v^b>_>eyFprcVx<;(+LJ?e1iR5<6n1KWiu%BIg!<*YW2JN%ia z4?HfeM$S6%;G-gs<>7NtbYhhQ4^0>Dqal0w(NkoPO)|%iJ@5pm8#(Um=wCncq%-3G zT;{Yx&wEaanN>gbyn_yL9TJ%nf+Q?PM->f5c9J?ymGh2_V>*digf6{PA#M1ulcq;x z<*6s4NR{c1ImN+aopSrU6J$%^pmQRiiE64>nP+m@5;^aL$=s1P3dznqOm4dBX8hEH z+ovpv)6SLG)w88&=)hAetG(jH;yMD_84sirwYgXsWyndwPU)yQs;@lWGpJC1%GQrO zlzMc|uF-mnQKA#cVkvfKzbgVyB(6OvU{IDZbJBKdp->of)BTuXDJp4`nI3ex( zG0O>Mh;LBj)VtX}?VS8gmFBj*^G)`+Gww#=v;%EkHQy{wNX7EJG)eckV`@w3NW2+7 z&OWtHJu5T4G(PV@?U9EhQGCN77(+AGqNIeHD#gpbbu(A)naReOg&|XRM$1iOlqah0XmOZ}5k#}STZDZ+}`S7Pdh9F}a?aD_x zVigY5StTB^0tY*gm%)Whqk)OypHBEeyq2hgdN;A4N5~Nmt&xO*{DL8(fQXX_wuBu# zEXc8@0f1M?FCqCLPB=(rlV`x;25@0P*$OfU0QdqS08o}ci~yH0vLFjC+~ODkU;-nU z!XRQyBV5+t4_^Sl2nbn&V?Y7NG2Y-HUDzXB_D~Qy06>li!9^3sXapt}gavUj2rf)u z0$i9xW7|4n)6x_UcwiDDRJ0-huxOAicJUxzEXW(lXhtojkU=C8E@jwYL8eg#6Trm;aOht_tXRr|tO5Y0&;}sk;tMpk(jTt?Kobt44I`L< z7}4b7O#;UfbJ(yXuanRYapKK1nji)jgXJFA0L^InVx41b=R5sDNc|xrnyc6Y{1UJ6%&@sx9kbgJ`9jk!LJ?hdXwyWPa z)X~X_3{XPySgAucT1FT46OgMIDI7Hl$C(<$k?@4$8gNm_gunp+#GocSlbOaCWPy8d}z zYDzF(6(L3_X+atT3bjV`u{jZE4iSh%$dsa=do>AUm^DA*oWpq$dBzP)&;?BVL5vjr z12|1-StGFE5yt&PB3a-;;L^dkgXjV=U&+lqP`A2mq$2JMA`qPk+F19i2_iRuC% z2Gxz9KUknf4;rMOEPw+h;Gkagb~cie>{*1+aTI!Z1RU1N32t@ETi+H0xUbbJDvNvE zgb2QO7+z;*T-# zV3uo~-bZE?0PHQWH2q;BrZ|5!ag( zwFw~}F|-dn+*P-F)I#<$l_zXwIAho#g+BG9EA8$0&etE~rEYo8yX=BA``OZt_Oylj zYK43jA)@uiz`>2))6TU!_gk%E=;%V5#5O)baz{?Y_)33hBL*hWU&IZ<;))-nsBMh7 zHTz2QgkaaG{!Xhwit2EMnBb8HX$){NaS&mzT&SFVq%S1d=$r6kao=c1PU^9(Oix7R z25I@rA1-fHf;P-B#`!hDxO0T`oESexh|m{uRuJ^=s+rS{t&&0KlTl|IVg zCS=h|R4f<`a!tWor5+I)M>!$FkK%#tA&b9kwh_XD&6|D9p{F?M zw+#UOb>1L!G%K_%>ses4+~xXvh6!!}%tr^b?Qf^M!&?tYiT3Z0fH(Wlj{*-aK|Uw= z;8GN?(s`Q)_Vf&iyBo_+{e_3I?1KkFR?bwuu1#oPs5-Ra-VX&;PXp6HRD z@LibC6@)DT-$F3m@FiYB6rJcXoBbUGDCHmF@s(wD)d z0TzHqQxO&f`z2EoSdrWL9qk3)L5v^INfa@()6jh$R`A33oC8i+ojIsh zO<15Gt)3D#geDdQDH_Bd>5rIIU*;_x0NkH8=$0N0LG_B8}R(E%pt-G^P_X@Me6>_dd4L*$u6=P@BHIt2FJVnJ{t_hsNS&>&iQ z5GA6Z`5BxYBE+uM+B59`L0hFF{l!2Puz>7&(;p$>9QGf!#bN+@;Ur$$Rm6iw{Ebb_ zRXK>FHnIRA3KQ!{;Bb{4H?AUNnPLX=;W2L4A0ppE2;?XcFwJY85z1B>j0- z*T8-9Qxx)g%4gLD1tCmElmT5da{jBx=R=Yy=5Sgh6tI zI%I`KzCso@r9;sF<7H;yD_Z11D3R=O0+fe zM}A`%&7~l!(G7g1N1DZ431Ll;8yiX=Xt|~7#bPwR;Bg`){h6jh>>@HL5f~ljLX;mu zlw&>_1VE9^BPIkravxLCLF)ZMai!!6?q9XlrvCwCT5V-l`sP(^8b^dfHZFucbObuU zlte(22qFYqHm50eCamcuNI@lga%3`IA$)q}LRe)zW&wF3lX(goKS~_;v82V#;$d;6 zgZfWL1W-a8OEbymMC{~2yaPv&!%+~@YaIkz){zl3WgtOY`F)mSu~%V%(`z|DDdgcMnYi>_sQ&YXN{U1VPX&tf-VCJ3D5oY|&A5x7`ELstgB1Cvzh3BwnO$0_dXoWWo z)Ql1n3!vAI?iFW-(u{IbD52M90@7Um5qo6;3%F64zTSq;U*5fFPeq}n_NdGK=xr+F zPo>dbVONy?rdc3aIM@U}bVPAVUzavpj|moB7$|Ov(WByOPr+K5vSvY$CVJgzjrs$2 zdJ&0z*sj7sk%cE^H5IVk0DPh6*!d&qxo46V6O*1rK8y&;nFQaI1Aj_{P${QD#OSXk zYoR{>Dxxatrph0h0;#JegefSgZbd1CiYK+clC?hLh00tDlH@WT#Kkd&zJ9A#s3=E3 z3yF3lY&l3mEY6kU1VAz7x^C%k30W0n6$?xqa5X^{ypc$96CI?i%IeQ-LL!OTD{c}U zNj8CUX~UwWUO{MsDSTNws+S2o*Jp-K8`W4gSuIv)7R{ibcuB@55Z0D9PLv(Mmd1}8-YkEoV5>+ozy5PYY zgj(k8&fc$CM90N~oSQU8PN1kk1P0edgxDg4HZ+=b8iW#6SxMnhK1G}!^=1IGT|43t zC%`PKKB&>@YTOdyDvX}yQ4((6SEyU`Y;9q@dm@P_L}kGf#ZP+9tkUP{r;jY z8$|w+a+sZRrdIDNI~te)up%00k>)Hk&vL_Z#7`c?u`qBG6~u_(?L&C~Qz8o6#gL{@ zdQ%qdoZ-P=%mGy59kJj|gOl!J?KXr}0u-@DSsw!Mv~^Y@QEpDigBTAdMu~H;}&^pc;0|+j`H*7 z+$tb$DtD1QgI?LV@>TxlvLZAVTg5t;M1J~?>z4E@Gss}eZats>^+u<)w*B-5@pDL* z(MSuF!Q$LB_w_aX^~F?gWv6u(akD>2CyW9>11WY)=fq@w!x$UHIZJFwIB-D-3mTTh ziqWEabx}wi1R=rS+WmBcEtBC%*IQe)V!=gLqp(55fPwB}7ok;K{ZapPmN%Ke$mPUL ze#5wyM0wkARt)#o758D0WO9Qeb6f9oYjymF;}})9EQ%l=_1G=~-WDOL6kj!5ytGZz zwPH0ZGDwo(+H_XXM{qO5Ja9xgn1w}pbR~P9O<#Cl9d$#zUV_>;ry9gF%5cSSXwJGf zGO&Q|VesAE_jHGLhL<8m3ET0~?^&42B16Pu@-$W~_uGR1=32}5+m7Q~Z*mrp>?AI> zvYOu|wjTwr5isGN>{+qDzS`kc?{OPMd>eK$byEleU@~8%iR#3xZUi`(L_QRdE=z>4 z@ptHLc~UDkXw5f=t6g-*?MUgj>UB2&5MpU#RkJ!$mh)&Hayg2-HgIW|6^Xf@mpM+r zP6C=UYoYT^__jYhW@R45Gq{`~(t&#A6CO1I7^0QYbsizf5!#g(=TTUWm5p@!LuBze zLS$hx#iAbVT_hC}gsm8oy5JUVB)9|y7*B-Op?Te!M5mj;r-%B_k@_Q;`bVd__2yg* zw7QwWx=qjeS8s0>48kDzDtm2HA#nmG?4s-q_;efpxSbY~nLWdTdvz+`co!qZZKrv2 zbi|J5ghjSoK*d_BsqrIgQ&z>#q?TEZ;oTbrpQ~4xw#(Tb&{=tnw?Qm>vm5hI``I8^ zk)Ulmbw@J5m(4R)`HIb7%wolaKWt5;e0NYOPN4cwky^roeA!X^I5vEBX%`w%F0SR6 z{_I+>MHWL>Zh1MQd4?O>I=KFtU8V-MA40pd=bV;NSN}8<3pnE*NYW$j`Wo~)G@8ZY z45~@c$9ZFgse`+?KYghmdWKVbzZZNM;k<{%I=6c+V-Zu=tJv2|v%xNXQ8j(_awOEV zJqwsug;5f2eNkMC(Y0~C*L$+RbVU2?b^}xYEJ8f?Mnw2T_`(`ATXDf%v&Fy~{B7C! zJ1(T75br31#5(TDxXk+DYt^3>MARBA)HBe*8yUeK)S9}#b|j`PAmPG_9p5f8UcJ}gh^V(o zAQ?Dle%=BAYBrNj?a^LuIQ+k%M3r9k0n&E9`Rvg@K=|_ofFJ-9H{c-iw~!%2Uu$9z zWbvq>zgJ~g7|a67AplGbpP2wq5e|T94UH6383UOK7AG0DG@)=|NthLJ=G2Mv9=LBm z)9n7$6O5YdEMZ z!LK(~c3HKP6I_BuUR8YMv0~VTF|sUN1>mYbOaOc_gcH&18kl>_!URRuM3%e^iB%~W z;}28MIn^EgX76*wccST)F0E}(lY(oWVPzb{F+-$g+q8&}2|UlyUV9 z<;&Mu?x;AW_jh_GHk||n(7*!`T!@$f6J%;0(cn4o!K2FAr$GW4#HqjwIs8l;nh3n9 z!x1m6$ETmdu@FFgPW%R(c}R@^QN|f7+E6DDY0R;-d0GVLKLF_wO&rp6EYe8zv>`^V z9BuTF7@-!Vk)Lx!3ktxZM2m;U4+G5b$P1%n(o4^jOm0jL^D*iuA}fk-zWNf(j|83Y z5iO}N@yt_Ch9>;!Jp@BLDx7-uEY#3LV{}KfZ&Euj$SV0kRMJT)C6S+W7H#yvb{1u) zpGrXuRa5}y2}d+?J~i;3Mb(LP)K_6OG@cf}aYrXNNubX@1LfQ)9?=GkRaithJ<7)h zsT55e6^CusSp)4MXHhK^lxNX!B79ccZApa3RMF(Qkex-vp_bco(QWV^a~37X)dTCf zHr#aSt=AuX*cEM^op9CvHO~BmD)uN(?Jc;peE2i=o(ucIR5WnfdDq~HDa8k!MZcN1 zK@-V^Csm3;cD5gQ)HIe*4&lLgoOnJC+2v>LNmev;BtFodZ{5@M<(&)7$6?WI#hb);82`x}d1_`|_tNZ9r}l5s zsXFS~!C?oV-H~7ah#z>`!#JJXaq9Q*1C1WaYmCWJI(bR?amVy_a+JsPZ^UUQpY7wW zC!OETIbO^M2@AKXZtoACaGNGp9M+*^hw!u*W*a_l&A!p-6960IJ?nN{2CVxMPA8q|rSBwWxBqFGnLwAUWK@LKE7nAN4?B z0L{U<9Ca^&;6R5wN;t!v?L!{u_}w?c;g0?_(0c*=hB@GIJ|X^%AM?;(0MnsC8tLPK z0W8Nmq&P*U&BKUZ)ZtJ#_%Ho@=6?Z1UH!y(Abq@}dF0ZVk@_Kz(2M@7Uk zVo{4l%H!VuGDnVf*yE3YQ69gRIdQ0C9zk?uy&%~RM2gTImMhXe zni#-w^wE-^6cs++kxAzj5{-uI;C(*mnSB5dWAaO#E7hY9c#N_?ecKZXk7&Dh&=Qz? z;s-t0QAbUZ5K;JON6B#L4nH64o0>URQZTUIuiJkaYztAX++!_)q~1_o+FC<%t<6CNg?eWDvz++-#N6o zM2C(@AM;>GM1i#qHc`qR?7$`j)fvW(9x5O5SO-Xj#Ey|x%9=106gu8f4~?!gPyM(@ zJk)Xj&dh~F9rOd$KHwow6}E#Op5!TsAo&h-1oS868>e~N+~vPJF;) zko#K)GY!R$A|}&4>WIfZTKUwayi=-F{c1fyg%5WSbARSwhdl5RSGgLfANY_5JIujW z3dOKl!aRp!tBMYH(1V?WwUj;Jp$?m#YLUX)M?5SVKyjoar%tu(XM7rum*UesjBQX> zd9_tJ(vh{}aH~p97}cq=_8gTIRt=-c+P0SVv_i?pJluhflRD8H@3^Wv=dli*MOyS2p4ttK}Ix(H|5O1jP7hLoI zisKyZm`7mSD-?ayqr1}`YC3>8BG`IRwh7Yfcfnx}|6+G_sa>sbdwbtd=|djsm~Q|h zybkrU7np@bu6NB#p>}MuUi*mdcNaAdbhJYr_Au>J_hAotv;!RmrY=}-{4DZ5HpT8q z2aRnUV4Uy+AM<#JI?7>J46-Av_9CKmi;R>R%i|tFE-^m1t69k26dupL(1LN<;Jn47 ztoQ^6dPgj`ejL{w>F_bR)?rk8b(9@^tt~p(q27E5c6Us6@;c1%VJCtEqbv@YZWWDB zasV10^uULgO}Gy*!-E~@fUyX1y3eJxw9i?#(>d1hj(K!>rA)3zJlc_ta*SF3W$A_M z5a|(5TA%mG@mMdRledq1AXgoUHuX2ik zfH9WquAxf=Qv1P4@|qP$?Z6Yl<_H!a-@SD@2GJ;H0(EoE1cox7>B*5 zR*rSVL)3CJY@bmssEr3&-3Di!u;~HoVn@2R>20}@n5`OT%Sg8v7h%8$u5OUOO4IRv zN2jCwZ`*=f>BYhO3?JDSop$-a%RPZ8y0NXDv9io{n`~KJ)uvyr{^Yk9*|&>FPKK zIU;XvzjFEZ9141AhOVQIT0HooEl1_q;SPAngC6$4M?Umn`v&o2ANtU}+2bMlb)@6> z%|~a%|8BJ$!-L+X&oVf~K@M}EL-y@>$MosJ_v>e$AH^R^Ke*5P@ZlcnVIJZE?bu=b z&cPgj5Ao=0%WiFoOzg=Z&jA-L{^qa#q!06|?;iGVANuc-0x%!{2JrI?5bfGw{Lp~` z#R2OiFW^)Sz&^>u+yTnW51+V!8-4=*K};RelX+GdR9#Ey05jM!$)0*C7EDo-7vuj}Ta13xeXp=t#20Uw(1 z1l=J4$&UqH@ZlgZ{c_~^J|p?Qgx@kN48;(i!a*FwVH^;V9LS*@%3&B7Q4tZ*5X0dP zQ;fg4lji)q`(>x6eJ<&x% z5fMkR6s18LPLUjNaU4{!7VB>ED9zH2aE7uC)?`h}xFHz-RN)fv01cKv1+G8`2N4tv zQ54646id+*%YhVeF&igQ80$_8Q*G6{P>9~^&`gdTS<4!CFch_s6uq$*!%-Z@(H&Fl z+>XrJVucSq!w(B3$K-7p^$}y5;R+7&=+03FH%iksG1iQ2Aye!Us(~VfYue619WJjP z22IH-0^xM*>Cz$5RZ^Eu;V*SMn#1 z4feLqCGFuOK@ul~?d&A-@M@78@L&{xk{mS-&^&J4U@}htk|_j|QnF3n3UBR%@e%-_ z8ER1|Iga37@hBVO-OLW{#*YDMvB;#r5h#)ep|9)zzRu><@;*im^h9q8tuQXLYzdYC zD8u3Ie9-==ukWr>$&jHf%7O{7Auyw|GI?<8@^1R-LGv0DB>OV_*n#}Wp#_Uf6<`1b ze6liGYw|Et`jk!&VI?bl`R`46IP_dQ(0O-=q4pIIp4+y=^`t))y2}{!m za2}p;4i(TGbQ2$Gtr-9S2u!aKx3L`N?+5J<|AsK^kh4&7j|o99{N_*{>M%O<(Y&a0 zETs)Q7ttKb@C@UD`o2^9!c#`d05S`r36_K*0&f7F5Czu(Jr^)KrL!*KQ#;SW2iJi< z)6h6!Ge0}z-QbWOZu30T!5PGWAkrW~-4PW37j!#;5dZit>^gH;WOFEJb7GRP`}D#4 z@*x=rq6n_92=@U5QtsNOM$2dGu;t6h?y-9}s~c5aC9P zF#B?p4VRQssDeL@KtveEN1fD2jr0x4luD8GN_R9%B?S}20|4ftA(|9O#k5S{P)yGh zOHF7+g`!25hAjvJN7S@lG^08K)qcpr2nw}X_OmUzG*OGo7k&c(Sb#$HbSLZwv-^ABveIZ8cL!p8r4*JiZ#ljFv7D^b>dQAW>f)SD_B)q4x=F6 zG*=<#4^(1PebrE&feu{4HB{mTymVOqfoT;Oqbj~4KV>y1Xw_x*penF~S%qa#38D+M z)o-5FEWUM5kf9`E027iVSlQf*0%=EQTt6#({tU0=mht72L8HDm?? zTmQ94h+z$Gl@l7`UCm-$2eyDfp-sy|Sod;TaiUs51{cZ#R2!C3IH4?tV`D>x4qg>x zW5gHs(+D`CA_gJ`7?xzWXBxP*EJ{^c=k=iK6>>zjEH1-k4J8Xym1ilYJ5)krf%ZUN z!DpMm6n26RTDEABCugeyQF)YNE5c%fwk_bmY4wC(WmaqH1u}HjYt_RPT%!thR%~4f z6v~1krnY9a3ZeXFFxtWd(Dq3G)V3}1wQjd1Rm&o4^>!#YAtkyXTL1Qw9zjx7qGrW%pI;gFls)bEn}CSil6B_IBe48CW287eZt|R$kxamYU6RuVxxd zA$m>WTBX+%Bv*J3#93qD28{=i?!*M79Y z2>i1lKmmT$Qgs)CbwOr&;+I7J0CoRYSjwObWcMM=;t~FHfGG$g6gO7gHn!llbNrWq zFXRuT)`CZ+fH`*+kTzxiHJF0f;%pODfBk`fDW-x$_&^2$S68@FIJj^F7XU2dg(+w- z#E%ihS7E!JQn~!mxu|7c#YRoO&EnI=7*WsGd#C&=^%^g z027Eoi|JrDa6yZE6&Lm(3z`595SAhSz>Dc%5LV$ok%5hop;)!JJRZS||6q-`*gS6F z2C{f&iQy5}APd&Ojp4vcd_jxTSY@>s6T+B{8#$4)c##`IlJ}S)$k>e0xFPhoj^B78 zIKhk4SR1@}T?b)i|DXzDz;8EC2;+_pc zp9zAW{Xw0hnHZ+Iny)#VxA`lcAyrw~AEvUk)%AqymyL~vn0 zV?cp{*?t_@S9!UI|K^ulS|>PRn9=qyaP@FA*dLsgAq;j?sUij>0{}7@SY6m)VHF}m zmv#Z5RQc5evb78l0;mC?3l1AzE{r1$Mi)7lN}r8?>oow80vqBf@W^BCr`k zvj3BIVHH4b8h-&~8U`UE${?{JTX%4nS!4QUJ?F8*dm+^Ja-n%4DjT}(dLi~xuR8(& z)Izi|d9@kBwEY1h0Cges8>4GGBc3C+cSAES8Z-R$i!7ye)n5_u!~ zg1!-?i(R6Y(|dcISX;|mchEb%kDO;Q)&$hzZkgO4CW9d&B8Q)WAQA(={oxs0D9=Ychngo$$1to5H<^h7yx2D%?YBg8)BUU-2}?OwFhB=q1JYDLd-XJ zS6|!+|2m_}6&)n9zzCeZJSt+%m3?<_z!c(~Av8myy_bgXeA@Tg+W8#7mtD0beAj#3 zEPfp~2qK3W0NdgE@TTV%7~n!(#2h+Q!>0@)W@*DJzjpSU5` z+#gB;bywjju6-eX+s#k>vnk@`eOl(j9V}=aCo(?JZ-C=ENyf-gDgjTZ^1^T-~tuqT#*vs|PvD3Bt=8LNlP9tt%eC z|2x*QS)RjvmIdm03E~y1;3H<YUdtAeQb;iD?_d>h1 zc2HB+XMbLexjnM;9Lr0B1u}%cwZXLkRqvsmF?ORQdir<)K=6;3ZTIsAi2LM`gXYVf zEApb{Vf$9Af98Sp=BNAj1wHu3MDvB>T0{5qYXUJA;_201>Pw%K0f2;eg1aN43+%u3 zb*Ci~oV)>oguqg!0QftIFrmVQ2n7mc2r;6>i4-eZv{>+>#*G|1di)47q{xvB|8W6$ zMlz+!l`LCo#00>Ewu1!$OiQRO1^{0QZvc38a1{U}2emZ;&}8O8Iso<@oaXZeN`wGT z9(-mnp-BLLp6bM;km)0aa0VUR+SQ-7sy7c-0RT$jAUgiao~@d8t<9o3c_#D;)SpnI zYL6yWXflRGP(KqA0{{mhoSg@=WX0O$ZrX#Bkx_hA1^|mao?V(gjXJgJ)vQ~qHPXbj z?Afi)I3$=Aq(c(4dq>RZJGk)SseM@`jy$<(tZNdo$c&gp48Ph!Z{`#eAu<*K${_3l z0J=Y9CICpiS{2v9R{)49Jh~Yd_dQCdy#>3lig&likyQ=s4|khAGY9~y|M;UHLT^pw zUt?!UN1b&DVP}GN-0g=DV--f^-&hD0l*M-r`h=2kRUE`vWC{`n(KMJC0YDX?D5YGD zHr|LMa%HeERgOLeXWKx)>E_Uns|{&nl1i>-Txm-_iDMQ3*2M`!&y-<86IqyG1b}pK zH`IZGRb|8~;8m5yGuo_CRZM#f0wVx1(D9{c9v)Q24V(Eh4ISk*(F6-s+=&n_Z^ekx zD$V(WiBEDqg3D2Dme=MhhgOm3qK#G+!<|=Vxn-9K)fr|`WV*K|m`#`k5gaUT3K=qw zlxKu5lL3&0FOu$oi%%w0IUZaP*=m9baQFpfvdS)-V}a^13++fj|B`^4NV`Q#5|Y?% z%dJ0;xz}yDr}g+0jSLavi@CCuQ3kmPJ%fvrIHIdAOVh9*W=eaUsE{_C9AyM1^@>Yy z!3HxV6n+OU?9a5-hIB2%6=^GR#mIeYamEF+tMNys7~w8RoEXv8S#T7)#<|BNHYKu-h;$RnB3#8lHf0%H@Hlst6QQmdp5Y*QyY zbIu;wd^62ge?1XaK7&p6YC@NN5HbUE1TJVX(J?mKa?gEG7cqoPcXC-vg;h;z4;#)D0HhStS zC#>vo62t@#bAOjEp00c4sf+%Wni(KsD2q*A>{}6a_j5_S$N608b1EO(_Y&>ET z+X%-vw(yB^1Y!s@5CH%@zycnqqaXhW$QWV(0DTOkAs1*ykLY5NjBKPM9|_4wN^+8v ztfVC=c?bYpa+93wq$fWK%20}Ol%)J500L3URH|~7tZbz#U#UtgU;&o2tR*O2!3QP8 zgbS3gg)M&x%wP(0n8Yk5EEfR)J{WVE%v>flG)FjSwoYTenr7=n2e@f)!ZIdUgDI>D z&Xy6wSm8vDH1l@OavqEYz*6Vruqn4^V1k=vFoomXiBE26;|Bc7XK&IuH-P$+TP#r6 zKmi9&u*vEz-L&UH9~v&Z;SHjw5h&U$N>OEX|0<(&OQ_U*@dp^OU=8`=s7aIc2LKH2 zq^H3s*<8v}Bi(>dF`b%6fA)$!_+(B~s;N(NRFHQ5v?VaTT0cku0DkmisN1>$tC%{p zo3>0FrdR`ajPQrJL3OLb87IuRnv$ZntRMN9hdb194swL!8*Ci~0NBb_ag+lc>wrf+ z^a0i%VKuKild8s$F;6r-LSB3=tZeq#Ps4^ptQphCJk~)DxJp*C!7#*RFAI)yxT7BY zP%LqpuqV&jNE_352D)&u3R8fB6PS3!ApWq2Jp9?o*YUch2~Lr3YP4b6SD1nmkNCrE%aDa87$F9fYLkIuBApw+rU_NZ z0&b1Fh6vLZ4s@bZY%4rl+D16Sx9!4i0cC_6#DE0_K5$h`5Chm`p$ruM!y`ChidR$@ zyDoVKC$Kb>J@nMRJWfeUwc2BeFxFuCXvbRHi`j69BOKsB@@2~rkIn|!G*zI>lslpg zWZ=RSn3xNI2W(=|+(1h#-h+!V0g6?;0vE(UhMH=GUt|yi7q3_aC|V0!H~_c_HvrRt zQ@mn23<49Th!>4Dl2aI&pbMBF|Lm0?-I1U|dSd0(89mB@zbcs_%$fvbrEFTxI8zY!zvCP>ue)roUqpRLWbNed8Bu}=Xi%bmI@NbvLmwVMaMn7 z?U5{ewZ~ar1tx4=1)a2s1x@Gz9R5IPLq9IdTgJp6;E-%T%Sk_bz{D!lR+LrH!B>oS zG`uga5U3;;<2VCaJ=i^2aIAwK=w69F;&G1krX#)_r%12uOAJ#y0uEWwcT9}X1wlXC z;_96BT)JR{0>{8nH-N*A|67i9f7s2t90xbF_JI!dqJ!kB*#|tvJy~}oJ#>pItTueX zi9K||Q424NDSW|npH9OUrWnN7{d|Ki?12;M+B(L6Y z-Vx$glRm=~e;9k4jIf43fTHg9{W>WAu!cC9rwe~D#Y7J;UrFbE?@;Q8Ixf48eFWzo zn4YXWKz|Tq4|VZ>$O4#dP=-O6g6EJ%M%(jL6&7Tn^Ph)3U%3r>(l2Qrt%s{Q3@;pd zh-0$tfIY%v>I+PiLElDz114~R?QClU7npDrr^lafsmv%_^em}AgSoaO&z;ZRx zcao(Jhern+hfmW03RYzvZ~zLt=YR}>44?o9J~2|=m4ZIC2bAG{m3A=wV1ejRayNnx z^2b)}kazpyPtWiOZ@~g(zzHZAgA^fcI}#(0P<%`1PNG+X9g_|m2!9qh4|cZ)v7t_- zV;;OBg(`syIu#>NGlm0&eCs!XI06q?s3iN~gW!;X92ZjMRgS2D>Iyn3y^cn1Y8`Bl!0X z;h=h`F%3${{}J*aS@n=~gKz`Wgb6XwDG@jv5Z5BOC=$;Qg{_f_1;{oek&5>62()NW zp13)CU;+)eb)uLY`XE{LR}p7oV)exoU560Gcw%XC7lbl{hfxtJ$5!Q#bj7GTs~}>X zI1&wFVh55^1{MQwFcnNO6_N6IJv9h6mn++5U?@fhJwlJ~(t^SFR9aCLotRRw=ntA_ zV9l`=2x%S(k&kV%j|Wj8{b&T=^cXP!HPb*~AybJ9(K%+p7%Qa^k8l&?q>3KF2}4&> z$cQ)v5f?G&jL6Xra77Lm5q7a9C$Y5$V!|CJ05y-0WB(^7H^Clvl2T1ziwlttl7$aQ zH;bdQ|0X6-kSg&Q^yQPW$3=wr~wGL9r0-%Eou(6UJ@fh{B5Gf^^;K37i=?_HNnf_oB`vDVQ z#U32#55}=4DFq$Wxsa8K5IJE2?y?mCKqHTlm0=k4XIUM%@SK;6hw5bqSk`pZA z4}C@>&!A)e;G0AVjv7%98H#kAaGaZi3^9O>Ezz710i6j!lyO0wDOFO;br3AT0_svA z-vpitVHqevo&GVSt8grc(H+9_nEa*~@Cgyw38nt<2U7Z@K`IyKxt~Q5BX@BF+OSV1 z;Aqoe0?tVcSQ0RwxeB4V9sszO{-P5`;2}mpp{B70F#w?vx-+0Cg%&Ct?*Lc&*AX0w z5S`N;Gl(oR=v1O-_>oF8nF{U0tcx*)vEyoy}vpC~=5$^eCkI5C7&}HC3802XRl(J}nVG;+r z79sMgAJVLKVHHequD5zJGw~RfGNlaJCOpBhGT|*h!K)Fw5EF}~=3xXOMkXbLp16Xb zd(pCiky2e@tyPeoZV{vjiX!cCUomi?TA9A!3xq|U=CTSTrWOD&qruV@b$hcIixP~{iR1UH0n!{rhae#vpH2!EBNd{N z!K-W0y#8UVG{_kFst_iSrwS3cA?k$(fw?M$70#KB0N^B#A+{}H{}RC|x+^oGn0UIh z(R}Z=x(6|Xzb2U|8mY%9Vv*`0#=#>TfxF)T4=pDY0=qM7A&4sxs?RC3l#3FAq7Zmz z1aw)#2J)SsnGi>Tq*n15GfI@*hMgCCGR6TEfg55~YQxLe7&?r?3BjW++o;{K5bPTf zmVppi5T9$IvnsU~Fas~z(7Ej4tSphi{~I-c%7;QxG2-|Q{d&OuK#NAY6WYKfJ@O2T z2EhnXAnjWbr1*Lq#~vEIGe%&)pc=fQ8o~=fouM%$;4u@}fsLel12<8(FtH!x`4+XR z4csv{w>lD ziA0FM2muqe>k@kaqFSso=IV)A%rNT!SGs$^LXpQQf~;~v2L^|hZHy2{5xEo5d`^pT z?BTyH!leQy@-( z9dPh0Oo7cEK_2R;%PfPmo><2c6A$jSq5Q%oV}r&fTZ#8*uY@vAeM}MjAX%q~aVF7= zXajWB3=#x;V01C6etaCna42#TT905EaVkV_lG3mAHO>i#IPmA z&D<8l%d6?T6~hmTSbfQYgK&ioN#`Hm{W#CS|0ZjY*_AgV>-dr>feiay+38&{BOReC zTrut-SqA(h75oj*hjG!ZI=%oNUC_Y>{uvj3t{Pr2f^D?bR5AVFgQoZ-kPTNrm~?_; zO~i0APF3IrP9V7k2l(yb5+mE9>r@$Y-E2h;itQuzporrDK03s(2RgATHm=LWFbGF+I%q!S65}Us zK8T4IF2l)I;~?QR!ithL$6L1+0Uk~u1tT&F3QazS&oF{@npJPEGmH`3tNDQ|1IFI~ z4q|L0>@ZpEYIh1gQ1V2XMR$S`x`G3_|7c9k=z(3jT5OIT(+`vW4eG$5w%>5>%=eI6YA@a*$v>4$fv zCY61_XCCAH3Q`Vf(;)0?Ky>JL?8t60xCZWH*xM@;4~ppH(mWge@DA|)4dbA?uxA3R z9aGb=3XP|4%073L*J@uEb~VQ-oVN<=j_wr`=Yaj^F_RC?hm+F28v9^ms8`*2C*oTr z?vEgsl%nj({0jXYQ^@cNnDBu%*96|q@C~0aj?Ua0bu;yTS>eDA`F}_6nfzb3q67P`~t3Z!tNZ-jV(@`JnY7 zr&*l!5$Zw)v-BF{$Va`qgE7fN;1L{SS@#0C#p1r4926_4d5_L-X(k z-tI%g^G+uJ@{jX*E`>(0|G$sM7^&YToo8@#01#!eG>KsXz(Is2V%%twrOO&Rdj>ti z#L3jESGar;L-uT%Hh&>SjwD&qL697|X)vjgR*6rJ+ z+R(zK`?epwb>hD5<=fZqU%-6Jxx-i2@L|M>vl{uM*zse?C$$-b3YCQ~rqhu5dNuRW zpFMEObg9y$CyUCd0hnOHBF5_*v1!PVVZ%m`Awz@+2@=GJk=Q+7x2CN6r)!2)x|D&l z=g%XHuY4g>>Q);Z|J6+n-k@WudSvk7#g8Zd*cX7t<<+lef7QsI_UiS`V^ zopV=@Uwi-qM63z;1T@ePaFsSIb@f&9=rDm8SZj?8831sxbyr@C%mR*H2h|3YEC66(k3A{_ zr9CG;^{^IU>paw?X-uKR2vzrBHCS%D_3Es&ycJii7zP>lTyzub0@!qo#03t6$}rWR zXO7dgy(66MaF;}B)vDTS531siS6XfN;9bw-6k&#|S|f{wBbHdJhb9j28B76a0+T<4BnybM)y6v`+ij?zukJ~5tWU2BLdH)v|a5v%o7S3tY>` z-Q$MLI}a2NB0UewlrD^=gYr2~SlXJZ9f#b6E+!X!cBa%^n08y$Le}n_#-m&t9 z4%Ey>bZG~+`I;C^%0P{U*z*oP^O!@QzE&0>20ciZJ@s#RsF4c)9_0o-^zMNX1V9Ydz=;liuwj^!VWYBFLmaB* z{|_?S-ZU_Qts4xYe6`Yo8@7jwL1e)oE`(tY&t*F&A_@*XjA9kJG6*X6ZWU!fAl-On zg*x;PiAroC9HwH$T~!Z_fl`wK*Z9VhV5)b-*aHbcVZvHDA%zyi#UHA$9WxFQj)>ad zAkoAfLnd-0kB9;7e5jou#!y^G@PiC#(a*db>rRC`OlqzrnkKL+4BhZK{So%qq2U1homRY1S z6!4i*tXbfu!3pie;17~SSPL>SNMvSfThmmJ8p#Qy4OZ}+76gR}`u2)7n7{;U|LB^< zTF{A20t{Ok)7CK8NtY%V!Jq$J4#8Git#FlTmxP}owRD=XMR4+Af5S~NhI zFahE;I=!NKO&gr}qc$^YF$YBSnNL$000vgkkFJH5^NOi}w7~*r)|7jzU{uxM884dl z%mJwQCrhI?HJyf~nMpO!A6PI@rdp1tP({~5pRzHJT5KB*kdsB<7Pq-YH7n1Upja-?StDqT!UUZ~@XP8|Us#RMXlLtPEF8eP29IQn2Dw2XJ*nCZhesPR zU{{E7RxednYH;Iw1cPbEisYHNBz^oN06Kg|IQ~|~ktrI4cg(P90HB_)I$+Rz+&fe_ z;KwGpWR1Dw9V?QRXG@Mu6{Jb!hRJ2fkdkl-1sDn?W0}b{f+dt||F~X(irG8>$gG+X zCYPWlDttf`q6a&10ym#ZPsucF?+Se2JiCQ>eGYW3=F+HDl+FZjNO1r&(Skz9WzXWg zuH`@+>BEe;#F+Nhp#?=pJ65`PnWzD$Gh?lNu1eH(+hBZ64VR*A`PIAvKrZL>NE_EV zijC00tgA9nm$D3ZQug&>%5Z~K7aL6`X!Ds)x#&jMkPy9Ac2$fTY?xXO&B?AdR;CfO zZey?6OZuft9r{K;93k9Sxpr6s_vdxPWl^$@w+)ADBwU*`k$P}Ky|MDu$RV|Ej}WrI z-%^Gc5Zd5Jiov~q^olablb)WegB2E@6}(1nvOU|N_AbsA{~@j6)n3j{Ec_2Zs0yIK)8?bD%>VT$+bH^6{U?jI^ZNY}Yn~ zfr>`B!3v5)Uf=!xcfbc;@PpTv4!lr@OvGZmX5B6H92`)dKC%uEkj96r% z4qgx$;f;TM;l74ZG3dkqz`$l|HJjO&rdke0gro#1m37$OB0#^ zbMkv7AMJfUfxE!>>vPLaJ;RiFwm*pEk0xUoSG`<$F0yiK8 zR1k)u(<$?spx!DHd~gSHkiYl~2X$bCf=fTOKn4?-6c3~ee1HdZcs~pb3~&I4a1aM^ z2nVpsycG0{aaf0W&6CTv0{d_pLU zLMenocit37q-5`2I=xM&i}O9w~9 z4{$(-c(BHj=q`=n7)a^APSAq@L_jNG0|v~k=`lwDJOb4jycTc*c5J{0T&GwhF&qmI zd_V{MIEQzz2VT^Qe2@oqK*7HlhhuC;ugC&JaY(KJ!wbw0a5#r{n1_5A56O#%b*RYv zIEQ%{#MwF)$P1Hg*o7Zt11FeA!E3(f{|g0I#34--J}oG|!2?Ca8^wR3F+l3a#NdZ@ z2#j=~hk^`?duRu}_=|F2Ly61^ApxGJw2FEFNc;$gbchG1jE{b>2X{CJi)4pefR{@Y z6lGwAM&LwG96T$;!o?e;y92_#GrlA60zGKOSG1)j%Oo?a3wcmTz95Hhq>Fm6#lKjG zqnyf<@P!amOqA${c-Y3iFo$>8${y*5dSD01l!Y7sfFzg%g$od7sD*R%0s*8)1#E-{ zB&d}9zW{W}A+$$Ah`xOEq*pqK9O=uo;0ME;%z8)+eHcm_v`oi5iK211;`E1j0L;EP z2YCR;CCLYO5W`@w176q*bFc>>|7p!rY)Qg90VjybU8v2Z0=`Vp1L8A2P;`Sv;7RqG zG#sG=Wg?4wSjxcs&BxG(-sB5;NKU30gj!0@d9csED2Krm6ntn0=G=>O*hm05%e=FK zwG=$V(}To&OX3m+v<$*3w98IVyiE!xei|pQum>3g2Y3*Vd)P*I@Xw`)Cua1(d@zTy zTSG{p2O9m0c1TRcFi-TvNx`du)O1fOYfV33N!SED@MHt%8&5+5HdWh-dhm;Kz(I~= z2ftVc7e$FIhygDoJ$U$waBv6cd=z}x&%GFj@JkG9*ag@0&cR#KCM8cy1HRG((!fhV z_GATjL?d8(se!r&8PwAC|DXpMY=V+$gHSi<#13^!G`coGJ&Amf!FY%dd?3&iO;nP228KYtV#|kZR0sU)6?*{5y+{YP z*akcWJe_P$QjEitTuC^6Nh4aLb`!Xe=m%cyPXNgWz$DH@MFtj-N)N;bz!V4k^cHvU zi*w-7uAnQwtH*nEQdso|b41V6L>?_jB!H8fsligdc+~(&RlXPpa!pQRfSwVg*1kZ8 zGUb+gAkcGA3v>Mx=ex_9T-U2OOPicf4~bV#$~Y8=gS}7(9-)W6XizW7#3>+^Cy}kL?QVXo8|B9jPQZ7We~D2rM=eiNYC)vh@XI*#m5e z2#N3q09}LmMV9oa6Ene%%$i>=%7Xd*S(E4oXxxi%C|Iu0+ww4i?0JOG&<++*3gaS> z{pH+|h^!sS6l9?T7NW2ic!Xpb8f>wdlt?UpfM5xpU}LEc+~tJ%-9cnog$vGL4K9iC zEnfgQUt>{U_HDNJ6^Zzr-}*%sbNK@RpaTgWgb)sie+7US8iYTx9_S?r+${;X#o@Ft zF3FK#ICu*06%XNc0OIu|;rxb#y$W1m0XV<}I6w_kkzr(^VQujT?4Szi9k<@KUZ#i# zz6giy{~d`9+O&%k!!_QD{2d7ZrWX^q0g#ynm6?~8a;CCnu-)DH_NS@@7sAMAcG(65^kx*g<+Mp@>qrn9LcDdpWzA}*j zj+0nsEJoqr9a>xJhs;A>rXaYpQIk*(iDV{;JdWOjupWO<;#bBB?Bxr24%ts zmf~tQ4FJGsY{rfq;pUN$3nkg0F%F5w4GD6-TyqWyp(Y8K7Kzd^V8kfnGghQ~;0tlc z3MCGzi6Du5#$-KCEC6Vg8z>!SMv13ohe0M}6S!x8>rKH&p9*KIHl#1~O(+CN; z@rNqlp+X_+(td{1E^2eWYbZ|YxL9nHXzY@BY=5|c$ablapzLhE>eC&GWJxPp|Dv#v zXl;JJ-LYQb)*6Y_rrPd?47RqVFx}X!IFAs_HoG?GeXf%?9PUN=>ywz$y(ovl)-c#< z+9gv+YqbhBp>Gx_ZM4p0=H}(W3Q(~oX<9`MKs#r9##t{xb2s1Hie-Lhe`0$_jZhr`Hlo)W6 zC>^CZ@H$EGs@CR^P~ut!4%}3VI;pQkiD(xu2^){`%D@ccVu9a)TVwfL@^)ug>P>0I z3R9pA*r7KZ&*r@TwkdO}dZBNV;D^4*@BhNw&{hYNRr3=a=%$$~0)3KpOmhvF zb}sQ0hL{avkPTXe^H>%&0R=S)mrd_hKL2q*4+%kEZbC1JXE=1} zZi3uC2#@1u{E71Irt>H=lI>?1>U>@e)680E0I_q`y8 zOV4!h5b8BpVv$H~@J@~4QuQ1c=YOUmxp)q3Hi11Lxay#Iix=zm|IT<@Jch3qjv{E_`30b{?FD*2OPuj0oYLUQ|MgmamrfA3xbiJmq1qa=nH;IZ* zEA46MS{v@aPL`5yVU4L~KnGB~2i_{kf-ImS(&&P=6=t_p8C%ulI*(klbi9H96;tuOPU4|E_rWz*a~WiD>}mw-1SL zJBZWY=(;C~MFAcVTrfgrc(71tHI8_|*Dy%Oc#(Jwk|-9;P<4>*s2~ffKaeLgc?5tk z!D5fUf<|r>EQkvLi)jT}R7qISM8t%|01%X@G2tTsWhAb_=nol-8_yC>s{&wE!5bC* zO=AH7fGLS^060we2Y|1H+HTM+m?db!K>*+sqyyljMu;2z9enEVV#b1dJW9=~bt~6` zu^fQa(seA^vS!bsO{;dT#B<-?)+<{Ez$=Me0BDm)QRB^%N*%%}Nypzak!9b$<;Qk0 zy+NOw^>Tc{AtEo+pb#Y;Wz{DuV_Y{{dhm=Ry{LKn()QDWOY=pk`(+ zcr#}~qigm65B?09K&JjM0mvfp6#z_&*CB4l*cw;|Ek1XRDpBd0sQ{V;558)Z;e@x5 z-Wa%t&RqbrZ|6<8kgmm5;bOvmj|`#`QvdO_ls|v@LmqvF&B)b0h=KE?mRoY!rI%YN^~jfFl3C_v zL4K3YTH5T<#7z4c=g%u;$Yeqmph#4kL__7~|Icm`8TF>1xTw^pKSMcJ5OayD#Sb{5 zO;_D@Uux%*jksJS=$mwegrQF+nDCRJF?3NN9Li~AWPAnLxN4;Z)d&EK{w1_meVWcR z*hXx+Inx?(&ROT3c>;Lognm|76i*t3r0X+uq^8;)t16{wL^5gWT(p|T;N2e@k(XbW zIyxYhndhRbRXm8HV-_(g%@o78sTSMUlgSEH5S3-kgV;Ig0vxcw0C%+Hzy~8t<~@jg zbI)ShAfr{p33JsBV&XJL9axi6>=-g}1`M&pK0d=+$BmI|?#U}-mk*ZxphXPGJf?$~ zJ1pazv(8+tVyDSF10D2O<{&l>%tCLr|1e@hM!GT5Pec85%288oR%ma@)A5yvIb5~Z zUu)@03^z;+w%Hxy!*IK2v!zcsMLVXkb!^jJx0X}eZMHjzk<;+ zNVwu#T8x;Qi*KdR)o*p(^xTkRo;g`~YaX;d3^RT3=Sz29I^j3PXu9f=x#h27s^9ZA zV(~mCw*yYFK}aaOY_q!WbTd;+Cwh2MLvh?PTn0Q(!( z`e#6#;bLsWI3NT4!H;r`?`E0P|Ht#*QaEzD8KFOf^HyIsisP#DrrXVO(REt^x<_v9WXnM2tBq zXE%9F?k(aN&1Ql(JTGvuiz{q_71-y#I3Ca%^z(xm@Fz$OJ~4N@>m4G$#65!06LQ`UH z8^S=v39ZlqU1pGj9;`(O|AASiedMtmY9v$R5JL}VVv{mDN%tRdk8Bg@7uMX`~Mn2OeIfjK}9$(T2 z1R0jKfD`jB$M6P)93;wvTuaO_ktyhbGNr4dLm*3`h9q`3UI-LZCtz zl(eY8P8teE9)XE0Z2|6J$~&3{Y#-gj7~F@cX2S;%Rr!sL z)}y3alSc)S+75Citc3i;B}>z%gRZ5Z4tO+=V)tmrP^~JCcYrl#ZM7d<<4R9IbmE?f zb&Oc+K-Rbh)R$@N(c=wUc%Nr#k&7PjeU3neuic&`s$|<4Q>DzAw8h%STAZrw&g}Mj!5ofrEO){$T^Qy4tU6e9`?8gKJbwbeduE!`p8E< z`h1Ui%p)G}XvaFxQ4TW4SRCq@hiz9kzneCai3>2m0T6(I1Sntu3}^r(d%8PjpgPs6 zUNx&*?dn&>+({9`C9_mamWBUmP6hkVmCiK3Gne;095(dCvNg4|=>~ z9p)GZ!7rZQaF7EX?1+aw)E&n_fqW)Y$TZ5UZuP4pvj)6o#cFw8I(^7v9nvPZIMA_i zub=oCbNLn8dz+7DE1J>iIEOjPQ4Vs9V;tfb2RX=54s)0T9qM35JKhlwdfY=F9cN>v zK+b~1`NNOIJ3Y#f$^3M0a*ZCdu z|M2hW(Y%Tqn1E5W-~GUdF$iqostPmXJ@Jcg{DEm>Cr#k?@tfbHSL^{O5OMc>op1f? zWB*5Lrb9)u@BN3FVhu7e!XH>PpEt39Mc-dP``h1`+_FIT?w`L_p23OsRN)b2o5-8G z(0%#?U;qLjo2bUl3E=$so|~9J8F0e?g#-YI$pK2>1kN8a+(jAe9|c|?^hqBatioYw zgelwr^J!oSnxOH$LTda02%aF?X+!vpU-^-RHn3m|+Tab^944?p{NbS2*&qJtUk?i5 z5MtjnT*VC-3K5=I0s_DSHlPwhVH6hMGlWDF7@!o676$@A40vD-V&NBp;k6Ni|A?$u z7%tTdil1u8pc%4Z8v@)bxXBu1pc^{V56(pX%^@A?;U0>YDR@X8@}U(mp%eaJAR6K! z-W4bS1sn_?A~KK`a^VL$;v`a{U~vLa&_N?s;_skg465HIf?_CI6eidR8~~yyVvZdG z03M=ZE4pGgUE+S6;w#e4ATnSf(qb;^Vh<^zN0i|%f{hlkKo@ zI^;xBB-^aQP}~3}cwt3$1v<(jM}lNX=8QXn${qluNM=a%F@!y$WJ|i_&P>B2*hm&6 zVM`9=8op#s>SWGbU%Q$l6OfPzZgz#{;q zIo6{jM&(v=W$)2}Of)1r-ef^`Wm%e~nSjDj48%$5qENEsS;A#p`Uo=ofsIrFCv0RZ zI;C9lWnYrXoHXGYWTPu)WnTK_VIrnyOamsUgo^xPC5~lcO6Fv830(q(5m+T9x+P_D zW@r8gE(At$q~jj$rDvMvX<~;dl!1|S0Vo)z5EdqC%I0i(hBko0|6MGgDZFM0Mka0g z=5KDsG;qRDn7|t7rVwr>a4P3=PR27ZW)qgdahhOhGG}#Kr(-N8akl1eZlFpIqgGyL zc#0=u2xohA0VZ5!^Y!LckcjsFhJ?e=6vL5{Y-N0Vq_|wp2;T3_D+(3PP#xx*ZNbbU~rQ2%`QdE(mGKG^%E3 zLxo=GDsZWXdTJIlL8#&isb)bL(7_;Jf+^qvG8~qEPU^8Dt5*2JA2flDxLXsvr()qyOpCOc%Gm;xsJ0UUIJ5xDCX z#_Jz&!Yf2;zItqXHeS?afpKVqFPOq0tU(n39}67p9z24uHtY>eD{Pz!P`t?vWC0yu z!YXv^$NKDVb_AfltBE@67G?n)ysR(0Y9&1bE`Y)xtO3Eki4l~+ADDub{%qFL=93C- z6K266T8Fobc2 zuI?HyC~5;IP$eBOB@V*pY8Gen9`E$#A}*}KsO|v?Lg5RpW(@o!^@1-gf`XeY| z`D$?;dU)J9yyw;%(^CC(~z0k#;yeY6Lz5QJ0sDI77!w`_}`@P|a$$cRWprt}2V35p2- z%Ey%fQ$%o;%!NkS2;}*KYA_1`C`xlk>Gj47n^nutq{f8KZ%Dv_+x~A7W4z1b zDRCu7VM>gkW~c<@mB>Q?h*Cu31Ivh1+;9x=26+UsKiG-_M??gF#HUomt7L_<;t6fc zL<%p=a+bUEs3z z8UYJnj{(nyFDC>rYr_IA#C+Xw{}m<#GB1jEcr)qxg-0}VOWcJtHwG>Y#5HU53F4{~ zO9gTy;Z^hm%TB{IJi{y8K(}awIhTh<+(lQkvpD+$CjdY#`@?0#hY_gsR>VNghDt&3 zV@KdJS!x4TNJRv%fiEbvQV&H!&~!~aRYKgbz(Qa!>u@qB3PW-N$S8#=)O0~y#BEq~ zWB9@jZS+}7;BEY_Stteb$#Z|;3iGk_uCj`_Xhlr>M?stLlp0J3zX|j8G;?GI>LLd& zYp_2ogfvGm$R&iz0`-jua&y#(2}BG-^Dtp-OA|!ky~M;T^W(m7Mz(D;T9bD1Wq~(S zhEhD{Tc-$cum?kM8%t9z|DFD7OYHPr7YnU~ghL?5Ct56R$LY-UbboxrMZ7{``-4&V zDs9)q4eQENL~s>g!gNn}Y7_)g)8Gxi@>EAja`=N~>mDvJD7`4g5fKCtl#&LCg{4b6q`SL2qy-5PP-#(6LL?O3m;cOr&72RnBcBE+&iqVodk+yr( zTt2=x^YxRK4wf(U*p-V@!oQ|5-Rm8L~I{xHRqqIe%6CzF`4RKN?G-D5cKI$(1 zhTpJ0;<)ABx%h>edxv?a(v?Sm{BXLG3VIZbeM5*Nm5{v8p_7*=;RI6EywB8ORjc}ZtY&^#4TT|P1v&11&Ik)eMzkyQ6SQkMQ5 z(hD|W0lQRcL{?mr`Nrxjj+K8t*JTk)urBHNgyHk@@ioxOF(77Hm8=;FT>q|2Rx5d< zGQBIip$aBZTWa2#AtZIsm>`+*`usV|ato)y68PxludtqVH>E)3WNf`|PfrzlLoZ}K zVYGcmea?;0pYxaFVO&bQTtewXn%@sNzNqyVUYm;(>#;lcj>r!pp8eJ!{x^JVNi*VH z5cl`rgnPV}=BoD$Y2U2PSRA$(&*W@gpl`M=eCS;-!ZSSc{hxOx3if&t#@<7Vi$%zB z)WH3}?;kLWVbti!xLL<#!tqZs}V&ro6r?%YTGkB4QotdBlqSAa$IM%g8;>svegdrrNWO8;))zmX3G zLB+zn6AH%*ξ@t0KO9X;ECe9)`RA!+lo7KdApKo;A$IGDja0{kbFLxFdT@yMC>? zZVtyJmNm>?_|NyA@bRCJpT6q{$brv3Lu{ccJBk}SB(6V5mA=cpymb0}P<+(Bfnqo~ z5)gT*hIhm<7&i4UF!oXSufUf#J?x@+u1kqk?RvO5>XDC9^AcILzX{YbU;wyzLIt!~ zXF7R?8N8%ipl|1(I}xZAefU@^mBr?K2j!ocu6Dv3tZYFlqA8(aTada-zxd*H=V0_)1%1@ z$v}EH%eJQ<+F$&k-;sDw&bGj}h>66_7in}4{@<*+`6+1skgifQaK)jEYKebEA?rTX zp(V`g@ocV)@5K$HxAQ`Y*7KDpvP}E&dq2N9J}lK{97vMp;<`BU{Q6`9nma7vOqnNa z%63QuEzzPMNDCGc`pXjF{dKX*tRsXq(B~`Gtcu=tYy!0d=#GZwtKZF6ig1n~iJxW= zb$DC8_pC~9z13+V8e!{5LaWP$mzXHoJy9HU0$;4@-Hrb(&^XkZn>QSG8S&%Sro(H zo$QvRw2lDNq_CUWw5rj*i$l;-clRXBxc!7lEAG-%-&tU z=cRDLHbh9HkMek)Ze|;H%QMCga27{9yVu=d%2aHJ7?R`gPZ~j~VOF`u@^yxR0@BZg|crFnsi0 zw#oeH^D&s=liyx`=BI$~eGH$2etpdR9P;-s!)7QETh?Ycxgu-}LuZ?{6~!6?+m7aX zowXe+)DPQX7p|j8^q|4 zpwMm){y8Z_6@=`^6NxN(g288F*uVjw5Y~^`IV&?FIGB(NdIM~MVR|A(Kb2vYf4{CU ziUpC^=ZgIJ5_N!sB{hEfYf(V-fS=m-x$Z=6ZjnN~z(~LF_|pT%Z3LRm_Ho}4D>UqC z&30KA1v^pK4@wfjBlX)?;>g11SA8R*!PpfFA4D2J`bb2V7dwJ7cDHR_dW8kFwq$QC62C) zN_9YI(Hfnd#a07r6v*dMW=A z`;4JT(aLi5akNhOM(rz8@nO~eXh_$dMmCuYJ9m)Le0a`0`}+C53ZqM{MZW!Trh2d3GJ9Lrgx|i$WU80Gw(z#(Q6>TGe-pTi9>jjws8TwW z!M!5lmVd10VR^r0>RZvj6Bh$^WQSy0k>rni0?I62?P2#88T>`@SuWat7Iq* zr|(`6qJJ(JPTF4Yz|_er1U!(Vf}ZMjZ_yq5O7sRX5hk& zMqAiv_e5QJXfQ+Rn;5Mit7)6|)2zX!>`5J8n{b^zhb^&xuRcG%fpNl_6C1g+i5G$7 z$ff>0#zFEatd_9@%#3h`8DT}@gl{n+f2rcr@1*rQg)S$9Q84&5pXJd;4orHM*2bAR zDtDu5{CBz6)$|P3VR-0X_%@|~6~1aKnZ4{n3_Rk>`$<=L>XW5Lh}AgRVmAtQC7R9g zujEWa`(sLL_h|MNhlVY&TaookxA(xoFU=slgP5f?fia4o7ShUCA)1I93V&P_Zg}m$ zn2J;CpFlynwBNElsBN)uayr-OYz#)WDm13gs(Q0(kv!4O-SI32XMCuXWFP(MPTLMT zZ8jrhWGo~%1iX0hj~@OMdCF0+v5rh)JuFGo)wF#`ihI#Ng;Cm25(H+C$1b z)WOKs33xZ*iEh&l;X_+xtip_JV;zesy!0l zI&4_0%J4kaBhwE8Q;e#Pv#~uaU>OTr|Ln7)a4DM}jPQ&)!R!fRyY3X* zrv94z6$C&#(VOHcO%V#d)=T5>xM|@sNTzs z(vn|--UgE7rMy;qIg{>VJLl?dfM7ZIs(8F%@zu2Gl8z2oy>rfnpBr+_p%#NUs)#~Oa+E)#oZ`e5R5`d@xkZ^zq{lo4%V%OkKJ2{Df6A4~4Ddjx7%K;-Y zL0;|nIe5ETe|tS3*7MoSRU#$exfA!eI*M)N!%P&yE}F<8I_h`QAM?S8Ry>R_N6qVJ zuU5#5yX_w5JxgGE%2clGBTOLa{t!NFkeBBcTp05)Je;ETX)vuJ@>$}?iHJqT1%{Q1 z%`7xT6kNLmisU}^Wh$mB*rMt?;XHqfbw~K$UVL_9F?MP%pY8t*KcI`6z=jtee>=&Z zAOzUv8}EVv?s+=ZQ^cHRB)T|i*QvrBo)^@FY!!Cl#%bo|gfb=QFk_nXk;CE6RB=oy zCxo%`GgKVB6{IlxVB#%WRTZWLJG$Pfrq&;nB-I75@aOVi8jFTQ!BfP8apqsA$g2hR zNFlg#IC;XOr=L7v79vFxsxpZj#HlG&KqzDWDRM=Vx`mgaCRQExgfSAm%Kt69dV$u3 zmtOgp+o%}8&SKkdx!6Qot#~2(kWQAGC|#|0NU_SB%Hy-*G|&2!5Nc#KF*R3ll>DhX zES%0mtj!Kg$Du;klALA7--bj(8o+7Yn#js#L@hOUs0W_`u<9F$@@!y!`@s~0grgsJe_O1gB>JE8Fra03B8K`AYMNiBOx83TxB>OWbeQXU_v z#Lr9pdE7ZQ>7TPEJJ9)$I^^cN3X!InCthr?w1}c690KH?z*Xl$3uW!IP?!E;+)f%aVEKu55hh4p zZ1}1hv=fZas)BE2q+(?ce~ozMgNuQK@6ELfBkjETq91CgwkQ@fm&wnw{}AP}J9z98 z9cM_*$@_+IpKq>OgSM-IkiZ+I>do7K&Q;iosyJuTYyh#V$mCAP=q_oREVduUY+TlCn^Egzg|XZG>FuSLYNYi+_ngK(f;g=?{IH}tFe6j`Nk&t#I@`ro+w_Q_q4fr;L2{m9$7n>&L2ybzRT(IxMShzAlT8)+ zjqfy8nfq5%rnRCJn)bn(bX^UCh+;?fa0_Lg{L(rJf|_T<;B!TKhg=SB&tKxd+FY2v zn1QOD@}t7Ix6N6LH9VXx@a!C?pKuu;2s}i~e%3=b(b=agy%5;OM2ZP57bPXk(_%8Z1kI#ztfr-f99$uy^2(Z(u zeF_A#w;{s*F~KzW%h}@>M&7}+-j4XrnLeCoLYYMHlKVFD^sh^R$V&SPi?2 z`2)m7z1#Uk`qu>4nnQwNb2?%{`G zm96(fE|;Ibq^ljp@u$JWLlDK14{IdTq+XO=9CZ)6D&Uv+xmGl{BW_>a&$neDY||yLa-M$~KX>|2XdYXW3DZJvl3|I!BMk6}E|E?|-jl@5t0Rf4=vS*8 zgPn-cE5t+=hX_gGMZ2%ZmG86@Hv8_N^;L{rR&d8M_FLx~^)ru1m}6U*DB#0^cE;pZ z_>gBAL$AC!kWIBwO;JuK=dlepu?UdLE8?M-y>I+n$90dsi>*|U{KIdV@{Is#)!X}K z5*J5Kaa>tTQ7HVvD8HzmnsH^@s&_EUU=PbM>zW}RO20M>e!6OzV3!P=bl<-;OHU^f zj%%)!7^=njWu75PXTHZp!y?&LEoS7 zQ0~-y!rmD)h9(1;=hQpr8bwIXT6*9>! z0GwZs9EV)RKz4JaeV1L0lQO;1ie=A7_sn8uT&hk_Hfg85B+}E(Q)#$1fdDz9C*Njy zulaDccG`<*r|PcgP&pp|)fs0RlA}PbO!E?PB{v{cBHmAmWV(HCoP!Kn;ZSicma=E^ zd5pZ#>=no7=V#%ld^)T=G$L7i^vra`x99o!h?fW6@ZUdlzzr$RDvVE_OIv<~u7RVu zV0!SQ+2D6kbR**sm&zNi@FA&A174(>Gk)g3SN(@vd@S~9%4x zEMq0z5}kP;6!m7ls!dDbd%gNj+4yxTH^}z90+NVN5ndcG<;Z0B%SD+W8Fp=O#)1>H zZ&QzWE4Mq7i+nwrHv4Ogp+6}QY;6+#a8`WkYo_VG#m6000CD0CE72$O#{N`vCyAaJySNo-hEQ z0iYZJ8Udgi07d{{769G@z-IvX1_0*(fD;1%LjZUT01*I?2mqY`Fbx2!0I+@g4*(eP z@JI;==m-h92nqGTU{eT$^grPMGJt`MteuPuM@AM-K_Nj!g-?x7^Ph0Fw~dB|hmH4jcyu9w#0rCytA2g`3-n zmsb(s1Nitf`1y_b`5)p01ULja1pgDxR9Ki&j1mwBB>oc)Cm|sqDNu6vE+7NQ$au-g z$>HSWwm%l zZUD7A^{l%);055kyu7_}FJ8O=0)fClTu@NZZDS5+j)=hFA|e0`fWcsXMn<|~U8AG- z17ZUn#65_M!{No_#mC3v;^SG9Sh!g+>YGt?k?E@ndmoB+_!Ishljus@clc^ z?uVVyPo=2ysPq4X1AYOxU%wJB5^q-(DUOsDN6Ug^;lyzY-~=Ra61Ov_hEvnU>6+k7 ztZ~+laF1MZuHHEBKwKaOhq)b^G+bI9F0T|>>fNA)xCmM@Juuu-|Jakj>SR72;Wn-eLHI_xI zxpeXsuSvD_WOLbcrpTiyTA8}+*<5L_ZyS>>6>kerw?Vnut(6NU=vVwUQ>|5t6^13+ z`PyyO%QfaLE+41bYTh^4jl{C*w7Z%lIg3=X~2O7OKXv>2zv7PO1 zJAB{xX!^BYPy6@vPOrmHvppThp9dpI+4XxnPj@C>2|SqV?K)D0%@=2D^>zO`Ty1&! zd9JVL@_2J34##fL-+P@}UhJzT{ht~w5c6(^64E&H^arRJ8T2VG=;2itKX zL z3PKbN;ylSJTUT5T>(qQ)7AiVped7gEv$P|sQx-O>IqPTE4x4B09tvW=`GdDwKCnKT zLO4T>U=RtY=vbEP#ug|iHApO$&#il#M6hvn`$x}F^`BC{k{~fIM941m>v~@$0@ndN|BaCmzdg0?fQ8scYWS?5aQToRJXI^NJU)Up)jdS8%T} zKuoSgwMC(C1b@y*yj&7dr+j~8B-O}r7CGftH&4hEqE@XVej-^CR~S;bKsfJPcQS@( zyjT5m+1yF*=Pb=$t)#lV3ClUH7qm8R8SrU2M|kPtN!kdR{r$D!{hplvJ+;5vrOF#L zt~qkdvq}C9cVnpTmPDiLyStknDUFew#K%I{yJ_#EeboZYgp-9soaHiiLTIgv+d^*i z7^p%Him$)bcD&x&gE}f+i((E$GAX=*@LS@?g2b@jJ-?#RyY&puZ%+L+rng&beSd1v z480X4d8c?+eCpU!@)WW*;d}h|a(n1s@{yO67ii~c%E=h_D5?Jp_k+R)dWf$f<-fmI zU+%oRqDU!tA?g}R3R$%y`LSHZTy7U}BQhBE@#}FIG#-w=dJn~Yt!2aunr0vY^Am>P z3A;8H5Ne`_Z{wiD$(rD^6k&?2=wbJcoG8xFH}upj!x-tAhZIJgP`YtlhS!g#X>r!d zLT0){LTZNj6qyrhH>ZXOMe9|bjb-4i+OLfKV2^6Fd$F6=nj_VV29)s4!Fng-kLB3AHFx9)QRFE7V|Cglodc?BNXn zikny^K};ky(R;82h5wZ_{1$mzAp#~TErh~=P&#}qLg4p-7T_?%93Pk)+)prkHd2*Jb6sz8bozNiMP+STmIx!vw&&1gl(9883WAjlOf||5_WASufz0&5y@P?Kt}3(W)aRi1dh(thJPPUSl|7X&Zi!2N;(|r zVPDGF>%D2NsbguG-UfGayboJ0CV<&;CdRNv(;5PXFdUqd7C`Z-TzpGHc^1(`V@f&y zra%mt2%*4%14Zfly)I$;ngpTSBvVp?bqd-h4H0Lyx~IvrDrW=>eG?MBv0*}K7rh64 zOHUxFYqoR|Lid-zuB67qfV*f$^HC{6uFLAJ3c-L*G{&8$>p{dOecp80k$XYU+moq` z_J7eF1{@Fr_kE)b&m#vRBRdR*I5m;{Y_!lSa_v0lY7}L8j9QDUNvhr>Wy!vyEF1Wl za&v23dJgI7kplfhLdftnjf25m)>?Jk2M+on#Aic7PRzVK^slkzZtE6l4z*fdRyud0 z*Nh>|L760e7oS1yOt~$TZp>M~CE=bRl>V~|j^gPDwfK+P$w!Yb;9A_#cm0kiKP6hg zCkmZ-y<|T^E8f*X?$^rX3nrJ)b0)$le((t8X?D|(_DqQE3CK6cw9P2b7E-jutsYTI zt@B5~U}Jen);eAKrY$vsXigw`y6<;Zk=g7MAyC}B-re^8UqUV=MR^3Y(o02dz@P(< zO|qF89booX6flHZC&b$exkB68J6Ms%x!w0&kM^oT#R-DD7jvMZif4a|$o;;($g3!n zR4$Kum-l;QL)~=GAYE6>kBOn1IL1Mk4+3f7cK(!}M3zt`6LjfO| z8*&AF-mWo-=|oAi(n~Z3Uz&vycT!!ONqt0t4^;7;PBhx_NlA9;IZ@zq-%waq7+FfF z&uy5!5grYI#|YrTPy|&ae0d`;pBgDKt%eI9Kv~dIYbPWThD1z_n#_Q)?KU-^F?|AcaZBrq>mrQu2RH4`hmm2Jx3_yW{-phCT)cR z*BfHs@5t&=;Bp#iR~#zT7Z&5^91<8F0x2P{1;9b*C=C>-2p#~!BULtMFjfVZqax{y z$;we+B@+e-cMy~}rdc`$M4xH&J`hfwkp1HjKK3O#iNAcsXa*>_XJ@~ z67@c0g3w9cDrBr5uISAC`=E@&SnB-e>ll&(wHiAo%#%)hqQ>i6CIW4l)L7iYw%UX2*No(@70Ct zB#anzV(@$=f2=HEF zzLyq&wS#Z2k^P%`_q9NXY>`pB1TPFeHoFjJ8Auv{yv6QaWcz4{fVDuPdR#i?7*Ky! z5x=EBvn-PJcM*)6Cpn2}qO89RcLX-}^>MB<3b`$DGgm=X>rx9g~6pEWzdyYD- zC4)P5y_J~Cis?%k@X1q9rCx*fg#_p_@Li@_YMTYV*miIwv*lmbJelYA%Y%i`?^J<3x|9)J8YG7+h+XV&x}f z*(U(-zo>^AP)O05T0^0xw_WA3Wqq1g-H8Xu)7XdX$!2AAByvoZ`?3w*5%v3cfg%0E|(f;Hvn=n7qV;(h7X=#@k39l57fyY{;tLQ(`dDrFa0zBPfEEtlUZx^V{axi zUy>#^W^sab)~o6Ui&)SiiAzMcHp!nR!9RkC8rDv`Mp!#xKD?n2ceiT(qE4u%rJSqD zMz!gDh-9W)GbKHs#IjmGx3%xAd3gb19^1HU>Cq%xqixluU={a!p|rPKQ^T-j{8IB; zI_JJzyWeo@9X#^exGt|zB73>u(RmAN_htvV4!->se}dNCFtY75A2)c%_$A4nM7_6M zr;2?$e^Ey&1Z*T1eDWJF!mb6w+{J6($rDC4+3pup*s@(go^pc6>t2}3+%2Ng#i>k| zDWD?*lnG5V0`T(UKizyP-H3->PcjJ@12N13M69Gey?7yX4w`Bxj0Q6GbA~#Z#S2HE{-Y%)l4MS zM=#SRV%WRKJn-A2pDwJ0ySh6B(ylY#ezrbvp*%nxMhaLK753ymf2tpHI6x8~IKV77 zv`)<|pc5g{9}MXP;GLv-LqG5a$?^K0V2h2`I+JRKIOW+`dZGD+{gZJOU;lJ*-3)*7 z7zS^V64b_xYL~O+5gd_@$g|LhSwoi^I|9%9D{^|IHb&4h)JpQu5*HnL%i1?OCR4ei z_vEQ{bVp$KiQ{rBY6~^t)?;?oA#3Z@Yj)^n2f`bjI=&mcZRYW3^2+4$ij+50wT*4w zguZx({U?9MgXF0IYc1IQO2L|eC%RNi(189>(gvJ6Nd=UoXdaTB2g!y)Qu83mM-%Zg zB%Pb2qmAmc8*kWk28vM4GKLeBfrA#wgST|j1}Mo#49IPe2wT>F^sS56&_)DR&E7cO z<)@>QOyr0HML;3>gCzZhU4^c-bEM==U7Ep8ZS?Z>H8UjH0N4#qwAQN0!cAm7Pt2VO zDRL)iXq=pARHrE-UkW7~-6U-oB&nE(qyS)VLt+a%LL$jg9YbQjWN=X^=|tX)AAe7N zVY^Q9AXJB}Vm@Uh8eeCS$U7N4c0_6wM7=aPRRo3j84?>0*1Wh!904WiKbZs)iqdq* zXRUqxHk$P^z;2-rY%k_3P-Mtp)(PAR$?^=5@GMa%l!RE}9oVI_7S(JlGPb8p-jEEo z-t-YET!bqu`cIR@7!o%T+WMWuD1I-|tsz&j_9fD7n1_-)I>2UJzr9j+%twLr4|1~` zd(3Qd?;Mhf$S(QMkQ7wKtw<~oH9Min4f-M0Zgu$xxu?w%7W+)EpLWMJY%Y;(lP*o5 zgWjyXWKFstAwT5T)BrwIxRmXk*B>VNQU)}zu}$QJ~-Xg_JBcS)CMZNv%Th{ZDrM>UsBzd+gsFODuFOVs(Zh%YyGYUa3JRAK$)dd`Gl!uSHA0 z;vRHz1tOmAQ(3AVcu$l0AC&tq9!_X$l%cc&RRqovo=3*K{LqW?| z&rP|BW|xTfF!+x$Y`7S-P0L-yC;&VVa5u!!v((D;;vT`NR_Q&Ro=QUyou=QWA$k_A z{txOwd!T)rm%+dL6xXxHnB!$!ctBvZN9eKl&u&}v-=o*L>ZTc+wz1EMQue*~DaCD} zDd+nH%k?7j<`a_c!Y>R>G@%|6#VNb)3`N@CZ4TNyihi@YEv*sDi0py!z7in5rU~1O zq<r{bCCltT``^*eN253@S-^&7b{R7s61ohT|bc zYx6ZNQy)#D7ls75JJ5w^(5pG^G2$;4bk_XOiYXtsvl8cMY#!c1{ne6AY2IW{oD|8R zmNKi!zqC{7oZzJkPHx{j>w9r#h|}i316?H2|Ip?r74?;L{6zeiQt|<`+MC2A%KA{l z@JqdWoxp_(ou&?)sOjDbJ>D-&;&}!rYqYUV$NgN%QitjXbWm~INa@HEXJW(t5STws z{z7@@4pbrJ;z^Q^diQ?mmy_hT4?5Ec5<{SfZPhxLzn z8nj=XU+twGy?SsRqgK7Yvj5!H(Zby|pCop2<$68}3O%MI|3cN{{Ck`sp#fFYd%y1L z$7%GP?dE~2R$PO7&y2dw18d2$qkKhce1x~F@V95rpi$>Fi)Dvz?p${NbDmUx7PfQ= z-5Q=B*rk1~^7&c|z(c_S)ITydwJgIel_+vnMGVYnTP>DO$aXZ#Xh$QFRbij^VBs@0 z3}S!>c7#UKM#vZf_!w;5pgwUhUNU~mEHxmK%qVe&w{M7wx%oJQapwji8vq9cNiiN zQb`dm)oTwkE^g9fM-Yg1u$Kf95eXOibEOD+v!L4$1sE}HzKWx1jU;z@Fax(tNwdx! zar?ap!MvO1J8lQSAb1`_Q>iQU86!T&UUvl6c^R#&tSWOEPs7Nk7>zOLX}5pXR*NH4 zmoQRin5`H2+bIrag3*(v8R=kWb_qEAdL%d;ruKJo;mX6&2mJg z>jDocT!33Z!?%>#$Gsj|rsLoQ0TsDrkLh2snX?!WewC+Gn?KR5J8QQt5TMA9VfQbN%@M?O9y!s$C)F{IJ4RCz4OW*7y`W=EbYb?yxF>=h5^^C{kcV zW`&3+2r?qMho6MjFS7AFeN$k38QUuQP}FA9>2vX>#w?!DJRTKYlo)bI{TD@yk2m64~%{!JPYHRFQOSN0ZU38$hkCzhm^Xg z@?3DYtT?`PZ!*dIm)EJOxIz>IMG(IlU(qBfuO#yi$6LchfE4axm>MPsSxfDtY0748 zM-NEIxl~07w~L>!E>bbBCr6viEFc@)H%*|!yETB{XfD0fwm<=_H~wbMuE&6#{ynR)ArX2jDV zd}X1_SmkIJh?iA#?V25`(~yG}j{v3U*i4wYN=zgoGE7}M7%A3@N2$s<3a3tqMz?yG z@v#!>p4^hy+!3TYhBPGQmw3r+0%ZgiDx8G;3=bNe3wo#27`eI1m3#5?b{~G^>Dm+x zGp8y1anJdFob!An#$NPpXxjZp8f1Y}+#o+`B6X^8>_{D@@cabkC$T+Ylpz6Qznwa- z7*&kgXeqhmY&dO(h&e|kA4p_E*`8F+oDYp4#c`Dp!0+Y6=;z%AK+QXNm_KSUUn>)( z)uLiup{2iHnXahwir;kSMXkd_)7pkURx`=R(j0k++QvNw36+H?t)fCQ&or>Wl$~V) zl!NM9Yl28g#Ku@*D20|6g+zSS91H%uGIMTut^Q))aU^BO=h+-rs#KXU=!pKNTgKf_ z<#U3Z#Vs#o%=z8K4ZignNVD@2c;_#UON|R8suDqcf)7w)pFE=v)EiqypF+3EISs$) zHCDT)fWI)}1sE)oO72eQ?AuigMF#A9oAWV$9`;m&lyY%>@NWQjz)VRsh+o+fKV%Rf zVbn?|4^UCIezQ?(2!!ajxsE8GF8zExxrB3yFaFkAB{pe3VN9R&RDO?FD65yZk|m*v ztit}?!z$|ujAy~mhbs<8vJ-YD*w3o*rNWY=dSfRMl}+_WzvsZ#Q2V@&-Y?)7vlt1P z=!|Rmvg;Ua5-536GSPPSX=t=0J8(uJhggFps!y@~wx4&gF!ky7nJ)-Qc6}tCoXMy* zUwDyn2e~lS(ya3K)k6U=qK>Ug!wr(co#buCnf&dskH-5)%`ORa%E!fPt3Olfh@Dsu z*4c73<`2}E&z4&BuexO?%!ff~Wjw#)E#aYq7oZIn_J6nJXPe=APS$8mdb#zF3Ef~} z-t=UX0cMqN3i|{t(3s0a9$JTc2_Cq6*KAzX*>&-*^U*d>w#AB83rrn_u7us5#~w%4 zOq2=q<=UKrwu=4JeO!X(B=RtVlWVIM)YG|6!8Cpav*0q|N-@e#!!ry_DON`1I${9hfkGSC6(po$C|U?9MK+%1kq; zB26uvY?xP490lJX{kSb;_O6ejC@NdJk9UX&mRx*iNZxj_H)x?-%&<;wgy2M61u0hW zPBjP1J9H-x9Q5eUWo!zLH*YvQ7xFttMoO%MYa>&Jl!&;v^z9#|LvhtKQFe(KL^SV+ zqH8pyrjaz4YG?EIs6@4GF)BUOZ}zS~D5l)RIg@?6wf-lOBIl^M>K9pixxT= z&VG~1L>SjAz>Ll1|D2tpJUH%NOE+;^UFpyNkPKz|Nb>!T8aRv?;y?>h!vpK4@w@hY z5zoDYWU1zF3>YH9X^g(P6PD)^Bac0KvlManiCmtvy(TPsKx^aGn-Ox;Aq8CV$aol9 zh^#E2_f6aeP5P;5UrKbMyj+lKd9e5-;z{;5o%9Z7Etwd#sGj`HG<0??I!BNu{h>gZ zU2z`a8{k~}3nb3Dba?F%U72dw2OT<#86Hb-mT1@J{+@9gZ*BHdhG#6%RDvoToU$dV zveMbTQ#I)+>l2p#8s+b_PnA=LiZrY_%k->u7*83FMe<|jwVZT%%IxJ7vwOngU%l*U zbM?@C*F52e;bgtZ5=w#h@4M#+ zy2EKs+3te}a0{GC&P7pUS6WLneaq8%ECGFL7G2Xb^ls9f;TDrD(o)FgF|F;q@mIHh-9Y>Q3hV&8(@+)HY{*DLd?9!Ap~~k{!!Pc1C#})rK|-_=HFVwv5+9~hyQaz1=e0(9m^ZlW z8`ZRmh1gsxW`m4wPUa#8E3Zh$ajv4AM|Gb`jHFMs8wDAiH?_x$W(79N%DPm+3M_mF zvg|g-xK4LE?l6O$4RHzD=nI!cb%b_HQ9sR#jIXu&k=jNdXY^5xnLl+GqYGcHna{)) zE+~qr*{Y77uaWS6%t`IdzYMCY)F%9`rel1Ye({|t65bn@ZNxXrZ=$BorALdohpy*C z11B2OR5~};!Yyg(EC;XiXqLFjUb%=WjngiRC=j*ja6ejNc+tBYE~9Cpp4}hRGJp|6 z_AR+i--B%5!!}a8na9uSE?w}os{fo1vd{ny-bSj_$~flDU^LH-8AO=Aaj7tTzf`{GvhrS=fO8Fw)o%-lQ5r8Tkatf>~@}C8#JJ>&LrR;)pG^S*s4VtW3A8>RT_j zS@Sl;zL>D|^#j%avJ7Yh1wpNvg&6oabxtily!{3$7z4Gxu*I&@%fgPRe-<-Z zA;pjtIIds60QneV$$!|fk8cYSZ@Kes5{z5Y-?eV4OHzzvGJ1Cp2ZU`0z}NWNw>;q> zk1IQ&j2-s74-wq<9JY3W6QB?P%6#2Y<@!0`S6px>*540%+c%Hs9ht|wG87026h8RF1^Vl@YxHav1pvA*I}ctE;r-2kL6G7~X2?7?@Wg>! z7$i;c&~_CM4&SqVwo7fhhkU(D-0r|GYKCZw3mybXhJbv~ATO}vgN*$r6n5@l`}?+b z_92uYat@1A^C#p<`iUSbdXURqC*zP^ufa86^mY*MV?Vj*O2YT)R1Q4{pr9;}FB;@& zzUy`2#6|(~G6(rpf`a{c#;hcWBa=cVumP|y-kp26ml@7f^~AQxJ08+l!VFO0D(Kz< zC@>kTst5{s=J2fXAV%Zs?NoS99Qdv}NBm0(O}BjscTjx{^5z8@>3_}7I7pv!_H};f zD@>WatTMNy>>d>#Snn9U^5nITi-zqJ!pC0%51bDhsk+`7FOkIuS7Lq9SdW7}>brK{ zaGRk$%OyRV-8RrITfP0+VOu@6v)l#QJ7{cUy;pf|H`6Zt>_? zPemtTwx_=lkih&5ruv2ivDp^{$=c-)Azg9kaG`8rVNv*qgtuE)-Z#G^kM6?F{}dJHJK@ zUZz(by}R-y@50F)jD?5%7|(b%G48h3{v)vQ=;-y~kMjQken5f0Tkyu^bB^lKPH!0X zPQ$rU_k9R{u!na*2jK2uRc7TD4qI3T4C;+vdKhO=ws0OE2X%-Cd?>|EIHd9vSn|k+ zeBNht9tU$EVlY13<&6#x2U-x%;3~%7@y=p!&})LOY9Hb1!ZwEy*J80AVs^0c)m>uh zpx&Pi+BJ6Okv{NI4(Nfdha!IwO3ep)cn5Vza{phx?^XU{c#sFfUES8jj@ez?dYIu1 zUWaFnW?v?U^OlE7R_{ApS5b&hjVC$dEoR3+4Wxkbw(EMVK4U8 z_3HqUln!eM+k!x0Duqx14KA!EG}aDQgzmSb@c z?Rm%tsPY9c^3yWKgLaBDeVB)KI0t^mV*d;$_7=v6V;2w$p5OYtYhV}Qa_8%Aw~Gof zR3Z%3y6A^`Fm)al2WTg4inkBKHf)XeUs->LS!k#Zjos`50kn zHFgJepzj}^^?I;(`{;)W|7tB3Zgpto;_i~ucIcuX_nuFOcc^s;QTTQ^^!*J6Oi+Us z(1R^D?{*mGppO#xC3-}L^(-a_bNBe@n6pTNp?KkfK`5n!*avz5dzcsC(3a$Rpz5|? z=7&CdAD(-82zZP08b<>FP^DM7;0Jj?2f_aajz9Mi;cBmrdjM905CDK3ume<};K0BK zc&P9TKL>V*hl1V{r5^Le7vXVOhyQtCdl%RF(BJq}C+%rpl#d=~bRg*-#(l@X4}Pcz zcQ^;l{|0X0-vkf-y5EzU#(GN^;BsIGX;+WiOoTF6lMir$IIsj-s0JQh>2zoZc{qGQ zIc+f~bM60MS*LaD=OwQpyl@pykKl)R00?v61`;edklZ?X`1~&YU`T^0c_mUc7bYUdDtu zZXG>-J(uRp7tdqSnBmN=gI7-<)2v#za)qd`9z3BRqZ$mCE?&N_YA5cS$BrCXnBvZ{ zi)U}&+Pr%AR^*3op1X8#kigI40`@acIzGmD`LbCJ5sgtX~^_38B!3f5j5P~g9qt{P7mVy({TJm)K%{lbEtvtp1z;kUu=A_OPKkW<}PuAFXzH3qKe4W)kn8!rU zJENPg{QsP^*KPY&?cd4x{BY~@&h*NGzlj_w6s>8)@TSAPZwQbl&{$OGkfDvW#6wW) z;L1KEN0N5PFC`DOmo|_=kq=JG9`+;1JG$aGm|V?p{fh_*C4vlUperBkI951%rWSFeF59aiYi=;S~)*ffO8qiw}GtMJj=rytUhUI?y-+~1f(E$S;#{IAdwAOq(2&2NJv5?5u2q)1d$IxrZV#gNs4+4ftMVFL-HntlzNMqqKd}?zo8CTq9lVxkq1vJ!csl* zks%9pBr9ktQhyA!m!AwJQytj`TwwAcbN~P`#`#BWdJ6_w@T5c{0RwAhVNn5fjS5J(KT;(#CARK1O+64f2Bmd!D@ZRwQ zN0Mhjb8;$7?Z~BA0tjPj-p%KS6 zoEQ*zy`*pbaaEEU830Hg??!VnE`yANxIJwZ@0ilgEL1lVbQmiFnQG!D3$n^r1_U(5 zT;?-Z^{R_%=MtI>2xR0f%?$zG8^z1!6iZ>eZSXJ*kDJ`(0-(A6h=d*o5ehtvwYleA z@weDJUiZFO&?gq8HrQxNY8;o(Eq*T`(%ZxZlEcd=!H1wImz52d`MP?fA)fVY>P@>i z)zhstqVWq=Powy_+599rm)vB$<^Tob#lv?25X^@_+6aJ%gd|}=L;p7?S=eih%ziC= zTIWKyAwDiA0P1SmC^JN_{^;+21N_jeyMvlPlB*rtYvA8jO?q7HK){-T2 zb?KaDVvpCnD$eX>PyJ$tz#G5<_J@4OOJDnXo3!9|$a7PBVLD2ezz%WkLt^`yip%@H zJt>`o2k4MPg&e(ObkLUE(GxsQTN^O=z@kPT-8;Vas)7_cL(J(|lt<*eSUu*q9CFog zOXLHrWk_tFn+b}Y*KN>g$MbnWO#J>=zTl<>SEJ_kCv1+L+uOFQlkQ998j z;&fs_-Kqefx*z{5wN?+HG9bjGbr-{_J>J&gZ; za=Hg|w9}V-_G7zs#zG_qLLMawvh0r~=(iybeU0XE-fDfDY&FCz%@NWu#JQQ1y1m}* zuE*=6-UWN@IL`^sBu)S7E8PYjH9|)=+Ma*DcsU8dn*>fFUBB+~M`GV|`onUE@ z%w>c=_(YW&(6AMON83NuQp@8m96G@p96*2_> zY10z05#J42@u8al(jP?RO%4!)K>?oYaRoov&pGe}=$M0U;RNhOQtgpoMWo`5Jzzsf z(xTXz2}T5@0if|AR~C*U00bXHWI-)bTvh!UAYRhpNft3J5+kaYBWj-+7Gp#>6dr() z;`tpUGKD_8B2VnYi=;#7;l%V6;w(XrrRE|y<4+F)bN^IC9 zmH|m6;11fN7XITz`U5Ybq%l$>8De8NlA9Z1BpN;yLMo(0`k2V+;J}$2Jf@s%StJ}` z>>Ga7FX6{ZaQXA66P|xpL9NiGGLpn-?Jro2w z1XV^rW?=f{U@}B_+FotKW_pHUbDrfj(%wwQ<$_k-)peX#{bx1}f`B^Y2m+-x!j^2I zC|%}-Y7WE&MZ{9JhEp=3%RIz85dQ=@FvTTd7|3bpE*XK7+MsVem;JdGC+*izjT>{m zod}_&OseJ8fe}RI%{O9QHDMGN*cJ+;X=D&5_!Wc;QA9oz1R7~Yl4d9tMwOF_Wn@I@ zBRT0T8G)6eRf=VTmRciuQs03-M4b{M;pIRmUDDQVX437aKWJ5JpdK5I>D^H(SJ2Lp z=7eaZgIctcL^)|Sp}>7z>Ew|nspjda-WP5<(r-%Aeq{m6R90)u5%3A+5-w^)pa2-M zSn8*G&+a(2Lc}Tf>zP8tTFR?A1yj9(k&Fr=54P58 z{b0XZQ^%@BkrG7K9Lgd^##IWbL;%d3@`OXdC&MynRAqq_j1>xy-E%F06cAEGt`iRvb9$oiaH)xUsY5J5c4xNj&`QR%aTS{?5v$WLsY?Vv(!@TTQN zaIM!a;m5|W@Sp(wTHgIOY{#K2qY5Jj1Mph7N6vbYYFH!9-dsb7#@b;7;ZmL{y{AL8 z<-*z2BE_!I8UIjq)~JX!Vkbzgvqr>CUQ}yvp6;=N>>S7bq@i>SH<`*Hv9H z`yXpgq$10us%*%m!=CT$v3BbT9vQp6qsAvdw{F+Rj|5oZpEF++S;M4((7 zFVY)l>?5O_6r4!SPOV;;X4#?y?VzSsnDO8KqC57jEi1751?wX&RPgyRx|)`zZf!a} z7Zt*94GIq(s2+?8XKL{<86%MX^4BdV;|I8%Lr8HpuSGu;1i~`JJs9a)$no#dkTgqE z83XYx&;KGLGpi9NXylo)68A0TZSxd6#0M|4lsPlY+;2LwWmVDW{-r0F8uOp!b61=# zRX)U_utsE1rb zU0a!r;Ry%a37<%i@`OAPL?Wkzy>aALWHhzWXGc3ANZXW1({yDD;YmxAO2aYk1*A2P zB~Z6kH|H@vvPSmxu?}X|*0J*Baw=E!G)^$|TC^io=UBd=OKAqM7z#%m5pr!udV@R$=TtS&Uq(NOQ?8&$J9MVl5H$dM>F50i{W1 zq)z*23`6uM+H6Ez%N^oGkEw8?=qe&Q#3ccs@16CBaZ}}mmoK+*aYjaR%OGpPKzb_U z9I-}0O40-oSUX*8Rv_g!5Uffd_!f7?dAnVD6B>IL;CmY!(#>~T+qW|1HxS4NoEL`Zp4#5@p0Ih6H*mULP}axSu)j?-QxPyYn) zIj3O%-X=E$7+tYHJowf|I5Ge$^BLX+GsK2dHfBpwb3t1Ax^iCd3N2rRX*P6MRCspb zvP5tzXTdiQrZSLafzhV8M9lV$krLp6uMtF3@(o{U_vdOc>;1~qgue!CYZE(7pko0z zEi!0UOzS{^15U(gqla@wK$?f&9-zmyd#AaaE#{y^(zjE2yhz4ocf)&Kh!#Ij>juBp70E9yWG zo6wKEEj49lGFij}nX>@`v~hiX(pNXaXt_@65;`~S-Rk{Xqr68#vQO%pjn@PL4+?Z6 zAdpfgc6%FqI{*~(Rsf8s;zWcfI9DXQz!Q8U#{JyaD6^ORSV5G?^Z&a5`a5ESR5krQ zj|IMdA0(f*Jwv?xvCBQZqdU@ZYVI%-$Eap_o)Bjh*4>{-{`{=6;jE5*9 zS$tIn1%MA(LLIES(IB)B04x@z#!#wG7|1@Ln0pZK2dxI>z7*Usq{?)IIrbB$a-2TI zh>hH2sLd;`q)M3{416%MMalqh`9mSaVBLs34?;7mjiE!Op#9}I&4ty+PO={wc_kJj z!_fv|WHDGXb0z@3GN)iI+fq-aX0wxJ3a9)wE%F<~g zWU_b*fF%<73A^PCJkO(j#!-nJk4(Y9pfVsTNJM|oK>y>Ta#Rw>u@^al#=VYo#L>qe z5hG?tAWfR*J9u6+Ql)bCAu`7yE$Y$9DNoYoFOQ%RkjgDztVg7g!fEn6eZE|29xl;L zlS?T*x>C(I1)HZ#aBe)0p6|p756(UL984QxSW}ZtKZ!9($Taym=extsQ|!BV(zNoU zLHqpiP(h&-4A74*-7=rWRML~96jy8!#~44t=ex!#ZPis*l^p3r9KBO1oO)i(Ro7j& zbmzNo%zN?9Mfu^?*kh5UlAn~y^Rb;@+36?QX{oLDym`X;jvQ%4yysVSinUhUadnNS z%x~QJh*W=Ae8@!+QT-^M?`|d6US6M7iOwGt{r^rKG4BmnU=i&h=T}I5l;_uQkQ~_I zhpB`o+wbH7lAT}0K^WqVIsWk;bABZ!+%6ZcxZ{&i_6Hx3zgx$ob=$4z#dtr;SEXoE z&Ka?MXwWqRAGbMedj)Tf=bRWiqNAn`GH;mR>D{ndWr5&b)GG>*LIRlhb}_8%Kx zI-IuEwL9i_&!gESdd~f(9Ia;$xSw}87ytfqAbDQ;8+3>-Ui!pRcYYnPos1{>ksBgv zooazE(caR;;f^FKF-`ZV)vwAyLN!_KhvaaF7`d3Te$>Nr$nW-^vKCaM)FOte4X>i(GGjq z@|FMm}!FLZb&;GxYL#qL#J?Bn|0u}*usiXZe~2P>ngMPA{f9awXt zJ8C)5s@20C>BM3?&TUqktm{e@=iRyd(1ZrdH)VX%CsZMROWZe0l{6tvZaPQ2Sfu&)G6s>9_%>k!qy=w zWZ8oq4mILOF)7tu9p|S4t>VZ9PE$>K2D2T5%mKf@?eKKM7yLj1`MC)2rgyO;SPF) zRJY2q2Rzi#){qI)Ui*kgQ=^#!W&*vJ0gm* zeavIYUJfTZ+93~naM!T=u!lU_fsP%2^|Ct4uyM^St8}D!%{Jx;KkzY+cc`NrtOX-G z>MZ86&be!7md8EVJW$?7IJ(ER6*}NiU?%~&cU5$fKH{-i@q(knSTQbI*7g7P zaS#8hS}+eCkYlthVOjxOvV&Zgp`%`{eJScA;y|-iyml$-u4o z>c(LXslOv0^teYpl8#|$+hZQ^NI5#nA$J&~c^)vwdT*I7`VE6)93Eds&gD^$d+6ip z1LMcfPagDjfWGUG$C!+7J95mq8NB9jI)1}Jj&n%e9q|A;K4@R;edNO)^pM9pgbsSu zufD_YSgr>l{9mF6JM!sRNAsV*58+pn?$veic=j zuKsQa>Xwfk+yNfsfgbDuAM!yT)bB_5VITA%AMSzk;sN{CAsyyV06XgFYHxAe3?5W& z`~MP79LT{O(82uL;T;f=9%PRJ9grXN?_}FxX5G1y%3>olgPp0S5O$2A#0dJZo}?R9m1`~ ztZB>KfzUWmytsiIMgkm~5FOOv^;j?p?O_TVFa~GvC2CLtv#8t?;n05FhZt9(0fg-N6SnFbv0# z){5{0;pFZfBkx>A=Ag?GFHyY0K^(+k9DI=+$e|p{VHkuF7=4i!!=V+;EXHih2>t3L2?R2&eJ!s*x9c5gWBp8l-_6 z$)O#`;TzNO@sdy3n5_@fDB{j7-B^nof?*Xfp%2tR2~GeD;4l~0ksaMp8@I6>vaua` zQ5=%-9@R0~z)jpFQISZj*SHQNF9C9I`PY<&h#S5*(K>9HNfT9*$RJ5iw|y zSKJKc08%8K#~HAIC!@*A&X6BH2->6#-TDnE%?uN&K`9;a9Mqxt>LJ(s%p!d5%?vLc ze$6G*5fd=MDf5i=ByZ{*jwq$-=jI_BenAx|VGqCn2twdBpnxR7vM$xJ{Qt<(9;mV^ zIU?UW4=H;u9l4O#&eTcsuk<0f0o~lk1<6o>#Ps^5OEPrv)hzF2@EqFT8|D_Facll zGhYN5UPCF802^%6FqJb7n@|N2upXcfHvLRCFOVHHkQ~I&%vPZVLVzxpQ@)Z<^3Ja_ z1h1J z(#xCy0D!>D))6Jk0R`)D1zRu=G0#79rTr+d`sCpPQ&AmO(LgT}LH{2jGZ}Oj9n>5& zkrU$~0V(tWEtE@QrZ|@%P!8hu0(2e-(H*?dLxIph4^$}|G#F8o4%b0NJ&`|Q@kNy- z3%8I%4>28{K@1*Z4Sp0OtIoOalev6jo0{R{s|VH|~R2JBAt{0$HII zej>F{s)OSbRsmAR9Q`eS^owQbOJg0^R$Qc2C-`+`sirCt7G~ju7}~%lI)NcTb~!kf zW|b%uMpY^1RX7RuBMjDUa6u`k)n}2V6O@82n09LLKwP7iN_>Guj36y6qAFmZXRFqO zsG(tX0$WE`ib|H?z-MZeqB_1-R|Y~Se6?+jra~fOY5(Q+MP6YbmVl=i0uQ|QZvRJZ z;r3CB)+3HqZ0I&3=m2nCrDc_ZY#DcBo?~qzmpe{jJ*0qbE4Q3NK`Fu_Pz$#s4wq_b z;x#^CbNQrmUqfY4_hGnIDHgYNPofiYf(ryzcCE=N68CSbGITj&bf@NKi_=|kcT09w zX^U52g4QaM*E1r)E7|~Zn^%z%A`-&(A(jACeb*v@*J!k1c_X3?rguuTp*SCcc*l2I zPQiT7R(7dj5K!O){5E}22^mm8d<|l1p|))Ard&;ScBtVL1o#vXc7RU-ed$+3{uKst zzzASqG^8L4xL|)1*jR|cfTchR3|M+m*Ix5t5dU7bf>}r#j9^9?A{0WHXi1iDXR+Xl zXMIH&O$H%*W7uBG04k=JE0!Y?Xtaivh%9)Qe|HD6-Uo(#cuEE#ZjabnZn$|_L2tEo ziIvD9G?!Snmm#`0Y>Zfne90U!&i7>pyRCMft8lD#bqSuaf$bRwnTie(m-gs)_*pDCMd2tsH7#R;TffyPY4}t<08rfZO z0T8la3Fsha4`L7^84nVH6=)P0J{cM8wUIj{5+ZpJJlT~?eiwTxNA7s?_F zXw+!|fi<+DDCod4MAZls78+RO12Q2IKsE<%*B87Z2NX6YKA;okLL1n15VT>SA3~fn z1P0vIA#xy?Phn>{7#Y0c1NgOFKo}t&q6|VI6gnXijDR#$+ALc7Azs=aJi4HX;h+6wLqm|owxkdJ1b zzj}7>d2@|pKAb{{{o!8^;${sbCjVd{Isib4<27AxR#&?Ma?2M0u(f4BKw;ShE8^N= zdpKSRn<0cZM2Pbtu4FyL85wA}u`S}UYxS__dO7SGuk*toK(?+Af)zf%ss(~|4Z^Vf z!Bz!B8*<>7rz8|Gmxa$dd>rDb1$l*uIDE{ywmqT~P8WB-7dDn!A~b__*B1tQ*dL}A ze``XCANxwCmUZ#qE;!;L+(TN|L$uk&2v|2EJ|I&dx+AI^08pTZ0byqagd)V*9|X3% z&zm9A`yuusxtF`S8w9!`+aRdHD`J;2VEZ6|n?}o5SI?sp^kPb=ArZWy3~<}G-$#uX zcC6!Bc6uAaEuwRim!J(|xc~pVv`YoG)#4#oCAlov;(MQCy#XMKsXA3FWhfulcC;~|j4&=DN~aQtFPyT$!secuHc)|U%>ffy3u zEsUTHDB@uW1O|+t4gbnwvAH0)apZ|@B4h3Rf7p4s-C4KKi?`t-+C2i#0arG1HVfz& z0KWaw{r4a~`mN2P47NKFdRTBt_0gXk#tnj9xxB^?-G8AW3yfgjJH#R~ozPt*2TnoN z4T2(?`hm;1)B!%V2Oh~?ynP=&Eo$Ne)O|VF-7}P9jSb?qe?ry|TWjOjAc)flSi?9j z-5(Iv)?Y%8Wg`m^SlhuTtmWL=%@usO9m8va+BY{|r9d^H!6AA+-bF*)IpVK(V<=QN zz^66~7W%;-KD(K{D9*y(|GK>|B9QwwEQ%s~S>YxK-XGYT)2E!dwSGh1+Up&@ECSr< z(LD!}e%sC4B>z~%O=9Aki$Y-sqAeWOAd=QV9OC6&f*9z)obkZ4VP1MTo?_>`h_n6h zdjbG-zH%SimhWBG#~$nfpx-A}C~IUW?-7OMSH; z;uWM|A-+EBJ6!WK9rgi0_G#bYDMItD1I0VvxH%$eMYbbAc1Y@8Ga%$5^!{Pp`&kXb zDFXZ?PGKNQI`SRAcqF`I7a!ZV|M62eaD75+01^@sxe+;Dn2*og>wpae}L}DnAph1KR84d!F zC1S)*1OFMa0$_w89RMhxiDZZi0F0L}0RbQ;k{AFw5m)+Rm<0esXaKmt1TaYCM~XC? zQcDD*DAT4?t6IH^HLKRGT)TSx3O20Rv1H4BMfQQG*|lujx-Dx81IB~^PK`x_=B?hn z5l;!_3plXg!GxQ}bXhpD;>BPy0r+6hpWluWqy7|`0|3*LDGx3Iuq1Lt0Du@$%bWu- z!-R;5zKn^~bks~Q@c_6;lTM}f3P9%>yMGTqzWn#vj4YXtKfl(va}(MXi zz4RG*xP%BuWYjoB8%JP)omvfbMin)g7y*D3p@{Ziop#=dr&VRNlxLrO@#l{~ z{@s3j{?I@w|r99-Bc)6 zmvzb#h*1*>oJ(Y~HQt6F&Q*Q)A#P{d53hZ;+l){C5>azgj=5WRQ;v+zUH@U6-eY)N zo_Xnra{9ajqvdJwGQ#vk{f?E zHQF%`eN(ze?^GrZ!N4B%+VlML_U<7Mx$?4mKQ%&=l0W?P-m@am2W^~>e*BOUla~C6 zfp0VslI+ia|Nj3EzyJzxfCMa{0R#93032|E3~ZnS9|*w+N^pV{bf6*t0EY@{aDyD| zpa(w)!VMw;2_Y<@2|LIKD~JF9I7ovcP6)#o%5a7>tf37zs0bF+;D$Wx;SAXzmp~D1 zEO(%R5s!$(Br0)T!>J?4uw52uL(`AOkN5q#+Oa$NoIdc+0aN z&lc&(+o^AN(8$DRP%yekYI0y7vE3#I<1p6=uaTdOm~cvIx=F6iO%8D%{7?zY2c1M; zu?$Qop-0ME4lG}~q?{@-M~zMd#eJger7;CWKeHWEb3}FxKk%O6elP$M&=J>{$C=>tK zndV5d64BlDqZ9z(M?X$_pd6eFrWMmD(zJ1kHiXCsgCHGEf2x&a0u`mr#9Tk}F%NgB z;~eA&$2Xoz3IIIysp2RHI@SS?dgx=Qi{U9(7iQC)AtMt(0ZAlM_SLjf1)SnkD@{_0 zH+{@w9psp*UF{kSMBH_+;5dgn>cJ1U?qvzL66|2KK@Dht3>U08MJPIviAW@35P$## z9_T=aHjs=9oXtWOqyXB`<{+G*1+8Z}%LS6P0ke3h>>v^miA;1N6sKSXE_^`@Xi$Tj z`r#N%0@t_QqaEYet8^2rt_lYB$wC?MJBUbh;uNpQEpOifjZRz&C4dO2cGLP1pBgr~VarE5oElxehC>|T z07r`VT8?-G_QPJ0LUCv;5^W#@7pKSsamagK1+ydvnR+lFBAkg(u;LZCAOjT2dU7_-XVgM~cUR zj&{hy9{HFtP5QA9eAELT>o^CBp{tI3yjd$*xc~8RcN~gH0HO?1$U%Z1Lu4+r0T2={ zvxS+)T`GT9(rw^EmoNQ+aF&77p+MPIkfGsSvebll_CR>^`gVX9+RWMa}%hzea$ummp9VGv|CGwyWE<4g>K4yOK;wRQxE zOt7MJ(qRQ2@Wo~~Guy4FY3r_w3blG5`>x}S-O`nt)r-%d{ zvS97U7=a67ZneDS2WjGP!3YV4L9%i{2M;4$t*8^Oc6qI_eW1g+==ir__5qJy-_;!> zk4PDeKy&7bb@$HUE5ZibVL_+&SO^AUYAUnyWR$w?=uz=HX(y z_FK`4sfRgW%yoo^6_(IA#UOl67bCP`5TS_rNp&8ILA2pj1L49TPJvIUFEq|o$9K>A zp^m+-V;}Ijhn?>#kCB(Un`IwF7Cx1OG9=;@7yl|Ul3s>np&$!)&%55ond^C5U19q; z{8Y^m_W0049CvL8!pVJ1Uu2>T);0niGJy+QyHy*w$V9rVRfbH6`13{&r`Y|jF@3Op zs^Ex+fcO!3yQX8_53i*!BEf}S!JrJw(mjoN9|_t&h6|AxKJjY_@uJgs^aR_7%)dd7 zg9`}jc2x(n@nCO9QX>>P9S9vlQU7`8_aUJ`2bqCT!6ksbWE19ha>z$9{qTP10B{C^ z56kyd>~L!`qe;*Z2@Zh*WzY!$n1ECPYkLwWk>GhF=skRQd}^~Z=}>=lq7Uyk4~qr} zixWSrG!XWpf@FaU9Tg{p(}VUCdNs#>j{*-dXeaw%f#4v2+GSAj6FRnnF+*q;#4re% zQ3fJsg}YO4dUu41vJYAJ4dpO_4}uS5cn)voD&*4%mi9GWs28}<2B~IJawt6M2Zj!) zFwyr7;XrtRK@B4474l$L^?+lEkOSM32{3Ri>L(xU)+Ulj7SND_gfWQT_c>-Uh;9Km zj<`;Ecsqdb0SqW-d{{90VEg%32=h|rTNA0Uowp%d7pk(G5GA3+EEMIGhg5BD9|2e zQdw)?_oz&yxc9|s8*I+0y8@ez4hB0mw8{s59i6B9h*7-r)l z_~;J`aw{KEA*+dwi^Losz%j4^0HT47Gl7ohNE5&*oYuk+u_+R>xe!zr5iT+!;PgB$ z0w%*@oFY+|S+SQ4p%bPk7RX>;qA8eRa~juKn5Z}~_b^rA02Yh+5|_dpY~l|Z(TyGE z50wd$yT}#w(Epw^)(M_DJjeh8viKIHNfoCl5+Df?tN9TM1zQY(0tGP~4`Z7z5fm=5 znm=NotUw2s^O~oWjMzq;Mv$H3C?X;{2qY?@6#9;B=^Z&1Cr6P3+5k=;;AYgoDiJ{p zmx42~ahITzeMea{QDFqQVFc)@7i|Co<0F_8%$sFie3ZCpv5e0r!?0&DR*rEb3Su_$A`{KnA}Em% zvicQY_f+)oVc9V}z5}aVfsN`#oFBn$K2{=~p_`|WUTek_F;Q5)5fXmd7o2Jw3o;tu zX{cRe5IV7{B`Rb;v96He5AM3C;7St+5fbKT1SXmgT2mRjqNKA(8B2kdr245+;F}Fm z5Z#F;F9I0M5w5c;HiJ5XxQZvr`VEMPrES$KyLF>_nh9(P132L#pb?8;p{pM@QIivn zf}s;ez+Ym9mgLPZrqDWGrcsjE# zVgIO0JEBhO4?p`7L3@%?@G2K|5)0cDD?uWS`XdO-BJQK52H~(`T1;<|2m(s61hu0V ztDcZzdM(+pJSt>4LMPhr5fb7e)d44Mixm_3RPVsD{bV}MgS2h|r>0pKzTgXJrJHoZ z3JB&K2D&eOI<#mRsOXv>JJE2pFFXs#Yq*0TDlSfa#o8Fnp?KX7^D7>9mpyY zwpbOZTc0@S53v!DL{XXq>nJ)glXFNJaciDAFtEhuFoe1=rnhW)d#R)m3D>)yVj7AD zCZ@QdAbiplhWich@L?wrtz%<8FiRQ<3bDC?uD;TaF=_-TAY2bBw1xE<8j7wuVgH~Y zscTnju3e)b6p^|ny1D^eirP`Y^c%lNVyQOarVOFGQ~@49VVnqIwg{mn(2#p;>mr@% z7UrwGViTn!s458oy$aKd-@vTY>kp2Iq28elo1z-fux8yG5k=CxSK)_=r(G^`zGGtq z!~3S|8>j92600c?lTsoEK@zg)3#qmccE3&sn<%kr|>i!A+qm?O_ub)0npVsU8;*b;yC{5fP3-xNQLl^f|&zQ>#NL zti@O`>o8S_OT8yS#A)IWHt|v7MwB=l5ub5US7CY_Yh5mql9Z#SPCA;P;s2EzNxNa% z9a6h1Z6H~!>ZL~+3bxUt48fFu;FL0fv#o(%ASf&U3&1%NA~#mRtm2is!MdfJmJQLU zx)=$wtjlOQH?v76aZD4wdLk(BEvFzPc+4V>@ij$K8YDpn31b_id=|V^$WLRjh)frX zOfc~vUGF(Fs&Y9qOs{V!jm1hU^n=7#!4Gy7h}Lx&!?SUrOcv9sUd17&?u!`2AS^Ob zSdmax1zLp)vMM0wD7smjObU*h925GYDojynT=``HEYir(-|Xkj%+BVM{^(6D8{29C=kfQWyA376=mSNZ^6xFVApq@Ct0w@ayXBj8!<^p zRcCl7{y9}>I$Tb}JZhQ+s(f5oXcj-xxMe{Mn!?zW9WjS2hmMgDz^pL*K!&c@ClEMQ z=zwECg4(zP4XV-xVf}Vd@O_3H}AQR4#i{?D%Z9K#P2vK1!22QcWkO-fFJvvU}5u+3LY+i(7=C2+EYQ!G})2ZLuhVTt|%+~4dBqi1;P$@ zwX2JUjGt6VYzcAzh=<5PfbiDgUXIjr`=rgX(>&7;X8sN8;NtIb5Bv61( z2%5qY-1iET{Z$hYejOa0by!pH|A!B`(cLvhhjfd;#%QELx(t z>I0(&qJjbfBGTO@7W?u2{dxX9&vl*iJn!fAy6+Ma%)oL?zZ98K>!%($-+%|JC-{w% zL7M81ao!*Opf3#Cod_mjb>v&{PlAwF1~UWX4WD)eXhp(C&)Evd*N-}#~q9Px6~zbpS?;>$Z_$8xD)ac&`UK>WwJFqcJN5WOHH6(qW)L^`AFEjuDxkAD_6LFZ10V zFJNrxOacY4QH_ zmptG1()f!*ZrXbrsmm||qMf<`l>TKH@8|<{h5m#YI19V=o@=!g<~@z^P$xE&2UF>g!cuj(c+w_yVcKtKH20Qisel= zq((sD-0wy4pGyj1FKVc;F_~Td*XC|VI$Q^(cNA{>OP}z6>Hgc;en>p@(Q@wIb&&t6su{t0SP&4aOh+q9p7jchgupX9h4tO!>7Gfo%JU| z)b2%4sCX{+IVG1yGBjM}a)L$1N2YVyrN`ZwTD4wAQ@~;I!h5DMp1`?u@zQtxR*i(9 z`_tJUeK3(owD`2XEA(jZwB2E;>x%&LvE;-499dUBGE1KtQYtoSyi=EBt4g@2MZIAO zR7>9?!Gs31JNr6P!*i%U;_1VMtj34Whr;1fSl=;#`uHWvGVXK>3YtPQPbSAkdJz(^2n`s_0MR0JA!#d z3VMRaih^KGlz_vRqEeV_Z$ROX#j9JN-E9)eGC$EOO{V#-_eKsk;DFsESiq*(%9C)C6umv`*woZduR) z16RA5>hhIKW#|a>nLRZbKoHPsyr7p=5ZtqW%`Bczj5NTb8e6f5Bl~}a?spPdp?2Iw&A`XRxxvldqUZnhDDmB@`@qY8~iT~-$ zQPoQ8lJ*?Zz>vo0_%>69Uz^JO{%ONpl{v72b)NTJ`AXn1N@Bf|*rq#dlVRs+WMJWN zBTtvKRQPMlMr8C|qig*d)(S)KB9WR~V#BhcK>S<7qbM?BQN&F#q5A>PtJ2?luy0+o z`-QiIeP=!>Mp?7^mpr?RyNX`-D{<&xx^I{93;6)N{VH*#clUXJ?&DXNS38UnfCfSK z+iYSNcVAnh*h8AGpl^XVp)=DZ9E<<$@JsCLUzkeUJSU&iiOK%9gieGrM;T_kma@|% z$bRqBu2t9ZJm~PceK9nfibf>NSU%5c=;Kd)aiaTbyt6*+VlN+B|89ZF)101CY8&-PiP)@C&<1Z;CyuBofPtC zs&NytD4WlD)1v%ykOq+Q(iBpi`7i7PRgz;2*^yw>U_NEF#=9DALEFM)1m$ZJ+Z@fK zpj@=LVQM1vFl5&H3)UvM+zZkB`1_fCj&L{NRX~d0UHBoXeP9%?ZGHlYZ8P92W9tVC1XW2 zCsz=x5(S47!!%AbocPGJ-#d`Po3T%U<53z(?kbl@2tX*s4j7qcNr<=4R1 zhH0`-1J1M&oDaz5Otl|$lUsGCWYL#kss@x#OIRtjB&`>@uV-`W7;j#MQuaC9=^w(a;g=VM^gpQU%%x zrkO~to)QI>RVf$x3uo$;dI{qQ6Mq7WDzWhmLB;MeLtBq>9tVEFKJ9~Bw1q3RF3DZV z(ze1}IsWznk9{khQ3N)?i=^EOC7>>#s6wl3g5mq z6@z&x+t+{ae5qIPV1DIyhlA3JAMWt*kH!4JqN5py=ksP?_REjk5f2>RPHtA;j)^l1 z6$3kSnxoj(C>R}|q_+AIk+zq)bxI3KT>=%TR#x2t3Z&d;(_US*gsDQ9`lPwNT zr2G?dP}enYEPtzj%lu6GF01GfCpa1j7OCGgCBohlnAu*f*);WzRb(PFeR?9@(tEw z7?*6I zJ`$ucm-k~AN?t0SK_YYW;b#EOtKN=u!aQ@;RG+mo&iuU+TdWYtlM8p`K_V1-s_LCTYf)ZqhO z+*deIaK)MlnmW|_8?Ck|H!n!E;6qqweeCzcgvj3Qd$y`OaBexej9nw9&gKk6YT!(E zEUt!YS87bD`G_I7fQV}!iMn^y>UJb?PvE6Addr-lI;w++YyE`Egs?UJGoF}pQx|pD z9Z^$K8TRyd5tFQI6-G78#VKYx{x}{VyYiHpSVt-uf@q~6mJB}`_@1k_(MtWfepzu< z19Z#zpbd$|s|m=hcV3m}Wk+|cbFStj{QB_YX>jd2*JamVJ?(yL1DhJ=YT6XGN~XBK z+fMjD+wtTL5Rj%WiOZC7$=RzJ8P)XxP$UF(F8g1Q2}lPmeQb`olRrRKMiz$zg+cx! zB__Wm+6B>qg2AAdvAxs>m|N&l^c3o`Mh^-7D=n8E3n}8R{GJ}7L^d3ZYYE1E0OR=! zgWN`6IN{L$F)`uG>jZaBdaUPPWy=gOrMG{IMeT^G({jQXE^Ye*@GZwLt_yWbDvWXljlRU5F`&X5P*~FVjb;0S;!kRhrorxNXjt5{8anP>ay>K*LoK#N=Xiv}eFap%r|DZ4&Xhx$q${Lh< zkri5ux=Z}(K5T$yA8$os4(&b zYfz;wCn98wbFe(g!g zRAD&5#ZeC>nndlOqN21bq)?G)eef;Kl`F(*q{c|Y8Qs+|IrAz-8Yw`32}o?pmkVnb0a1{F%6 z=*m%=aZqFf)_ID^af&Ro1x3d8;Xb2Y9}$6|)VPE7hf;joz5CZh-`DTy7bNST^$uNS zzotCT$AKz)4Ho0b<8wZg--e3|I6ZkakDsD0H!vZ%_hN&n2wJ0ZB@kNfadX^VQLsVp zdXM^bPus>Fmy-#0L-?Bu;sFMJ_JmtY>a_m9Nrmbz6XWQt4PW* z#ORtn^FsaSM}Bniv1qu5?b`5+v2B_$B}eXlPrH$#RQ~~@#%5?LvjJ5h3}VjIjo872 z(@vwUlsXNSAT6wngiti-sD{fli8u6+_!I&?0t9f44sLgfO1$n89U^+nr%e2& zqQy*<(FigD1AenND)Hzzslg~=>Nwmd>PvG}gT~W$194H>;a=)`t*G=FOqeys-^;uw zcKmdBSmr`EN8CJ5yh?!lZR|^ob)oS(4%9PwU;TS?QWW&l5U7U{byda`xq}Hsn?BwV z?KT(-=QK071p8ln{3yqDC#WXIuC&nmbGh*B`t?jXE68v5zK^vufaCn2X;q;Cx|3BH zi~AI+OXmNIVa?Z|;O|_<&O{+%>QeZtW=I!p15vpN3RDHE0GlQ|fKx@lu_H;wF-ghGxWUxM4~fnk#mA<6$IcXvt7LYx zjuvR3>Sp5XaR4*!lzY7(*BJQr!Wr=>$2Lhg`9dGF3pL`EybbY;^)-;GRuu-qn)gPg&Sol&^u!^7p& z>*W)!A8=dnxdZsZ(&;MP>5&(sDr+`kz&SO?5%8)+1QoxMUQP>gKq-A=QNS#CJw2@? zN?iH&?Zbj~_lLmi2N_)gjFriTl|-HM`HWiyObJYnRx(p)o1AOfH?z=D^Ozvy^3T#` zWr??<#BNzgr?f675Bx7Ar^b`$KBY%W4eM)*8fr{wm28*r?tmB!#18Dh)|DLByG?>2 ztgv(Rb{B1afY~Y5X6xb;?C3K#dYk3PH4ZiQ?cW>e+ucdn3_#A(NWQtR8|4g z#)X!W5!PZK7k9fldfmt9Ssa1uHNd)(XK;g7x!R8EL>p;m`FWzB;2Qmlz{y`oS%o{z z1HgGxO}h97s?b)pJZma0Z$_G5JE>sr)B%r({;k|PG}05DnD#K)Xs!Pq1z^479^j_q zi1w7QvpoknOuIf}wNGvE9P8UkZP`uR*-ZtP-P|D$V%}EoMwQcoGQAkH^Rm;twzU)1 z-w^_^@x+0uZIrNW#K5|0fE6-lgJ@@qdZU|7$#5mfod7rr{kA-j|E1;=vah zLM<;`oKZ?Ak6TdXQYeyJtE6I<)i|LDE!q>aOhR{|q24_FFWdBdixTAWX_{$wipk2F zO8PX{PPA2jne}3O~gA$22H`tK*SVEo2jeu5~{X8>DwKhD^z4EI- z@fF{DRL=#PDiG|1qr8rGz-s$t|GES~Ek@g!-_v}6fsL%7$lJ|@_M?l>ExYemUw=wC z?wo2e{)t+MVx+PKhvsHJzX3FgTc%B-?Ec&9-1269f{uD586#xx*B_aRH{p|*`pNa^ z=@VQcgcl=KQ-WySTYMAgf8^m#3%(OC@eiY=1EbA>`*KLEGj z&s&A|928^!!D7_|_ptwb+XB(;0ueiNzfdbaJanO7!7(txl|aBe!z6g>wr?1@PZO9d z^`NM)9(`FB^WrDS2jF`dbbydOFF!$(*=@f!rC#RrOl3_#Y#s05Q4F6VcRCfXqI5|8 zC!TcuKquU+CI1&oBI|-Vj!#8rqyY>~*B6*HA=dK-*i_vlhWIUx^OfU2hBt@fuhE+i z!ALJ6AV=L@I7H*TKBtrsXky&Tsh;AIX9R$<(w%H-z=s_~a3XJNN1aLDUR+iT2DnnA z0REXBnp8x|P3A^RW6r!r77{3!76}ucu2ykV)_pN@n=v(d2G0@=7oDx!?ss|hvx5@( zfc+gXRd=fS0mI4Af1$e#;eIp~qc)OW=rr3MalH27Wu)Yf{^-ucz9{LMAJ{Y+KBG+Q zGW%G|clvq-I55D+{cn{J>1xt|5V(6kvq5^sIg3+L3~#?cO1nQdl@qT)_r*yNu#xL0 z%o#sYd3ShZYp@4NU8_^_==WRC-p-qQWz(8;H#h9ojcdJ5Mp@c1k16Rf+8}dW4bB>8 z`dpxA#TUBW#Iu+pUee}~TA~uLFivUQrUNo807AmP69zjS>uX1_` z53F!YD8{dFLonh(F!<*Vo z$ce~V4!GuVZ2GqOu!)?3$ zf}hfUKxfQJrCW9SL_Yje zjZ-Ty8xLgAZ?iIb8^-N`{!ubbR5FgSyxyQ_XTZ$T%a{4pAym8_CJ=~o+#jGNu>I zr7S+a;b=*psvMPaoHp`p9p&3HcUL!~JC{n|#ch5Z6{;xSr!pVOe zli{g*Z7*@tDD_dD=%4H;jDd>D&3b$YFO?f|Y1i?^mN1|&spKgcGc8B>VeEJ$PQuAb|Xz6xm15KikFBnB@&eD>%1Yhd;0SfdN6#2&KaG=AFT*v{u=sH|vZ z$G3{5h=#F9>!0!SzRHREr=fE>TE*0Nxy}`zWbgDQxBoI)3Jq~iVj~_@@g0`JXuD{O z%bDKgCYDZLicO(!)RKAEG~^&uV4!qmIwC2d$6+6x$LL6^J-lEnu_au#upzC~gdUPc z<#HW%10%vQQY029+}9t$8otHGw4NqB+#E`X+18KY31xZbSB2ES@9S~T=C4D=l&f`z z$w)l1+Hy9J$w6~WVXl1PY1iE)M>h@QjBcV zQ5XxxBvnFMBw?uf5@`Iq zal5H{Q}gs1>wM24`tzZ;OHu2Nka>2>`_S&hCr;tO-4C>rQAP5OvP}(l8%kMY;>^lE zXNWmr)+kus7)HmF*s9GC;4Pbw>zpSi-v^GSsh`0uIgh7ACqAaDbvAdFC!G+hf?l#T zZwzPFH=71VH4+2pcCVDn$=RY&}6S_|U_ZRNXNp}xv&;Eb^|G>5^ex8VXFY3@`{FHihBCDdRZ~ejLxbde_ zhVm)Phjv->3%^GFxOb_z?@q)%iR;zuopnP!lD4<<$((+ZD-}F>F!*?RjYVu-33PYG zG&`=KOXAPKnCfyBbtryGakMHV4GNKbb3lp!q;72=3led-gE+X!;O4PlhM4Gq_ zO`hNLD7CZPZ1o>C+~?@AyNj*VVFk&P(#$B8Ejqi8g~oNs#A}4)&PXB%gn--{AYymc zEo*zs_QFyArQPA)gCUi$c#^vbFW{0qJlaqmo>mA&^r3eDE!Iva=}DZ!!5?uh^d zw6`_j7G&qzUHp&V%ZmS9-a|z0U3~NIL8;`#9wnnfxSg1l%9Vh~zvfn+A;^vu=S03pdh1LYbevG5x+&_U%$=yWi4khPrFLA84kl7Z+p}-Qr)sB@ zXrj`0?cM(2K=+ngsY5e|Ryf82m2ipF5yL~=2UqL8y4ND>`G;l@aq+O+%`iR<`Y@b+D7CJmR`J10r+%GtOISx4C;d1W?Z&wQ(NUgsp$=@kqOF8qET=0wllrXu z^&WyCf4t;}%lkWDm_EClCJ;kc8kS)-vz#fqSFTPus|S$ReU(QgJijeRe=Dad)H+lB zUZL6Jl*lY|W*rnHSbcl?3{S8^C*#5p@@$AVopI$E!$%h4{t03$2$Rn$bGid40b=r( zo29}EPsWTRyQ*ZZc47DAkWqv}itb%`c{=MtWd5Z){;EyyRi7vij?Cqb9=Eu6NjZ|sx@EYyZ6T3 zaDsoL4E`43vh<dpX{qI3_IOyjY!1g1vPN zYJ&d1w*m!8#X4Y4i==AlRi1%x;BM(vTvA(HI#a(%V6+(;b)(;vom*E>Ll+;hsD1hHOw#PS<}=N@NB|HDVWA6gFA zN2@S$N0FFbgD6|Shuv%^$uw9%G^0NXk#`Wd?IDgWwH?%`Xncs|gM{TZhENG!D4&KH z+xL+QJDBPn^PKvKfO=g~Vo6V)LO)@dYeI<@tcbWSl#&o4tj`uiz~Y|BC{AEB08mx} z!5_EdMoB3*kaVnr^tv$SAQ)Q*0oTf48Uikuf)(jasxmPQZh44LPv0$OCy~G4$=d=F zf5RJf2_77g$kP%@ta$NL5^@#5n8g#r`ELoL-p_vTKIXcbT|&fDEA6%MT&fOMKpAt{3euEq8w3r zVd}TzOG_j@vm>_TB`gjuKl9>P+Rb5ae22B8b|>KC$Kpk<;eDR_LlR!k9YvqLAwVVV zLkbeRv6n?m1P@|+$=Fn({f`(>U8(i2O3lSz~+1MQ=6usXuh+PyKUOT|t zqzr2B&<7E4Q6}N~eVn1x_04|#0Gz|*<>Zk!bpgS{CBKjQ{!v?^FR)0)0s;uSN%Cax z&tD1&K#YZTu;pl%ehMbIk~D5~xCq~}s}&^|hjCECg?h1_ws{J)L}Kpn2#Y)8Y%mUY z!nC%-l?Xw(l%&v`T~J(xB(1*}aMbJOz_^;lsQF6GGg{ z@J^OYA%p}CZfLJU7`?W-11x{TpA|Kv3qzi4Z>sY_vEa@Ri~_I>?V3(BC6TKc&$KnHZ|lV55Z?OHs%Se8aa8 zvM^@QF$#NKA!1m)koqTIi499IZ4Y>`U(P4wWR1#LK7%ARNl zPJwTX!zt_EiWjy3Tpq%~mMw(%wTgYsfD_i5Dh1>zh!TF)U8^Omn*|Ked_tt22EBmbtQd=< z&u)V&>9e_~y?xcFAwGzx4P`nXR{w*8RFasulOvF+s^7GPkUU0f=f>HP_r1YHEL#4o ztHx;JrI4NVEo2*xSo%CgI+?^=DUA1zrVya96r70okz&q5LWz`kKy$kdG~=(YRscp! zT7o#P;dXE3^$m%w9wKtpS_68CN&CY@o(vzWO&XNJt&R~^*%ywqL$Or@5=7yykvwho z`YydGD|w?Skn|OJc%zbhzFGL=+A&nz0F613f#I+`xrFsLsmsQ|0yU0SoyU{%3eh6f zdq0J-5{ZW+iQ}3Ym9YaLz|e7@Ob%k;tEQ_{CguNL+jl<|2G-6Ly$bpC!`BIMqCH~J z;_f~s9f2evclPq&Am9#CjMULHfB44G?~OV*MQ0{$*xN5M@+}T|X-G_BtP$Wtlp3=R z({9>r+##me_7psc6f{YLH_6j9H_F}OoaRZRpM){RjzJp8S2)${~kcpi`Ew^I6hDW917Cx-Yb%Y57QBElp(A{wh_`rM6f0c1KuH zkVp@Y`Y9Q7p%ot?hmDb#r{55U@i#iAuWK{eL-Zl~c%ZtrN`Nd@Tqe6FW&K)Tjx zPl)$-V43^+xAne(@IIT-I9{xzWVObR?SCr|Im8&B(Mh%NLWlgRDZ&_T*55PB5C2nt zleiMd>uR?ARr6-+(NoE(6D2z}+5icvhq?>5|BTooNnG3bTbOY1&0jiAJU_B^yk2_m z+s!a+h$O#E;6FlBooB0`6Za%wYd;cBjmv@p4u;%b6F@b2B45`6*d2EKMgGyR@q-_v zN7trAV*G@ZigzHB_bxO3D>QXFZrfT|!|uDW$^|`QQ+<@7kH3EZsD&+ngY$K*I{_pN zK19B>?n%f6q!iEqU}m|fATo&6;$Qi&Yg$(XhC#%tBZ|&VJBvd(;^LalT{o9cuShPM z-b24o%;rtlFM3bI5?SxLj%WrUJ@={;R^{t+Axe=jXe)G7>Y>?l?#5r_g$*C_5;YnV za57%2X0PntMO^-7^0RJr8!wWNW%jr26z}Xyw;(08d`F!6hK-S&1>eybd&h@wRFt7o z#=vB@~K@lGCy(jwT)jy7?pw%IH`n&QCncEAY>Ki#`q{h`o^+9ws+GWs=*(w>T z>LCxkrOi-N$~!MuVq-4%7v4tVxf9}MaMoJbe1Wm+Q{fQt)z75aFlfmUlT%LOmfUE= zG&PG=SH^IUOq(?n)p#L$K*WdGCw{WO@rQPpR*N%tnEpOL9ot+E#W9wd;x~|XQ;Vz4 zRM;6H%7M{i`11`_$Tk_R90yZ%oih1J^;~mo=-IuB>}B-$vhe(8x3?L}0tH4g82(%Y zDFHE!%EStkLL?JMKtka#Bl$n0J+b74QCa>+mBU_!PpX%bWNh{o*cwWaVY~b(IMqFi z|F+t^h;j^#PDh6B36vLn*1yxoXwG06D^Q>d{4ye!qvI;HFf)}(8U?sUMmh`~;+f{G zJ<1EMa2CEfJ(443QXtkDai;lC&T_7!GCKf&t|456wnW!Ao_W#hd3n*IkHhCtJ*S{( z5`#PM{?cblePeAm6sff)bl{u0CyR1)^HW5HPiuW$67O269_0v~$sOZKZE9X1e#KS1 zl0>D(x|+h|SiG9XmB6}&7N{s*OTW|4x}GV&T)h7H9{z@PBS(w7WFzms8rx=prDMrv zkwXI8RW>(!Ce2 z`q}rJI+siLoBM9q4_ZdJpB}V*P~$l4n00)5_~v^8$5Gd2#nYqi!+wtA-pl2u$Nm3q zI8Fvgc*;(&RO+0k!%R+Pr=whtIL~kb&&tjw?)=wVV**b^n)+Q&2pfdAuCGSols)G&9l)Z)(E{HwP z7{1zj1&W2RaxeD%Oz+^uP3WG<#@DW9Z0|$?IDDWm<4HNlxkeYLy!rk6@NLWs!?5jt z)yJ!WJB$F1KyUJ)YEs+^uT7AXPDMT4uj%6ix&p|Tx+KEdpx2LxgTXSytaNX|J36$d zBo+W2`O59S8fy{& z;{_&-|Ec~6_nK)TR?zCyDB=U=JP+Dycb=tu)%j`8l z8n~7c@NyKQd`IhaO-w_Lai@mEDDj{F8FQHX4f)hH45M(Zk6Q~oCLiz6$P;3DVr$WO zN*TRhY^!oEQU)jeCmw$Qpv+0kU5^I=RV$UPa=Y1GGg9#%C?S;4Sq*9G7TzXn zL4>oiL1@LfdndmjP&qhk(m_yhTKfz?%E-dV|C*P)sY*R%m&VA`1v)hJRp#lg;bo^J z1qSKDZ!KcvdN53W~$;tYxmAUny81wXkl4~o4X(GbI6Ps>h9`pl>IQsQ1`K%Fl^dFg; ztKfR!;tjQDN?)hcSh}aMFV!FWE~2jk?c1hB!^x7K{Tg{**)N<)_xSmMIP?q8430+M zLD=e1 zcS|0UkJu(%n9qyH?LB4JiS4FCPV_rkJ@36+@Cy3Kkn}N5YAQ8}scYbsZHFoAesc)Bf0l@0u9K@-vO9r@&w;etoF0?e`|LiUs?j4L zhlK1U33~+_9#>^Rk5O=5gyObveWH2{e~mUr_kStCuvE9ev&43opTdKW77D=p zkw{lgpZeEQtIK;@f{fN5M!g;iVurnhNljTrdGcX_FaxQsKN=$(>uB81)$-;BViw2s%PX(a zQ#JI`nDIYj^{?8l|FDga6FvUVMpw||bw8S)Vc9>qu|N4gL9HZD1<42mpoBcGsjSr( zO=5IFVyk}jrwEC+x84xQxG5A8_Kn+uoFnk6i+}4#N<%G^%HOU{pAf& zhB_?EaPrL}4zhydmC+;Pg(9^-KsO3n2rz^l$m zLd&j|x^ce~j2#&W>ZwJiL)QMX!)Xkvo7mwTaRTZ)H2ljg|D2n7m3ARY(_hKq4f?r| zxh8rtFt8-S@RSHpI^|R0Jtp3vrD-kUG^HG$qU0c?qJUFLW67zicf3qXLROdwU=0bm zkfdp$%BmluIA8$RrSWjQffffh79R;tF zer9RL9r`sgk%_$HTuVO)ert{6T`}dR!mJx%DWr3C+w_~8pn1*!=_D@9&$rOAs!%Ez zGAxNab0g~@1~yuTe^USlCQu4`XdNz3A1f4;++i}%sNDulI*z2L6AJ%4(y_`vO9v$; z%)A&J&S_BD*Sin{o0HX8ZwdOg5`s2V&QxgSt}_ESI@FxzKoKZwXXLX8#!! z8zP?X#j6Enr-4Q{S#al^-e2YEW!wZA)S*np4#)jLl& znh){&CZ&5PcPB7vr&Uoo49DET&pyM$-SMtv>K4`^ zTlz{pd7H{yMk_KPZ8((UpUaVqtBSFhR=cwnm#Zon{-i~f2%Seg&WVteUSnFj!!3dp za+j8ON`(lgNN^hd$5}0}aiH-?EbYb{S|+O7ltT5drRVQQ9&#cQ`a!u2bBIop)Ea%+ zB%QMQrySR0x^biXc%5h5aIsS{isB3d**ybkXa(WyE);&3Ay{J*^dt>b*)NnXAzCxV zfV9np5`uHmMCdZTsd=iY`Pz&epc0h?Rc7gsk73_ZjG@9z_t2|mBy!NB84aRoo-P>y zrSY&hY_Yu=TwSA~!#|4hGe|kt@I!$Km7cXrl2yjJRV+Qfgb4>tp)!-D1xH~NcX*9B z+<2tGh68K*ir6gUoX*!wFz42;4ksK-SAL3@vjSJEcR}crKuY(}^~(kr*S*jN*Z9#w ziHZ*_jyZh1YYsfFyJ@0kttU#-&|I0}`*qUJPE9Hg7>tSNoSu*+F`q&v4!e=?zrxif ze6u!L-cb!G(|=VB32@0p6p3-+U9y&YiEU$qh5~hHrq8t{NydK+<)OA=X6AFxxvl6K zIs}GAs+RV(HK9oNm+|Q!Q+avjC)rMSTD+YIy_nMD(yjp5spmWM1l5RUc!WDmk_9i+ z)kVYU)f{j49OPn+jR&f}V=ka1!trXyetK4qD*}2Ge$0Q#yFxD@058h~nls(QLz)|m z=1v{o|M*J|5Vv{Y$9UtwI zh&+f;^2I)a2Spm01l$?n;7f}4&;>u=Yop9~9rF_kuaZ)wVVBVpzkCqPQ=Q1T=l<=p z^ZJ#j&;w^8-^b6XxX^JdEZ>8Ub<_gBEqa!2 z$jC1v0=#_^&I?a^eI6-YEZ~kG>JKRPqLNk5+)c&j>&53HQ3q&;B-CkSr*=}YTD#3V zmee8zcf|~TfHmovHMGeZdSGRc*pxsu^KVPh96OH`fF~wVaOXnb6g+&yO~)QY^q1Z} zGDIhg)3tuFgM{H{BFhu@1+wAc_29v|M1il_Zt}8JEDVPMvDMxfO4-Nl?U$Au2>(qdo6ZGuRehgj;8MvtQYzR4}QZ=Ed^_~d@|6Rx!Mhu+7ocyrx>oyAIxCwMf3X$xQIqQ{m!Fv5`zVMdJ00 zs2z&5qQKsXjBJdg1idKTj4sBrAq>h(KR_+gtV^y$L$j8o7f)F6Ml8ZqL_)2S(s?+O z)W-YObdF_LX18z5fLO{cuoU&4g{Iqp(Kc^Y9GMAJoQU}Tw~!o9Pjd6Jx?C6;#^hlT zMa<$+QXrBl5}7)pSs{g%p;T|Wn}4R8YCa}Y zGs@yFJkkAC^(%a9UEpu2JHJx?Q4Cl`L0L|I!r*xuyc3%kXC-69x6ksW+SgK(pWe!O{~P^Lup)2}nc;}NL;w31j! zH;Mil4ah?KQe)q_^HEf~)SHOD6SF!hu(JsrSG@0vbsEMp*zf0{2#LU1wWPtj7M*z>4C3SB+63=9uEl=%ss$QH&}=NmxNOhY01Fp}XC?G1Il?|l zCXH!8r>|7`iY48S|3=L|gk8VT6n1^j=6joho05i49jdmDv9Jtt@_#OCYJ@km*-wkb zL#5)gX;MCzdd0w8g*Z1q8NQgQ>h*4L>plN+%l0B+geqoM3hxikU7iid@_ikL8!%JP ze|aHr^g!emwVHlgF)jC=^dh^-peBVE*QBzaT>6XDlygSU?nUuMe3xl1r&vgy&#!)& zE^4{ZILeYVB|4yLdjpldg=zKoA(pt7#R$nP|Hy1Os1du)_$TS^2tojW_ZKmN0e}a1 z@f1c{7Ur5T6D=7DDIx*@0K|y^QQ#A?9wF-X0RUJ5Kp6nE0l)+RoB$vI0HSYo%m7dd z0IvX`7XT&zU;zNu0bm~hE&u>e1^_Yupa%dR01ycP$^S>vc}K(bv~he{+_l>3UG(T7 zR&UGdo#;f=M33IfV%gPu)WoXMOOR+0?CPQfAtH$uB}hm}r0(l?-tRf{-#z!t+-K&_ zoSAt(Ul1q@1bPYrZG%83|9=7mA|XjI4PvFH7NVvmQeRQU zXh|7J8UGWp<*G0;UT0=DWMc!dgWj>Tf8^lc{7*zKt}t%yJ49}7MIIi0et&g-egOdi zqJRL2Ac>$LQAp^uu<$*x>(@m@)Ip*kQBhqnF-tKqSE9JMfP{d=eng<|B2Xi`!>iG)aB-; zkLy2KwVW1dLObjtLHukD; zCUGVwCliyCL1+*fP5hRU;+yUpogRHB;|@tCN#=hd5;HTobGU_agkYy%A zS65eeH<8%gJuomp92giJB!ZrU%3qW-4m0BML?RwfAQ0Ln+9oH7#L3C}sd~^X2r-M8 zot?cscN??>l312l{!c{WRaswu`}S?;M(4%`5%eCU@?M3wwRP2fc6NxnySvl})H0uB zh@UZ=0I5G$^l+CpsUB6eLh22aG# z5NDQ%OK*v9cZfSj#G`ZKIgv=b>iQ5O=r1Lgeg|$O*>8=@X|kgVkAVpoR_J$D>trM; zdo5$NU+W}bKv*(qttnxDa8afc`#oN`s;YLIIXssR2wVX5z#_jnr)&uW~6swxrA?UWZH|)&w!zbUy>*V+cpAI)oa!S=is5lD$eRda=A2pdlf&Wv zs5zdk8myh!eQXN1*j3BMXArNM4GsSMcCgfZXMQN|*VyNs^7dU8;6@)`8hxCJp%PuH z82$bRM6olK@g~TRf(s<_MmYEWETsn{k5QU-0m6)4AvcI>tTV|(@n-=Q2re?V?r-=f z!rUIs)IAC@Y#gJa4Y20_kfpfSu#>I&{a_~tu?@MCtGzy!;iwOnwT&mU0E4Vu8ogji z8?IN5I1~=Hb>f)KT^Dofe)&;LOG0g>2D>47o#uLG$2gYtm-{UTf%uOfv?N&Iy1v13 zORsgunM4n8XcnJ?Dy(pmh&EE{(*eTSZy65{l`+U=78lXOYt@1lK4{thVq9Xwp&^Cl z|NKacG|`|1COVr;BE+*ys?8oI()_B!JLIky$!)^{j>7Ba0&5(raJbLzSk0{b(Q(&; zuDE1ZmZ?gEOu&|9aUHmo1SIE@CS-X_UXEO>asf?_Vk)Rmw(6q@YvO1Lt~T@y8mC1! zBKUOJ3Im$z1lz9Q!#Ki-YAbBZskfK8p$DcXaYneZ4-3*+aUK+i<4AO6eMWjh+14Ig z=y%h!b#K(Xf%N0ea(#C=w#nokgJ64u6%FL&bI!dZx}tS2uyGPu=iTe~ge(2HX`Jjv zme@AmlaK4l`}kd~*xhTrczx_SD;E>tN0^!bEw^lxSOj0?Xo03rmE zqq=7;KgTvo9PdJKu`uQL)~k zFzs)3Xx7xM(_sQxLd{@jqn2En4t2PGk9Kfi-y>4i$zcj*Y3bK7ye{UQ^sJ%oj-U?e zyDc{<=FBlL)}--}*C43+fo_8Rt8w3!K&(keD#fNxrMwjbVD48P!(IFErFN1^LJNvw zd1~wnaaJx@O^MdTm-Bt7I>`@=xTshlmbkm4&-+ohSy_PUjQ z{Hr3(aUnmBP8{oXgDFD?2H)F*yH1Mrc~e+`W7-`8uLje5O`h1luPPI^otn1ke_~!5 zTc^{p`G9QtrW9}tVhNadSy;6ls{388ny+F8fyWTH>B;J&av&c(lCZa!I%TbT(xD+oQ`^7v-5Z zL>m2mPpD0%u-F=Vsfv?yCB&pbOX-$j2FPt7;(c5JdaFZy`JSyvEkI zL1`cn8>mY`9(anC@FW2W`pL<$iX!^=442B(jcWJ(C_HX9S!izP6%4}2v=yFUqSwzG z^i6#70)6K1LP*IuCsE`g=9Oy0TJ?6v6cXR1t+H?kn67n`-Grgd59yVPNnUauG%o|p z3I%4@Az@QI>!9FBW^@Vk*3BEHY;&xUCNZbOcYmW2Ii%B9PZ9wRHO{bS@V2r{uxJ~Hk&QhJw#mXM$hpa}Y7 zV3X#)Hukrno3R`9gh`vj=Q7*CmSY~z+ z%(H_7%_?+?pYIu96t`U?>C@FBH3tl{ed+2q$>ldRVHum+qArP4&RjJ__NB{Coqn0D zyIjevy(k?~#ZHLkMk0-Ki56Y>06mQhFPI~+gKDq?V~(|(&dm;WX*@=Fq1fxkt`LbPcV2Vch=3#Wwiqm1{<{$8#C7xef^8ZAkbaQd*`J5dBbF4fd|?j`lSI|syJ}b*IliVk27b!YI-$4|D%j>V#=AO2SOLx%glh74lbwsjEJ_oNd zy;-aouZzl`unR5IyC~Ah%gIdPadLcn`MRAd8b!`^WXCdoF#d?*X+HDd+YN|Wr267R zCAV{myX9;@s>)n5RloGjCwwPmAv5vSy2nXg&m+q+9lHl_XJ;KJvW17lA_McJEw`j| z`jpt6m26K(#Nfk7FI)VYl6Z*B+UkYDp#68Cz{1%$~kd9#mA!@OU` z`mXb)SiBCyvg+MJlf5vxGR^$%==|5`6OSS*e}JhQSKTJ1)WF96=dmtFYL4<#6L0+> z!;ox!37akCheOlLQQ3vh49Vky&az30$4*rToM}vdNYeL3lJ-;Q2lsAZF1m8Hp-8{sOmvJP zGM8Tlh<}x!>Q&evNVAA5!v@`FVVhx<%il!e8DD~SH4W~R1HlxrT}OlA)!zoAXfIL8 z%<;C2R4R*+4#J)brz(~m4zRUE9uYRuNr43%M23(2x@G_Yg@jxtN~|RX({GFFJ>nfw6z{hziq6&T`FyF%6=Zzd#&V2y~ybwf-mo-245z4>GRZ{a#${) z%8TP17gIwnF&}I(*?j1JdVp&!=F?-v69AjJ7L!CQVe>|--u@c*do>{|GHvJwHgyX+ z#7DjlBtR!oC5q|gkJFM#pkgu<7)Zj6PAOLX^bY;-`b*YzY09^*$zSxSV{TIPvSkeE z2XsxcE+8NR3+z^x37w}L%9k0l4S^#itdpl&1voS}7A+76SzgOrS9F{AVqKinXnuz7 zQ6v@T4SyS%ZT>ndM3edj9=wH3af5&cFxlsmS!N{ygiT39946#4)dq&SV9Twl%Vy2X z`3gx*fKU&0QpAho(o))y8PHFjsu-ejvr4eXKXN%J^L|Y7EVwYL4(ANnr7iX52}k*H zm+~wlGU2D$vux>b-u&xI1ysy=G_EOw#Dc7$wRFqVbokZstvLUqYgX~S{C)`Mb4Z~f zyNP_7#&j>1wltGHG}FkwXlS_*98@@HSM(a2W%IMhShO&cKF4sgz)*?u&%JDyz6W=& zKfu!KOHMsdS`<@t$qSAuc4W%y!!bCg6>IEBc5W9ZDH(h!N$gn6jcClJOe#r!Q?k00 zRluts9#!0iV@TRBDd|IuV3SuH3L+_?H!Sl@mC6>})0c-i+=oj_DGP2Il(qeg4tT>_ zdx|VjDpU1-z^X}8Lw0pYOx2hRtwlg_(iP<$lm+G$<@cBOpO8PXq7k z>AItU^xVo}L{X9^&87jgU>NXqA@|j0FzlefAfUszhbo3O(rl%cpxl3bvgt73(~$L*pCIHEa&lTUeVpq^SM#g=>xmLHBo0k+{T<);nl>7^ZdG%;?fON})0K$s%dNaVu-RY}7;+A$2Qu?taBB+Y_M}L3t(Gzl4#;{K%LBwHj z^lw)2u~Q1{I!Kl{=u-fN@6EmZV{4=TnfWr)Phv^m@d^Yf& zB9^K%;A633a;FaFyw*wkik3V=_2>xsA7;~*Ap`BJt5iLuDokc)%(ld1I(sLWLubhP zg4p&Bmw`6k(0r1Wfx*&++_iHw*kYLOc-(Hyd>EnY?^ooR*afo~(#z;OaD;hk@+DF; zfOto<;V>bmLEo;)EI(a03&M*KLlRY(!ZF>$QSg%im?5I8S~dMQ6IET_EJ6t$B0=AB zOt(KUlAS5m^@rm$W`;Cx<{yIQ&#GF%z$kLb?RgEuJj*PZ%&e!eHW`|=aeY>qvz*2E zd0XGu8Wu8dq*@$sEGxN#Gy@$(%<#;Oh2 zN3Z!#FO!G-8h@w2EZnTC6)#aS?8)K`B<{V%^ZCK_*SSGMZaa%vyOzE%8FGO&u^@w>N>_FAsVHHpRiH4IguSU^3N zlwetDdBg3c_K;y+&ZpBUcipj;rX`R6uyOr9HTvMzt8_yJfz7GnbCp9S-CoJH0+rQE z4jgAI;{GJL-bU~%;cxz?E-YuPAUty?= zp7%3OX`~5Fc-KRVcRlOK=aj}n&djpZ#MPO z;m@VWCltDK!UPnqM03O)zK3k9w2a#uErJ~o8M}hYmA77M@K<@)!uGRo?qi07PwIzG zm84f+vfcT&e_+qxF5TbQQJMNE`;Igz|2w?DNucTiQ-~?VRu^1YMiui6jfDaj2j~@#$7{Y*pO=$ij zfq+|DrUDF>fejNS&?PCB;=!Xcnd7m4wVLV&ci$f;l7smcPT)}L^x-`eur-+rMGuo- z$TDxTZ0lIA;}DQ&F})8IP^1ylXzSDEO=!oz139bB?Zrb^o&yi(-LH?V96HIN#x5ng z{3hzs;*RtYpR`JXS9T;fI}y~IZeL=qwu46>QK`*s>23K3;UJAJXMjbsxCNmwe!XJ- zbU*P7CB>NJyf^#)tR2p8I{f@!saq_Ju}+5a-jgx%;x{lpxZ3oGenCyD0=2H+60P16L^<>AM6 zQ<OhASrUAlK)R7#_ktX!4C7Y4*`vSR`v={U&qkY9BP5V5#w zZ3%MGul#p4!G;T8le?q}HoviwP>Ydl5jrTba->eVo<1MgK16-<{S}VBwi$Z=2PQ`+ z+K?v0EPYR>x9kD*k2j+@?i~%0FXmtf#to%z<9$?sgCubRz6gR1FZpr8xY`W(G(r0K zKDiB|(!>-}Us=(ZOSRAk^ZvaD3WAhqB$3nbXqTAtY3fk1@`F<_z?I1s_FV4lfGN*8JEfVzbBO5+Bbcq7Wa>g!|K4iT1P2>GR1nUiFc`4nkL)#-w-3hQDVi=HmWaJ(bJ;tH*Saf7@rv z-DEdQfV#9>$M$PGqmg^qjhCZI*@c4v0epo=G7#0mR#l3Y`;JY zf$$GLJG1>x!t7TG+~eljqA-a%A$2e4$+ij+&Nd1mrz%YiToEur<;bNnl-}A zs@?rT1wDvzx~C>h$oi+3-2%GW5`Bsu&qe90DWpdrT_$NV70KDp<-QPP&D4DJN7Ivs z-pZP5#*MZsq!U{dwI{ZmwP(zZat+1t5WajYe}?qcl?*R8l8?EH|9NInxK4Ll-7+f* z9FnZm?yNjtnzrrUA$ZH`DfK`o7`$0(2+6?YVf;qkDjgmP0G3tz2=!R z0(a`3I{0QK-9@MVLw|L+Ux%hBRD?P4DGs$Db<^pcwWtb@pO~cOhU)3p;onhPiz>RB zNSaFTT9Q8PGvWF3)FqkjwHFN!11Xa$Mo1znCrx+LEuYvS;Hd;*)}&E2myi^(RS#%| zY5cu(#J}(*7hFJ)iB=~H>D0#c-Yu66BM1G4`CQ(ud_40Jc1O~ZQPuwxgvv?&uA(sT z*TLA6Oo;gXBygOg2ye0Ii{$X`7%g<4Q%Xt@9`K`zvO`uqdU==g$;}B~`W&lzPs=ps zPihT@XSk`CxF`erC$n;M#swbt$;s*v=9T_wdrz-n{&X4WdadJn1STp(eDwP)`wAPs zN0Nz)Y2?E|r_z7)4XFkSKi}_*|914TEChBwdw=5Hbsu@hgRdJ2?TzNoS|&Sp6+yfn zCbl-_Zq|V(qeeGgao@EMx@Vmf-g$u9ANrkriQ~JN>%SO^#h@(EROdd-394?9c2!?Q zt2*7tBEAj&(Ah+btJDskUb5=g@HQ12&&L9>P9>io%YWrth=tjMle}PnN1PX9wX91a zEeVRK*nPuLxZFBH%0}MPpHx4nY$?QVAHl-mM`O5v{o+v4CuTtHTM+mN)7?n{B8ZrG z+L6%d_Fiv(BjQ{7+vu^Au;2r)`D9u2HqI;g*?T$Z^qiUbjM`jbA(CVOzV-%(s~WoT z&@d1!&OxK|wSJdcnpSEm<)}?@Lgaa%z|C6smsm}K!c!DoNXDz!US2)5AY+L(hlh1i zT9Xs>cxGX)M(o%_YRsKCd()!0HNC{qQkEdpm|wIaCIqVRX#Mk7_YTC5QB>74rI>`s zdv|gD{f(ZVrP3CQd_x=cxapnd^V&00E>5t3kz8dc3pHBg7CTi>r!GiXPcsWV&WP`u zuhy>>*xcPO%;l@@eJEnSZ2sZqbI)0u_-3qvp9pEpvYb?RkfxNkF+xs^$p?hXjOgCh zt+A?`^AyEIr<2p5vC=-Lup(NOn&1TwQxRo`{5xQl`jvX zbzF3mAtbC&)XS>u8(w31&rYQj3PP*|^R4(sj>@&QmK<3loDS!^g9h94TT6*Qw?Ew2 zH#8c!E0>-4w~l%Ilpf{wUA?~4ZVVeGwiK4@j~nLQea}7}nHusWOfJ*&+b8?Y#rjs4 zI?3=PdH992ocVdzwOy?1GZXdCRSIH{bg#c{S^EU9>AQI3>+#C|KzGkS2)z63!K&&k zH%-)w4AC*@#F7cUNIqyl#68yW9VRNQALV&JMiEmt&o>*|fWJMSG^W=0G@w~Rd}{Uc zA>s0$#J!qb_E;)a<}GfqKFDv*)7&72`=no5$MYXK=XFY`TfsHogpen(W;X52qWc9{ zpud(eu3fJse13m?B(F_j^8VmUY2w2JL6DsZ0sFFDWUfG7_tYY%f^AYIk%%J`y>$1T zQuGF5JF)klIneKgKN`0vb}j5kMS5LKMsJH35PyBFUJi6~A9j5sYa|?T` zz3R{(boY&5SH7X28~J{puzj~x^?Lms(VfpjN966L9?I`8A3Vd8rm{%=J+9(gB1{(8 zO!92$HeY2M3`}Kghnqpi{#k|*)W6!wJou>uj(<^Fket-& z*K~^O)OjFRKauw6{a!$)+Om+FUW|d*W54%Qj~e*Zt=}lrlg{scHYy?JraPh-J#Yd~ z>eKEzKJ9sL)8?@(QgK}-koU$9U+flsc>MP1L#GAdxd&ZE9dHWCqWMP7dsUXKk0ws% zm==$9E{4iu?q#!j(vRMC%#D0_6xOkO(~s&+U$Aj?!RHst z-?OSdpX|1jG5ys3t!o~`nClC7tLm|1Do_f0+IzLi8pTDPr{5 zZTO!BK>b)VCu$PR^CeJE<#kndXf2q#U1LINR_6DciGLRdUeA@+7KyYG!vZ{COZ3z} z=jFtm?#h?EU0X}sN9evX{0V7DbhW|2ZyjL0i){Hry!BE5w)28)F z2&i-+`)Cg_aggV~W`+-EUeFST*1*FcRm|JpL!@xcZ$=1aiJ@uzoVgm>_(>q(`@sYE?g^0Eeuh?r3lKJpRNEjpbdRZ1V`LjEtEC;v=E{AQZHwe`XC6 zQ-X=G!-TM5?q!aLW{wB)0--Lz6*@Q83xBix#{3(yfadJLT0pc`D|lEdj7&dF;;NEp zX(mjtG!otd#4hN5IRxT*F>!0R0JlbC4GbSO^0HFQR! zh2josJV+5ETL{Fg0bPqgOfC?tifOje3&@>$CTOfd1_U?EMD)&dFbNzaHnU4$1G)6V z5t!i1iBW>_Gu7GtP(a6N`lw{^YWG6wXsUq#qF;EKHdHO7SL?k5CJc*-ahc9G z@4WDqIn_710``W$46A2#K9Id&pf)9|1-2#Tc8uqC;ifTOa~HFkf0kZ&`sucyr-Bfe za4ax*&Jtf@7|dk$OkzG*1pf_=_;AbYd#2fs*m=NnZof-|W<&FJ`Pf~nmsL!dHePTD!$1gN=45dmx_D8s02D38VCOl)=D2#)d0O?WLrsVh zmRXx%qTWx%`5DUi4on0BL$g9iHrChS8cMBA=ZMw13Qm0Ck}($$h6S#2-UYkrg}#|K zbx&7w0U?kvXJZYPL25N-HT52(efQ|K9-Y93x%NU6%|4S;I7Z|7*n2XaaFi8=w$4Js z$O7kc52vXYT(?lWhV3mFv)hc5oY+|0)o{aDyqgr&(RW4zQ3wr+(BUoI@Ln^Z-;ED* zF(}^94`#EB%GDDns;|4ke>BuS3|zVbLc5SpmU7wnQV(st1yp-7ZCb6C3J4DbR~&Y& z!Xklam)W`zaD~WV&L$?x1qj|GgrCnxTCE=D8o$^uF(Jc*1OhQ$NZElZH>NCNd@m-_ z4img!g;1Tm8c1VK`cAF6n9Rw@KumCP@+^VB|Vy8m@a!+!VGDk%V=Q97|E3+_3xyo^#w{i|J(U+^n zOhD0Jvu8!QU#QLcKaQrvVH_06BpKkx3qaHr7`IiY-w9X~bW+tc@(2aOP?n}1YtbkN z%0rC8%1g&%^DuLLSR6Bn^~~JKn`skFMBthL-`jB5sss~-iB~d_?G)_9^eP-QHLJ#a zeRm_+*v?T9z&*ErH<@Ar;=QMt`5!&}4G=win?R2I7dsJ!eYveO)$?b=SP&8*(RTls zGCmgwU3h=L<1OxC{@dA?yBAu@A|#+w5;gOb{7mXt6h`skQiuyC5;T*XH|>0}5qUBC zA61;!`DhZDs{7E-yhikUQ~t&XcjXwotWEI%(^>?JZ}^W|{rxXRz{2Rwp3OZSwkh;& z@_(bjF2IAaY?@nSDKgYz9bn^*6rV`~-WU^}i}@19MxwL1aiNuDPA~kC<}i@$9%$+vycM5}u8(L9cY)86sGAKP>?FFUc)V!IIJr zSN@Zw$IGPIT=5NdSAks-gbYom#YsY<5g2s|OgQYFZ<m@NPFM-%PFv>C-?#m8 zKscM5tkgBc5LMz$IF~e>)k937H#} zXW6$QahJ*7iYm#BHU~~j?AKTJKJz&e!$klME+7K8Kd}HrmUxLiHPu=Xj_t7XrN9TH z0M!SXagZGiWBZ5>+xXz6m^sfIfM;-K&96w%z%Ci@tr z>`&mv^t7eNUG7T{9ur@E78Exx8~qN6cfm{q0JZTI&%bL0oTB7JWdV`8w-Sj6!S zr!UbZ0W8~z#nVbTC4iD7=J+#D`-58{(g9Q9fC)J!>@;B3^3Ks`tzaBbvw#T$Aw#f0 zfwQzU%ZZc|Kue^I6Zymz54`{PB+l+b-uLb5VT_gO=WyKpJgC&>Yizto&`kU(_!A}) z;TYKa7MF_&Q3npQ+v8A}5_y1WmGX+m!=KIsMxGS&pc!0jLyFB$w zN&>?UK#7t+Mkzi7<&mPk>3m38z)|Sg^C^| zUO)JT6(B*x>h5a=V=;f_AzGEWmU^rzha#9A*{n%mg&OnmDWz9KVJku?o1G8Ddpym^ zE45alZ$T<%`MnSePB5ouyNTU711uY|t|;NRl4#6lio7+jPA*JBkQx%77xZi-T~?5- zz0r`|i{xGm4Gj>9jWA*V&MXoRQvB{=mci^2o*EeLg~{@6|DHS??vJ4f(17T;6cXjU z${sus|HF?>>n*7;AP;DvIZw=@KgHN`GA1mf7T~h7UZhcd(F$?7$QnMA;=>-=5!J#g&vSM6R)62UnpL;@JFa5O)oT#}DAKhw8eQ z1dkXb*vt}NNvR!i4u6sMU|%c5>#QR1vJe%S_$4B#BL-p+UH3Sue4p1xfrW^E_Nz~i zs`ENzUN7h2+4pDqJQDzD@;YSd2QaMwnfZYk)PM+`#to5H%-o3h4Jqh361sc^tf&H_ z8W69|LE7&nE0oim6@_XX*k>k1koZb z0&NPI=e&b9l(rx?O_M4O50K5~8#Ja`J3+ifWLC~iTPjn`_DT2k&z!UO z5A0sGl;%1cmu$Yj=sIK+fb$59`zdR~tVWR_R3n+Ra8*8YK|N3Yle20|^;^tY>?FL* z%k(syN#qnDkXB02)zRYXT%4wsx6C#P@5hCD8Bni?psoz9h>lGKQEZn4L8)td=dYYo z*E2r24rn`5C{%>+8VJqwT4(*T6XB(i@X4t7+uPo_nuN~+5z!liD;jAh50xqDvZh?c z8A*SaQhBU9`R%+uX>apdw~Q2!<(jb<66hby$(c zw9Ni%#MddEN-Hb>)pz;>*84f;>kZvWO#k$C-`ExB^(G-=LwnW?cSSWP1)~=O7w_k$ zY3FP7ku2P{yeCt~bc>Cig$h*AqW$(Ro6>_^U9KpySIKf8-PeiUryhNl8Xwub=8Gj( z!fc&mm`iK5&_N44TVrptw%R~6;mt5xD?tsqHFTEm!&|ZPD`goV|_t7$xC0})4N`5 zh5SJ)RHl!@s_co!YkNY&7|U(sFF0u~G^lA=m@N;m(dq+%j_ZaiYVS1uDJi(6()R1t zy!Sq@h*Vt1wBb>$pYZP8wyCd%-dZmxj2JsXY~>3vQ|jr9=~GP#cKRhGC_SyzlhRzb z<=0e*Zm7YYls4H9XkVR4^lhZe1ot;L418`t1`l4|l>v8zae752521-n*M|i2!yJ%IZj3 zW;1Zj)1r;~Bu=p$tkT0;e+wBv3ee=Y^}4}0g?&8~Z7rm8`}EIUa5 zVqLgxbd>hN{*C?F2(6!=k9eMR6hAaSSjldKb>%BW+k6ysR{uTlAQjtqLK*bQ`|O`x z^Yhnrj~s5PDD`}9(hV2eh4SdV;(WdV_2!a1Uw2xKCoUGeN$GhpBVWG2A+uy2nFp=s zpG$$TN$=Qvn#(JQDKxz1pFB0KZliusOBYg9eP^7X^~KM{$I)y-bAK8Ac^c)LreVuC;gLYSsXfVX#}Kiyl{o(+0yRUFhyESKCc)qpuOL`LCcr{- zE8faWb6w_Uu=U_J*E_7n)}pp3sTG+dW4>ypaSlbjcHJ#OaGevNF5^xXLVJ%E0EAARTP zGx6(()mAi=nQ~lpM)D~){MD#vPIDDm#oK$0t6AujtKR?xvSrZ}CxtB&AF4W(usywXdPp{_eba4 zz6kI=TcWUErYObj*VdE_tP)a03|F|xllK|S!Cj;QqB>9;yZi9RcuZKABi#K0<95ma zK^+{LV7Zwse;V=z?IplGM#8V7>!NpUoMs}}1QuRkXB9Bf|AuT!t5-oQV4&i+#G$&# z!0O58$+5Sq{&Rfp-ub1I!n*{itd$nr;`YN=p)~CK$!je-d{g~zt{r9?++gf>C_xMK z7{@{g&u>!jH~#)upnG$nl{b`M>8JFKN~*`XPu5ypRd+a7Z(6S)Cjv4AJ2g3FCsyG1 zVXW^yu?6W1YB*cPpI&l$Oy{e(xoInCsBZ6k&RCM=;eC?qxB;nfe)IO_7kWkIjku7g zaha2_pGxa0QD;yZ{sAlZZ$wY>;c63!<4Hfa=%*n!_dY;=|God{b+Cv1HYPE{=Rt1Q z?!aME?}xK^zdDta18del11jp_&unHtXl;9Uii+pY+%f*f{ZS&QJcAxNG5zD-PtN-< z)5IQh|8;7+88vq9zSh8T@2;iM3*tW)bV)5(TE+Y;2KmK zo0hZ|>O>E%r2MsTc-gwHV6~hKSy5<7E{~$UxjSz2a?yC}c47DC*s!Vp3ITdb9@Rk! zc6(^q@rHO-9v@0>v^zaOLZtf@XGb~0t7M;O-Oc!XrNV54FhU8*CMmkj&UilKpp%u`OLpkOMfLK^bwAWSNAF>4|KKg zuPnbnn)a6YZ6L$@?8)L`LNuG);gu(`TaPW$r~ zB>*ja7+J7=`X=tokyop-$K->dYVLdy)W17dBNuS@wEVW9>oA)+raE`5NMjW~sjwlH2^&C)=Z!)vh3c=a3qq)a7Na z8irr5Z=ATQukhFZA@WhAK5)m<00AtO+H;JroL+jnKM>qdKxgDkmGtn^{#?n;`e>?Y zXtI>WmPTU`saCG;`tZ)yZ3P~jM+=iGPnFExrm~@+i8&H;maF-%tUQIz7?<5NMtJL4 zb@FFhQ0x`M(5@nMU6W&8Sf8R%5X~NtziL2XX2L_a`I^&|_KtL#xCn_md!^L+?4yG% zfmGfa%^i92Jc*_U8;iHEi=uURZ{6CyedCo!x9F~}K8;L$jh?UoTmRB8vPz5mTb5FF zwXt=N>d4{3Xct$sWiHgZzq;3nQx`X@AiXd_t;=dbcB^FUpU3zP%tX_T{J9-NP8O3LfjHsPjQ?ru4GtiPwp z<4U1cX<%q1*s4>Qo42N)8eg}@ZHc~u{zrJ`*6`sCxVVcC^LXj+Ce0#^3-QqSM@sLD zEQJKavrUof=*7Yn|GV}!b+@&;gG!mQ6hm#Cc2#zMhb!f7IOox^6*;7{$Vk0_I~Pfw@nRW*eDL->hJVJUBt*x3@Z}|d`koDnmO4~YYG#hz#UGt z>MhhL;Gz^>-qfQd1P|jRmjyO4!*R|uBaQBdlxaJI=5~C&} zx^z0NGdl&8_YfFs7LYtjJvTXCV9Ll$@d~ue^b(VMMNf08Y2ji(y)KCPWLz}nlaFdF z35FGbChSVEZY`iuhEaoKsz>9Ak7!dlSeO~=@Wl|zCT8o$@t<5nF$8rn!27Wl!I$x@ zNjuiJd~`o|`d<7K!bGY>7PBax$NRQw%Na>djn*e^1i`LW+uX^>l!0vj&+#ME#vjAg zhR0mS`j2}-NG?d4)j@jh0)h+n5IFOZI^Shwv5)zLZ=CS7U1r{A)>F0%}k0ww^pXY;lVk^Cacvvm=h#6Gmz^ z!Pi<=vv9B`F|t7Z-HhAzCN$%Qd3h7Y&KnzeaO++9dg|3D)c!6H-MpZ0pRig4U;q3G z#xgZk)cIbr{@rJ-vXu(*WzB9Z*R7<^=3m{tAI({b!#$dp|aFE^eAGT%?hW58vQ3mQqJsI zqgdvK-dN?|KS6V0ADjuAI zWSvO`*JUPDcXG|k7%6s|6v}YVpQ)+o7%=$B*nfAUrpx~6F7`KkNf(Sn=+XU zTBiRLOc@XUc{lB6y4lOTAQOpTz9oLadxzJT$oN+Y0xCx5o(4r?+8NcU*CL*XzX&dM z>dY>h{zv{9H+oZQ|B2L+_J)5)GwAE(+_n^*tPE?H;VJm`2pK>;@$SH9a>&B1=h+>%6GI$MGNwyuNoKR(OZ@tzU;;5&x%*6MqD%D|p^0EE z_efXK<4pC7tj2YSs+BA;&e(?{nE88`=CyOL4dk_tYzo))tNKldeBIh+GK#kv72CCy z6Ov`k;Ce)Db-~Vz*8!#H$eRORD)3Of$8hc)aQ2wCc5ymKWvb#ZUaq;@gf2v5IGOj5 zAelj>T0!AMK$?7jTQGL1b$1!#y5~sXN)jO!8W%oR=PUon5Tj^Cf{fJ;B1q7(HblFez@EjslK;73yE{ZpiY6CAFXL|0;1ZfcP z&PUps1L^80xN&B(IkD%CQCMm1SHlH7xYNN*Rv!L?h*tr@Z!oXufqRUto_b=QYLI<* z!4s6JdzuN!w-fM6vtXYLdA}}gjajfI5a8N1t70s#J(X-Z@$~*GTCE{DfR4dF;UX{@ ztg?x}|DoI6=!4HGy?FuH=0*hHw`7&I9z=hsYRgl_?&JZ>2>Vr}GWD}-q7g6s<)V_2 z$_sekD{I+oCYAqT=okbaJHL zmt<8OI0|S}S&rB^#)lR6UK_zj4D>pE!>djbBhnrxqvpdBD|M9ykvYG6;R*_33cV($ z_&C-)U)d; zYmr5I(MbnjRjz)98}fXPmzmcWRhatfmt-^+z$#u!XroK<9i+-}_r3CDmCT=wOHpdQ z$(8q`RAM8q?`HKsV~o*8Uv@D*w7pMNKW4fD+4`2K*jtmKK6@3tzPBxSUi=faj^vgYc_pUb`Op3b zu0T=0dm-n8JgkGfhk3o@GOQ0g&!NLR)C0TwyBPPwJ;Z}L2s==RgF3MHF7E?8G<$Wn z13d`3!@CN$3y8L_#%y!EIIKgUBRf981DcmOIIM%I6EZ)<_^4->I>f_0Kzhj2gtET_ zIy8D9uLHE_GCsJ2r-!vUu!B6{1JCz-g7gDE$OAi=!_8NxhkG-ppTpH#H9EKhJsdmK zN3uP@gE}C5cEPzc??XJG`$xqzQa8|_&wu~Mn2HPJEQ~JkBrJU z{4UqIt|NX}mxDUMLp_N7=llFUQ2jYbeu)$NJfJ= z%GkYgla4!}Jvf-d@O!>xLwr04Jn#=b@8hyQ$b&lI{(PImI>5f{Te;LfKIvzF$J_cr z@54M`zxpROI<$j4*n`|#`##u%JhTHkXnu+BJ3!3a=Z|1Pg9i~NRJf2~Lx%nAxjP5$ zTSbc(F;*m(uAMx4109AGIg(^alP5{?!#D4px^m$%#=KaLoj{Z~aW>p1Z{0YVGRL9o zcn@Swqeqb@HP{ayy_V!CX7rZ||G-f;Y?K<*7Y|+3pB2HCQwOh}zN}}__RBZVT{@p% zEs9%5uOC{w4fV}qR}OB>pzO%zt9zGlVTLMMt}`dDFGg_Z+|?Uwm@-0$@6a)BjFB8V z!1VAHtvs4E!F^5Pu|r30Ge&Xf-r1X6x^zByds|~1cW&Lgc|>UkC#|wxymslzdCfg; z?7i8-3GbuV&YbebVL8St`y6_Al>6?nRkwKa)T&0W8jff#H@J=NJeu{cRBgS~=iY7( zhb~^e`Gotq2Of3u%k7(R%JB@IdF;`LyZ?sEryhCSSqDGsHVca#dHjQrw0-J%$1MYO zGe@0v7QDwn4?XNhpYY&m|7W}cNgOAgcj^(1#nD!3=Ny@swKG*BE9QDhKF zG5tAh!go}ADNS|y!xO-J;!!Nh_Qo-X$amtQ=bn6Y6m+3}PP1nocq&Av95lK7=CyX_ z=}=PAn3BgwPCEr>oCMcFQJ#8KOmftc_<3={7}wDh%1$p5=bVo4d2m+B_OYi>bK>On zKyuC@(w%r1#77@lhaHL1eC(k|o_A!_6xCG4dWRnQs;w*^dfY*W#voNybe(6T-ACAR z{kis@GWk^J9CDK7|1unNHgt|4s@N1RFCY62Qk-&{Wyc+O0N1uG| zu{0iT*GXsJgloL99%|{mt5|p}{8m|T#39EVbkb?{9f;|Hm12wenHQ&iGTs=Ud+M1d z9!%K@`5bfjRn{+|iopZ_= zhwqm=Yp2Rc?OP&SBQ z^xE_e!~sh;|6QhV#1Utl=gA?boN|VlKKkdI52tt2eGPc=(CuQ19+vS_o#@&%d-G4<$nyrl7gPMyR@bTh%>Zc#a@|Xh7z=) zhfXuT!coI@SYs)t~{I5ij9V_WP<$6}J0MonlUjn_%kqT=(tIL_)t z^T=!9Aqjvg{IHLe%+Mc;bR5MjtbK+ zp;@mNX^UHy%Nz!k_k<_(5p@YVk&&Fq%4ntx8Id$+B%^?hYnrl;T634H>LD$0jx%LU z!j~QQ1-I6;A_gU(Bs+idKzBXNPQgoxD{lhJ(AA1)M$4DmRH(V_Jplm8RHO4`)-0Pf zYGTi%Cp0NFw8$Y%Y37J15;x-v01zQV%o}1k1V^}t9S%|-wdqOhl`VOEEO+YI-AP}V z|I$b1vZWY>sp?o4kC{H|VmEbaLDq0ib4o!Q2dR}v=OHFZO6?b1}jL-^PuN}V)2~XIF0FJ zWBEwNBa=X}#O-5jgl+3(6QbC~Huh|pcw{f~5n9GtjIy!A>}k8B+0lmfY>#;4BP5$R z(pDC=wRH$FqzXwTP(yYZqwH!~ix1%rm$jp%ENyL@+`DXI1dw#aK~&4w;0{-~#bs=1 zEju5CqC}y!RRkoFpAKqAIXCX zQ+N}ReC~)*Y?MPpQot!bEj0Krf)z-?oIIGYi*f87SI|JJx!A8^Ux`@MVmOk($T5A( z3&}AYnaLsHF(hfw|CmO){VFav-FT&DA%k3x#(=C;Kf2694$jM-eAQ@?=5EGW*o2}d)T z4;h{`_<#{rC6EFkaqjb_L0uARBw4BrCNuGpTj_JgBGgXh!a3X9Y80=4|IxCpRxDmE zg;Xqr4}P}wuN9(dbH@7AhfOZ6&xr?MN0ZIxH1)Cv9E;qt8rn)q1&ysh?P_n^A*49x zB;3p4l6aV{uE^~OxE)7tOEt~uc5j5Gs%&?A2NdfagOBH(?}JPN0LXr%uMJ&h)ZRI_ zC)W4W_FU_OD|gW6MEAl+qm4=MI1bN-_`Ib(iBUgN3hxy+NyzO~Z8ZFxakzN7w2|9K z8vNvKrHaaT-EzB7;}9ztL9b;#Z)B{%UpDS& zztSN(Zg&R04h%WQ186w6z07`bnn#XE8i{?r#?dXkbOREf*9K8LyL2veS_zM5L}P~ z9`am}LwKJLif{$t{ecYjBLl|TSA;(%;r&CbfBOa@1`l{3`<{&$CLs>EAP(FQ4-yX; zv@iV*f&sH{6T*-Ev~Lq2kPjTtATH4M$j|)H&mj2E{@#xu|3Ja}(oY+_FYyXO5uWW3 ztbhjOjU+Z<^C$xu2w?<9fDlyh`rb_K*sj8+4n4G!A}kOaLL0cC(C#7^qDt`K zPYM6*@OF^EiqIg2(58M-7jXg-Kc^Fa?u|ut>?^**zyq~E3Mu5fCeD4ApVUY42;GM zk{p$w-K1>!%PDObQLp>kmOEGB1?AZoH8ZW1VYOy0DC z1}L&1N)rB3P8(6e7~d`@s6i2UYz;VaB-O0;|6EKY{UI1XN-ynlB)n}TuaF>6k}MOF zAfk#P3nI<{psK!-AQlY(d@U!pu_t{}GJz5)X)O|ovc<})6bphF-tzlsP3uN35u*|S zXpkDTku(9oG)Dk65fcCulOPz=F$1z7BGcE*4FH~t54b=f$}$EgLDO228c1OcpkWiZ zKml#@5EOv{xnL?+pf+)$`(#W9{}Rxc56AS9FX_oIx6>iSbL$pu1gdJ?(9<7Cf!KJ= z_B!#AjO;O?fyPR)((r%_J}n`B(iG7x5zlK4l%U0$3*1J64<^AH3t|(jz#CH&6y^{b z%C8_0R6z-XK^>GJ-V-69K|Yr-H~pbL{{a9rs|+D6vmmGfBawkB#sC_kVx_yCLxnyyK`j2Cn4JC&5~p43#0!rY9M2UpArIushdi&im{AZYC@y^+<- zjR)54EGvx*zOXK_G*{1*AR=r8|KL=>ym2b;5c!r8+#;;wTtUS2)E}gBL4(uhs1*RL z6iAe>vAfzo;uP+}BFhn0IKMkV1=F}fC^D7}OCFztlqjd!W(M)}m#zue# zS`5J6(H|nLQvnqo0U#EvK+>{xTM6@G5!7X2R%XXlTLIu=L-Qv!_K}Qr(Z)<6!VKWz zwIJq|AnH|O?{(2=k>A#pAyVPh`p{t)_Q>$^$W%4pvNkKE?JapQ$u{rR%2oaT^dKND z^&T_A5YHeeQyYZRGDX%P|DfT?S}ejQ&*uPuWxaJadEhGjVP-A0AhdziZk8OiOmOvd z#k$oX*48UQ)NNIeXc+>-EG}u46#(eT37F^+vGEEV{wroptwL=dgGFyxqHWY6A zHX-m}PMw!9ca~?TQW4IpdLi?XT1-!S_aJ~*a${|He^591j3K=3^E6jwIhScY7XUz) z79$K)4^XkgB?A*u=u(oAya z6po%{!;IPhjLYJTGrg@8tzd79Zxhzb3P=HK3mC^x zwaElnxfC{QPOLZOIN`Ve8nl7nCXo;n0l#P< z`)F+iI<*{Wku{r33Z$Uw{5BVoj8>Ph=QaVr_-qn#fyN+-AiQvm z+xU&+c#het!SWc{q`+qn0uP8Sk|!1!CLu!kO*)KZeg2l>mWDqTbvr7lJtsj5VK+Y;{|g zYwH>M=6UxP+MwZam?4@p_L;!i)1O7++WgG;)=LUDL2M`5#Q<2%zBsv_`Km|kok@DCfibFykgVG|<*XX4&mybc`3vJ< zt=T%o|A>Kg0qr}Hpc3?Yulc&K{rax~JFo?NunD`c0ecPrz_1m2u^GFu9s98%JF+Ev zvK!kG0H6*iJF_)=vpKu7J^QmYn+cczv`M?PIU5kNzyttb2i(9BO#8KAJGNzewrRVz zHQNzhAO~!Fw`F@xjA9RPAh?BlxQV;Cjr+KfJGqs6xtSZeTL1u>JG!NNx~aRmt^2yM zJG-^Jx&fdCw)?xmJG{kvyve(~!CM4Gz`WIay}w%qQUC-100dG%2H3m4?fbs*JHPdN zzsFkyK41p;JHYYVx1D;f7otrKyfEC_#g>!CR-mR6oX%D(sNwm|eA-nDH^SRmh;wYK z|M@hoHJr}OZE6dAtV7%&dilh&V!;!P8bIOIMj#HfRK=yNt`pkA(c0s3oS0XD#A*B_ zSiHWL6UNsdPk&s^B+{HG?ZTII$G3O|RQSk&oWl`JmOtFcrCiN&8lx-R#A*DKvpgq) zT)com^7<_f#yQNHY{FZO$yxQsdBAVgoFvA)wqk)00PK(Ddd|J`Y(ve>*{UB-0RT)( z!+C&}`Ftemysfk$6?8Np55cArJ;!X^)&QN8)IvN)#v)7s0D6U0$bwvidO#=LBo=+j znDddQ^s7rf#ghEjE`5UXVjWoHD};d&;NmeNW3Ad*3R3o}wSgL-ftqu{6;h!Q|2JV0 z6yXpEfe`or5AYxk_-+iSoeQ|Y>ag7k05IFF&f2Mc4EQb%pq&qz9Xcmr6F`9!m@XGw z6dI(tT<)Bg7ZIdeoyoQw)}5T5?x7ucNO{fy9z?y@tuuACVc26K6&!K>@L&zhZ|X?S z;UHM*5c%q;J>28q;-MW6ysz24FXJ!%;-`HKt{st;fCs4V;ms)z*N#{<{tzYs6jC7; zhyjm-0~$aN!jKOb ze(I!Z1$cms*Pta`eg<737P653l$taO9vO%s7h=H`NFmvoec{b7aHR@I|B1EzHX#+Z zdF(^t&ev-MvX0exUZH*5(PF(?=z%Z3;&+aO*F(xaEX04*K~(&g3!466bAc2lfe>bt z2S~1DrCA}`pX)?)%4?xBF2Gx(m~6*d9$UoetpYz*+=5bFN!{}<|| z{t)nB_5F>{V1E-_;iU=F75IRc`(F23jT(4B$$wtF>OuL5elCokA-3WiWFj2`USI)Y zOdNj%3mQC#FrmVQ3>!Lp2r;6>i4uvWnuLc74FDWRC>g^?5vg3r|0tR~i87_iiO6!1 zQp9JBk`WvM(BQ$52cB3fd;0tdG^o&_M2i|d3Umw&YDk+p9hwTj)2S5qq5Fn4>o;`t z`Ze7LZ=BYw?x>nQi#F(*foa>iRjCaaC_-a!a-?AekD^q-n7;iBSRplFsT9>+QIp3Q zLZE)Aq~r8-o% zrjj^^NI9kr{ApZLhvFznJmyd;Xu+F1e-3?RR8Y~QcdmZ2pSslR_PM=RF10-Du0t_7^?k zjMa{U7OsQ`dKY>I%qDAW1b`BFY?8}k8=feUHo0tq$8k2Tu}Nkq&Pd~d)bLF9{$K^Z0a7BG{HK@$yYOj06~RxYICBp(7W!Wfgtcx9MkLZ*>l zW17X|h5Lw=6*>A;2AWptoQWbIInsF)HKfFWm`4E9U>6$*5elyRm*byRyu@X_U39390l8dsE|K>snPF8HPsI=N9NuYt-`qXQH z@BsVTKAnQ|=C{BdqJ+8%#j?hGhY;hgq{JM8(;7V0%dd?%_M1|;|LqeRR^@2=8a@So za}L0*Dbg;$pztt49ML*FZ5$);Kpn;&M_1;@6A4^jk>7;lU7oG>B#&10h&-7hJoLK> z4Kz%WEo8`;m=HBP`vi@xZ~4sgQ1tfXvryDbV)G_2KkXVJBY+8YLnNoCk5)oHG)7Hc z13(Btd;q|fK}>hVg@3sKFo{8sN>MaJ=VVn*)qycxuPZ#z(6N1A*W`*nmw8PC9{!E3 zwiLSUTaY3k0>A~B#CSad4Sd*&ctOw@%Fzmn|0J|F0DTj5+06d9`1L@ck1qIo2030+ z!lILeNXU~mF%=8j1zjdZ&I{$E6+YItRkOO${6qhW2-~j;4K}M=Zga^bx&mX9g!X#off)&gM7yz(>_u`Nld~pG_f544YEL|9Yh;6u-!p~ zcL;yDLJCJq$ob}jhq4Vt5fzKu{l0jV|89X%Ay+F;II>bKii}`X1c}OJlt>UMl&>I= zs$Tor<_|^uPks!!3Rc!JMzd^U1>Zu35~RSqpG+=n3As!a{n53|kkB9JbD}@iCL4cj z0swe0Nc(cf!6?2D02-R)=>T9!CidULS)u60En_7S0u>fs3sI-NI?mS45T$xawB72(;#KU&ONdcj+3nMAU$Ma z#0bKJ74Qrigy6{?DfvGyI;0-p+@@1NK}c;Og9cjkA0rXsNV5T;dXv1&1oMc;6{zzd zdbC|A55ffoDo~;h?58X608LB||MYz;kU|VkH%nFmAYRoYT}MBv(6}|UbyU+v2MMw^ z9%w_D%M0EJ{z;4;Dq)jRoZ``90VNA+!+9aXqY|`flsM3!n)Yn!Lqg`%ZdT_VSvitP z##s>I4FpnPf)o!F@koE%@qBPINk6obE~gR&6eG}^jrB)g`*}%aveaz=faBN{@y3L> z!mj?f(FhK)K+P5}TEG~=2r@LhSfC0I(7TL0)gn{1>h>X0gXT7+8Xc2uO^s~aWZ*cc zOM;}QFmL(t&1K!N4gK6;r$Y+@6J36XLo>W@^=giB-C z*V>j@5SJP=umy>V({#Y#qS4ly2Hj^O53+qsrxM1t5xWgOOEpVYT-EX|?s%(^k2xm$$ zzHS0aHlbJh{x>$vLPWSL<_z|f<+-LzP6>Qya+CWvIInR5i|F--Hd28H5IOC=39&GJ zf49O(Lh5)7No@O!NXW{bu0T$1GIPv~O2MT$hFnI7h2duf7va$$qXjWS7FZCa6vTm1 z^l-(p`j`=mHO@#;|3p5Etm4-GQ>`z4PJ_N-KN=UryIxab(X?@yfwZrz$FZbDw1<^< zKpI>}-j(pkTok!B5|zLJ#>z}tG*_&ktuDnZq5bjY))7{s#bcd{BI^%C`9miZbZkfs zlGtPqgxQttH9~&vOA^1wiz7fPLe#wM1QJBG#_Q{(38Lu9_3tQa~%u22spt!y~qWMwb@7XFYm*$XOYW z(^1kshC50L=O&7*0XlJoOfp^%@7iNP;n#w+mIqAv@+LEBN>m+I8;HqhXQkM|e6sx6 zCqMjXt)Tdi;$4e-?`5PIj1yD)gXao~0|4d&siV1(#nHM+K{RHn&lhBqnkNb&&UyB0 z_IAZhKhQd|qH0Ye{@lguM<8(r5&WPPDdT|>u{H#ED>Bi1J;7;SV<2?( zXXr%?sB#wzHVF}B6LC;Q98?nAFIHeWs6#lJm4z`!K_Qid zcfo|S15KXyeg%jSeYI|};35k|Jfy`%9Ki)lVM3%df3Qas)bJ5E7)X)^Y3-&U^n(@P zzz0U13o$v?hbwHw1>*RPHOF`-;Xts~U0lF{ z=NLV+w`#9uA^fl~6%rhlk`?Fxj||~J=6Ea6;CXR?i?XN({m>~WLmQ8n z6`!J#rbt@+LJa-&RS$WSX#ou~^I8D;lixy%vG-LQ!i`u#4vFX*_OLMH;FLvK|5`cO zFH3VA*W!~@c@)slEJY=aT&XS57<(RcA{W^vQ>htHSwCV~l@I4I#6SoeAq8ONmP)1w ziUU4Gd6&|{2P43jx)mEJ;zw994jnleEu$4?CzdW%kr)FED#TBuu$6uJiF>jUN^l4% zX_?Y$c?aS!A$76@mVjAlK+pa|}mDpRsoxw%BKpe5wP1&Y9#(rJ`5P>W7CZ{&fR zS`iM6=@$GDo9q9sh{8Smw~Bbj;SO0 zaA|Eqo=&k3?Qlk8u@2g3nJ%eY^f?J2v|ii^e6bLl(-;ezU?-0w1@P&iAL=fj>5Dg$ zBnO%r;lK_HY83QPm|8Io_Mo3DuyiVRABte3$Pq+*F{CgT;CDD4JH14{s_O=imo{Ez<{8z2}48(d~hD`x;F>g{}D9MvTMKx3L6W=aI3NK zIDf+hlo_#4gLj~LgDyq3NzxDWunwJ~v28mR<=_rZ8n&A0ky|&iE0CT48nB=+35P(i zYcK{@AO(A4tRtI2D_}D>!2>*S12$j-I-mnVa0EwS1V&H~ zV6SUXK!nOoijBM8^p*pGv>b6-S4xH+#vWpM^RFV$+ zH%fq;6A`pSTeL^ZfT=(VO1rc}qzIq|1QB4p*NeRofCh(f2-7RGGfT6ikP0@7vpUPO zK06s)`!(&N4Y6D~P!8vi4(#v_ z^KcL9I=?#9cHh`af3pTZixgduy%+on^q~!7Qxg!Y4X%JJ9KmEA+Q7TAto?{xC9JFv z`;5pSGlfgClW_$XoV~N~AGrV`kI7&l+`0hd3KXouG%~RexsE|>H*0#12N_W-;5~n1 zrhfqiG>pBs03aKTVGltKslW#=;=@2}#0N6ADoFz%`^6t~ug{nZ{WVh1(^oaI0_+$V zOCZJ9y9)v$qFJTIE@B0T5S?X=B(w{UanPN9+%Ezli$RGKQh+U47b3@d7C$h@)>{mL z0u8ju5qsRngnS{;Km+ybk!48d||0;Z7eNiz6q3jej5Xljs431LCoC7(E z0LP-79w)4leFc}e>@K8yc4I@Fmy$9AQkA)q>9Tk|FPw3SF}Te2|^ZK?1LA4ckJ_ z8_^}_+|I61XH>a1JNnOfvLq`84DCls#S9&<90A)h_M1*Ld^yJ6rAkIMY#f3 zh|#!mziNe*MdirNkq2_124v6$EWiRw@B=l#(ikJrvhtjpaM2%46kN#4H;4zzh|{+s zDlXPFuT{_gp$2jgJx;=7s$$bWeHPQ)|CjAU&`!-H$kJ2^jTm3M)6kIzgTS6o5+!lK zDpZXW>fDzdP1df$F4x2?$I+8a(gk^7O}WsVVLjGq4HN<0mo0+VpaL<0osNduf|5T~6WCm0F^uv9jqq-?4ZpYD z+k(ZK+Syl6yxd7LNKYe|8({^Yu+R#Xmhw z_kE70-QQu7!u5g-+SNhWZ7d`(3iFxAm(0fy{-OQ#-X*ySb2Q;h@|}S)RH#A&?yW5m z(BU3A%XWduBHoz?n5=|X-YJeEQnxJvP7`aO!?+Rwt5B3RZimq9$K)-$hp$r0=6HT~wk=;SKgS7%=8|1miw zE+B$V%y!f2u{^&@O%qH#>&iFgI#mfp=+ZY;YH0phH}#J+@gU6R_M z=SEBoBhcf{o*p21C0Dk@^!!D)aP5T5oQf^hyl!25BJX|4MWRjaw9!|L9w1n~(1A1y z5s=TgoY*uG+RrHJU(7^on(y?{S7ELorg9wJjPDNf3J=}Ro(x3K?{NC`d zAv^_cA@O|5A}=zl5CI=e+n}x7+DzuSYy}uM^4Q^UrmoOhtyZJZ@=lH1A{_Ix{q3U6 zL6Ls*&OxI7ItgyA{|FTiHJ?BNeNEp&FW$OE@7LUZ)SmQ^p_~EY3PWvyMb-%{(A!ln zeLvh(36IMQ-}U72_4IMskdxVv)(JJx-wMvf_8my=GwvWShHnoYEtVdv{T@&6bd>M| zQm*$Pp2>ET>=>ONI4<~RA*0w4+=H!rjj;HVuH+pQ;!xx5N(Bn9unQ24=%7%} z{v71)0!rIMj2Sg< z&(&bB-F=fuASra9TD|slbDz?qzPoP1C4kcQYC^ArS zRsa|=W)d)?QKfS1$8X=jj?h-DncCIsSFmBlW{d@Z)L67>)vjgR)~&yN`0B}Xw=Uhe za^u2*`_{Lw4@Q3df)hv196ELE-p#X@kKbE=+CW(WK;sZH$e9;zE4GSKo+~VDjwW5& z^l8*Ge~{_ZY1T8%v1QLDwrS1M6;6rG4%_dZymsl#jpHlWuefsP*1@y4Z=p6RN-pxI zUfp_B|2`uyU+3Q4`}d*y?8R$`Zk%!S=?k?=-+pkpbnfQeD=0QroDn19!~!q&*{n|b z=;DXe3N-LQ1QTS5geBB4WRzR1>2E>`6RPbt0Jhp>w+bK9N1k=gb8key!a)a~dk#`+ zyAE4)aV;sNnC?XyYqW8weCBEAoN!3=@kb!P87G~0>Txcrh$fO^6KZbs$eLR|8EFXx ztF-b;EK9>Fk4AX%$w@HR0%npiM#ylD4Qa&39dd$nGaPjy0f4_S>$DRiWJciQPComb z5FdEZ>CsL1zyT*5al{E{ui^+zZ=7}J(I+U$D5HT84*ficvsD_ABeX0-6?IfnTdJXp z|JG(vMpIUU8nevG;sAxk@Ai2|L?91Uv{4di0YH*<@M(3}8&ffhSY(q0YoB<~;nl}* z$T>$HcjBSPo_q4yr@n`*%IBVX=7EPD;cx{~oOSBq7NcD717HnTSXJhg0K2H6RDAQ* z7vBa4`6NPR14bwpMYu4N23cvPE}wKrB!|fP^jR07xaxst9dn+oMiRfunMYudyCY-- zs6saRSgP! z5zs^%bqO;3y#@aGE7T^I+yP*Bd`Ly%#v%{C`Gp)h8%sSJhMsj4uYVJC2r%?SK@4`M zAMv=EV9fCj?scdi^%kU%1kX6*2%!+I=nrFU;)+{zOFR-Nm~+TuXE(}+ zJ8&foKlq|A=CFr%Kqr#X@i0rKdr%O$NTL4(D2Ak}K~82@yE|$|8?GRP5dnZiS13^^ z`AD8%$nnIt=%a~#kw+kfs0aYW%aWU%$~+oroN_R*PxxrZ7yEJ!$jR>&pSi;8M75eh zypNMkD#j}7$G-f@a+YfuWFZA81wy!DMnJI#ufPEh+1%r1?%3r7@l Msp+iFb8_T z5iDfUqnhYxhXlcrM;OXbRDIORD{LsuKD}`yqzfAq90bJFq0=kW|40QNl<>=74pSrb z0Oh12#}4DHL!Qp#0u2;4P=vw*J>hW27?FjKN%mzNDf$XXbhrdLV#$YfJR3rh^`D~c zafUkKiASSCPkRc`3Wtb=J{fWkUJ2AX^x#!HOlmmj5bmb(8QN@#~LV*S;@bEK=u+*gn$p^3E(H(qz@*BvU>dBzN0~wWdmif4u zI*x=kdsMD3>F5elt8}Vz0#dEhd!wh=sUWHDH7HfRssOp5h+H6~A9Agvck+?IDhd`^ z$XJ2&B+Es7Bv2fXJe)g>a}IC*#IC4e!PvTF$IDugXfQ-t|7rr8TA)-l4ig!{VRHdc zzmWAiUG>X2LJQkF5hFLiz2Rd8gARup?jB-2hg3#elYXI&xY10X>qzt3>HY*Mvjsp2 z=+GB+P>3FaVPFmjOTsE_eCM2{|#2*Pvn+>-fb7i7`TGY-2y~xW_miff7=I5NfQ! zOiaPUjKZv9gXno7cFysbk(}cZ&|puQA=*e7j1mWnSR?ol3}@MeVkOY2&s`>jKR)ei z0kZI(z0OEK_-0dCe>lX+M%z5d3luy6?GT&jbDSrUp+k^@4-;9mKYpeJN7on_gdq1l z`9N+(@L0^`X7?e${O2?S;Kl4-1Dx3nY9_C&FX2e8Xjd|e*(hNJlOT8zGt?v`1C!n4 zrqxnLU_c(MfZTk5)uj(YgGszOQ}-OfbhPoV|2hx6;6_}5&_oRic2|bou1NU88NQHc zOJUm8KFGCKknL^Do6(01cPhqRZUCH{5a~Wd&Z zPKZk*nF|D;1Rf?b=Go+7C9!6>Zw)l!cj*#d}h`R%}<3os|qrPI}8~wAt8gYeXW58@0 zj1J=nfg7ewd$BMZG};rh-IIc!;gHM=KZNMKh5&}$A%;|AA=4X*zv~Nds4$FZy?|pr zgSas*iob@?G;N@P-rKz%Gzd@tfYX}hba5E8AJ$ZFrv4iLxrFQ z4tz6c;=P10zEv9ou=7BJm@R*>g5I-&*powl$VC9~o%ZtwLsU8hY>1DmM1|-?h4{qX z6Gc+|zWzH!GBiLlM2I)jqD?SGg8;_$Q;2Fb2qmBa_{&BRgTu_o!<5)VK2#|1;s_Je zla^wLYaByFG?ug2z#DT!hv28aC0>bq(@$eqlwo*bBLoH~#+h(2qQkqonvG(}WAh@#^?*gK*o z2{X^TF)e(cI0WF^ z%5D)1zN`^o&@=jqOd8pT!ElwJYlymBhD9W<%ow9HR6FiGi zB!yA5xZK>$40%O=|ES6-{LSDLPK7YRRa=P8REW>?w$P-3(JalfTuh*0MgbzkcgvK6 zaLag{I(@7&J^9XEtPI^WlQF1Fx8O%uvrL4zuLZ;iZAb;L6Cm9)N;n+AFr$X@xd`4d zK96jOMwvhZoe*N+!8`2G7Rd(%5{jZs2w-TEG8r{cls+081NJ<}gCIeMIL>I}MuULO zpdit{AcxvSiQCK!IgEqN#72+2N*p`Ap`1R#JA@wmxI?%$p&0=}kWzYNJ|Z2@?Sw?nDS)w89k_CQ((lQbn>;jY3sb(=vpnHzPf7 zt-3a=(>%*hXtY)Y97W*cjX#ZmKxIIE`v!{Ah-w@^0F2PXoY3uz1ICO-FLj7ds?<%L z5Nen+?Mzq-;RnbJifiLWBJDNGioAoMhVu)@j9m(GO~qIBha0o3JQ+<`d{CemuT%Qb zlL*o)|D!>8MYw{cfvhwL&Vh<0rM>*1245VsL&%T5n}iXNxHyDa*WA)V?9PHsh>1mr ziuH$!mAs8b&yE!!R$Wu3Oh<$0Rg2@Ca>OL{Q&%Wt(L7bhP4v^f>`*65&>Wl-gy^$j zI!JS5)Kgpm9@rB}c^qoAlaD^LVQut}kT*KG3{y`$Rtv|8Ci z$vC(=XvmXz^+_-dvRE97vs=BOkilI|h%hrXpt(9%ps^~vF~W7Yy7E z2{ZZBU+_3Px}p?c5}-Ixu~@*6+L~3!JqWLz+*ItcT%e3n3G5Z#&t z$??qvB*ig4Akrn%hWzMN9HRjkWxP0pK1D=`iI|ag{a~cs-RbKxE&aR!7Bd4zU@lPL z8YzOds_?hau1#6oUmwxBh?@L~$m2fi5xqu^j!=Au#-QJDM&TZi_wkivW%veGn^F4&{}ArGMd{W^Nv*f+{`SRDBQ(PzH~1&aG|!j%f3~b9M_S zLp#0=oO?*1ui9mq@S#QkqK;Z;Vgadenu+sez_SAfgar>L69*X7=eKCbHMmHG{tAAO zD=Kmxfbd1vp|D;cq4hw#` z*S?r!z|n_x{)URS=+b~CU3zKms3lf%CD(FTF8+q_0uPmKW1p^$WIoEJ7K?r$F(F~w zfa!;Kc$RV2=>id(ciM%JerlosCy}y}ueQHQ$}jFn>Td`K4h8Gjc(YD(>#sQJHL7N1 zkq5E*2650}sd$Cf3Y9I{2`iK9q3GxxnxT9a(T5g{bSP=IkO$-7Yr>X|LsbHoo^0dk zAakHO2wQ2hOe$X1TAQdI!fAi`627zs?e+siDNCRM%a9FojNpe` z84+>No*S19?*X6V2BYK-lm5o2(lM<7SFifMZ!F1gFEMj}|DdMyPzTBG3Vm>g&*pAG zw+?9F9Ccpwjkt$(FeMl_2X>GLe30>}5D$1zha*Q1a(D+K_Y>Bsm+H=v*un0p0v*x$ zt2ugZ>4J2H@TBw*2X-iQ9C-)$77TXKbXgaRYm2Vc*K zxR{4`PzQ6M^z`_kR63dYY3-SS009^P0w{n2Fn|L<01rrj1W*74SO5lKfCg}Y2Y`T+ zfF(+(cYD8ge9w1%-*0X< zi**=vh1U!)27ocn_>9PhcQ}VQUz~9;hjd^EcYp_Z|G1WWzz2ND2Yt|oeb9$|$Oo4P zmwAYXcW8%oK!#;<#FPzMxw zcIpA@H2C`@3xEY6eAG{Umf-27s{F2~hjjrbAlTo~{JM8|F>?Lx00qc{aNBQ)e#i%U zcn6Fjj?E{LXh8>dhzEPf`)DdN(sz2*e}1XQXmi^AwEy@J%3*wQI1#(1e(HdRdMjY= zzuxaUchBCC&c8qyJ8=dNu3Tu3xuSR(Uq<+O}`w&aHbT zF)IN0c-}31IPv0si7Aq@dAZo9#-BrvE`2)n>ejWATCoD^_3qxkSB}Mp26F%_@DSRp zgc$ht?%%_YFMobfZRwT7qOX5He~?|jmd67jXZfRzW_W1RpMncA*r0BIYBXV3Bao7zi6^3%qKfmyq67ekB)FoCGh&6Da?Fh|1}MRCv`JV} zfJmc|LlRk}ky~w|9d}1E+2lv!k;mM5uEh8eG`~5SrIuTAnI$#p@sLWFW6Jm+Sj-7X zAW3SH*`}Ls!YLm#aWIx&oOkNCVR-;}82==me*zk)pw$tR*%<5%nrL)3mY3s>ib5Kx zq?0}ti*j-JNvWogL0R6ERc;!psH2i9R4OiFTB@p9nHk`KtHK(qtg@2A0|0nHGpnvc z>Dl3jy8;`mu!DkPD<8QIn`}pn;;18!%0e5hv`2129v+!aJF9rMN{I)R*@7FcxF{A2 zK&s=C+A5l_qT8;!@9tMCg^0qdXs^umS+BnP@_XH>0F1$Aziu+SoTJYUoUp$V?=y3c-wmFEvQaYUA57!WX8iLQ(H}($eWSe zwb*0-tnvsW5aKi0v&FpG%xl9Px5?B@LLL`l$~{@onhl+|-+yC_H2^dyf-K;IL0vJ| zhcn)Iy~q#()*5y?e%05Sg&&7yYRyYi_9V9Spmx8w_i?H=EOrEy{xgU0l)}xG<|#NWSuU( z_urRF%_hFlKnT5~%RUzE;IrTUsK(o>MCh9TzFF|@!y_WGO(qo47&aj~etmF6 zAxt6_tN12rkV1isQN;N?!oLp2s(zZepB2k!#xZHb75xK%4mGkE8VD>_I1Ec0&uGUx zW+@khu+<9UKnj<+C@h4SjW(cx3}WOW7OqGIDL}D_NhsnFgx~`o@DPVJU~&v&-~tz{ z;K@%~K?=|fMq~kkqcZxW0!;}z zM>XS7F06qMir7Rda)As``ABgM9V$_q1dKySAxfG7l1j9J3~4HY4{Jy&Syu4THQ?b8 zn;?Zrb=jUmA~F@Am;@oNNz(;3hX+xiZ zfWj5Pcw&Z}n*WPbY~m1j7(*b%vH~f@;SivRh1%k3*~>bJ7<&Q$7YwHjV5CA3aj?Q% zAG-!ZOrocRb&?{+pa@1P0uNl^79F|3hfTPmvY6d1Z^1K(%_fu!N~Hl8@Q_+y7!*#> z$OS155r@5sMG0#-L@Juu+v{TYIUL&T2)OWCVkB2m)W`)W2r&lQDj^P&h=t3t>)!WP zW}KJYC|Y^221Vp{TAw9_d80&wHLN$h@EtILs{)KotRYHLzym0dTQ6$FLJI zjC<^3fB)!7gPX9i$#S6x-}KRjKVI^HwE>FH@quzx!=eJOCuJsM8MC>F!wAafn7DOC zfrP_wmdh+mQYdF~j7wUYzij3=(YbFNcA*Ap6vMp7ywX$M(A{rQ*(zJTn-BjDt!??gau(P z+I3NoK_@)4m(bc4VS4zG=#dZuEO7wEkpcGT@=Y>plSZ{>rO?WE^Ufcx7#xH9H-cQ1 zxBr!cdw0Y{?T^La4I^~2kP^@aZ$bndBT;(RVgMj(a~<5-{#cn5v^pU!TtwFz=`}`9 zZElPR8vqj(H~@|f0PZMf4Ai{Tv%^x?i)a=A)HX_QRsig6FiQJFvop_#uxY@fdA~X%BN%l?{C(*)aAoQjsi#PyR3eOU>)}1GwYf5~vX5z; z*CiMD3ZQMckOP2T#`XuYElr_}*eXHL{`f^2zI3DfR^mAyJpQQh(QFUp)U4Ks6h<&~ zs$b-{S4O(hJ2LB=w-QuZuR7JOFoG(P0swX8dPrzM+NdWTU=H`ktjCTKC9D8ci~mNu z%s0OF0%i=@Xf zkEI4Rpb-muuDjBe=g5RaviV0-9TWigx<5V~BPGzF>$d%T-&gS4hsUu%ckjrh;{c5R z|6gWcg!+jI4aDC?2wPqOz$9G6+|`{wEDb0;5wxYk`oY}#)m~Hp!w9@w^m*X$gdUN# z1hlzc=zR=d#aj5Ko=shZH<=&gJ>ErVLyjoZt-Ri5xE^tx8s4m!k31Zx`Gf8;U&b*9 zMvzwD0YK9vpf=FJ2v~>%p4>%1Lt{w+1GXGBL|Xk_$q)kG2q4}Ef+6Q{!T<0r1++Dj z3Hn1fvD1XCz}QjVu&EOzjU4!e#QDWw0|J0&_=uVmTpUsoVG&mvVx4Bif)BDDh4`VE z2pb=G#HCrmCO{%2x(E@*fi`^B{w1Kiah5+o;gSR)Qia7B=v^M-0sXDm7EXmWY)2Tf zBJ8Z9RH)z09U^-W-9_jg8K$22W#C32pGF`48KbmZD;;fIk>w z>vhB?Y6LKNM%rNn$2A;TV=a;z7hKxNZJj=r9{KUkNf=T}5W^Qz2A}msg9^rPeW!t9sMw^(Jlf8p*<6N#DA~9I4K#-BfI>+g zXNamO%}@ihjX{DsPOZ>D2d-$1QjN87mMj7e-Q1HRP@jzgsm34!A9UFvq$t(I0+(sc zj|ORzW{i_5M-}4P(dZF*sF{;usn`(1A=ru(C<2WBO#d3;0rCJ)mYS)S9)gSXQH%Wy zj-?pDlDWL6C}`1&k;0Mg3yy6X8}Wg~%xR#;4KOIen}*nkEetfI0wH+b;Edap z3TmWw&4EqWu&Eb&y$dp=!Xfl2V^|oZN@}Q*jfrupe~kelkb;9L3xwrWqgtwnF{!A^ zs@0g7Xf>Ad-IhAS%3VzY9*|a8h}vzb>Z}s0?GQs~Jr`PyfD)W4Zh=>#Py=Z7s&F}$ zZEb?C7HhV8kN1_=8Wfyk`4u0eRxW^QnOGFL$=0_r+gd3?s>Z6e!fWb8Ybs2FUO^hO zW))8j6om2zG(;0AkX0Pa7Fw-qXgce=#%sbtkN;r-Y*}#u>{ZoOWfeg66hR?uWIR(e zA(kQ#!eY@ES&(RPX_a7=tHPQr5Bcj$k(I~7pU`-e72wn=y%IptQY@TQE~V7PhJ+<{ z-yucQB9X!)ajZ3M5w)zjMTw)HGA0E0^bEhsqBM=C-j z9m3!)=DQ8<;0CToLgFmZk}cURFd#$D-tFX4Zsl6;7s7xs_yEtZtJ@4>%wmA%I@saZvXAt?(O1k?&|LD@^0_??(YI`@Cxtn5^wPu z@9`pU@+$B0GH>%b@AE=$^h)paQg8KI@AYDD_G<6;a<90M!i7qcL49xIT31RqA5WUZ z_in^XmF(+zCP*lPc4!BUc5nS=$pU?+Z+gzjx+NnD9lV#Ew4Axe;kNMO); zEuj56u$F*WcgEe$hJ`D*!X(%V`mO{3pTq!b@LB3;>D``4{1%5k@Cg$MS}lqGQt<6n zpnRI|{t57iWCVwVa59c?NR%)N^Kc3SYmBh4f1o81!f-556U|PeXrbztaWF=R)eQq^ zSm4bvRWL>vS!C&GE>M#q+-F9_LjT9ouRrW?50kNsoK0qoTo8YRf!yA2kzs9CRYvUH zSa?Pg>!x{tX%z3Dk1SBrq{mpS0jPYMSYQM+C>5Vf%@~*QBA>|F=$;q)gBp7T8%N$z ziq3>6L3N$l`!2=`q{1%6f}=HYXYOwys~-J%UiCzb`q|uPxrkV<3=fb3B7ubrUW!F2 zhh~a#12b|jW60UGU+qb9N3ii&oX>A5L6w-t1r8)fa4?-1r$}7MVy)lRtxUxHnkfT- zg*YaUB(Ov-<-Cb!KfnUfW=@fwT)d8`2CWCpM;b6ZN|M(Ex+ z4bE{+bLe^3IbQRnz3)Zj`S9DIx$3s9G}|DZ*x)tm9G0RCSW;j~VRb$eX& z9fw6J%V1Eu$j4A|kSK>|>fSgj_1$HS|4y?eTIDrU@w2!{euM?(%;rAs-baM>SR3|r zlr=P<6#U-7ISpM;BEg)hdAXq2?dSpf&?;kbTie-8PXTx;F5W@li z*xunyP|=gjee=JWhq#VM4*wQpjoWK;H*6QRGA$616|?oo4_8#r;6R0Ms0lEc!`@h3>DIy=xJiC_=U9i#k@F#6qNRk zFIOR|Zm8mH)8qIGl%IS>9#_T%)%cJHiB>AbmUUTfh>tugLX01Uk(Y%0t^f>6#TVPw zh4P1$TbNZq`TvZ(6pvJo{OBz2^37e=23Qa%Qfzc(#>j*KIb~GmX0YFH88=bvTxFUt z+XT`AwINY}@=fnEU^sehcpT-VLUz&sgD0;P zBbE*81~h2;P%Oq8yp+!HiKngJo<(9%?Gz;X$*|hFiPve-k#In-{6aIBdvtg(kFILND*=5L6^4>NF^P8W! zw^s!;Pk3PnTduEQ7^B3lX9ciF#e~e`gvj-LzGD!TZC_U5NJl%!-%WniQ;DMQ^1hk3ejlkeau=_?lJ=2HgS@gW$rg_XGvDAHc z)nC1xhXu0mCIPDiGJwKaKm$m6q7;I{E6LLNX#6DH+~3Qh)nh$MP%+$#g}4!u@OQf2z1mx@y#I@3r07NPPd`JN>2;h)EVh0EMbNJAoHYp4Ntbpim#E6R!c^IU& z5oE}Vxd13}sEx*mj+;cbgeZ`YD~4Pkf}#k3rNcuvkv-f=u_s55E{Q%QX)D z2S>!ZSP%dqrfAg?TLZu^NP#>CH+K9Oa%9PqDOa|98FOaMn>ly(?AYL0hfozg#LHs@ zA7CBZR&=Ee7~ZTpE2g5REfyZp1!;hSrl_K5J^;oLQwYk{LudwVQ;Ri8G_qW|z-6`1NLXG~gcT_zbDf^PCh{dY-aCq_K$sO|c4SWUBK6^+l zxgwelxY-`sY6LFu0DuM>2mt_(h$f1ozKKeUsHu^3g9Nb4gGu z02+KS!U-#s2mlOY(U7V9j3Wj)hxWWFF+>$*)E~SiIwTdm;&Vu*J2N|M5`PX$Y$I(9 z@^HXbU40eSSY@4+)>>`lY)5T|%h5FE?gGFeu=tzHAgfjlNv#zfnn>BbHr$A$09(S# zx`(g>pj3wh?KN9}Y%0&C^BnprFcuB!)*y$rQbNG{Bo!5;WFKM{)&dnnazR0z^NN~8 z{UMPtP$3+5$9qFOOQ3{KglH#auSIt~nw?wjd3X0^vLz zf@!05+ja6GC_{o~V$4Wn3z(Ap*@`VH4Pw{aL)F5Vww$7!Qq165f|+8=X7;CMlW?X? zn|-+ex!RNQRaam8005?6wg7%eCK{M>7Nn@{n;PVYz7|;QObt@D=DaR)g%qMCD+owa z5fcfooNoOa@W2HhobbX8k8I?QAT$s!^t>%iN0#3W($CjI3D>pHYUp6 zjZlVYE5S&jNmaI>T4Q*DS{LZkiv(K1bcGc0aN!gk24HmpTmSbAENBX%-k|U)!;V$& zD+-^O@n5=HcZhg*$YrjwNrmv@Awz9RP;%J{O-Rz66!~p*rQ1jZu1DbHCdN2SHLo&1vsfAMDU)ps|7x1m#pBd644{*Fzuv5Qsq( z;t>5|5FSWj43nsu+m`4L`#A6+4=T-YcnA@|+yq7bI7C+<)U9z%#31^)2^o;NoQn$KKw4A%NOGZ9zVuqQpb4E;RVlK)Do508?=NW@wt6D|M;MP$<% zAPcg(=rIf@4*>=kiIX&@@luSRaV0F@^{mOT@I(CD$|A8fLbgDXA{CS!7x{P*WL^ZC z3^5&ED%cTUZiY~J5GPp1fQQ)Or(8r-=Q`QhPItajBn1Nr86k3%i=4?HfC7n3lu#yr z0;7V|n1tf;R!^uHBy>znof+xaI&r1woFRE=k{G&4f9Qyn%Tc932`aFD{u7|}(pV{> zr$~R0ge8ug$RDTQ^dbRl zRoozC9k3`%GN;^0si5M;2l-7*2L(VvbD`CQZvTd(k}p14KIEn$xkn$ zs=bDU6l%Gtat>o9h%gpAH?1RZ3OXh>rjw$g1(svwG7(iiB(r9;?5zyBKeU<@rOx!n zgp3Ih$8J_+h;)d3)EFw8o{^KB*^15Vq(;gdCQL#Vu5B;E+IjtNm9es2bgiqi4|c>u z!%?I5u*zBDfCL(yZPj)gTP({gq3DsbV!%KY)X7R zC~sR?!AF3>#qyLGA}0owiT@htHe=?^4~g?d==>PR3TQ@TPHS!*d1Q!uwQM?m2#?zf z07wtFB1lg0xYUwRZ9GMqn8_hXqV-@`zZ%xD{$YeaJK>9b2Yr(m4w!VQ7A4?;5PPu> zU0$gO?3AWoE5O4fCV?Fq2_~sF0N_P03cH!@>={`d>nQ?zz*~oxF%aTfz4ewsrfZp1SGIAo*8AT1-gnQ? zx>rq14_@||P+)L@izTTRG5-R4kTHBX4z@EQs$9WKg8=eZd$d+Xak@V0jEoptk-Ln? zUB((mq^~q!NnkysjmR^E2XSOPLmn?umTT{ATvuy-)p!W9L?w2C|j|a*Fq(ExGUhg1cud)Jg z0243*%Y*RtXvsiLu=vXR?&;n9P6S7g1WV8}s38}Yr=Gq^%wmBUh(Ix}|La3qUjARKdt_GWk24lenQw0h+BMir|jLLBAa=`~D z!w4%V@p$3~ry~e0P7gz(4{Hq>pkWP93eyJUCmPWNBT*73krELr6$lB}2C))jW&c)y z5Fin;lt4x{@i9b^2+OSROfiT?WD{D^67XRijXh4FRaS{Up06U?%%-H|{ literal 0 HcmV?d00001 diff --git a/psyneulink/core/components/functions/function.py b/psyneulink/core/components/functions/function.py index 00017dbb734..b5d551ea9e0 100644 --- a/psyneulink/core/components/functions/function.py +++ b/psyneulink/core/components/functions/function.py @@ -166,13 +166,13 @@ from psyneulink.core.globals.registry import register_category from psyneulink.core.globals.utilities import ( convert_to_np_array, get_global_seed, is_instance_or_subclass, object_has_single_value, parameter_spec, parse_valid_identifier, safe_len, - SeededRandomState, contains_type, is_numeric + SeededRandomState, contains_type, is_numeric, random_matrix ) __all__ = [ 'ArgumentTherapy', 'EPSILON', 'Function_Base', 'function_keywords', 'FunctionError', 'FunctionOutputType', 'FunctionRegistry', 'get_param_value_for_function', 'get_param_value_for_keyword', 'is_Function', - 'is_function_type', 'PERTINACITY', 'PROPENSITY' + 'is_function_type', 'PERTINACITY', 'PROPENSITY', 'RandomMatrix' ] EPSILON = np.finfo(float).eps @@ -1201,6 +1201,48 @@ def __init__(self, ) +class RandomMatrix(): + """Function that returns matrix with random elements distributed uniformly around **center** across **range**. + + The **center** and **range** arguments are passed at construction, and used for all subsequent calls. + Once constructed, the function must be called with two floats, **sender_size** and **receiver_size**, + that specify the number of rows and columns of the matrix, respectively. + + Can be used to specify the `matrix ` parameter of a `MappingProjection + `, and to specify a default matrix for Projections in the + construction of a `Pathway` (see `Pathway_Specification_Projections`) or in a call to a Composition's + `add_linear_processing_pathway` method. + + .. technical_note:: + A call to the class calls `random_matrix `, passing **sender_size** and + **receiver_size** to `random_matrix ` as its **num_rows** and **num_cols** + arguments, respectively, and passing the `center `\-0.5 and `range ` + attributes specified at construction to `random_matrix ` as its **offset** + and **scale** arguments, respectively. + + Arguments + ---------- + center : float + specifies the value around which the matrix elements are distributed in all calls to the function. + range : float + specifies range over which all matrix elements are distributed in all calls to the function. + + Attributes + ---------- + center : float + determines the center of the distribution of the matrix elements; + range : float + determines the range of the distribution of the matrix elements; + """ + + def __init__(self, center:float=0.0, range:float=1.0): + self.center=center + self.range=range + + def __call__(self, sender_size:int, receiver_size:int): + return random_matrix(sender_size, receiver_size, offset=self.center - 0.5, scale=self.range) + + def get_matrix(specification, rows=1, cols=1, context=None): """Returns matrix conforming to specification with dimensions = rows x cols or None @@ -1215,6 +1257,7 @@ def get_matrix(specification, rows=1, cols=1, context=None): + INVERSE_HOLLOW_MATRIX: 0's on diagonal, -1's elsewhere (must be square matrix), otherwise generates error + FULL_CONNECTIVITY_MATRIX: all 1's + RANDOM_CONNECTIVITY_MATRIX (random floats uniformly distributed between 0 and 1) + + RandomMatrix (random floats uniformly distributed around a specified center value with a specified range) + 2D list or np.ndarray of numbers Returns 2D array with length=rows in dim 0 and length=cols in dim 1, or none if specification is not recognized @@ -1222,9 +1265,6 @@ def get_matrix(specification, rows=1, cols=1, context=None): # Matrix provided (and validated in _validate_params); convert to array if isinstance(specification, (list, np.matrix)): - # # MODIFIED 4/9/22 OLD: - # return convert_to_np_array(specification) - # MODIFIED 4/9/22 NEW: if is_numeric(specification): return convert_to_np_array(specification) else: @@ -1272,7 +1312,7 @@ def get_matrix(specification, rows=1, cols=1, context=None): return np.random.rand(rows, cols) # Function is specified, so assume it uses random.rand() and call with sender_len and receiver_len - if isinstance(specification, types.FunctionType): + if isinstance(specification, (types.FunctionType, RandomMatrix)): return specification(rows, cols) # (7/12/17 CW) this is a PATCH (like the one in MappingProjection) to allow users to diff --git a/psyneulink/core/components/ports/port.py b/psyneulink/core/components/ports/port.py index d73ab06be0b..5320aabfe4b 100644 --- a/psyneulink/core/components/ports/port.py +++ b/psyneulink/core/components/ports/port.py @@ -779,7 +779,8 @@ def test_multiple_modulatory_projections_with_mech_and_port_Name_specs(self): from psyneulink.core import llvm as pnlvm from psyneulink.core.components.component import ComponentError, DefaultsFlexibility, component_keywords -from psyneulink.core.components.functions.function import Function, get_param_value_for_keyword, is_function_type +from psyneulink.core.components.functions.function import \ + Function, get_param_value_for_keyword, is_function_type, RandomMatrix from psyneulink.core.components.functions.nonstateful.combinationfunctions import CombinationFunction, LinearCombination from psyneulink.core.components.functions.nonstateful.transferfunctions import Linear from psyneulink.core.components.shellclasses import Mechanism, Projection, Port @@ -2953,6 +2954,12 @@ def _parse_port_spec(port_type=None, if isinstance(port_specification, types.FunctionType): port_specification = port_specification() + # RandomMatrix (used for Projection); try to resolve to a matrix + if isinstance(port_specification, RandomMatrix): + rows = len(owner.sender.value) + cols = len(owner.receiver.value) + port_specification = port_specification(rows,cols) + # ModulatorySpecification of some kind if _is_modulatory_spec(port_specification): # If it is a ModulatoryMechanism specification, get its ModulatorySignal class diff --git a/psyneulink/core/components/projections/pathway/mappingprojection.py b/psyneulink/core/components/projections/pathway/mappingprojection.py index 557c1b3dbd4..c6a3871c268 100644 --- a/psyneulink/core/components/projections/pathway/mappingprojection.py +++ b/psyneulink/core/components/projections/pathway/mappingprojection.py @@ -19,7 +19,7 @@ - `MappingProjection_Deferred_Initialization` * `MappingProjection_Structure` - `MappingProjection_Matrix` - - `Mapping_Matrix_ParameterPort` + - `MappingProjection_Matrix_ParameterPort` * `MappingProjection_Execution` - `MappingProjection_Learning` * `MappingProjection_Class_Reference` @@ -98,10 +98,8 @@ ` can be used. .. - * **Random matrix function** (`random_matrix `) -- a convenience function - that provides more flexibility than `RANDOM_CONNECTIVITY_MATRIX`. It generates a random matrix sized for a - **sender** and **receiver**, with random numbers drawn from a uniform distribution within a specified **range** and - with a specified **offset**. + * `RandomMatrix` -- assigns a matrix sized appropriately for the **sender** and **receiver**, with random values + drawn from a uniform distribution with a specified **center** and **range**. .. _MappingProjection_Tuple_Specification: @@ -185,14 +183,14 @@ In addition to its `sender `, `receiver `, and `function `, a MappingProjection has the following characteristic attributes: -.. _Mapping_Matrix: +.. _MappingProjection_Matrix: * `matrix ` parameter - used by the MappingProjection's `function ` to carry out a matrix transformation of its input, that is then provided to its `receiver `. It can be specified in a variety of ways, as described `above `. - .. _Mapping_Matrix_Dimensionality + .. _MappingProjection_Matrix_Dimensionality * **Matrix Dimensionality** -- this must match the dimensionality of the MappingProjection's `sender ` and `receiver `. For a standard 2d "weight" matrix (i.e., @@ -204,7 +202,7 @@ `receiver `'s `variable ` (equal to the dimensionality of the matrix minus its sender dimensionality). -.. _Mapping_Matrix_ParameterPort: +.. _MappingProjection_Matrix_ParameterPort: * *MATRIX* `ParameterPort` - this receives any `LearningProjections ` that are assigned to the MappingProjection (see `MappingProjection_Learning_Specification` above), and updates the current value of the @@ -286,6 +284,7 @@ import copy import numpy as np +from typing import Union from psyneulink.core.components.component import parameter_keywords from psyneulink.core.components.functions.stateful.integratorfunctions import AccumulatorIntegrator @@ -304,7 +303,7 @@ from psyneulink.core.globals.preferences.preferenceset import PreferenceLevel __all__ = [ - 'MappingError', 'MappingProjection', + 'MappingError', 'MappingProjection' ] parameter_keywords.update({MAPPING_PROJECTION}) @@ -355,10 +354,11 @@ class MappingProjection(PathwayProjection_Base): the context in which the Projection is used, or its initialization will be `deferred `. - matrix : list, np.ndarray, np.matrix, function or keyword : default DEFAULT_MATRIX + matrix : list, np.ndarray, np.matrix, function, `RandomMatrix` or keyword : default DEFAULT_MATRIX specifies the matrix used by `function ` (default: `LinearCombination`) to transform the `value ` of the `sender ` into a form suitable - for the `variable ` of its `receiver ` `InputPort`. + for the `variable ` of its `receiver ` `InputPort` + (see `MappingProjection_Matrix_Specification` for additional details). Attributes ---------- diff --git a/psyneulink/core/components/projections/projection.py b/psyneulink/core/components/projections/projection.py index b26b6847cac..796cd14c281 100644 --- a/psyneulink/core/components/projections/projection.py +++ b/psyneulink/core/components/projections/projection.py @@ -106,13 +106,13 @@ * **Keyword** -- creates a default instance of the specified type, which can be any of the following: * *MAPPING_PROJECTION* -- if the `sender ` and/or its `receiver - ` cannot be inferred from the context in which this specification occurs, then its - `initialization is deferred ` until both of those have been - determined (e.g., it is used in the specification of a `pathway ` for a `Process`). For - MappingProjections, a `matrix specification ` can also be used to - specify the projection (see **value** below). - COMMENT: + ` cannot be inferred from the context in which this specification occurs, then + its `initialization is deferred ` until both of those have been + determined (e.g., it is used in the specification of a `Pathway` for a `Composition`). For MappingProjections, + a `matrix specification ` can also be used to specify the Projection + (see **value** below). + COMMENT: * *LEARNING_PROJECTION* (or *LEARNING*) -- this can only be used in the specification of a `MappingProjection` (see `tuple ` format). If the `receiver ` of the MappingProjection projects to a `LearningMechanism` or a `ComparatorMechanism` that projects to one, @@ -122,7 +122,9 @@ `. See `LearningMechanism_Learning_Configurations` for additional details. COMMENT + COMMENT: # FIX 5/8/20 [JDC] ELIMINATE SYSTEM: IS IT TRUE THAT CONTROL SIGNALS ARE AUTOMATICALLY CREATED BY COMPOSITIONS? + COMMENT * *CONTROL_PROJECTION* (or *CONTROL*) -- this can be used when specifying a parameter using the `tuple format `, to create a default `ControlProjection` to the `ParameterPort` for that parameter. If the `Component ` to which the parameter belongs is part of a `Composition`, then a @@ -422,7 +424,8 @@ from psyneulink.core.globals.preferences.preferenceset import PreferenceLevel from psyneulink.core.globals.registry import register_category, remove_instance_from_registry from psyneulink.core.globals.socket import ConnectionInfo -from psyneulink.core.globals.utilities import ContentAddressableList, is_matrix, is_numeric, parse_valid_identifier +from psyneulink.core.globals.utilities import \ + ContentAddressableList, is_matrix, is_numeric, parse_valid_identifier __all__ = [ 'Projection_Base', 'projection_keywords', 'PROJECTION_SPEC_KEYWORDS', diff --git a/psyneulink/core/compositions/composition.py b/psyneulink/core/compositions/composition.py index cd5da94cc78..3a646d39f35 100644 --- a/psyneulink/core/compositions/composition.py +++ b/psyneulink/core/compositions/composition.py @@ -2734,7 +2734,7 @@ def input_function(env, result): from psyneulink.core import llvm as pnlvm from psyneulink.core.components.component import Component, ComponentsMeta from psyneulink.core.components.functions.fitfunctions import make_likelihood_function -from psyneulink.core.components.functions.function import is_function_type +from psyneulink.core.components.functions.function import is_function_type, RandomMatrix from psyneulink.core.components.functions.nonstateful.combinationfunctions import LinearCombination, \ PredictionErrorDeltaFunction from psyneulink.core.components.functions.nonstateful.learningfunctions import \ @@ -5591,6 +5591,7 @@ def add_projection(self, projection=None, sender=None, receiver=None, + default_matrix=None, feedback=False, learning_projection=False, name=None, @@ -5599,7 +5600,9 @@ def add_projection(self, ): """Add **projection** to the Composition. - If **projection** is not specified, create a default `MappingProjection` using **sender** and **receiver**. + If **projection** is not specified, and one does not already exist between **sender** and **receiver** + create a default `MappingProjection` between them, using **default_projection_matrix** if specified + (otherwise default for MappingProjection is used). If **projection** is specified: @@ -5648,15 +5651,23 @@ def add_projection(self, Arguments --------- + projection : Projection, list, array, matrix, RandomMatrix, MATRIX_KEYWORD + the projection to add. + sender : Mechanism, Composition, or OutputPort the sender of **projection**. - projection : Projection, matrix - the projection to add. - receiver : Mechanism, Composition, or InputPort the receiver of **projection**. + default_projection_matrix : list, array, matrix, RandomMatrix, MATRIX_KEYWORD + matrix to use in creating default; overrides default for MappingProjection. + + default_projection_matrix : list, array, function, `RandomMatrix` or MATRIX_KEYWORD : default None + specifies matrix to use in creating default Projection if none is specifed in **projection** + and one does not already exist between **sender** and **receive** + (see `MappingProjection_Matrix_Specification` for details of specification). + feedback : bool or FEEDBACK : False if False, the Projection is *never* designated as a `feedback Projection `, even if that may have been the default behavior (e.g., @@ -5733,6 +5744,7 @@ def add_projection(self, return self.add_projection(proj_spec, sender=projection.sender, receiver=projection.receiver) # Create Projection if it doesn't exist + projection = projection or default_matrix try: # Note: this does NOT initialize the Projection if it is in deferred_init projection = self._instantiate_projection_from_spec(projection, name) @@ -5918,7 +5930,7 @@ def _instantiate_projection_from_spec(self, projection, sender=None, receiver=No proj_type = projection.pop(PROJECTION_TYPE, None) or MappingProjection params = projection.pop(PROJECTION_PARAMS, None) projection = MappingProjection(params=params) - elif isinstance(projection, (np.ndarray, np.matrix, list)): + elif isinstance(projection, (np.ndarray, np.matrix, list, RandomMatrix)): return MappingProjection(matrix=projection, sender=sender, receiver=receiver, name=name) elif isinstance(projection, str): if projection in MATRIX_KEYWORD_VALUES: @@ -5930,8 +5942,8 @@ def _instantiate_projection_from_spec(self, projection, sender=None, receiver=No elif projection is None: return MappingProjection(sender=sender, receiver=receiver, name=name) elif not isinstance(projection, Projection): - raise CompositionError("Invalid projection ({}) specified for {}. Must be a Projection." - .format(projection, self.name)) + raise CompositionError(f"Invalid projection ({projection}) specified for {self.name}. " + f"Must be a Projection.") return projection def _parse_sender_spec(self, projection, sender): @@ -6384,7 +6396,15 @@ def _parse_pathway(self, pathway, name, pathway_arg_str): if isinstance(pathway, Pathway): # Give precedence to name specified in call to add_linear_processing_pathway pathway_name = name or pathway.name + # MODIFIED 11/3/22 OLD: pathway = pathway.pathway + # # MODIFIED 11/3/22 NEW: + # # If Pathway has default_projection_matrix, use tuple_spec to specify for handling below + # if pathway.default_projection_matrix: + # pathway = (pathway.pathway, pathway. default_projection_matrix) + # else: + # pathway = pathway.pathway + # MODIFIED 11/3/22 END else: pathway_name = name @@ -6557,20 +6577,47 @@ def identify_pway_type_and_parse_tuple_prn(pway, tuple_or_dict_str): pway_type = PROCESSING_PATHWAY if isinstance(pway, set): pway = [pway] - return pway_type, pway, None + return pway_type, pway, None, None elif isinstance(pway, tuple): - pway_type = LEARNING_PATHWAY - if len(pway)!=2: + # FIX: ADD SUPPORT FOR 3-ITEM TUPLE AND SPECIFCATION OF DEFAULT MATRIX HERE 10/29/22 + # # MODIFIED 10/29/22 OLD: + # pway_type = LEARNING_PATHWAY + # if len(pway)!=2: + # raise CompositionError(f"A tuple specified in the {pathways_arg_str}" + # f" has more than two items: {pway}") + # pway, learning_function = pway + # if not (_is_node_spec(pway) or isinstance(pway, (list, Pathway))): + # raise CompositionError(f"The 1st item in {tuple_or_dict_str} specified in the " + # f" {pathways_arg_str} must be a node or a list: {pway}") + # if not (isinstance(learning_function, type) and issubclass(learning_function, LearningFunction)): + # raise CompositionError(f"The 2nd item in {tuple_or_dict_str} specified in the " + # f"{pathways_arg_str} must be a LearningFunction: {learning_function}") + # return pway_type, pway, learning_function + # MODIFIED 10/29/22 NEW: + if len(pway) not in {2,3}: raise CompositionError(f"A tuple specified in the {pathways_arg_str}" - f" has more than two items: {pway}") - pway, learning_function = pway - if not (_is_node_spec(pway) or isinstance(pway, (list, Pathway))): - raise CompositionError(f"The 1st item in {tuple_or_dict_str} specified in the " - f" {pathways_arg_str} must be a node or a list: {pway}") - if not (isinstance(learning_function, type) and issubclass(learning_function, LearningFunction)): - raise CompositionError(f"The 2nd item in {tuple_or_dict_str} specified in the " - f"{pathways_arg_str} must be a LearningFunction: {learning_function}") - return pway_type, pway, learning_function + f" must have either two or three items: {pway}") + pway_type = PROCESSING_PATHWAY + matrix_item = None + learning_function_item = None + for i, item in enumerate(pway): + # Ensure that first item is a Pathway spec + if i==0: + if not (_is_node_spec(item) or isinstance(item, (list, Pathway))): + raise CompositionError(f"The 1st item in {tuple_or_dict_str} specified in the " + f" {pathways_arg_str} must be a node or a list: {pway}") + pathway_item = item + elif (isinstance(item, type) and issubclass(item, LearningFunction)): + pway_type = LEARNING_PATHWAY + learning_function_item = item + elif is_matrix(item): + matrix_item = item + else: + raise CompositionError(f"Bad spec for one of the items in {tuple_or_dict_str} " + f"specified for the {pathways_arg_str}: {item}; " + f"its item(s) must be a matrix specification and/or a LearningFunction") + return pway_type, pathway_item, matrix_item, learning_function_item + # MODIFIED 10/29/22 END else: assert False, f"PROGRAM ERROR: arg to identify_pway_type_and_parse_tuple_prn in {self.name}" \ f"is not a Node, list or tuple: {pway}" @@ -6583,13 +6630,22 @@ def identify_pway_type_and_parse_tuple_prn(pway, tuple_or_dict_str): pway_name = None if isinstance(pathway, Pathway): pway_name = pathway.name + # MODIFIED 11/3/22 OLD: pathway = pathway.pathway + # # MODIFIED 11/3/22 NEW: + # # If Pathway has default_projection_matrix, use tuple_spec to specify for later handling + # if pathway.default_projection_matrix: + # pathway = (pathway.pathway, pathway.default_projection_matrix) + # else: + # pathway = pathway.pathway + # MODIFIED 11/3/22 END if _is_node_spec(pathway) or isinstance(pathway, (list, set, tuple)): if isinstance(pathway, set): bad_entries = [repr(entry) for entry in pathway if not _is_node_spec(entry)] if bad_entries: raise CompositionError(f"{bad_entry_error_msg}{','.join(bad_entries)}") - pway_type, pway, pway_learning_fct = identify_pway_type_and_parse_tuple_prn(pathway, f"a tuple") + pway_type, pway, matrix, pway_learning_fct = identify_pway_type_and_parse_tuple_prn(pathway, + f"the tuple") elif isinstance(pathway, dict): if len(pathway)!=1: raise CompositionError(f"A dict specified in the {pathways_arg_str} " @@ -6599,8 +6655,8 @@ def identify_pway_type_and_parse_tuple_prn(pway, tuple_or_dict_str): raise CompositionError(f"The key in a dict specified in the {pathways_arg_str} must be a str " f"(to be used as its name): {pway_name}.") if _is_node_spec(pway) or isinstance(pway, (list, tuple, Pathway)): - pway_type, pway, pway_learning_fct = identify_pway_type_and_parse_tuple_prn(pway, - f"the value of a dict") + pway_type, pway, matrix, pway_learning_fct = identify_pway_type_and_parse_tuple_prn(pway, + f"the value of a dict") else: raise CompositionError(f"The value in a dict specified in the {pathways_arg_str} must be " f"a pathway specification (Node, list or tuple): {pway}.") @@ -6610,11 +6666,13 @@ def identify_pway_type_and_parse_tuple_prn(pway, tuple_or_dict_str): context.source = ContextFlags.METHOD if pway_type == PROCESSING_PATHWAY: new_pathway = self.add_linear_processing_pathway(pathway=pway, + default_projection_matrix=matrix, name=pway_name, context=context) elif pway_type == LEARNING_PATHWAY: new_pathway = self.add_linear_learning_pathway(pathway=pway, learning_function=pway_learning_fct, + default_projection_matrix=matrix, name=pway_name, context=context) else: @@ -6625,7 +6683,7 @@ def identify_pway_type_and_parse_tuple_prn(pway, tuple_or_dict_str): return added_pathways @handle_external_context() - def add_linear_processing_pathway(self, pathway, name:str=None, context=None, *args): + def add_linear_processing_pathway(self, pathway, default_projection_matrix=None, name:str=None, context=None, *args): """Add sequence of `Nodes ` with optionally intercolated `Projections `. .. _Composition_Add_Linear_Processing_Pathway: @@ -6654,6 +6712,11 @@ def add_linear_processing_pathway(self, pathway, name:str=None, context=None, *a learning-related specifications are ignored, as are its `name ` if the **name** argument of add_linear_processing_pathway is specified. + default_projection_matrix : list, array, function, `RandomMatrix` or MATRIX_KEYWORD : default None + specifies matrix to use for any unspecified Projections (overrides default matrix for `MappingProjection`) + if a default projection is not otherwise specified (see `Pathway_Specification_Projections`; + see `MappingProjection_Matrix_Specification` for details of specification) + name : str species the name used for `Pathway`; supercedes `name ` of `Pathway` object if it is has one. @@ -6769,8 +6832,11 @@ def _get_node_specs_for_entry(entry, include_roles=None, exclude_roles=None): else {pathway[c - 1]}) if all(_is_node_spec(sender) for sender in preceding_entry): senders = _get_node_specs_for_entry(preceding_entry, NodeRole.OUTPUT) - projs = {self.add_projection(sender=s, receiver=r, allow_duplicates=False) + projs = {self.add_projection(sender=s, receiver=r, + default_matrix=default_projection_matrix, + allow_duplicates=False) for r in receivers for s in senders} + # MODIFIED 11/2/22 END if all(projs): projs = projs.pop() if len(projs) == 1 else projs projections.append(projs) @@ -6835,8 +6901,10 @@ def _get_node_specs_for_entry(entry, include_roles=None, exclude_roles=None): # Unpack if tuple spec, and assign feedback (with False as default) default_proj_spec, feedback = (spec if isinstance(spec, tuple) else (spec, False)) # Get all specs other than default_proj_spec - # proj_specs = [proj_spec for proj_spec in all_proj_specs if proj_spec not in possible_default_proj_spec] proj_specs = [proj_spec for proj_spec in all_proj_specs if proj_spec is not spec] + # If default matrix is not specified within the pathway, use default_projection_matrix if specified + if default_proj_spec is None: + default_proj_spec = default_projection_matrix # Collect all Projection specifications (to add to Composition at end) proj_set = [] @@ -7040,6 +7108,7 @@ def handle_duplicates(sender, receiver): pathway = Pathway(pathway=explicit_pathway, composition=self, + # default_projection_matrix=default_projection_matrix, name=pathway_name, context=context) self.pathways.append(pathway) @@ -7060,6 +7129,7 @@ def add_linear_learning_pathway(self, learning_rate:tc.any(int,float)=0.05, error_function=LinearCombination, learning_update:tc.any(bool, tc.enum(ONLINE, AFTER))=AFTER, + default_projection_matrix=None, name:str=None, context=None): """Implement learning pathway (including necessary `learning components `. @@ -7130,6 +7200,11 @@ def add_linear_learning_pathway(self, ` in the pathway, and its `LearningProjection` (see `learning_enabled ` for meaning of values). + default_projection_matrix : list, array, function, `RandomMatrix` or MATRIX_KEYWORD : default None + specifies matrix to use for any unspecified Projections (overrides default matrix for `MappingProjection`) + if a default projection is not otherwise specified (see `Pathway_Specification_Projections`; + see `MappingProjection_Matrix_Specification` for details of specification) + name : str : species the name used for `Pathway`; supercedes `name ` of `Pathway` object if it is has one. @@ -7179,6 +7254,7 @@ def add_linear_learning_pathway(self, loss_function, learning_update, name=pathway_name, + default_projection_matrix=default_projection_matrix, context=context) # If BackPropagation is not specified, then the learning pathway is "one-layered" @@ -7196,6 +7272,7 @@ def add_linear_learning_pathway(self, self._add_required_node_role(output_source, NodeRole.OUTPUT, context) learning_pathway = self.add_linear_processing_pathway(pathway=[input_source, learned_projection, output_source], + default_projection_matrix=default_projection_matrix, name=pathway_name, # context=context) context=context) @@ -7251,6 +7328,7 @@ def add_reinforcement_learning_pathway(self, learning_rate=0.05, error_function=None, learning_update:tc.any(bool, tc.enum(ONLINE, AFTER))=ONLINE, + default_projection_matrix=None, name:str=None): """Convenience method that calls `add_linear_learning_pathway` with **learning_function**=`Reinforcement` @@ -7262,6 +7340,11 @@ def add_reinforcement_learning_pathway(self, specified, that projection is the learned projection. Otherwise, a default MappingProjection is automatically generated for the learned projection. + default_projection_matrix : list, array, function, `RandomMatrix` or MATRIX_KEYWORD : default None + specifies matrix to use for any unspecified Projections (overrides default matrix for `MappingProjection`) + if a default projection is not otherwise specified (see `Pathway_Specification_Projections`; + see `MappingProjection_Matrix_Specification` for details of specification) + learning_rate : float : default 0.05 specifies the `learning_rate ` used for the `ReinforcementLearning` function of the `LearningMechanism` in the **pathway**. @@ -7292,6 +7375,7 @@ def add_reinforcement_learning_pathway(self, learning_function=Reinforcement, error_function=error_function, learning_update=learning_update, + default_projection_matrix=default_projection_matrix, name=name) def add_td_learning_pathway(self, @@ -7299,6 +7383,7 @@ def add_td_learning_pathway(self, learning_rate=0.05, error_function=None, learning_update:tc.any(bool, tc.enum(ONLINE, AFTER))=ONLINE, + default_projection_matrix=None, name:str=None): """Convenience method that calls `add_linear_learning_pathway` with **learning_function**=`TDLearning` @@ -7325,6 +7410,11 @@ def add_td_learning_pathway(self, ` in the pathway, and its `LearningProjection` (see `learning_enabled ` for meaning of values). + default_projection_matrix : list, array, function, `RandomMatrix` or MATRIX_KEYWORD : default None + specifies matrix to use for any unspecified Projections (overrides default matrix for `MappingProjection`) + if a default projection is not otherwise specified (see `Pathway_Specification_Projections`; + see `MappingProjection_Matrix_Specification` for details of specification) + name : str : species the name used for `Pathway`; supercedes `name ` of `Pathway` object if it is has one. @@ -7339,6 +7429,7 @@ def add_td_learning_pathway(self, learning_rate=learning_rate, learning_function=TDLearning, learning_update=learning_update, + default_projection_matrix=default_projection_matrix, name=name) def add_backpropagation_learning_pathway(self, @@ -7347,6 +7438,7 @@ def add_backpropagation_learning_pathway(self, error_function=None, loss_function:tc.enum(MSE,SSE)=MSE, learning_update:tc.optional(tc.any(bool, tc.enum(ONLINE, AFTER)))=AFTER, + default_projection_matrix=None, name:str=None): """Convenience method that calls `add_linear_learning_pathway` with **learning_function**=`Backpropagation` @@ -7376,6 +7468,11 @@ def add_backpropagation_learning_pathway(self, ` in the pathway, and their `LearningProjections ` (see `learning_enabled ` for meaning of values). + default_projection_matrix : list, array, function, `RandomMatrix` or MATRIX_KEYWORD : default None + specifies matrix to use for any unspecified Projections (overrides default matrix for `MappingProjection`) + if a default projection is not otherwise specified (see `Pathway_Specification_Projections`; + see `MappingProjection_Matrix_Specification` for details of specification) + name : str : species the name used for `Pathway`; supercedes `name ` of `Pathway` object if it is has one. @@ -7392,6 +7489,7 @@ def add_backpropagation_learning_pathway(self, loss_function=loss_function, error_function=error_function, learning_update=learning_update, + default_projection_matrix=default_projection_matrix, name=name) # NOTES: @@ -7618,6 +7716,7 @@ def _create_backpropagation_learning_pathway(self, error_function=None, loss_function=MSE, learning_update=AFTER, + default_projection_matrix=None, name=None, context=None): @@ -7631,7 +7730,10 @@ def _create_backpropagation_learning_pathway(self, # Pass ContextFlags.INITIALIZING so that it can be passed on to _analyze_graph() and then # _check_for_projection_assignments() in order to ignore checks for require_projection_in_composition context.string = f"'pathway' arg for add_backpropagation_learning_pathway method of {self.name}" - learning_pathway = self.add_linear_processing_pathway(pathway, name, context) + learning_pathway = self.add_linear_processing_pathway(pathway=pathway, + name=name, + default_projection_matrix=default_projection_matrix, + context=context) processing_pathway = learning_pathway.pathway path_length = len(processing_pathway) @@ -8649,9 +8751,6 @@ def evaluate( buffer_animate_state = self._animate # Run Composition in "SIMULATION" context - # # MODIFIED 3/28/22 NEW: - # context.source = ContextFlags.COMPOSITION - # MODIFIED 3/28/22 END context.add_flag(ContextFlags.SIMULATION_MODE) context.remove_flag(ContextFlags.CONTROL) @@ -9616,14 +9715,15 @@ def run( details and `ReportDevices` for options. animate : dict or bool : default False - specifies use of the `show_graph`show_graph ` method to generate - a gif movie showing the sequence of Components executed in a run (see `example - `). A dict can be specified containing - options to pass to the `show_graph ` method; each key must be a legal - argument for the `show_graph ` method, and its value a specification for that - argument. The entries listed below can also be included in the dict to specify parameters of the - animation. If the **animate** argument is specified simply as `True`, defaults are used for all - arguments of `show_graph ` and the options below: + specifies use of the `show_graph ` method to generate a gif movie showing the + sequence of Components executed in a run (see `example `). + A dict can be specified containing options to pass to the `show_graph ` method in + order to customize the display of the graph in the animation. Each key of the dict must be a legal argument + for the `show_graph ` method, and its value a specification for that argument. + The entries listed below can also be included in the dict to specify parameters of the animation. + If the **animate** argument is specified simply as `True`, defaults are used for all arguments + of `show_graph ` and the options below. See `Animation ` + for additional information. * *UNIT*: *EXECUTION_SET* or *COMPONENT* (default=\\ *EXECUTION_SET*\\ ) -- specifies which Components to treat as active in each call to `show_graph() `. *COMPONENT* generates an @@ -9648,7 +9748,7 @@ def run( * *MOVIE_NAME*: str (default=\\ `name ` + 'movie') -- specifies the name to be used for the movie file; it is automatically appended with '.gif'. - +_ * *SAVE_IMAGES*: bool (default=\\ `False`\\ ) -- specifies whether to save each of the images used to construct the animation in separate gif files, in addition to the file containing the animation. @@ -9660,7 +9760,7 @@ def run( `projection ` in the Composition, if it is not already set. .. note:: - as when setting the `log_condition ` directly, a value of `True` will + As when setting the `log_condition ` directly, a value of `True` will correspond to the `EXECUTION` `LogCondition `. scheduler : Scheduler : default None @@ -9781,6 +9881,8 @@ def run( # Set animation attributes if animate is True: animate = {} + if animate is None: + animate = False self._animate = animate if self._animate is not False: self._set_up_animation(context) @@ -10117,7 +10219,7 @@ def learn( specifies the number of training epochs (that is, repetitions of the batched input set) to run with minibatch_size : int (default=1) - specifies the size of the minibatches to use. The input trials will be batched and ran, after which + specifies the size of the minibatches to use. The input trials will be batched and run, after which learning mechanisms with learning mode TRIAL will update weights randomize_minibatch: bool (default=False) diff --git a/psyneulink/core/compositions/pathway.py b/psyneulink/core/compositions/pathway.py index da18203bc84..98b423201d7 100644 --- a/psyneulink/core/compositions/pathway.py +++ b/psyneulink/core/compositions/pathway.py @@ -115,20 +115,22 @@ *Pathway Projection Specifications* ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Where no Projections are specified between entries in the list, default Projections (using a `FULL_CONNECTIVITY_MATRIX`; -see `MappingProjection_Matrix_Specification`) are created from each Node in the first entry, as the sender(s), -to each Node in the second, as receiver(s) (described further `below `). Projections between -Nodes in the two entries can also be specified explicitly, by intercolating a Projection or set of Projections between -the two entries in the list. If the sender and receiver are both a single Mechanism, then a single `MappingProjection` -can be `specified` between them. The same applies if the sender is a `Composition` with -a single `OUTPUT ` Node and/or the receiver is a `Composition` with a single `INPUT ` -Node. If either is a set of Nodes, or is a `nested Composition ` with more than one `INPUT -` or `OUTPUT ` Node, respectively, then a collection of Projections can be specified -between any or all pairs of the Nodes in the set(s) and/or nested Composition(s), using either a set or list of -Projections (order of specification does not matter whether a set or a list is used). The collection can contain -`MappingProjections ` between a specific pairs of Nodes and/or a single default specification -(either a `matrix ` specification or a MappingProjection without any `sender -` or `receiver ` specified). +Where no Projections are specified between entries in the list, default Projections are created (using a +`FULL_CONNECTIVITY_MATRIX`, or the Pathway's `default_projection ` if specified) +from each Node in the first entry, as the sender(s), to each Node in the second, as receiver(s) (described further +`below `). Projections between Nodes in the two entries can also be specified explicitly, by +intercolating a Projection or set of Projections between the two entries in the list. If the sender and receiver are +both a single Mechanism, then a single `MappingProjection` can be `specified` between +them. The same applies if the sender is a `Composition` with a single `OUTPUT ` Node and/or the +receiver is a `Composition` with a single `INPUT ` Node. If either is a set of Nodes, or is a +`nested Composition ` with more than one `INPUT ` or `OUTPUT ` +Node, respectively, then a collection of Projections can be specified between any or all pairs of the Nodes in the +set(s) and/or nested Composition(s), using either a set or list of Projections (order of specification does not matter +whether a set or a list is used). The collection can contain `MappingProjections ` between specific +pairs of Nodes and/or a single default specification (either a `matrix ` specification or a +MappingProjection without any `sender ` or `receiver ` +specified; see MappingProject MappingProjection_Matrix_Specification +). .. _Pathway_Projection_Matrix_Note: @@ -231,9 +233,14 @@ `. Sets can also be used in a list specification (see above; and see `add_linear_processing_pathway ` for additional details). .. - * **2-item tuple**: (Pathway, `LearningFunction`) -- used to specify a `learning Pathway - `; the 1st item must be one of the forms of Pathway specification - described above, and the 2nd item must be a subclass of `LearningFunction`. + .. _Pathway_Specification_Tuple: + + * **2 or 3-item tuple**: (Pathway, , ) -- + used to specify a `learning Pathway ` and/or a matrix to use for any unspecified + Projections (overrides default matrix for `MappingProjection`) if a default projection is not otherwise specified + (see `Pathway_Specification_Projections. The 1st item of the tuple must be one of the forms of Pathway + specification described above. The other items must be a subclass of `LearningFunction` and/or a `matrix + specification `. .. _Pathway_Specification_Multiple: @@ -293,6 +300,9 @@ those `NodeRoles ` is assigned to a corresponding attribute on the Pathway. If the Pathway does not belong to a Composition (i.e., it is a `template `), then these attributes return None. +* `default_projection_matrix ` - matrix used as default for Projections that are + not explicitly specified and for which no default is otherwise specified (see `Pathway_Specification_Projections`). + * `learning_function ` - the LearningFunction assigned to the Pathway if it is a `learning Pathway ` that belongs to a Composition; otherwise it is None. @@ -322,6 +332,7 @@ from psyneulink.core.globals.context import ContextFlags, handle_external_context from psyneulink.core.globals.keywords import \ ANY, CONTEXT, FEEDBACK, MAYBE, NODE, LEARNING_FUNCTION, OBJECTIVE_MECHANISM, PROJECTION, TARGET_MECHANISM +from psyneulink.core.globals.utilities import is_matrix from psyneulink.core.globals.registry import register_category __all__ = [ @@ -416,10 +427,17 @@ class PathwayRole(Enum): class Pathway(object): """ - Pathway( \ - pathway, \ - name=None \ + Pathway( \ + pathway, \ + name=None \ ) + COMMENT: + Pathway( \ + pathway, \ + default_projection_matrix, \ + name=None \ + ) + COMMENT A sequence of `Nodes ` and `Projections ` in a `Composition`, or a template for one that can be assigned to one or more Compositions. @@ -431,8 +449,15 @@ class Pathway(object): specifies list of `Nodes ` and intercolated `Projections ` to be created for the Pathway. + COMMENT: + default_projection_matrix : list, array, function, `RandomMatrix` or MATRIX_KEYWORD : default None + specifies matrix to use for any unspecified Projections (overrides default matrix for `MappingProjection`) + if a default projection is not otherwise specified (see `Pathway_Specification_Projections`; + see `MappingProjection_Matrix_Specification` for details of specification) + COMMENT + name : str : default see `name ` - specifies the name of the Pathway; see `name ` for additional information. + specifies the name of the Pathway (see `name ` for additional information). Attributes ---------- @@ -452,6 +477,13 @@ class Pathway(object): Returns an empty list if belongs to a Composition but no `PathwayRoles ` have been assigned, and None if the Pathway is a `tempalte ` (i.e., not assigned to a Composition). + default_projection_matrix : list, array, function, `RandomMatrix` or MATRIX_KEYWORD : default None + matrix used for any unspecified Projections (overrides default matrix for `MappingProjection`) + if a default projection is not otherwise specified (see `Pathway_Specification_Projections`; + see `MappingProjection_Matrix_Specification` for details of specification). A default_projection_matrix + is specified by including it in a tuple specification in the **pathways** argument of the Pathway's + constructor (see `2 or 3-item tuple `). + learning_function : `LearningFunction` or None `LearningFunction` used by `LearningMechanism(s) ` associated with Pathway if it is a `learning pathway `. @@ -500,6 +532,7 @@ class Pathway(object): def __init__( self, pathway:list, + # default_projection_matrix=None, name=None, **kwargs ): @@ -547,6 +580,16 @@ def __init__( self.learning_components = None self.roles = None + # Assign default_projection_matrix attribute + # self.default_projection_matrix = default_projection_matrix + # Parse from tuple spec in **pathway** arg: + self.default_projection_matrix = None + if isinstance(self.pathway, tuple): + for item in self.pathway: + if is_matrix(item): + self.default_projection_matrix = item + assert True + def _assign_roles(self, composition): """Assign `PathwayRoles ` to Pathway based `NodeRoles ` assigned to its `Nodes ` by the **composition** to which it belongs. diff --git a/psyneulink/core/compositions/showgraph.py b/psyneulink/core/compositions/showgraph.py index a4fbe76eb7c..b2d54019bc9 100644 --- a/psyneulink/core/compositions/showgraph.py +++ b/psyneulink/core/compositions/showgraph.py @@ -107,6 +107,22 @@ - `CONTROLLER` : purple - `LEARNING` : orange +.. _ShowGraph_Animation: + +*Animation* +----------- + +An animation can be generated of the execution of a Composition by using the **animate** argument of the Composition's +`run ` method. The animation show a graphical display of the Composition, with each of its +the Components highlighted in the sequence that they are executed. The **animate** can be passed a dict containing +any of the options described above to customize the display, as well as several others used to customize the animation +(see **animate** argument under `run `). + + .. note:: + At present, animation of the Components within a `nested Composition ` is not supported; + the box surrounding the nested Composition is highlighted when it is executed, followed by the next Component(s) + to execute. + .. _ShowGraph_Examples_Visualization: *Examples* @@ -828,7 +844,8 @@ def show_graph(self, show_dimensions, show_projection_labels, show_projections_not_in_composition, - nested_args) + nested_args, + context) # Add cim Components to graph if show_cim if show_cim: @@ -907,7 +924,8 @@ def _assign_processing_components(self, show_dimensions, show_projection_labels, show_projections_not_in_composition, - nested_args): + nested_args, + context): """Assign nodes to graph""" from psyneulink.core.compositions.composition import Composition, NodeRole @@ -922,23 +940,40 @@ def _assign_processing_components(self, COMP_HIERARCHY:comp_hierarchy, # 'composition': rcvr, ENCLOSING_COMP:composition, - NESTING_LEVEL:nesting_level + 1}) + NESTING_LEVEL:nesting_level + 1, + }) # Get subgraph for nested Composition + # # MODIFIED 10/29/22 NEW: FIX: HACK SO NESTED COMPOSITIONS DON'T CRASH ANIMATION (THOUGH STILL NOT SHOWN) + if hasattr(composition, '_animate') and composition._animate is not False: + rcvr._animate = composition._animate + rcvr._set_up_animation(context) + rcvr._animate_num_trials = composition._animate_num_trials + 1 + # MODIFIED 10/29/22 END nested_comp_graph = rcvr._show_graph.show_graph(**nested_args) nested_comp_graph.name = "cluster_" + rcvr.name rcvr_label = rcvr.name + + # Assign color to nested_comp, including highlighting if it is the active_item # if rcvr in composition.get_nodes_by_role(NodeRole.FEEDBACK_SENDER): # nested_comp_graph.attr(color=feedback_color) + # nested_comp_attributes = {"label":rcvr_label} + nested_comp_attributes = {} if rcvr in composition.get_nodes_by_role(NodeRole.INPUT) and \ rcvr in composition.get_nodes_by_role(NodeRole.OUTPUT): - nested_comp_graph.attr(color=self.input_and_output_color) + nested_comp_attributes.update({"color": self.input_and_output_color}) elif rcvr in composition.get_nodes_by_role(NodeRole.INPUT): - nested_comp_graph.attr(color=self.input_color) + nested_comp_attributes.update({"color": self.input_color}) elif rcvr in composition.get_nodes_by_role(NodeRole.PROBE): - nested_comp_graph.attr(color=self.probe_color) + nested_comp_attributes.update({"color": self.probe_color}) elif rcvr in composition.get_nodes_by_role(NodeRole.OUTPUT): - nested_comp_graph.attr(color=self.output_color) + nested_comp_attributes.update({"color": self.output_color}) + if rcvr in active_items: + if self.active_color != BOLD: + nested_comp_attributes.update({"color": self.active_color}) + nested_comp_attributes.update({"penwidth": str(self.default_width + self.active_thicker_by)}) + composition.active_item_rendered = True + nested_comp_graph.attr(**nested_comp_attributes) nested_comp_graph.attr(label=rcvr_label) g.subgraph(nested_comp_graph) @@ -2722,6 +2757,7 @@ def _set_up_animation(self, context): if not isinstance(composition._show_animation, bool): raise ShowGraphError(f"{repr(SHOW)} entry of {repr('animate')} argument for {repr('run')} " f"method of {composition.name} ({composition._show_animation}) must be a boolean.") + elif composition._animate: # composition._animate should now be False or a dict raise ShowGraphError("{} argument for {} method of {} ({}) must be a boolean or " @@ -2737,10 +2773,10 @@ def _animate_execution(self, active_items, context): else: composition._component_animation_execution_count += 1 composition.show_graph(active_items=active_items, - **composition._animate, - output_fmt='gif', - context=context, - ) + **composition._animate, + output_fmt='gif', + context=context, + ) def _generate_gifs(self, G, active_items, context): diff --git a/psyneulink/core/globals/keywords.py b/psyneulink/core/globals/keywords.py index e08ab37b0ca..5bd996546bf 100644 --- a/psyneulink/core/globals/keywords.py +++ b/psyneulink/core/globals/keywords.py @@ -46,9 +46,10 @@ 'DIST_SHAPE', 'DISTANCE_FUNCTION', 'DISTANCE_METRICS', 'DISTRIBUTION_FUNCTION_TYPE', 'DIVISION', 'DRIFT_DIFFUSION_INTEGRATOR_FUNCTION', 'DRIFT_ON_A_SPHERE_INTEGRATOR_FUNCTION', 'DUAL_ADAPTIVE_INTEGRATOR_FUNCTION', 'EFFERENTS', 'EID_SIMULATION', 'EID_FROZEN', 'EITHER', 'ENABLE_CONTROLLER', 'ENABLED', 'ENERGY', 'ENTROPY', - 'EPISODIC_MEMORY_MECHANISM', 'EQUAL', 'ERROR_DERIVATIVE_FUNCTION', 'EUCLIDEAN', 'EVC_MECHANISM', 'EVC_SIMULATION', - 'EXAMPLE_FUNCTION_TYPE', 'EXECUTE_UNTIL_FINISHED', 'EXECUTING', 'EXECUTION', 'EXECUTION_COUNT', 'EXECUTION_ID', - 'EXECUTION_PHASE', 'EXPONENTIAL', 'EXPONENT', 'EXPONENTIAL_DIST_FUNCTION', 'EXPONENTIAL_FUNCTION', 'EXPONENTS', + 'EPISODIC_MEMORY_MECHANISM', 'EPOCHS', 'EQUAL', 'ERROR_DERIVATIVE_FUNCTION', 'EUCLIDEAN', + 'EVC_MECHANISM', 'EVC_SIMULATION', 'EXAMPLE_FUNCTION_TYPE', + 'EXECUTE_UNTIL_FINISHED', 'EXECUTING', 'EXECUTION', 'EXECUTION_COUNT', 'EXECUTION_ID', 'EXECUTION_PHASE', + 'EXPONENTIAL', 'EXPONENT', 'EXPONENTIAL_DIST_FUNCTION', 'EXPONENTIAL_FUNCTION', 'EXPONENTS', 'FEEDBACK', 'FITZHUGHNAGUMO_INTEGRATOR_FUNCTION', 'FINAL', 'FLAGS', 'FULL', 'FULL_CONNECTIVITY_MATRIX', 'FUNCTION', 'FUNCTIONS', 'FUNCTION_COMPONENT_CATEGORY','FUNCTION_CHECK_ARGS', 'FUNCTION_OUTPUT_TYPE', 'FUNCTION_OUTPUT_TYPE_CONVERSION', 'FUNCTION_PARAMS', @@ -111,9 +112,9 @@ 'SEPARATOR_BAR', 'SHADOW_INPUT_NAME', 'SHADOW_INPUTS', 'SIMPLE', 'SIMPLE_INTEGRATOR_FUNCTION', 'SIMULATIONS', 'SINGLETON', 'SIZE', 'SLOPE', 'SOFT_CLAMP', 'SOFTMAX_FUNCTION', 'SOURCE', 'SSE', 'STABILITY_FUNCTION', 'STANDARD_ARGS', 'STANDARD_DEVIATION', 'STANDARD_OUTPUT_PORTS', 'SUBTRACTION', 'SUM', - 'TARGET', 'TARGET_MECHANISM', 'TARGET_LABELS_DICT', 'TERMINAL', 'TERMINATION_MEASURE', 'TERMINATION_THRESHOLD', - 'TERMINATION_COMPARISION_OP', 'TERSE', 'TEXT', 'THRESHOLD', 'TIME', 'TIME_STEP_SIZE', 'TIME_STEPS_DIM', - 'TRAINING_SET', + 'TARGET', 'TARGET_MECHANISM', 'TARGET_LABELS_DICT', 'TERMINAL', 'TARGETS', + 'TERMINATION_MEASURE', 'TERMINATION_THRESHOLD', 'TERMINATION_COMPARISION_OP', 'TERSE', 'TEXT', 'THRESHOLD', + 'TIME', 'TIME_STEP_SIZE', 'TIME_STEPS_DIM', 'TRAINING_SET', 'TRANSFER_FUNCTION_TYPE', 'TRANSFER_MECHANISM', 'TRANSFER_WITH_COSTS_FUNCTION', 'TRIAL', 'TRIALS_DIM', 'UNCHANGED', 'UNIFORM_DIST_FUNCTION', 'USER_DEFINED_FUNCTION', 'USER_DEFINED_FUNCTION_TYPE', @@ -413,6 +414,8 @@ def _is_metric(metric): LEARNING_PATHWAY = "learning_pathway" NODE = 'NODE' INPUTS = 'inputs' +TARGETS = 'targets' +EPOCHS = 'epochs' # Used in show_graph for show_nested NESTED = 'nested' diff --git a/psyneulink/core/globals/utilities.py b/psyneulink/core/globals/utilities.py index b75ae1965c7..0adb9969835 100644 --- a/psyneulink/core/globals/utilities.py +++ b/psyneulink/core/globals/utilities.py @@ -359,7 +359,11 @@ def is_matrix(m): try: return is_matrix(m()) except: - return False + try: + # random_matrix and RandomMatrix are allowable functions, but require num_rows and num_cols parameters + return is_matrix(1,2) + except: + return False return False @@ -498,13 +502,11 @@ def iscompatible(candidate, reference=None, **kargs): if is_matrix_spec(reference): return is_matrix(candidate) - # MODIFIED 10/29/17 NEW: # IMPLEMENTATION NOTE: This allows a number in an ndarray to match a float or int # If both the candidate and reference are either a number or an ndarray of dim 0, consider it a match if ((is_number(candidate) or (isinstance(candidate, np.ndarray) and candidate.ndim == 0)) or (is_number(reference) or (isinstance(reference, np.ndarray) and reference.ndim == 0))): return True - # MODIFIED 10/29/17 END # IMPLEMENTATION NOTE: # modified to allow numeric type mismatches (e.g., int and float; @@ -1037,30 +1039,42 @@ def get_value_from_array(array): :return: """ -def random_matrix(sender, receiver, clip=1, offset=0): +def random_matrix(num_rows, num_cols, offset=0.0, scale=1.0): """Generate a random matrix - Calls np.random.rand to generate a 2d np.array with random values. + Calls np.random.rand to generate a 2d np.array with random values and shape (num_rows, num_cols): + + :math:`matrix = (random[0.0:1.0] + offset) * scale + + With the default values of **offset** and **scale**, values of matrix are floats between 0 and 1. + However, **offset** can be used to center the range on other values (e.g., **offset**=-0.5 centers values on 0), + and **scale** can be used to narrow or widen the range. As a conveniuence the keyword 'ZERO_CENTER' can be used + in place of -.05. Arguments ---------- - sender : int + num_rows : int specifies number of rows. - receiver : int - spcifies number of columns. + num_cols : int + specifies number of columns. - range : int - specifies upper limit (lower limit = 0). + offset : float or 'zero_center' + specifies amount added to each entry of the matrix before it is scaled. - offset : int - specifies amount added to each entry of the matrix. + scale : float + specifies amount by which random value + **offset** is multiplicatively scaled. Returns ------- 2d np.array """ - return (clip * np.random.rand(sender, receiver)) + offset + if isinstance(offset,str): + if offset.upper() == 'ZERO_CENTER': + offset = -0.5 + else: + raise UtilitiesError(f"'offset' arg of random_matrix must be a number of 'zero_center'") + return (np.random.rand(num_rows, num_cols) + offset) * scale def underscore_to_camelCase(item): item = item[1:] diff --git a/psyneulink/library/compositions/autodiffcomposition.py b/psyneulink/library/compositions/autodiffcomposition.py index 28b4e6cf81d..7a001ae3b75 100644 --- a/psyneulink/library/compositions/autodiffcomposition.py +++ b/psyneulink/library/compositions/autodiffcomposition.py @@ -26,10 +26,11 @@ Overview -------- -.. warning:: As of PsyNeuLink 0.7.5, the API for using AutodiffCompositions has been slightly changed! Please see `this link ` for more details! +.. warning:: As of PsyNeuLink 0.7.5, the API for using AutodiffCompositions has been slightly changed! + Please see `this link ` for more details! AutodiffComposition is a subclass of `Composition` used to train feedforward neural network models through integration -with `PyTorch `_, a popular machine learning library, which executes considerably more quickly +with `PyTorch `_, a machine learning library that executes considerably more quickly than using the `standard implementation of learning ` in a Composition, using its `learning methods `. An AutodiffComposition is configured and run similarly to a standard Composition, with some exceptions that are described below. @@ -44,14 +45,16 @@ An AutodiffComposition can be created by calling its constructor, and then adding `Components ` using the standard `Composition methods ` for doing so. The constructor also includes an number of -parameters that are specific to the AutodiffComposition. See the for a list of these parameters. +parameters that are specific to the AutodiffComposition. See `AutodiffComposition_Class_Reference` for a list of +these parameters. .. warning:: Mechanisms or Projections should not be added to or deleted from an AutodiffComposition after it has been run for the first time. Unlike an ordinary Composition, AutodiffComposition does not support this functionality. .. warning:: When comparing models built in PyTorch to those using AutodiffComposition, - the `bias ` parameter of PyTorch modules should be set to `False`, as AutodiffComposition does not currently support trainable biases. + the `bias ` parameter of PyTorch modules + should be set to `False`, as AutodiffComposition does not currently support trainable biases. .. _AutodiffComposition_Execution: @@ -59,7 +62,8 @@ Execution --------- -An AutodiffComposition's `run `, `execute `, and `learn ` methods are the same as for a `Composition`. +An AutodiffComposition's `run `, `execute `, and `learn ` +methods are the same as for a `Composition`. The following is an example showing how to create a simple AutodiffComposition, specify its inputs and targets, and run it with learning enabled and disabled. @@ -224,6 +228,7 @@ class Parameters(Composition.Parameters): # TODO (CW 9/28/18): add compositions to registry so default arg for name is no longer needed @check_user_specified def __init__(self, + pathways=None, learning_rate=None, optimizer_type='sgd', weight_decay=0, @@ -233,7 +238,6 @@ def __init__(self, disable_cuda=True, cuda_index=None, force_no_retain_graph=False, - pathways=None, name="autodiff_composition"): if not torch_available: diff --git a/tests/composition/test_composition.py b/tests/composition/test_composition.py index 01dd5053c73..6377b1555eb 100644 --- a/tests/composition/test_composition.py +++ b/tests/composition/test_composition.py @@ -1012,6 +1012,64 @@ def test_various_pathway_configurations_in_constructor(self, config): assert all(node in comp.get_nodes_by_role(NodeRole.INPUT) for node in {A,C}) assert all(node in comp.get_nodes_by_role(NodeRole.OUTPUT) for node in {B,D}) + config = [ + ('([{A,B,C},D,E],Proj)', 'a'), + ('([{A,B,C},Proj_1,D,E],Proj_2)', 'b'), + ('([{A,B,C},D,Proj_1,E],Proj_2)', 'c'), + ('Pathway(default_matrix)', 'd'), + ('([A,B,C],Proj_2,learning_fct)', 'e'), + ('([A,B,C],Proj_2,learning_fct)', 'f'), + # ('([{A,B,C},D,Proj_1,E],Proj_2,learning_fct)', 'g'), # set spec for Projections + # ('([{A,B,C},D,Proj_1,E],learning_fct,Proj_2)', 'h'), # not yet supported for learning Pathways + ] + @pytest.mark.parametrize('config', config, ids=[x[0] for x in config]) + def test_pathway_tuple_specs(self, config): + A = ProcessingMechanism(name='A') + B = ProcessingMechanism(name='B') + # B_comparator = ComparatorMechanism(name='B COMPARATOR') + C = ProcessingMechanism(name='C') + D = ProcessingMechanism(name='D') + E = ProcessingMechanism(name='E') + F = ProcessingMechanism(name='F') + if config[1]=='a': + comp = Composition(([{A,B,C},D,E],[2.9])) + assert all([p.matrix.base==2.9 for p in D.path_afferents]) + assert E.path_afferents[0].matrix.base==2.9 + if config[1]=='b': + comp = Composition(([{A,B,C},[1.6],D,E],[2.9])) + assert all([p.matrix.base==1.6 for p in D.path_afferents]) + assert E.path_afferents[0].matrix.base==2.9 + if config[1]=='c': + comp = Composition(([{A,B,C},D,[1.6],E],[2.9])) + assert all([p.matrix.base==2.9 for p in D.path_afferents]) + assert E.path_afferents[0].matrix.base==1.6 + if config[1]=='d': + # pway=Pathway([{A,B,C},[1.6],D,E], default_projection_matrix=[2.9]) + pway=Pathway(([{A,B,C},[1.6],D,E], [2.9])) + comp = Composition(pway) + assert all([p.matrix.base==1.6 for p in D.path_afferents]) + assert E.path_afferents[0].matrix.base==2.9 + if config[1]=='e': + comp = Composition(([A,B,C],BackPropagation,[2.9])) + assert B.path_afferents[0].matrix.base==2.9 + assert C.path_afferents[0].matrix.base==2.9 + assert comp.pathways[0].learning_function == BackPropagation + if config[1]=='f': + comp = Composition(([A,B,C],[2.9],BackPropagation)) + assert B.path_afferents[0].matrix.base==2.9 + assert C.path_afferents[0].matrix.base==2.9 + assert comp.pathways[0].learning_function == BackPropagation + if config[1]=='g': + comp = Composition(([{A,B,C},D,[1.6],E],BackPropagation,[2.9])) + assert all([p.matrix.base==2.9 for p in D.path_afferents]) + assert E.path_afferents[0].matrix.base==1.6 + assert comp.pathways[0].learning_function == BackPropagation + if config[1]=='h': + comp = Composition(([{A,B,C},D,[1.6],E],[2.9],BackPropagation)) + assert all([p.matrix.base==2.9 for p in D.path_afferents]) + assert E.path_afferents[0].matrix.base==1.6 + assert comp.pathways[0].learning_function == BackPropagation + def test_add_pathways_bad_arg_error(self): I = InputPort(name='I') c = Composition() @@ -1607,7 +1665,9 @@ def test_composition_learning_pathway_dict_with_no_learning_fct_in_tuple_error(s C = ProcessingMechanism(name='C') with pytest.raises(pnl.CompositionError) as error_text: c = Composition(pathways=[{'P1': ([A,B],C)}]) - assert ("The 2nd item" in str(error_text.value) and "must be a LearningFunction" in str(error_text.value)) + assert ("Bad spec for one of the items in the value of a dict specified for the \'pathways\' arg " + "of the constructor for Composition-0: (ProcessingMechanism C); " + "its item(s) must be a matrix specification and/or a LearningFunction" in str(error_text.value)) class TestProperties: diff --git a/tests/mdf/model_basic.yml b/tests/mdf/model_basic.yml new file mode 100644 index 00000000000..a9b8ad2af8f --- /dev/null +++ b/tests/mdf/model_basic.yml @@ -0,0 +1,313 @@ +comp: + format: ModECI MDF v0.4.3 + generating_application: PsyNeuLink v0.12.1.0+135.g211a8db3af.dirty + graphs: + comp: + metadata: + type: Composition + simulation_results: [] + variable: + - 0 + results: [] + has_initializers: false + retain_old_simulation_data: false + execute_until_finished: true + max_executions_before_finished: 1000 + input_specification: null + node_ordering: + - A + - B + required_node_roles: [] + controller: null + nodes: + A: + metadata: + type: TransferMechanism + termination_measure_value: 0.0 + output_labels_dict: {} + has_initializers: false + max_executions_before_finished: 1000 + variable: + - - 0 + input_labels_dict: {} + input_port_variables: null + execute_until_finished: true + termination_comparison_op: <= + termination_measure: + id: Distance_Function_2_1 + metadata: + type: Distance + enable_output_type_conversion: false + variable: + - - - 0 + - - - 0 + output_type: FunctionOutputType.DEFAULT + has_initializers: false + changes_shape: false + execute_until_finished: true + max_executions_before_finished: 1000 + metric: max_abs_diff + normalize: false + function: distance + args: {} + input_ports: null + termination_threshold: null + integrator_mode: false + on_resume_integrator_mode: current_value + output_ports: + - RESULTS + integrator_function: + id: AdaptiveIntegrator_Function_0 + metadata: + type: AdaptiveIntegrator + has_initializers: true + max_executions_before_finished: 1000 + variable: + - - 0 + output_type: FunctionOutputType.DEFAULT + changes_shape: false + enable_output_type_conversion: false + initializer: + - - 0 + execute_until_finished: true + args: + offset: 0.0 + noise: 0.0 + rate: 0.5 + value: (1 - rate) * previous_value + rate * variable0 + + noise + offset + clip: null + integrator_function_value: + - - 0 + input_ports: + A_InputPort_0: + metadata: + type: InputPort + require_projection_in_composition: true + variable: + - 0 + has_initializers: false + internal_only: false + shadow_inputs: null + execute_until_finished: true + max_executions_before_finished: 1000 + weight: null + projections: null + combine: null + default_input: null + exponent: null + shape: + - 1 + type: int64 + functions: + A_Linear_Function_6: + metadata: + type: Linear + enable_output_type_conversion: true + variable: + - - 0 + output_type: FunctionOutputType.NP_2D_ARRAY + has_initializers: false + changes_shape: false + execute_until_finished: true + max_executions_before_finished: 1000 + bounds: null + function: linear + args: + intercept: 2.0 + slope: 5.0 + variable0: A_InputPort_0 + output_ports: + A_RESULT: + metadata: + type: OutputPort + require_projection_in_composition: true + variable: + - 2.0 + has_initializers: false + execute_until_finished: true + max_executions_before_finished: 1000 + projections: null + value: A_Linear_Function_6 + shape: + - 1 + type: float64 + B: + metadata: + type: TransferMechanism + termination_measure_value: 0.0 + output_labels_dict: {} + has_initializers: false + max_executions_before_finished: 1000 + variable: + - - 0 + input_labels_dict: {} + input_port_variables: null + execute_until_finished: true + termination_comparison_op: <= + termination_measure: + id: Distance_Function_2_3 + metadata: + type: Distance + enable_output_type_conversion: false + variable: + - - - 0 + - - - 0 + output_type: FunctionOutputType.DEFAULT + has_initializers: false + changes_shape: false + execute_until_finished: true + max_executions_before_finished: 1000 + metric: max_abs_diff + normalize: false + function: distance + args: {} + input_ports: null + termination_threshold: null + integrator_mode: false + on_resume_integrator_mode: current_value + output_ports: + - RESULTS + integrator_function: + id: AdaptiveIntegrator_Function_1 + metadata: + type: AdaptiveIntegrator + has_initializers: true + max_executions_before_finished: 1000 + variable: + - - 0 + output_type: FunctionOutputType.DEFAULT + changes_shape: false + enable_output_type_conversion: false + initializer: + - - 0 + execute_until_finished: true + args: + offset: 0.0 + noise: 0.0 + rate: 0.5 + value: (1 - rate) * previous_value + rate * variable0 + + noise + offset + clip: null + integrator_function_value: + - - 0 + input_ports: + B_InputPort_0: + metadata: + type: InputPort + require_projection_in_composition: true + variable: + - 0 + has_initializers: false + internal_only: false + shadow_inputs: null + execute_until_finished: true + max_executions_before_finished: 1000 + weight: null + projections: null + combine: null + default_input: null + exponent: null + shape: + - 1 + type: int64 + functions: + B_Logistic_Function_0: + metadata: + type: Logistic + has_initializers: false + max_executions_before_finished: 1000 + variable: + - - 0 + output_type: FunctionOutputType.NP_2D_ARRAY + changes_shape: false + enable_output_type_conversion: true + execute_until_finished: true + bounds: + - 0 + - 1 + function: logistic + args: + offset: 0.0 + scale: 1.0 + x_0: 0 + bias: 0.0 + gain: 1.0 + variable0: B_InputPort_0 + output_ports: + B_RESULT: + metadata: + type: OutputPort + require_projection_in_composition: true + variable: + - 0.5 + has_initializers: false + execute_until_finished: true + max_executions_before_finished: 1000 + projections: null + value: B_Logistic_Function_0 + shape: + - 1 + type: float64 + edges: + MappingProjection_from_A_RESULT__to_B_InputPort_0_: + sender: A + receiver: B + sender_port: A_RESULT + receiver_port: B_InputPort_0 + metadata: + type: MappingProjection + has_initializers: false + execute_until_finished: true + max_executions_before_finished: 1000 + weight: null + exponent: null + functions: + LinearMatrix_Function_0: + metadata: + type: LinearMatrix + enable_output_type_conversion: false + A: + - 2.0 + output_type: FunctionOutputType.DEFAULT + has_initializers: false + changes_shape: false + execute_until_finished: true + max_executions_before_finished: 1000 + bounds: null + function: onnx::MatMul + args: + B: + - - 1.0 + parameters: + weight: 1 + conditions: + node_specific: + A: + type: EveryNPasses + kwargs: + n: 1 + time_scale: TimeScale.ENVIRONMENT_STATE_UPDATE + B: + type: EveryNCalls + kwargs: + dependency: A + n: 2 + termination: + environment_sequence: + type: AfterNEnvironmentStateUpdates + kwargs: + n: 1 + time_scale: TimeScale.ENVIRONMENT_SEQUENCE + environment_state_update: + type: All + kwargs: + args: + - type: Not + kwargs: + condition: + type: BeforeNCalls + kwargs: + dependency: B + n: 5 + time_scale: TimeScale.ENVIRONMENT_STATE_UPDATE From ae08e47b618a8ed83ad4ef5b05b5276dc27850f3 Mon Sep 17 00:00:00 2001 From: jdcpni Date: Fri, 4 Nov 2022 06:31:20 -0400 Subject: [PATCH 042/127] Feat/add pathway default matrix (#2519) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * • compositioninterfacemechanism.py: - _get_source_node_for_input_CIM: restore (modeled on _get_source_of_modulation_for_parameter_CIM) but NEEDS TESTS - _get_source_of_modulation_for_parameter_CIM: clean up comments, NEEDS TESTS * - * - * - * - * - * - * • Nback - EM uses ContentAddressableMemory (instead of DictionaryMemory) - Implements FFN for comparison of current and retrieved stimulus and context • Project: replace all instances of "RETREIVE" with "RETRIEVE" * • objectivefunctions.py - add cosine_similarity (needs compiled version) * • Project: make COSINE_SIMILARITY a synonym of COSINE • nback_CAM_FFN: - refactor to implement FFN and task input - assign termination condition for execution that is dependent on control - ContentAddressableMemory: selection_function=SoftMax(output=MAX_INDICATOR, gain=SOFT_MAX_TEMP) • DriftOnASphereIntegrator: - add dimension as dependency for initializer parameter * - * - * - * - * - * - * - * - * - * - * - * - * - * - * • test_integrator.py: Added identicalness test for DriftOnASphereIntegrator agains nback-paper implementation. * - * - * Parameters: allow _validate_ methods to reference other parameters (#2512) * • Scripts: - Updated N-back to use objective_mechanism, with commented out code for version that doesn't use it once bug is fixed - Deleted N-back_WITH_OBJECTIVE_MECH.py * • Scripts: - Updated N-back to use objective_mechanism, with commented out code for version that doesn't use it once bug is fixed - Deleted N-back_WITH_OBJECTIVE_MECH.py * • N-back.py: - added stimulus generation per nback-paper protocol * - N-back.py tstep(s) -> trial(s) * - * - * • N-back.py - comp -> nback_model - implement stim_set() method * - * • N-back.py: - added training set generation * - * - * • N-back.py - modularized script * - * - * - * - * • showgraph.py: - _assign_processing_components(): fix bug in which nested graphs not highlighted in animation. * • showgraph.py * composition.py - add further description of animation, including note that animation of nested Compostions is limited. * • showgraph.py * composition.py - add animation to N-back doc * • autodiffcomposition.py - __init__(): move pathways arg to beginning, to capture positional assignment (i.e. w/o kw) * - * • N-back.py - ffn: implement as autodiff; still needs small random initial weight assignment * • pathway.py - implement default_projection attribute * • pathway.py - implement default_projection attribute * • utilities.py: random_matrxi: refactored to allow negative values and use keyword ZERO_CENTER * • projection.py RandomMatrix: added class that can be used to pass a function as matrix spec * • utilities.py - RandomMatrix moved here from projection.py • function.py - get_matrix(): added support for RandomMatrix spec * • port.py - _parse_port_spec(): added support for RandomMatrix * • port.py - _parse_port_spec(): added support for RandomMatrix * • utilities.py - is_matrix(): modified to support random_matrix and RandomMatrix * • composition.py - add_linear_processing_pathway: add support for default_matrix argument (replaces default for MappingProjection for any otherwise unspecified projections) though still not used. * - * - RandomMatrix: moved from Utilities to Function * - * [skip ci] * [skip ci] * [skip ci] • N-back.py - clean up script * [skip ci] • N-back.py - further script clean-up * [skip ci] * [skip ci] * [skip ci] * [skip ci] • BeukersNBackModel.rst: - Overview written - Needs other sections completed * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] • N-back.py: - replace functions of TransferMechanisms with ReLU - replace function of Decision Mechanisms with SoftMax - more doc cleanup * [skip ci] • N-back.py: - replace functions of TransferMechanisms with ReLU - replace function of Decision Mechanisms with SoftMax - more doc cleanup * [skip ci] * - * - * [skip ci] * [skip ci] • composition.py: implement default_projection_matrix in add_XXX_pathway() methods * [skip ci] • composition.py: implement default_projection_matrix in add_XXX_pathway() methods * [skip ci] • test_composition.py: - add test_pathway_tuple_specs() * - * - * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] • composition.py: - add_linear_processing_pathway: fixed bug when Reinforcement or TDLearning are specified • test_composition.py: - test_pathway_tuple_specs: add tests for Reinforcement and TDLearning * • composition.py: - add_linear_processing_pathway: fixed bug when Reinforcement or TDLearning are specified • test_composition.py: - test_pathway_tuple_specs: add tests for Reinforcement and TDLearning Co-authored-by: jdcpni Co-authored-by: Katherine Mantel --- Scripts/Models (Under Development)/N-back.py | 10 +++-- psyneulink/core/compositions/composition.py | 8 ++-- tests/composition/test_composition.py | 42 +++++++++++++------- 3 files changed, 38 insertions(+), 22 deletions(-) diff --git a/Scripts/Models (Under Development)/N-back.py b/Scripts/Models (Under Development)/N-back.py index 903a82b862a..205cded643a 100644 --- a/Scripts/Models (Under Development)/N-back.py +++ b/Scripts/Models (Under Development)/N-back.py @@ -155,7 +155,9 @@ def construct_model(stim_size = STIM_SIZE, input_retrieved_context, input_task}, hidden, decision], - RANDOM_WEIGHTS_INITIALIZATION, + RandomMatrix(center=0.0, range=0.1) # Matrix spec used to initialize all Projections + ), + RANDOM_WEIGHTS_INITIALIZATION, ), name=FFN_COMPOSITION, learning_rate=LEARNING_RATE @@ -245,9 +247,9 @@ def construct_model(stim_size = STIM_SIZE, if DISPLAY: nback_model.show_graph( - # show_cim=True, - # show_node_structure=ALL, - # show_dimensions=True + show_cim=True, + show_node_structure=ALL, + show_dimensions=True ) return nback_model diff --git a/psyneulink/core/compositions/composition.py b/psyneulink/core/compositions/composition.py index 3a646d39f35..c55da9b5d29 100644 --- a/psyneulink/core/compositions/composition.py +++ b/psyneulink/core/compositions/composition.py @@ -7263,7 +7263,7 @@ def add_linear_learning_pathway(self, # Processing Components try: input_source, output_source, learned_projection = \ - self._unpack_processing_components_of_learning_pathway(pathway) + self._unpack_processing_components_of_learning_pathway(pathway, default_projection_matrix) except CompositionError as e: raise CompositionError(e.error_value.replace('this method', f'{learning_function.__name__} {LearningFunction.__name__}')) @@ -7509,13 +7509,15 @@ def add_backpropagation_learning_pathway(self, # Move creation of LearningProjections and learning-related projections (MappingProjections) here # ?Do add_nodes and add_projections here or in Learning-type-specific creation methods - def _unpack_processing_components_of_learning_pathway(self, processing_pathway): + def _unpack_processing_components_of_learning_pathway(self, processing_pathway, default_projection_matrix=None): # unpack processing components and add to composition if len(processing_pathway) == 3 and isinstance(processing_pathway[1], MappingProjection): input_source, learned_projection, output_source = processing_pathway elif len(processing_pathway) == 2: input_source, output_source = processing_pathway - learned_projection = MappingProjection(sender=input_source, receiver=output_source) + learned_projection = MappingProjection(sender=input_source, + receiver=output_source, + matrix=default_projection_matrix) else: raise CompositionError(f"Too many Nodes in learning pathway: {processing_pathway}. " f"Only single-layer learning is supported by this method. " diff --git a/tests/composition/test_composition.py b/tests/composition/test_composition.py index 6377b1555eb..f3d39374c72 100644 --- a/tests/composition/test_composition.py +++ b/tests/composition/test_composition.py @@ -8,7 +8,8 @@ import psyneulink as pnl from psyneulink.core.components.functions.nonstateful.combinationfunctions import LinearCombination -from psyneulink.core.components.functions.nonstateful.learningfunctions import Reinforcement, BackPropagation +from psyneulink.core.components.functions.nonstateful.learningfunctions import \ + LearningFunction, Reinforcement, BackPropagation, TDLearning from psyneulink.core.components.functions.nonstateful.optimizationfunctions import GridSearch from psyneulink.core.components.functions.nonstateful.transferfunctions import \ Linear, Logistic, INTENSITY_COST_FCT_MULTIPLICATIVE_PARAM @@ -1017,17 +1018,20 @@ def test_various_pathway_configurations_in_constructor(self, config): ('([{A,B,C},Proj_1,D,E],Proj_2)', 'b'), ('([{A,B,C},D,Proj_1,E],Proj_2)', 'c'), ('Pathway(default_matrix)', 'd'), - ('([A,B,C],Proj_2,learning_fct)', 'e'), - ('([A,B,C],Proj_2,learning_fct)', 'f'), - # ('([{A,B,C},D,Proj_1,E],Proj_2,learning_fct)', 'g'), # set spec for Projections - # ('([{A,B,C},D,Proj_1,E],learning_fct,Proj_2)', 'h'), # not yet supported for learning Pathways + ('([A,B,C],BackProp,Proj)', 'e'), + ('([A,B,C],Proj,BackProp)', 'f'), + ('([A,B],RL,Proj)', 'g'), + ('([A,B],TD,Proj)', 'h'), + # FIX: Set specification not yet supported for learning pathway: + # ('([{A,B,C},D,Proj_1,E],Proj_2,learning_fct)', 'i'), # set spec for Projections + # ('([{A,B,C},D,Proj_1,E],learning_fct,Proj_2)', 'j'), # not yet supported for learning Pathways ] @pytest.mark.parametrize('config', config, ids=[x[0] for x in config]) def test_pathway_tuple_specs(self, config): A = ProcessingMechanism(name='A') B = ProcessingMechanism(name='B') - # B_comparator = ComparatorMechanism(name='B COMPARATOR') C = ProcessingMechanism(name='C') + # if config[1] not in {'g','h'}: D = ProcessingMechanism(name='D') E = ProcessingMechanism(name='E') F = ProcessingMechanism(name='F') @@ -1044,7 +1048,6 @@ def test_pathway_tuple_specs(self, config): assert all([p.matrix.base==2.9 for p in D.path_afferents]) assert E.path_afferents[0].matrix.base==1.6 if config[1]=='d': - # pway=Pathway([{A,B,C},[1.6],D,E], default_projection_matrix=[2.9]) pway=Pathway(([{A,B,C},[1.6],D,E], [2.9])) comp = Composition(pway) assert all([p.matrix.base==1.6 for p in D.path_afferents]) @@ -1060,15 +1063,24 @@ def test_pathway_tuple_specs(self, config): assert C.path_afferents[0].matrix.base==2.9 assert comp.pathways[0].learning_function == BackPropagation if config[1]=='g': - comp = Composition(([{A,B,C},D,[1.6],E],BackPropagation,[2.9])) - assert all([p.matrix.base==2.9 for p in D.path_afferents]) - assert E.path_afferents[0].matrix.base==1.6 - assert comp.pathways[0].learning_function == BackPropagation + comp = Composition(([A,B],Reinforcement,[2.9])) + assert B.path_afferents[0].matrix.base==2.9 + assert comp.pathways[0].learning_function == Reinforcement if config[1]=='h': - comp = Composition(([{A,B,C},D,[1.6],E],[2.9],BackPropagation)) - assert all([p.matrix.base==2.9 for p in D.path_afferents]) - assert E.path_afferents[0].matrix.base==1.6 - assert comp.pathways[0].learning_function == BackPropagation + comp = Composition(([A,B],[2.9],TDLearning)) + assert B.path_afferents[0].matrix.base==2.9 + assert comp.pathways[0].learning_function == TDLearning + # FIX: Set specification not yet supported for learning pathway: + # if config[1]=='i': + # comp = Composition(([{A,B,C},D,[1.6],E],BackPropagation,[2.9])) + # assert all([p.matrix.base==2.9 for p in D.path_afferents]) + # assert E.path_afferents[0].matrix.base==1.6 + # assert comp.pathways[0].learning_function == BackPropagation + # if config[1]=='j': + # comp = Composition(([{A,B,C},D,[1.6],E],[2.9],BackPropagation)) + # assert all([p.matrix.base==2.9 for p in D.path_afferents]) + # assert E.path_afferents[0].matrix.base==1.6 + # assert comp.pathways[0].learning_function == BackPropagation def test_add_pathways_bad_arg_error(self): I = InputPort(name='I') From 50a5cd7bd2d74de00c0ec1f9f39a777f28cb66ed Mon Sep 17 00:00:00 2001 From: jdcpni Date: Fri, 4 Nov 2022 19:56:13 -0400 Subject: [PATCH 043/127] Fix/comp warning no afferents (#2520) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * - * - * • composition.py: - _check_for_projection_assignments: postpone "no afferents" warning until context is PREPARING * [skip ci] • composition.py: - _check_for_projection_assignments: postpone "no afferents" warning until context is PREPARING * • test_composition.py - add test_missing_afferent_at_run_time - add test_missing_efferent_at_runt_time --- Scripts/Models (Under Development)/N-back.py | 26 ++- .../N-back_WITH_OBJECTIVE_MECH.py | 0 psyneulink/core/compositions/composition.py | 2 +- tests/composition/test_composition.py | 27 +++ tests/mdf/model_basic.yml | 156 +++++++++--------- 5 files changed, 118 insertions(+), 93 deletions(-) create mode 100644 Scripts/Models (Under Development)/N-back_WITH_OBJECTIVE_MECH.py diff --git a/Scripts/Models (Under Development)/N-back.py b/Scripts/Models (Under Development)/N-back.py index 205cded643a..1b64bd54f02 100644 --- a/Scripts/Models (Under Development)/N-back.py +++ b/Scripts/Models (Under Development)/N-back.py @@ -39,15 +39,15 @@ - epoch: 1 trial per epoch of training - get empirical stimulus sequences - put N-back script (with pointer to latest version on PNL) in nback-paper repo - - get rid of objective_mechanism (see "VERSION *WITHOUT* ObjectiveMechanism" under control(...) - - pass learning_rate as parameter to train_network() + - get rid of objective_mechanism (see "VERSION *WITHOUT* ObjectiveMechanism" under control(...) (fix bug) + - make termination processing part of the Composition definition (fix bug) + - pass learning_rate as parameter to train_network() (add feature) + - fix warnings on run - validate against nback-paper results - after validation: - try with STIM_SIZE = NUM_STIMS rather than 20 (as in nback-paper) - refactor generate_stim_sequence() to use actual empirical stimulus sequences - replace get_input_sequence and get_training_inputs with generators passed to nback_model.run() and ffn.learn - - make termination processing part of the Composition definition (fix bug) - - fix warnings on run """ @@ -58,7 +58,7 @@ # Settings for running script: TRAIN = False -RUN = True +RUN = False DISPLAY = False # show visual graphic of model # PARAMETERS ------------------------------------------------------------------------------------------------------- @@ -151,13 +151,11 @@ def construct_model(stim_size = STIM_SIZE, gain=decision_softmax_temp)) ffn = AutodiffComposition(([{input_current_stim, input_current_context, - input_retrieved_stim, - input_retrieved_context, - input_task}, - hidden, decision], - RandomMatrix(center=0.0, range=0.1) # Matrix spec used to initialize all Projections - ), - RANDOM_WEIGHTS_INITIALIZATION, + input_retrieved_stim, + input_retrieved_context, + input_task}, + hidden, decision], + RANDOM_WEIGHTS_INITIALIZATION, ), name=FFN_COMPOSITION, learning_rate=LEARNING_RATE @@ -224,7 +222,7 @@ def construct_model(stim_size = STIM_SIZE, control=(STORAGE_PROB, em)) nback_model = Composition(name=NBACK_MODEL, - nodes=[stim, context, task, em, ffn, control], + nodes=[stim, context, task, ffn, em, control], # # # Terminate trial if value of control is still 1 after first pass through execution # # FIX: STOPS AFTER ~ NUMBER OF TRIALS (?90+); SHOULD BE: NUM_TRIALS*NUM_NBACK_LEVELS + 1 # termination_processing={TimeScale.TRIAL: And(Condition(lambda: control.value), @@ -486,7 +484,7 @@ def run_model(model, run_model(nback_model) if REPORT_PROGRESS == ReportProgress.ON: print('\n') -print(f'nback_model done: {len(nback_model.results)} trials executed') + print(f'nback_model done: {len(nback_model.results)} trials executed') # =========================================================================== diff --git a/Scripts/Models (Under Development)/N-back_WITH_OBJECTIVE_MECH.py b/Scripts/Models (Under Development)/N-back_WITH_OBJECTIVE_MECH.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/psyneulink/core/compositions/composition.py b/psyneulink/core/compositions/composition.py index c55da9b5d29..ec87a10a58a 100644 --- a/psyneulink/core/compositions/composition.py +++ b/psyneulink/core/compositions/composition.py @@ -6198,7 +6198,7 @@ def _check_for_projection_assignments(self, context=None): projections.append(node) continue - if context.source != ContextFlags.INITIALIZING and context.string != 'IGNORE_NO_AFFERENTS_WARNING': + if context.flags & ContextFlags.PREPARING and context.string != 'IGNORE_NO_AFFERENTS_WARNING': for input_port in node.input_ports: if input_port.require_projection_in_composition \ and not input_port.path_afferents and not input_port.default_input: diff --git a/tests/composition/test_composition.py b/tests/composition/test_composition.py index f3d39374c72..1120e94d108 100644 --- a/tests/composition/test_composition.py +++ b/tests/composition/test_composition.py @@ -40,9 +40,12 @@ from psyneulink.core.scheduling.condition import EveryNCalls from psyneulink.core.scheduling.scheduler import Scheduler, SchedulingMode from psyneulink.core.scheduling.time import TimeScale +from psyneulink.library.components.mechanisms.processing.objective.comparatormechanism import ComparatorMechanism from psyneulink.library.components.mechanisms.modulatory.control.agt.lccontrolmechanism import LCControlMechanism from psyneulink.library.components.mechanisms.processing.transfer.recurrenttransfermechanism import \ RecurrentTransferMechanism +from psyneulink.library.components.mechanisms.processing.integrator.episodicmemorymechanism import \ + EpisodicMemoryMechanism logger = logging.getLogger(__name__) @@ -4101,6 +4104,30 @@ def test_manual_context(self): comp.run({t: [1]}, context=context) assert comp.results == [[[2]]] + def test_missing_afferent_at_run_time(self): + A = ProcessingMechanism() + B = ProcessingMechanism(input_ports=['OCCUPIED', 'UNOCCUPIED']) + comp = Composition([A,B]) + warning_type = UserWarning + warning_msg = '"InputPort (\'UNOCCUPIED\') of \'ProcessingMechanism-1\' ' \ + 'doesn\'t have any afferent Projections."' + with pytest.raises(TypeError): # Caused by error on B at construction (with only one InputPort "occupied") + with pytest.warns(warning_type) as warning: + comp.run() + assert repr(warning[0].message.args[0]) == warning_msg + + def test_missing_efferent_at_run_time(self): + A = ProcessingMechanism() + B = ProcessingMechanism(output_ports=['OCCUPIED','UNOCCUPIED']) # Comparator Mech has two inputports, + C = ProcessingMechanism(name='C') + comp = Composition([A,B,C]) + warning_type = UserWarning + warning_msg = '"OutputPort (\'UNOCCUPIED\') of \'ProcessingMechanism-1\' ' \ + 'doesn\'t have any efferent Projections in \'Composition-0\'."' + with pytest.warns(warning_type) as warning: + comp.run() + assert repr(warning[0].message.args[0]) == warning_msg + class TestCallBeforeAfterTimescale: diff --git a/tests/mdf/model_basic.yml b/tests/mdf/model_basic.yml index a9b8ad2af8f..5997bf12e54 100644 --- a/tests/mdf/model_basic.yml +++ b/tests/mdf/model_basic.yml @@ -1,19 +1,19 @@ comp: format: ModECI MDF v0.4.3 - generating_application: PsyNeuLink v0.12.1.0+135.g211a8db3af.dirty + generating_application: PsyNeuLink v0.12.1.0+52.gc8cdaf69a8 graphs: comp: metadata: type: Composition + execute_until_finished: true simulation_results: [] - variable: - - 0 results: [] has_initializers: false + input_specification: null retain_old_simulation_data: false - execute_until_finished: true max_executions_before_finished: 1000 - input_specification: null + variable: + - 0 node_ordering: - A - B @@ -23,79 +23,79 @@ comp: A: metadata: type: TransferMechanism - termination_measure_value: 0.0 + input_labels_dict: {} + execute_until_finished: true output_labels_dict: {} has_initializers: false max_executions_before_finished: 1000 variable: - - 0 - input_labels_dict: {} + termination_measure_value: 0.0 input_port_variables: null - execute_until_finished: true + integrator_mode: false + integrator_function_value: + - - 0 termination_comparison_op: <= termination_measure: id: Distance_Function_2_1 metadata: type: Distance - enable_output_type_conversion: false - variable: - - - - 0 - - - - 0 output_type: FunctionOutputType.DEFAULT + enable_output_type_conversion: false + execute_until_finished: true has_initializers: false changes_shape: false - execute_until_finished: true max_executions_before_finished: 1000 + variable: + - - - 0 + - - - 0 metric: max_abs_diff normalize: false function: distance args: {} - input_ports: null - termination_threshold: null - integrator_mode: false - on_resume_integrator_mode: current_value output_ports: - RESULTS integrator_function: id: AdaptiveIntegrator_Function_0 metadata: type: AdaptiveIntegrator + execute_until_finished: true has_initializers: true + output_type: FunctionOutputType.DEFAULT + enable_output_type_conversion: false + changes_shape: false max_executions_before_finished: 1000 variable: - - 0 - output_type: FunctionOutputType.DEFAULT - changes_shape: false - enable_output_type_conversion: false initializer: - - 0 - execute_until_finished: true args: offset: 0.0 - noise: 0.0 rate: 0.5 + noise: 0.0 value: (1 - rate) * previous_value + rate * variable0 + noise + offset clip: null - integrator_function_value: - - - 0 + termination_threshold: null + on_resume_integrator_mode: current_value + input_ports: null input_ports: A_InputPort_0: metadata: type: InputPort require_projection_in_composition: true - variable: - - 0 + execute_until_finished: true + shadow_inputs: null has_initializers: false internal_only: false - shadow_inputs: null - execute_until_finished: true max_executions_before_finished: 1000 - weight: null + variable: + - 0 + default_input: null projections: null combine: null - default_input: null exponent: null + weight: null shape: - 1 type: int64 @@ -103,30 +103,30 @@ comp: A_Linear_Function_6: metadata: type: Linear - enable_output_type_conversion: true - variable: - - - 0 output_type: FunctionOutputType.NP_2D_ARRAY + enable_output_type_conversion: true + execute_until_finished: true has_initializers: false changes_shape: false - execute_until_finished: true max_executions_before_finished: 1000 + variable: + - - 0 bounds: null function: linear args: - intercept: 2.0 slope: 5.0 + intercept: 2.0 variable0: A_InputPort_0 output_ports: A_RESULT: metadata: type: OutputPort require_projection_in_composition: true - variable: - - 2.0 - has_initializers: false execute_until_finished: true + has_initializers: false max_executions_before_finished: 1000 + variable: + - 2.0 projections: null value: A_Linear_Function_6 shape: @@ -135,79 +135,79 @@ comp: B: metadata: type: TransferMechanism - termination_measure_value: 0.0 + input_labels_dict: {} + execute_until_finished: true output_labels_dict: {} has_initializers: false max_executions_before_finished: 1000 variable: - - 0 - input_labels_dict: {} + termination_measure_value: 0.0 input_port_variables: null - execute_until_finished: true + integrator_mode: false + integrator_function_value: + - - 0 termination_comparison_op: <= termination_measure: id: Distance_Function_2_3 metadata: type: Distance - enable_output_type_conversion: false - variable: - - - - 0 - - - - 0 output_type: FunctionOutputType.DEFAULT + enable_output_type_conversion: false + execute_until_finished: true has_initializers: false changes_shape: false - execute_until_finished: true max_executions_before_finished: 1000 + variable: + - - - 0 + - - - 0 metric: max_abs_diff normalize: false function: distance args: {} - input_ports: null - termination_threshold: null - integrator_mode: false - on_resume_integrator_mode: current_value output_ports: - RESULTS integrator_function: id: AdaptiveIntegrator_Function_1 metadata: type: AdaptiveIntegrator + execute_until_finished: true has_initializers: true + output_type: FunctionOutputType.DEFAULT + enable_output_type_conversion: false + changes_shape: false max_executions_before_finished: 1000 variable: - - 0 - output_type: FunctionOutputType.DEFAULT - changes_shape: false - enable_output_type_conversion: false initializer: - - 0 - execute_until_finished: true args: offset: 0.0 - noise: 0.0 rate: 0.5 + noise: 0.0 value: (1 - rate) * previous_value + rate * variable0 + noise + offset clip: null - integrator_function_value: - - - 0 + termination_threshold: null + on_resume_integrator_mode: current_value + input_ports: null input_ports: B_InputPort_0: metadata: type: InputPort require_projection_in_composition: true - variable: - - 0 + execute_until_finished: true + shadow_inputs: null has_initializers: false internal_only: false - shadow_inputs: null - execute_until_finished: true max_executions_before_finished: 1000 - weight: null + variable: + - 0 + default_input: null projections: null combine: null - default_input: null exponent: null + weight: null shape: - 1 type: int64 @@ -215,23 +215,23 @@ comp: B_Logistic_Function_0: metadata: type: Logistic + execute_until_finished: true has_initializers: false + output_type: FunctionOutputType.NP_2D_ARRAY + enable_output_type_conversion: true + changes_shape: false max_executions_before_finished: 1000 variable: - - 0 - output_type: FunctionOutputType.NP_2D_ARRAY - changes_shape: false - enable_output_type_conversion: true - execute_until_finished: true bounds: - 0 - 1 function: logistic args: - offset: 0.0 - scale: 1.0 x_0: 0 + scale: 1.0 bias: 0.0 + offset: 0.0 gain: 1.0 variable0: B_InputPort_0 output_ports: @@ -239,11 +239,11 @@ comp: metadata: type: OutputPort require_projection_in_composition: true - variable: - - 0.5 - has_initializers: false execute_until_finished: true + has_initializers: false max_executions_before_finished: 1000 + variable: + - 0.5 projections: null value: B_Logistic_Function_0 shape: @@ -257,23 +257,23 @@ comp: receiver_port: B_InputPort_0 metadata: type: MappingProjection - has_initializers: false execute_until_finished: true + has_initializers: false max_executions_before_finished: 1000 - weight: null exponent: null + weight: null functions: LinearMatrix_Function_0: metadata: type: LinearMatrix - enable_output_type_conversion: false - A: - - 2.0 output_type: FunctionOutputType.DEFAULT + enable_output_type_conversion: false + execute_until_finished: true has_initializers: false changes_shape: false - execute_until_finished: true max_executions_before_finished: 1000 + A: + - 2.0 bounds: null function: onnx::MatMul args: From 5cd211504bbfd4c3da2dabe3f6ae124d1e5b6934 Mon Sep 17 00:00:00 2001 From: jdcpni Date: Sat, 5 Nov 2022 12:13:29 -0400 Subject: [PATCH 044/127] Fix/softmax and relu derivatives (#2523) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * - * Merge branch 'fix/comp_warning_no_afferents' of https://github.com/PrincetonUniversity/PsyNeuLink into fix/comp_warning_no_afferents  Conflicts:  tests/mdf/model_basic.yml * - * • transferfunctions.py - SoftMax.derivative: fix handling of dimensionality of return on sm function, and of maxval determination - ReLU.derivative: use output arg to be consistent with calls to Logistic and SoftMax from BackPropagation * REVISE CHANGES IN PREVIOUS COMMIT; NOW: • BackPropagation: - pass both activation_input and activation_output to derivative; assume implementation of derivative will decide which to use • Softmax.derivative: - resize sm and use appropriate index on value of np.where for determination of output max - accept input argument (but don't use it) for consistency with other derivatives and call from BackPropagation * - remove mdf_basic.yml --- Scripts/Models (Under Development)/N-back.py | 39 ++- .../nonstateful/learningfunctions.py | 6 +- .../nonstateful/transferfunctions.py | 54 +-- tests/mdf/model_basic.yml | 313 ------------------ 4 files changed, 63 insertions(+), 349 deletions(-) delete mode 100644 tests/mdf/model_basic.yml diff --git a/Scripts/Models (Under Development)/N-back.py b/Scripts/Models (Under Development)/N-back.py index 1b64bd54f02..a7546baa6e9 100644 --- a/Scripts/Models (Under Development)/N-back.py +++ b/Scripts/Models (Under Development)/N-back.py @@ -32,12 +32,17 @@ TODO: - from Andre - network architecture; in particular, size of hidden layer and projection patterns to and from it - - softmax temp on output/decision layer? + - the stim+context input vector (length 90) projects to a hidden layer (length 80); + - the task input vector (length 2) projects to a different hidden layer (length 80); + - those two hidden layers project (over fixed, nonlearnable, one-one-projections?) to a third hidden layer (length 80) that simply sums them; + - the third hidden layer projections to the length 2 output layer; + - a softmax is taken over the output layer to determine the response. + - softmax temp on output/decision layer: 1 - confirm that ReLUs all use 0 thresholds and unit slope - training: - - confirm learning rate: ?? 0.001 - - epoch: 1 trial per epoch of training - - get empirical stimulus sequences + - learning rate: 0.001; epoch: 1 trial per epoch of training + - state_dict with weights (still needed) + - get empirical stimulus sequences (still needed) - put N-back script (with pointer to latest version on PNL) in nback-paper repo - get rid of objective_mechanism (see "VERSION *WITHOUT* ObjectiveMechanism" under control(...) (fix bug) - make termination processing part of the Composition definition (fix bug) @@ -57,9 +62,9 @@ import numpy as np # Settings for running script: -TRAIN = False +TRAIN = True RUN = False -DISPLAY = False # show visual graphic of model +DISPLAY_MODEL = False # show visual graphic of model # PARAMETERS ------------------------------------------------------------------------------------------------------- @@ -83,7 +88,7 @@ DECISION_SOFTMAX_TEMP=1/8 # express as gain # binarity of decision process # Training parameters: -NUM_EPOCHS=1000 # nback-paper: 400,000, one trial per epoch +NUM_EPOCHS=10 # nback-paper: 400,000, one trial per epoch LEARNING_RATE=0.1 # nback-paper: .001 # Execution parameters: @@ -91,6 +96,7 @@ NUM_TRIALS = 48 # number of stimuli presented in a trial sequence REPORT_OUTPUT = ReportOutput.OFF # Sets console output during run REPORT_PROGRESS = ReportProgress.ON # Sets console progress bar during run +REPORT_LEARNING = ReportLearning.ON # Sets console progress bar during training ANIMATE = True # {UNIT:EXECUTION_SET} # Specifies whether to generate animation of execution # Names of Compositions and Mechanisms: @@ -151,10 +157,10 @@ def construct_model(stim_size = STIM_SIZE, gain=decision_softmax_temp)) ffn = AutodiffComposition(([{input_current_stim, input_current_context, - input_retrieved_stim, - input_retrieved_context, - input_task}, - hidden, decision], + input_retrieved_stim, + input_retrieved_context, + input_task}, + hidden, decision], RANDOM_WEIGHTS_INITIALIZATION, ), name=FFN_COMPOSITION, @@ -243,11 +249,11 @@ def construct_model(stim_size = STIM_SIZE, nback_model.add_projection(MappingProjection(), stim, em.input_ports["STIMULUS_FIELD"]) nback_model.add_projection(MappingProjection(), context, em.input_ports["CONTEXT_FIELD"]) - if DISPLAY: + if DISPLAY_MODEL: nback_model.show_graph( - show_cim=True, - show_node_structure=ALL, - show_dimensions=True + # show_cim=True, + # show_node_structure=ALL, + # show_dimensions=True ) return nback_model @@ -370,7 +376,7 @@ def get_training_inputs(network, num_epochs, nback_levels): context_lure: stim_current != stim_retrieved and context_current == context_retrieved non_lure: stim_current != stim_retrieved and context_current != context_retrieved """ - assert is_iterable(nback_levels) and all([0` at either **input** or **output**. - Either **input** or **ouput** must be specified. If **output** is not specified, it is computed from **input**. + COMMENT: RESTORE WHEN TEST IN DERIVATIVE IS RESTORED + Either **input** or **output** must be specified. + If **output** is not specified, it is computed from **input**. If both are specified, **input** is ignored unless paramValidationPref is set, in which case an error is generated if **output** does not correspond to `function `\\(**input**). + COMMENT + Either **input** or **output** must be specified. + If **output** is not specified, derivative is computed from **input**. + If both are specified, **input** is ignored and derivative is computed from **output** + .. technical_note:: + allowing both to be specified is supported for consistency with `BackPropagation` `LearningFunction` + which uses output to compute Logistic Arguments --------- @@ -1042,17 +1051,18 @@ def derivative(self, input=None, output=None, context=None): Deriviative of logistic transform at output: number or array """ - if output is not None and input is not None and self.prefs.paramValidationPref: - if isinstance(input, numbers.Number): - valid = output == self.function(input, context=context) - else: - valid = all(output[i] == self.function(input, context=context)[i] for i in range(len(input))) - if not valid: - raise FunctionError("Value of {} arg passed to {} ({}) " - "does not match the value expected for specified {} ({})". - format(repr('output'), self.__class__.__name__ + '.' + 'derivative', output, - repr('input'), input)) - + # FIX: BackPropagation PASSES IN SAME INPUT AND OUTPUT + # if (output is not None and input is not None and self.prefs.paramValidationPref): + # if isinstance(input, numbers.Number): + # valid = output == self.function(input, context=context) + # else: + # valid = all(output[i] == self.function(input, context=context)[i] for i in range(len(input))) + # if not valid: + # raise FunctionError("Value of {} arg passed to {} ({}) " + # "does not match the value expected for specified {} ({})". + # format(repr('output'), self.__class__.__name__ + '.' + 'derivative', output, + # repr('input'), input)) + # gain = self._get_current_parameter_value(GAIN, context) scale = self._get_current_parameter_value(SCALE, context) @@ -1426,7 +1436,7 @@ class ReLU(TransferFunction): # ----------------------------------------------- specifies a value by which to multiply `variable ` after `bias ` is subtracted from it. bias : float : default 0.0 - specifies a value to subtract from each element of `variable `. + specifies a value to subtract from each element of `variable `; functions as threshold. leak : float : default 0.0 specifies a scaling factor between 0 and 1 when (variable - bias) is less than or equal to 0. params : Dict[param keyword: param value] : default None @@ -1451,7 +1461,7 @@ class ReLU(TransferFunction): # ----------------------------------------------- from it. bias : float : default 0.0 - value to subtract from each element of `variable `. + value to subtract from each element of `variable `; functions as threshold. leak : float : default 0.0 scaling factor between 0 and 1 when (variable - bias) is less than or equal to 0. @@ -1603,14 +1613,19 @@ def derivative(self, input, output=None, context=None): """ gain = self._get_current_parameter_value(GAIN, context) leak = self._get_current_parameter_value(LEAK, context) - + # MODIFIED 11/5/22 OLD: input = np.asarray(input).copy() input[input>0] = gain input[input<=0] = gain * leak + # # MODIFIED 11/5/22 NEW: + # bias = self._get_current_parameter_value(BIAS, context) + # input = np.asarray(input).copy() + # input[(input-bias)>0] = gain + # input[(input-bias)<=0] = gain * leak + # MODIFIED 11/5/22 END return input - # ********************************************************************************************************************** # Angle # ********************************************************************************************************************** @@ -2735,7 +2750,7 @@ def _function(self, return self.convert_output_type(output) @handle_external_context() - def derivative(self, output, input=None, context=None): + def derivative(self, input=None, output=None, context=None): """ derivative(output) @@ -2745,9 +2760,10 @@ def derivative(self, output, input=None, context=None): derivative of values returned by SoftMax : 1d or 2d array (depending on *OUTPUT_TYPE* of SoftMax) """ - output_type = self.output_type + output_type = self._get_current_parameter_value(OUTPUT_TYPE, context) size = len(output) sm = self.function(output, params={OUTPUT_TYPE: ALL}, context=context) + sm = np.squeeze(sm) if output_type == ALL: # Return full Jacobian matrix of derivatives @@ -2764,7 +2780,7 @@ def derivative(self, output, input=None, context=None): # Return 1d array of derivatives for max element (i.e., the one chosen by SoftMax) derivative = np.empty(size) # Get the element of output returned as non-zero when output_type is not ALL - index_of_max = int(np.where(output == np.max(output))[0]) + index_of_max = int(np.where(output == np.max(output))[0][0]) max_val = sm[index_of_max] for i in range(size): if i == index_of_max: diff --git a/tests/mdf/model_basic.yml b/tests/mdf/model_basic.yml deleted file mode 100644 index 5997bf12e54..00000000000 --- a/tests/mdf/model_basic.yml +++ /dev/null @@ -1,313 +0,0 @@ -comp: - format: ModECI MDF v0.4.3 - generating_application: PsyNeuLink v0.12.1.0+52.gc8cdaf69a8 - graphs: - comp: - metadata: - type: Composition - execute_until_finished: true - simulation_results: [] - results: [] - has_initializers: false - input_specification: null - retain_old_simulation_data: false - max_executions_before_finished: 1000 - variable: - - 0 - node_ordering: - - A - - B - required_node_roles: [] - controller: null - nodes: - A: - metadata: - type: TransferMechanism - input_labels_dict: {} - execute_until_finished: true - output_labels_dict: {} - has_initializers: false - max_executions_before_finished: 1000 - variable: - - - 0 - termination_measure_value: 0.0 - input_port_variables: null - integrator_mode: false - integrator_function_value: - - - 0 - termination_comparison_op: <= - termination_measure: - id: Distance_Function_2_1 - metadata: - type: Distance - output_type: FunctionOutputType.DEFAULT - enable_output_type_conversion: false - execute_until_finished: true - has_initializers: false - changes_shape: false - max_executions_before_finished: 1000 - variable: - - - - 0 - - - - 0 - metric: max_abs_diff - normalize: false - function: distance - args: {} - output_ports: - - RESULTS - integrator_function: - id: AdaptiveIntegrator_Function_0 - metadata: - type: AdaptiveIntegrator - execute_until_finished: true - has_initializers: true - output_type: FunctionOutputType.DEFAULT - enable_output_type_conversion: false - changes_shape: false - max_executions_before_finished: 1000 - variable: - - - 0 - initializer: - - - 0 - args: - offset: 0.0 - rate: 0.5 - noise: 0.0 - value: (1 - rate) * previous_value + rate * variable0 - + noise + offset - clip: null - termination_threshold: null - on_resume_integrator_mode: current_value - input_ports: null - input_ports: - A_InputPort_0: - metadata: - type: InputPort - require_projection_in_composition: true - execute_until_finished: true - shadow_inputs: null - has_initializers: false - internal_only: false - max_executions_before_finished: 1000 - variable: - - 0 - default_input: null - projections: null - combine: null - exponent: null - weight: null - shape: - - 1 - type: int64 - functions: - A_Linear_Function_6: - metadata: - type: Linear - output_type: FunctionOutputType.NP_2D_ARRAY - enable_output_type_conversion: true - execute_until_finished: true - has_initializers: false - changes_shape: false - max_executions_before_finished: 1000 - variable: - - - 0 - bounds: null - function: linear - args: - slope: 5.0 - intercept: 2.0 - variable0: A_InputPort_0 - output_ports: - A_RESULT: - metadata: - type: OutputPort - require_projection_in_composition: true - execute_until_finished: true - has_initializers: false - max_executions_before_finished: 1000 - variable: - - 2.0 - projections: null - value: A_Linear_Function_6 - shape: - - 1 - type: float64 - B: - metadata: - type: TransferMechanism - input_labels_dict: {} - execute_until_finished: true - output_labels_dict: {} - has_initializers: false - max_executions_before_finished: 1000 - variable: - - - 0 - termination_measure_value: 0.0 - input_port_variables: null - integrator_mode: false - integrator_function_value: - - - 0 - termination_comparison_op: <= - termination_measure: - id: Distance_Function_2_3 - metadata: - type: Distance - output_type: FunctionOutputType.DEFAULT - enable_output_type_conversion: false - execute_until_finished: true - has_initializers: false - changes_shape: false - max_executions_before_finished: 1000 - variable: - - - - 0 - - - - 0 - metric: max_abs_diff - normalize: false - function: distance - args: {} - output_ports: - - RESULTS - integrator_function: - id: AdaptiveIntegrator_Function_1 - metadata: - type: AdaptiveIntegrator - execute_until_finished: true - has_initializers: true - output_type: FunctionOutputType.DEFAULT - enable_output_type_conversion: false - changes_shape: false - max_executions_before_finished: 1000 - variable: - - - 0 - initializer: - - - 0 - args: - offset: 0.0 - rate: 0.5 - noise: 0.0 - value: (1 - rate) * previous_value + rate * variable0 - + noise + offset - clip: null - termination_threshold: null - on_resume_integrator_mode: current_value - input_ports: null - input_ports: - B_InputPort_0: - metadata: - type: InputPort - require_projection_in_composition: true - execute_until_finished: true - shadow_inputs: null - has_initializers: false - internal_only: false - max_executions_before_finished: 1000 - variable: - - 0 - default_input: null - projections: null - combine: null - exponent: null - weight: null - shape: - - 1 - type: int64 - functions: - B_Logistic_Function_0: - metadata: - type: Logistic - execute_until_finished: true - has_initializers: false - output_type: FunctionOutputType.NP_2D_ARRAY - enable_output_type_conversion: true - changes_shape: false - max_executions_before_finished: 1000 - variable: - - - 0 - bounds: - - 0 - - 1 - function: logistic - args: - x_0: 0 - scale: 1.0 - bias: 0.0 - offset: 0.0 - gain: 1.0 - variable0: B_InputPort_0 - output_ports: - B_RESULT: - metadata: - type: OutputPort - require_projection_in_composition: true - execute_until_finished: true - has_initializers: false - max_executions_before_finished: 1000 - variable: - - 0.5 - projections: null - value: B_Logistic_Function_0 - shape: - - 1 - type: float64 - edges: - MappingProjection_from_A_RESULT__to_B_InputPort_0_: - sender: A - receiver: B - sender_port: A_RESULT - receiver_port: B_InputPort_0 - metadata: - type: MappingProjection - execute_until_finished: true - has_initializers: false - max_executions_before_finished: 1000 - exponent: null - weight: null - functions: - LinearMatrix_Function_0: - metadata: - type: LinearMatrix - output_type: FunctionOutputType.DEFAULT - enable_output_type_conversion: false - execute_until_finished: true - has_initializers: false - changes_shape: false - max_executions_before_finished: 1000 - A: - - 2.0 - bounds: null - function: onnx::MatMul - args: - B: - - - 1.0 - parameters: - weight: 1 - conditions: - node_specific: - A: - type: EveryNPasses - kwargs: - n: 1 - time_scale: TimeScale.ENVIRONMENT_STATE_UPDATE - B: - type: EveryNCalls - kwargs: - dependency: A - n: 2 - termination: - environment_sequence: - type: AfterNEnvironmentStateUpdates - kwargs: - n: 1 - time_scale: TimeScale.ENVIRONMENT_SEQUENCE - environment_state_update: - type: All - kwargs: - args: - - type: Not - kwargs: - condition: - type: BeforeNCalls - kwargs: - dependency: B - n: 5 - time_scale: TimeScale.ENVIRONMENT_STATE_UPDATE From 04ded0f81e6ecc02daf571ee7997e177e31a15a4 Mon Sep 17 00:00:00 2001 From: Katherine Mantel Date: Sat, 5 Nov 2022 15:36:14 -0400 Subject: [PATCH 045/127] Composition termination conditions QOL (#2522) * Scheduler, Composition: fix loss of termination conditions when new Schedulers are created after adding objects to a Composition, only user-set node Conditions were saved. this also saves user-set termination Conditions * Composition: allow specifying termination conditions in constructor Co-authored-by: jdcpni --- psyneulink/core/compositions/composition.py | 8 ++++++- psyneulink/core/scheduling/scheduler.py | 8 +++++++ tests/scheduling/test_scheduler.py | 25 +++++++++++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/psyneulink/core/compositions/composition.py b/psyneulink/core/compositions/composition.py index ec87a10a58a..724e3f2f403 100644 --- a/psyneulink/core/compositions/composition.py +++ b/psyneulink/core/compositions/composition.py @@ -3761,6 +3761,7 @@ def __init__( show_graph_attributes=None, name=None, prefs=None, + termination_processing=None, **param_defaults ): @@ -3891,6 +3892,9 @@ def __init__( show_graph_attributes = show_graph_attributes or {} self._show_graph = ShowGraph(self, **show_graph_attributes) + if termination_processing is not None: + self.termination_processing = termination_processing + @property def graph_processing(self): """ @@ -3915,10 +3919,12 @@ def scheduler(self): old_scheduler = self._scheduler if old_scheduler is not None: orig_conds = old_scheduler._user_specified_conds + orig_term_conds = old_scheduler._user_specified_termination_conds else: orig_conds = None + orig_term_conds = None - self._scheduler = Scheduler(composition=self, conditions=orig_conds) + self._scheduler = Scheduler(composition=self, conditions=orig_conds, termination_conds=orig_term_conds) self.needs_update_scheduler = False return self._scheduler diff --git a/psyneulink/core/scheduling/scheduler.py b/psyneulink/core/scheduling/scheduler.py index 3db80e0551a..3eb3c4c272f 100644 --- a/psyneulink/core/scheduling/scheduler.py +++ b/psyneulink/core/scheduling/scheduler.py @@ -53,6 +53,7 @@ def __init__( # TODO: consider integrating something like this into graph-scheduler? self._user_specified_conds = copy.copy(conditions) if conditions is not None else {} + self._user_specified_termination_conds = copy.copy(termination_conds) if termination_conds is not None else {} super().__init__( graph=graph, @@ -118,6 +119,13 @@ def _add_condition_set(self, conditions): } super().add_condition_set(conditions) + @graph_scheduler.Scheduler.termination_conds.setter + def termination_conds(self, termination_conds): + if termination_conds is not None: + self._user_specified_termination_conds.update(termination_conds) + + graph_scheduler.Scheduler.termination_conds.fset(self, termination_conds) + @handle_external_context(fallback_default=True) def run( self, diff --git a/tests/scheduling/test_scheduler.py b/tests/scheduling/test_scheduler.py index 1cdbdc8462f..0c09eb33590 100644 --- a/tests/scheduling/test_scheduler.py +++ b/tests/scheduling/test_scheduler.py @@ -1209,6 +1209,31 @@ def test_partial_override_composition(self): # two executions of B assert output == [.75] + def test_termination_conditions_after_recreating_scheduler(self): + comp = Composition() + A = TransferMechanism() + comp.scheduler.termination_conds = {TimeScale.TRIAL: AfterNCalls(A, 3)} + B = TransferMechanism() + for m in [A, B]: + comp.add_node(m) + + comp.run(inputs={A: 1, B: 1}) + + expected_output = [{A, B}, {A, B}, {A, B}] + assert comp.scheduler.execution_list[comp.default_execution_id] == expected_output + + def test_termination_conditions_in_composition_constructor(self): + A = TransferMechanism() + comp = Composition(termination_processing={TimeScale.TRIAL: AfterNCalls(A, 3)}) + B = TransferMechanism() + for m in [A, B]: + comp.add_node(m) + + comp.run(inputs={A: 1, B: 1}) + + expected_output = [{A, B}, {A, B}, {A, B}] + assert comp.scheduler.execution_list[comp.default_execution_id] == expected_output + def _get_vertex_feedback_type(graph, sender_port, receiver_mech): # there is only one projection per pair From 259ccef19a15e0ccefb2ffa6602cc9fff8f086c6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Nov 2022 23:58:37 +0000 Subject: [PATCH 046/127] requirements: update matplotlib requirement from <3.6.2 to <3.6.3 (#2525) --- requirements.txt | 2 +- tutorial_requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index c0f11406a12..8900f425577 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ graphviz<0.21.0 grpcio<1.43.0 grpcio-tools<1.43.0 llvmlite<0.40 -matplotlib<3.6.2 +matplotlib<3.6.3 modeci_mdf<0.5, >=0.3.4; (platform_machine == 'AMD64' or platform_machine == 'x86_64') and platform_python_implementation == 'CPython' and implementation_name == 'cpython' networkx<2.9 numpy<1.21.7, >=1.17.0 diff --git a/tutorial_requirements.txt b/tutorial_requirements.txt index 738edadb115..ebda6d54f3a 100644 --- a/tutorial_requirements.txt +++ b/tutorial_requirements.txt @@ -1,3 +1,3 @@ graphviz<0.21.0 jupyter<=1.0.0 -matplotlib<3.6.2 +matplotlib<3.6.3 From e74105590c05da075ed51ffdb81e7686119a7bc8 Mon Sep 17 00:00:00 2001 From: Katherine Mantel Date: Tue, 8 Nov 2022 18:22:42 -0500 Subject: [PATCH 047/127] maint: ignore mdf test autogenerated files (#2527) --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 84bfbe22d2a..4993664944c 100644 --- a/.gitignore +++ b/.gitignore @@ -196,4 +196,5 @@ tests/*.pdf # mypy cache .mypy_cache -/tests/json/*.json +/tests/mdf/*.json +/tests/mdf/*.yml From 786cdd9e6b0bc905c1958f6417aa4ef9a2cbccd7 Mon Sep 17 00:00:00 2001 From: jdcpni Date: Tue, 8 Nov 2022 20:58:43 -0500 Subject: [PATCH 048/127] Feat/autodiff various (#2524) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add autodiff save/load functionality * - * Update autodiffcomposition.py * Update test_autodiffcomposition.py * Merge branch 'devel' of https://github.com/PrincetonUniversity/PsyNeuLink into devel * • Autodiff: - add save and load methods (from Samyak) - test_autodiffcomposition.py: add test_autodiff_saveload, but commented out for now, as it may be causing hanging on PR * • Autodiff: - add save and load methods (from Samyak) - test_autodiffcomposition.py: add test_autodiff_saveload, but commented out for now, as it may be causing hanging on PR * - * - * • pytorchcomponents.py: - pytorch_function_creator: add SoftMax • transferfunctions.py: - disable changes to ReLU.derivative for now * • utilities.py: - iscompatible: attempt to replace try and except, commented out for now * - * - * • autodiffcomposition.py: - save and load: augment file and directory handling - exclude processing of any ModulatoryProjections * - * - * - * • autodiffcomposition.py save(): add projection.matrix.base = matrix (fixes test_autodiff_saveload) * - * • autodiffcomposition.py: - save: return path • test_autodiffcomposition.py: - test_autodiff_saveload: modify to use current working directory rather than tmp * • autodiffcomposition.py: - save() and load(): ignore CIM, learning, and other modulation-related projections * • autodiffcomposition.py: - load(): change test for path (failing on Windows) from PosixPath to Path * • autodiffcomposition.py: - add _runtime_learning_rate attribute - _build_pytorch_representation(): use _runtime_learning_rate attribute for optimizer if provided in call to learn else use learning_rate specified at construction • compositionrunner.py: - assign learning_rate to _runtime_learning_rate attribute if specified in call to learn * - * [skip ci] * [skip ci] * [skip ci] • autodiffcomposition.py: load(): add testing for match of matrix shape * [skip ci] • N-back: - reset em after each run - save and load weights - torch epochs = batch size (number training stimuli) * num_epochs * [skip ci] * [skip ci] * Feat/add pathway default matrix (#2518) * • compositioninterfacemechanism.py: - _get_source_node_for_input_CIM: restore (modeled on _get_source_of_modulation_for_parameter_CIM) but NEEDS TESTS - _get_source_of_modulation_for_parameter_CIM: clean up comments, NEEDS TESTS * - * - * - * - * - * - * • Nback - EM uses ContentAddressableMemory (instead of DictionaryMemory) - Implements FFN for comparison of current and retrieved stimulus and context • Project: replace all instances of "RETREIVE" with "RETRIEVE" * • objectivefunctions.py - add cosine_similarity (needs compiled version) * • Project: make COSINE_SIMILARITY a synonym of COSINE • nback_CAM_FFN: - refactor to implement FFN and task input - assign termination condition for execution that is dependent on control - ContentAddressableMemory: selection_function=SoftMax(output=MAX_INDICATOR, gain=SOFT_MAX_TEMP) • DriftOnASphereIntegrator: - add dimension as dependency for initializer parameter * - * - * - * - * - * - * - * - * - * - * - * - * - * - * • test_integrator.py: Added identicalness test for DriftOnASphereIntegrator agains nback-paper implementation. * - * - * Parameters: allow _validate_ methods to reference other parameters (#2512) * • Scripts: - Updated N-back to use objective_mechanism, with commented out code for version that doesn't use it once bug is fixed - Deleted N-back_WITH_OBJECTIVE_MECH.py * • Scripts: - Updated N-back to use objective_mechanism, with commented out code for version that doesn't use it once bug is fixed - Deleted N-back_WITH_OBJECTIVE_MECH.py * • N-back.py: - added stimulus generation per nback-paper protocol * - N-back.py tstep(s) -> trial(s) * - * - * • N-back.py - comp -> nback_model - implement stim_set() method * - * • N-back.py: - added training set generation * - * - * • N-back.py - modularized script * - * - * - * - * • showgraph.py: - _assign_processing_components(): fix bug in which nested graphs not highlighted in animation. * • showgraph.py * composition.py - add further description of animation, including note that animation of nested Compostions is limited. * • showgraph.py * composition.py - add animation to N-back doc * • autodiffcomposition.py - __init__(): move pathways arg to beginning, to capture positional assignment (i.e. w/o kw) * - * • N-back.py - ffn: implement as autodiff; still needs small random initial weight assignment * • pathway.py - implement default_projection attribute * • pathway.py - implement default_projection attribute * • utilities.py: random_matrxi: refactored to allow negative values and use keyword ZERO_CENTER * • projection.py RandomMatrix: added class that can be used to pass a function as matrix spec * • utilities.py - RandomMatrix moved here from projection.py • function.py - get_matrix(): added support for RandomMatrix spec * • port.py - _parse_port_spec(): added support for RandomMatrix * • port.py - _parse_port_spec(): added support for RandomMatrix * • utilities.py - is_matrix(): modified to support random_matrix and RandomMatrix * • composition.py - add_linear_processing_pathway: add support for default_matrix argument (replaces default for MappingProjection for any otherwise unspecified projections) though still not used. * - * - RandomMatrix: moved from Utilities to Function * - * [skip ci] * [skip ci] * [skip ci] • N-back.py - clean up script * [skip ci] • N-back.py - further script clean-up * [skip ci] * [skip ci] * [skip ci] * [skip ci] • BeukersNBackModel.rst: - Overview written - Needs other sections completed * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] • N-back.py: - replace functions of TransferMechanisms with ReLU - replace function of Decision Mechanisms with SoftMax - more doc cleanup * [skip ci] • N-back.py: - replace functions of TransferMechanisms with ReLU - replace function of Decision Mechanisms with SoftMax - more doc cleanup * [skip ci] * - * - * [skip ci] * [skip ci] • composition.py: implement default_projection_matrix in add_XXX_pathway() methods * [skip ci] • composition.py: implement default_projection_matrix in add_XXX_pathway() methods * [skip ci] • test_composition.py: - add test_pathway_tuple_specs() * - * - * [skip ci] * [skip ci] * [skip ci] * - Co-authored-by: jdcpni Co-authored-by: Katherine Mantel * Feat/add pathway default matrix (#2519) * • compositioninterfacemechanism.py: - _get_source_node_for_input_CIM: restore (modeled on _get_source_of_modulation_for_parameter_CIM) but NEEDS TESTS - _get_source_of_modulation_for_parameter_CIM: clean up comments, NEEDS TESTS * - * - * - * - * - * - * • Nback - EM uses ContentAddressableMemory (instead of DictionaryMemory) - Implements FFN for comparison of current and retrieved stimulus and context • Project: replace all instances of "RETREIVE" with "RETRIEVE" * • objectivefunctions.py - add cosine_similarity (needs compiled version) * • Project: make COSINE_SIMILARITY a synonym of COSINE • nback_CAM_FFN: - refactor to implement FFN and task input - assign termination condition for execution that is dependent on control - ContentAddressableMemory: selection_function=SoftMax(output=MAX_INDICATOR, gain=SOFT_MAX_TEMP) • DriftOnASphereIntegrator: - add dimension as dependency for initializer parameter * - * - * - * - * - * - * - * - * - * - * - * - * - * - * • test_integrator.py: Added identicalness test for DriftOnASphereIntegrator agains nback-paper implementation. * - * - * Parameters: allow _validate_ methods to reference other parameters (#2512) * • Scripts: - Updated N-back to use objective_mechanism, with commented out code for version that doesn't use it once bug is fixed - Deleted N-back_WITH_OBJECTIVE_MECH.py * • Scripts: - Updated N-back to use objective_mechanism, with commented out code for version that doesn't use it once bug is fixed - Deleted N-back_WITH_OBJECTIVE_MECH.py * • N-back.py: - added stimulus generation per nback-paper protocol * - N-back.py tstep(s) -> trial(s) * - * - * • N-back.py - comp -> nback_model - implement stim_set() method * - * • N-back.py: - added training set generation * - * - * • N-back.py - modularized script * - * - * - * - * • showgraph.py: - _assign_processing_components(): fix bug in which nested graphs not highlighted in animation. * • showgraph.py * composition.py - add further description of animation, including note that animation of nested Compostions is limited. * • showgraph.py * composition.py - add animation to N-back doc * • autodiffcomposition.py - __init__(): move pathways arg to beginning, to capture positional assignment (i.e. w/o kw) * - * • N-back.py - ffn: implement as autodiff; still needs small random initial weight assignment * • pathway.py - implement default_projection attribute * • pathway.py - implement default_projection attribute * • utilities.py: random_matrxi: refactored to allow negative values and use keyword ZERO_CENTER * • projection.py RandomMatrix: added class that can be used to pass a function as matrix spec * • utilities.py - RandomMatrix moved here from projection.py • function.py - get_matrix(): added support for RandomMatrix spec * • port.py - _parse_port_spec(): added support for RandomMatrix * • port.py - _parse_port_spec(): added support for RandomMatrix * • utilities.py - is_matrix(): modified to support random_matrix and RandomMatrix * • composition.py - add_linear_processing_pathway: add support for default_matrix argument (replaces default for MappingProjection for any otherwise unspecified projections) though still not used. * - * - RandomMatrix: moved from Utilities to Function * - * [skip ci] * [skip ci] * [skip ci] • N-back.py - clean up script * [skip ci] • N-back.py - further script clean-up * [skip ci] * [skip ci] * [skip ci] * [skip ci] • BeukersNBackModel.rst: - Overview written - Needs other sections completed * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] • N-back.py: - replace functions of TransferMechanisms with ReLU - replace function of Decision Mechanisms with SoftMax - more doc cleanup * [skip ci] • N-back.py: - replace functions of TransferMechanisms with ReLU - replace function of Decision Mechanisms with SoftMax - more doc cleanup * [skip ci] * - * - * [skip ci] * [skip ci] • composition.py: implement default_projection_matrix in add_XXX_pathway() methods * [skip ci] • composition.py: implement default_projection_matrix in add_XXX_pathway() methods * [skip ci] • test_composition.py: - add test_pathway_tuple_specs() * - * - * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] • composition.py: - add_linear_processing_pathway: fixed bug when Reinforcement or TDLearning are specified • test_composition.py: - test_pathway_tuple_specs: add tests for Reinforcement and TDLearning * • composition.py: - add_linear_processing_pathway: fixed bug when Reinforcement or TDLearning are specified • test_composition.py: - test_pathway_tuple_specs: add tests for Reinforcement and TDLearning Co-authored-by: jdcpni Co-authored-by: Katherine Mantel * autodiff: Use most recent context while save/load * tests/autodiff: Use portable path join * autodiff: Add assertions for save/load * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] * • autodiffcomposition, test_autodiff_saveload: - merged from feat/autodiff_save * - * - * - * • autodiffcomposition.py - fix path assignment bug * - Co-authored-by: SamKG Co-authored-by: Katherine Mantel --- .../{ => N-back}/N-back MODULARIZED.py | 2 +- .../N-back/N-back.py | 537 ++++++++++++++++++ .../N-back_WITH_OBJECTIVE_MECH.py | 0 .../N-back/Nback Notebook.ipynb | 188 ++++++ .../{N-back.py => N-back/Nback.py} | 146 ++--- .../N-back/SphericalDrift Tests.py | 34 ++ ... MEMORY (fnn)_matrix_wts_20stim_2500ep.pnl | Bin 0 -> 92975 bytes .../N-back/__init__.py | 0 .../N-back/ffn.wts_nep_1_lr_01.pnl | Bin 0 -> 28527 bytes .../N-back/ffn.wts_nep_6250_lr_01.pnl | Bin 0 -> 28463 bytes .../N-back/nback.results_nep_1_lr_01.pnl.npy | Bin 0 -> 1664 bytes .../nback.results_nep_6250_lr_01.pnl.npy | Bin 0 -> 1664 bytes .../WORKING MEMORY (fnn)_matrix_wts.pnl | Bin 0 -> 92719 bytes .../Models (Under Development)/ffn.wts.pnl | Bin 0 -> 28527 bytes .../Models (Under Development)/ffn.wts_01.pnl | Bin 0 -> 28527 bytes .../ffn.wts_nep_1_lr_01.pnl | Bin 0 -> 28527 bytes autodiff_composition_matrix_wts.pnl | Bin 0 -> 1071 bytes .../nonstateful/learningfunctions.py | 1 - .../nonstateful/transferfunctions.py | 4 +- .../functions/stateful/memoryfunctions.py | 2 +- .../modulatory/learning/learningmechanism.py | 5 +- psyneulink/core/compositions/composition.py | 11 +- psyneulink/core/globals/utilities.py | 10 + .../compositions/autodiffcomposition.py | 159 +++++- .../library/compositions/compositionrunner.py | 6 +- .../library/compositions/pytorchcomponents.py | 9 +- .../compositions/pytorchmodelcreator.py | 8 +- .../autodiff_composition_matrix_wts.pnl | Bin 0 -> 1071 bytes tests/composition/test_autodiffcomposition.py | 57 +- 29 files changed, 1065 insertions(+), 114 deletions(-) rename Scripts/Models (Under Development)/{ => N-back}/N-back MODULARIZED.py (99%) create mode 100644 Scripts/Models (Under Development)/N-back/N-back.py rename Scripts/Models (Under Development)/{ => N-back}/N-back_WITH_OBJECTIVE_MECH.py (100%) create mode 100644 Scripts/Models (Under Development)/N-back/Nback Notebook.ipynb rename Scripts/Models (Under Development)/{N-back.py => N-back/Nback.py} (84%) create mode 100644 Scripts/Models (Under Development)/N-back/SphericalDrift Tests.py create mode 100644 Scripts/Models (Under Development)/N-back/WORKING MEMORY (fnn)_matrix_wts_20stim_2500ep.pnl create mode 100644 Scripts/Models (Under Development)/N-back/__init__.py create mode 100644 Scripts/Models (Under Development)/N-back/ffn.wts_nep_1_lr_01.pnl create mode 100644 Scripts/Models (Under Development)/N-back/ffn.wts_nep_6250_lr_01.pnl create mode 100644 Scripts/Models (Under Development)/N-back/nback.results_nep_1_lr_01.pnl.npy create mode 100644 Scripts/Models (Under Development)/N-back/nback.results_nep_6250_lr_01.pnl.npy create mode 100644 Scripts/Models (Under Development)/WORKING MEMORY (fnn)_matrix_wts.pnl create mode 100644 Scripts/Models (Under Development)/ffn.wts.pnl create mode 100644 Scripts/Models (Under Development)/ffn.wts_01.pnl create mode 100644 Scripts/Models (Under Development)/ffn.wts_nep_1_lr_01.pnl create mode 100644 autodiff_composition_matrix_wts.pnl create mode 100644 tests/composition/autodiff_composition_matrix_wts.pnl diff --git a/Scripts/Models (Under Development)/N-back MODULARIZED.py b/Scripts/Models (Under Development)/N-back/N-back MODULARIZED.py similarity index 99% rename from Scripts/Models (Under Development)/N-back MODULARIZED.py rename to Scripts/Models (Under Development)/N-back/N-back MODULARIZED.py index 87ee56c5221..74dcf24e51d 100644 --- a/Scripts/Models (Under Development)/N-back MODULARIZED.py +++ b/Scripts/Models (Under Development)/N-back/N-back MODULARIZED.py @@ -151,7 +151,7 @@ def construct_model(num_tasks, stim_size, context_size, hidden_size, display=Fal hidden, decision], name="WORKING MEMORY (fnn)") comp = Composition(nodes=[stim, context, task, em, ffn, control], - name="N-Back Model") + name="N-back Model") comp.add_projection(MappingProjection(), stim, input_current_stim) comp.add_projection(MappingProjection(), context, input_current_context) comp.add_projection(MappingProjection(), task, input_task) diff --git a/Scripts/Models (Under Development)/N-back/N-back.py b/Scripts/Models (Under Development)/N-back/N-back.py new file mode 100644 index 00000000000..6504493494a --- /dev/null +++ b/Scripts/Models (Under Development)/N-back/N-back.py @@ -0,0 +1,537 @@ +""" +This implements a model of the `N-back task `_ +described in `Beukers et al. (2022) `_. The model uses a simple implementation of episodic +(content-addressable) memory to store previous stimuli and the temporal context in which they occured, +and a feedforward neural network to evaluate whether the current stimulus is a match to the n'th preceding stimulus +(n-back level). This model is an example of proposed interactions between working memory (e.g., in neocortex) and +episodic memory e.g., in hippocampus and/or cerebellum) in the performance of tasks demanding of sequential processing +and control, and along the lines of models emerging machine learning that augment the use of recurrent neural networks +(e.g., long short-term memory mechanisms; LSTMs) for active memory and control with an external memory capable of +rapid storage and content-based retrieval, such as the Neural Turing Machine (NTN; `Graves et al., 2016 +`_), Episodic Planning Networks (EPN; `Ritter et al., 2020 +`_), and Emergent Symbols through Binding Networks (ESBN; `Webb et al., 2021 +`_). + +There are three primary methods in the script: + +* construct_model(args): + takes as arguments parameters used to construct the model; for convenience, defaults are defined below, + (under "Construction parameters") + +* train_network(args) + takes as arguments the feedforward neural network Composition (FFN_COMPOSITION) and number of epochs to train. + Note: learning_rate is set at construction (can specify using LEARNING_RATE under "Training parameters" below). + +* run_model() + takes the context drift rate to be applied on each trial and the number of trials to execute as args, as well as + reporting and animation specifications (see "Execution parameters" below). + +See "Settings for running the script" to specify whether the model is trained and/or executed when the script is run, +and whether a graphic display of the network is generated when it is constructed. + +TODO: + - from Andre + - network architecture; in particular, size of hidden layer and projection patterns to and from it + - the stim+context input vector (length 90) projects to a hidden layer (length 80); + - the task input vector (length 2) projects to a different hidden layer (length 80); + - those two hidden layers project (over fixed, nonlearnable, one-one-projections?) to a third hidden layer (length 80) that simply sums them; + - the third hidden layer projects to the length 2 output layer; + - a softmax is taken over the output layer to determine the response. + - fix: were biases trained? + - training: + - learning rate: 0.001; epoch: 1 trial per epoch of training + - fix: state_dict with weights (still needed) + - get empirical stimulus sequences (still needed) + - put N-back script (with pointer to latest version on PNL) in nback-paper repo + - fix: get rid of objective_mechanism (see "VERSION *WITHOUT* ObjectiveMechanism" under control(...) + - fix: warnings on run + - complete documentation in BeukersNbackModel.rst + - validate against nback-paper results + - after validation: + - try with STIM_SIZE = NUM_STIMS rather than 20 (as in nback-paper) + - refactor generate_stim_sequence() to use actual empirical stimulus sequences + - replace get_input_sequence and get_training_inputs with generators passed to nback_model.run() and ffn.learn + +""" + +from graph_scheduler import * + +from psyneulink import * +import numpy as np + +# Settings for running script: +TRAIN = True +RUN = True +DISPLAY_MODEL = False # show visual graphic of model + +# PARAMETERS ------------------------------------------------------------------------------------------------------- + +# Fixed (structural) parameters: +MAX_NBACK_LEVELS = 3 +NUM_STIM = 8 # number of different stimuli in stimulus set - QUESTION: WHY ISN"T THIS EQUAL TO STIM_SIZE OR VICE VERSA? +FFN_TRANSFER_FUNCTION = ReLU + +# Constructor parameters: (values are from nback-paper) +STIM_SIZE=8 # length of stimulus vector +CONTEXT_SIZE=25 # length of context vector +HIDDEN_SIZE=STIM_SIZE*4 # dimension of hidden units in ff +NBACK_LEVELS = [2,3] # Currently restricted to these +NUM_NBACK_LEVELS = len(NBACK_LEVELS) +CONTEXT_DRIFT_NOISE=0.0 # noise used by DriftOnASphereIntegrator (function of Context mech) +RANDOM_WEIGHTS_INITIALIZATION=RandomMatrix(center=0.0, range=0.1) # Matrix spec used to initialize all Projections +RETRIEVAL_SOFTMAX_TEMP=1/8 # express as gain # precision of retrieval process +RETRIEVAL_HAZARD_RATE=0.04 # rate of re=sampling of em following non-match determination in a pass through ffn +RETRIEVAL_STIM_WEIGHT=.05 # weighting of stimulus field in retrieval from em +RETRIEVAL_CONTEXT_WEIGHT = 1-RETRIEVAL_STIM_WEIGHT # weighting of context field in retrieval from em +DECISION_SOFTMAX_TEMP=1 + +# Training parameters: +NUM_EPOCHS= 6250 # nback-paper: 400,000 @ one trial per epoch = 6,250 @ 64 trials per epoch +LEARNING_RATE=0.01 # nback-paper: .001 + +# Execution parameters: +CONTEXT_DRIFT_RATE=.1 # drift rate used for DriftOnASphereIntegrator (function of Context mech) on each trial +NUM_TRIALS = 48 # number of stimuli presented in a trial sequence +REPORT_OUTPUT = ReportOutput.OFF # Sets console output during run +REPORT_PROGRESS = ReportProgress.OFF # Sets console progress bar during run +REPORT_LEARNING = ReportLearning.OFF # Sets console progress bar during training +ANIMATE = False # {UNIT:EXECUTION_SET} # Specifies whether to generate animation of execution + +# Names of Compositions and Mechanisms: +NBACK_MODEL = "N-back Model" +FFN_COMPOSITION = "WORKING MEMORY (fnn)" +FFN_STIMULUS_INPUT = "CURRENT STIMULUS" +FFN_CONTEXT_INPUT = "CURRENT CONTEXT" +FFN_STIMULUS_RETRIEVED = "RETRIEVED STIMULUS" +FFN_CONTEXT_RETRIEVED = "RETRIEVED CONTEXT" +FFN_TASK = "CURRENT TASK" +FFN_HIDDEN = "HIDDEN LAYER" +FFN_OUTPUT = "DECISION LAYER" +MODEL_STIMULUS_INPUT ='STIM' +MODEL_CONTEXT_INPUT = 'CONTEXT' +MODEL_TASK_INPUT = "TASK" +EM = "EPISODIC MEMORY (dict)" +CONTROLLER = "READ/WRITE CONTROLLER" + +# ======================================== MODEL CONSTRUCTION ========================================================= + +def construct_model(stim_size = STIM_SIZE, + context_size = CONTEXT_SIZE, + hidden_size = HIDDEN_SIZE, + num_nback_levels = NUM_NBACK_LEVELS, + context_drift_noise = CONTEXT_DRIFT_NOISE, + retrievel_softmax_temp = RETRIEVAL_SOFTMAX_TEMP, + retrieval_hazard_rate = RETRIEVAL_HAZARD_RATE, + retrieval_stimulus_weight = RETRIEVAL_STIM_WEIGHT, + retrieval_context_weight = RETRIEVAL_CONTEXT_WEIGHT, + decision_softmax_temp = DECISION_SOFTMAX_TEMP): + """Construct nback_model""" + + print(f"constructing '{FFN_COMPOSITION}'...") + + # FEED FORWARD NETWORK ----------------------------------------- + + # inputs: encoding of current stimulus and context, retrieved stimulus and retrieved context, + # output: decision: match [1,0] or non-match [0,1] + # Must be trained to detect match for specified task (1-back, 2-back, etc.) + input_current_stim = TransferMechanism(name=FFN_STIMULUS_INPUT, + size=stim_size, + function=FFN_TRANSFER_FUNCTION) + input_current_context = TransferMechanism(name=FFN_CONTEXT_INPUT, + size=context_size, + function=FFN_TRANSFER_FUNCTION) + input_retrieved_stim = TransferMechanism(name=FFN_STIMULUS_RETRIEVED, + size=stim_size, + function=FFN_TRANSFER_FUNCTION) + input_retrieved_context = TransferMechanism(name=FFN_CONTEXT_RETRIEVED, + size=context_size, + function=FFN_TRANSFER_FUNCTION) + input_task = TransferMechanism(name=FFN_TASK, + size=num_nback_levels, + function=FFN_TRANSFER_FUNCTION) + hidden = TransferMechanism(name=FFN_HIDDEN, + size=hidden_size, + function=FFN_TRANSFER_FUNCTION) + decision = ProcessingMechanism(name=FFN_OUTPUT, + size=2, function=SoftMax(output=MAX_INDICATOR, + gain=decision_softmax_temp)) + ffn = AutodiffComposition(([{input_current_stim, + input_current_context, + input_retrieved_stim, + input_retrieved_context, + input_task}, + hidden, decision], + RANDOM_WEIGHTS_INITIALIZATION, + ), + name=FFN_COMPOSITION, + learning_rate=LEARNING_RATE + ) + + # FULL MODEL (Outer Composition, including input, EM and control Mechanisms) ------------------------ + + print(f"'constructing {NBACK_MODEL}'...") + + # Stimulus Encoding: takes STIM_SIZE vector as input + stim = TransferMechanism(name=MODEL_STIMULUS_INPUT, size=stim_size) + + # Context Encoding: takes scalar as drift step for current trial + context = ProcessingMechanism(name=MODEL_CONTEXT_INPUT, + function=DriftOnASphereIntegrator( + initializer=np.random.random(context_size-1), + noise=context_drift_noise, + dimension=context_size)) + + # Task: task one-hot indicating n-back (1, 2, 3 etc.) - must correspond to what ffn has been trained to do + task = ProcessingMechanism(name=MODEL_TASK_INPUT, + size=num_nback_levels) + + # Episodic Memory: + # - entries: stimulus (field[0]) and context (field[1]); randomly initialized + # - uses Softmax to retrieve best matching input, subject to weighting of stimulus and context by STIM_WEIGHT + em = EpisodicMemoryMechanism(name=EM, + input_ports=[{NAME:"STIMULUS_FIELD", + SIZE:stim_size}, + {NAME:"CONTEXT_FIELD", + SIZE:context_size}], + function=ContentAddressableMemory( + initializer=[[[0]*stim_size, [0]*context_size]], + distance_field_weights=[retrieval_stimulus_weight, + retrieval_context_weight], + # equidistant_entries_select=NEWEST, + selection_function=SoftMax(output=MAX_INDICATOR, + gain=retrievel_softmax_temp)), + ) + + # Control Mechanism + # Ensures current stimulus and context are only encoded in EM once (at beginning of trial) + # by controlling the storage_prob parameter of em: + # - if outcome of decision signifies a match or hazard rate is realized: + # - set EM[store_prob]=1 (as prep encoding stimulus in EM on next trial) + # - this also serves to terminate trial (see nback_model.termination_processing condition) + # - if outcome of decision signifies a non-match + # - set EM[store_prob]=0 (as prep for another retrieval from EM without storage) + # - continue trial + control = ControlMechanism(name=CONTROLLER, + default_variable=[[1]], # Ensure EM[store_prob]=1 at beginning of first trial + # --------- + # VERSION *WITH* ObjectiveMechanism: + objective_mechanism=ObjectiveMechanism(name="OBJECTIVE MECHANISM", + monitor=decision, + # Outcome=1 if match, else 0 + function=lambda x: int(x[0][1]>x[0][0])), + # Set ControlSignal for EM[store_prob] + function=lambda outcome: int(bool(outcome) + or (np.random.random() > retrieval_hazard_rate)), + # --------- + # # VERSION *WITHOUT* ObjectiveMechanism: + # monitor_for_control=decision, + # # Set Evaluate outcome and set ControlSignal for EM[store_prob] + # # - outcome is received from decision as one hot in the form: [[match, no-match]] + # function=lambda outcome: int(int(outcome[0][1]>outcome[0][0]) + # or (np.random.random() > retrieval_hazard_rate)), + # --------- + control=(STORAGE_PROB, em)) + + nback_model = Composition(name=NBACK_MODEL, + nodes=[stim, context, task, ffn, em, control], + # Terminate trial if value of control is still 1 after first pass through execution + termination_processing={TimeScale.TRIAL: And(Condition(lambda: control.value), + AfterPass(0, TimeScale.TRIAL))}, + ) + # # Terminate trial if value of control is still 1 after first pass through execution + # # FIX: ALL OF THE FOLLOWING STOP AFTER ~ NUMBER OF TRIALS (?90+); SHOULD BE: NUM_TRIALS*NUM_NBACK_LEVELS + 1 + # nback_model.scheduler.add_condition(nback_model, And(Condition(lambda: control.value), AfterPass(0, TimeScale.TRIAL))) + # nback_model.scheduler.termination_conds = ({TimeScale.TRIAL: And(Condition(lambda: control.value), + # AfterPass(0, TimeScale.TRIAL))}) + # nback_model.scheduler.termination_conds.update({TimeScale.TRIAL: And(Condition(lambda: control.value), + # AfterPass(0, TimeScale.TRIAL))}) + nback_model.add_projection(MappingProjection(), stim, input_current_stim) + nback_model.add_projection(MappingProjection(), context, input_current_context) + nback_model.add_projection(MappingProjection(), task, input_task) + nback_model.add_projection(MappingProjection(), em.output_ports["RETRIEVED_STIMULUS_FIELD"], input_retrieved_stim) + nback_model.add_projection(MappingProjection(), em.output_ports["RETRIEVED_CONTEXT_FIELD"], input_retrieved_context) + nback_model.add_projection(MappingProjection(), stim, em.input_ports["STIMULUS_FIELD"]) + nback_model.add_projection(MappingProjection(), context, em.input_ports["CONTEXT_FIELD"]) + + if DISPLAY_MODEL: + nback_model.show_graph( + # show_cim=True, + # show_node_structure=ALL, + # show_dimensions=True + ) + + print(f'full model constructed') + return nback_model + +# ==========================================STIMULUS GENERATION ======================================================= +# Based on nback-paper + +def get_stim_set(num_stim=STIM_SIZE): + """Construct an array of stimuli for use an experiment""" + # For now, use one-hots + return np.eye(num_stim) + +def get_task_input(nback_level): + """Construct input to task Mechanism for a given nback_level, used by run_model() and train_network()""" + task_input = list(np.zeros_like(NBACK_LEVELS)) + task_input[nback_level-NBACK_LEVELS[0]] = 1 + return task_input + +def get_run_inputs(model, nback_level, context_drift_rate, num_trials): + """Construct set of stimulus inputs for run_model()""" + + def generate_stim_sequence(nback_level, trial_num, trial_type=0, num_stim=NUM_STIM, num_trials=NUM_TRIALS): + assert nback_level in {2,3} # At present, only 2- and 3-back levels are supported + + def gen_subseq_stim(): + A = np.random.randint(0,num_stim) + B = np.random.choice( + np.setdiff1d(np.arange(num_stim),[A]) + ) + C = np.random.choice( + np.setdiff1d(np.arange(num_stim),[A,B]) + ) + X = np.random.choice( + np.setdiff1d(np.arange(num_stim),[A,B]) + ) + return A,B,C,X + + def generate_match_no_foils_sequence(nback_level,trial_num): + # AXA (2-back) or ABXA (3-back) + seq = np.random.randint(0,num_stim,num_trials) + A,B,C,X = gen_subseq_stim() + # + if nback_level==2: + subseq = [A,X,A] + elif nback_level==3: + subseq = [A,B,X,A] + seq[trial_num-(nback_level+1):trial_num] = subseq + return seq[:trial_num] + + def generate_non_match_no_foils_sequence(nback_level,trial_num): + # AXB (2-back) or ABXC (3-back) + seq = np.random.randint(0,num_stim,num_trials) + A,B,C,X = gen_subseq_stim() + # + if nback_level==2: + subseq = [A,X,B] + elif nback_level==3: + subseq = [A,B,X,C] + seq[trial_num-(nback_level+1):trial_num] = subseq + return seq[:trial_num] + + def generate_match_with_foil_sequence(nback_level,trial_num): + # AAA (2-back) or AAXA (3-back) + seq = np.random.randint(0,num_stim,num_trials) + A,B,C,X = gen_subseq_stim() + # + if nback_level==2: + subseq = [A,A,A] + elif nback_level==3: + subseq = [A,A,X,A] + seq[trial_num-(nback_level+1):trial_num] = subseq + return seq[:trial_num] + + def generate_non_match_with_foil_sequence(nback_level,trial_num): + # XAA (2-back) or ABXB (3-back) + seq = np.random.randint(0,num_stim,num_trials) + A,B,C,X = gen_subseq_stim() + # + if nback_level==2: + subseq = [X,A,A] + elif nback_level==3: + subseq = [A,B,X,B] + seq[trial_num-(nback_level+1):trial_num] = subseq + return seq[:trial_num] + + trial_types = [generate_match_no_foils_sequence, + generate_match_with_foil_sequence, + generate_non_match_no_foils_sequence, + generate_non_match_with_foil_sequence] + stim_seq = trial_types[trial_type](nback_level,trial_num) + # ytarget = [1,1,0,0][trial_type] + # ctxt = spherical_drift(trial_num) + # return stim,ctxt,ytarget + return stim_seq + + # def stim_set_generation(nback_level, num_trials): + # stim_sequence = [] + # # for seq_int, trial in itertools.product(range(4),np.arange(5,trials)): # This generates all length sequences + # for trial_type, trial_num in itertools.product(range(4),[num_trials]): # This generates only longest seq ( + # # num_trials) + # return stim_sequence.append(generate_stim_sequence(nback_level, trial_num, trial_type=trial_type, trials=num_trials)) + + def get_input_sequence(nback_level, num_trials=NUM_TRIALS): + """Get sequence of inputs for a run""" + input_set = get_stim_set() + # Construct sequence of stimulus indices + trial_seq = generate_stim_sequence(nback_level, num_trials) + # Return list of corresponding stimulus input vectors + return [input_set[trial_seq[i]] for i in range(num_trials)] + + return {model.nodes[MODEL_STIMULUS_INPUT]: get_input_sequence(nback_level, num_trials), + model.nodes[MODEL_CONTEXT_INPUT]: [[context_drift_rate]]*num_trials, + model.nodes[MODEL_TASK_INPUT]: [get_task_input(nback_level)]*num_trials} + +def get_training_inputs(network, num_epochs, nback_levels): + """Construct set of training stimuli used by ffn.learn() in train_network() + Construct one example of each condition: + match: stim_current = stim_retrieved and context_current = context_retrieved + stim_lure: stim_current = stim_retrieved and context_current != context_retrieved + context_lure: stim_current != stim_retrieved and context_current == context_retrieved + non_lure: stim_current != stim_retrieved and context_current != context_retrieved + """ + assert is_iterable(nback_levels) and all([0", + "image/svg+xml": "\n\n\n\n\n\nN-back Model\n\nN-back Model\n\ncluster_WORKING MEMORY (fnn)\n\nWORKING MEMORY (fnn)\n\n\n\nTASK\n\nTASK\n\n\n\nCURRENT TASK\n\nCURRENT TASK\n\n\n\nTASK->CURRENT TASK\n\n\n\n\n\nCONTEXT\n\nCONTEXT\n\n\n\nCURRENT CONTEXT\n\nCURRENT CONTEXT\n\n\n\nCONTEXT->CURRENT CONTEXT\n\n\n\n\n\nEPISODIC MEMORY (dict)\n\nEPISODIC MEMORY (dict)\n\n\n\nCONTEXT->EPISODIC MEMORY (dict)\n\n\n\n\n\nSTIM\n\nSTIM\n\n\n\nCURRENT STIMULUS\n\nCURRENT STIMULUS\n\n\n\nSTIM->CURRENT STIMULUS\n\n\n\n\n\nSTIM->EPISODIC MEMORY (dict)\n\n\n\n\n\nHIDDEN LAYER\n\nHIDDEN LAYER\n\n\n\nCURRENT TASK->HIDDEN LAYER\n\n\n\n\n\nCURRENT STIMULUS->HIDDEN LAYER\n\n\n\n\n\nCURRENT CONTEXT->HIDDEN LAYER\n\n\n\n\n\nRETRIEVED CONTEXT\n\nRETRIEVED CONTEXT\n\n\n\nEPISODIC MEMORY (dict)->RETRIEVED CONTEXT\n\n\n\n\n\nRETRIEVED STIMULUS\n\nRETRIEVED STIMULUS\n\n\n\nEPISODIC MEMORY (dict)->RETRIEVED STIMULUS\n\n\n\n\n\nRETRIEVED CONTEXT->HIDDEN LAYER\n\n\n\n\n\nRETRIEVED STIMULUS->HIDDEN LAYER\n\n\n\n\n\nREAD/WRITE CONTROLLER\n\nREAD/WRITE CONTROLLER\n\n\n\nREAD/WRITE CONTROLLER->EPISODIC MEMORY (dict)\n\n\n\n\n\n\nOBJECTIVE MECHANISM\n\nOBJECTIVE MECHANISM\n\n\n\nOBJECTIVE MECHANISM->READ/WRITE CONTROLLER\n\n\n\n\n\nDECISION LAYER\n\nDECISION LAYER\n\n\n\nDECISION LAYER->OBJECTIVE MECHANISM\n\n\n\n\n\nHIDDEN LAYER->DECISION LAYER\n\n\n\n\n\n" + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "nback_model.show_graph(output_fmt='jupyter')" + ] + }, + { + "cell_type": "markdown", + "source": [ + "### Train the model:" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "ffn = nback_model.nodes['WORKING MEMORY (fnn)']\n", + "train_network(ffn, num_epochs=100)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Run the model:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "run_model(nback_model)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.7" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} \ No newline at end of file diff --git a/Scripts/Models (Under Development)/N-back.py b/Scripts/Models (Under Development)/N-back/Nback.py similarity index 84% rename from Scripts/Models (Under Development)/N-back.py rename to Scripts/Models (Under Development)/N-back/Nback.py index a7546baa6e9..abd8173c02a 100644 --- a/Scripts/Models (Under Development)/N-back.py +++ b/Scripts/Models (Under Development)/N-back/Nback.py @@ -35,19 +35,17 @@ - the stim+context input vector (length 90) projects to a hidden layer (length 80); - the task input vector (length 2) projects to a different hidden layer (length 80); - those two hidden layers project (over fixed, nonlearnable, one-one-projections?) to a third hidden layer (length 80) that simply sums them; - - the third hidden layer projections to the length 2 output layer; + - the third hidden layer projects to the length 2 output layer; - a softmax is taken over the output layer to determine the response. - - softmax temp on output/decision layer: 1 - - confirm that ReLUs all use 0 thresholds and unit slope + - fix: were biases trained? - training: - learning rate: 0.001; epoch: 1 trial per epoch of training - - state_dict with weights (still needed) + - fix: state_dict with weights (still needed) - get empirical stimulus sequences (still needed) - put N-back script (with pointer to latest version on PNL) in nback-paper repo - - get rid of objective_mechanism (see "VERSION *WITHOUT* ObjectiveMechanism" under control(...) (fix bug) - - make termination processing part of the Composition definition (fix bug) - - pass learning_rate as parameter to train_network() (add feature) - - fix warnings on run + - fix: get rid of objective_mechanism (see "VERSION *WITHOUT* ObjectiveMechanism" under control(...) + - fix: warnings on run + - complete documentation in BeukersNbackModel.rst - validate against nback-paper results - after validation: - try with STIM_SIZE = NUM_STIMS rather than 20 (as in nback-paper) @@ -62,8 +60,6 @@ import numpy as np # Settings for running script: -TRAIN = True -RUN = False DISPLAY_MODEL = False # show visual graphic of model # PARAMETERS ------------------------------------------------------------------------------------------------------- @@ -85,22 +81,22 @@ RETRIEVAL_HAZARD_RATE=0.04 # rate of re=sampling of em following non-match determination in a pass through ffn RETRIEVAL_STIM_WEIGHT=.05 # weighting of stimulus field in retrieval from em RETRIEVAL_CONTEXT_WEIGHT = 1-RETRIEVAL_STIM_WEIGHT # weighting of context field in retrieval from em -DECISION_SOFTMAX_TEMP=1/8 # express as gain # binarity of decision process +DECISION_SOFTMAX_TEMP=1 # Training parameters: -NUM_EPOCHS=10 # nback-paper: 400,000, one trial per epoch -LEARNING_RATE=0.1 # nback-paper: .001 +NUM_EPOCHS=3 # nback-paper: 400,000 @ one trial per epoch = 2,500 @ 160 trials per epoch +LEARNING_RATE=0.01 # nback-paper: .001 # Execution parameters: CONTEXT_DRIFT_RATE=.1 # drift rate used for DriftOnASphereIntegrator (function of Context mech) on each trial -NUM_TRIALS = 48 # number of stimuli presented in a trial sequence +NUM_TRIALS = 48 # number of stimuli presented in a trial sequence for a given nback_level during run REPORT_OUTPUT = ReportOutput.OFF # Sets console output during run -REPORT_PROGRESS = ReportProgress.ON # Sets console progress bar during run -REPORT_LEARNING = ReportLearning.ON # Sets console progress bar during training -ANIMATE = True # {UNIT:EXECUTION_SET} # Specifies whether to generate animation of execution +REPORT_PROGRESS = ReportProgress.OFF # Sets console progress bar during run +REPORT_LEARNING = ReportLearning.OFF # Sets console progress bar during training +ANIMATE = False # {UNIT:EXECUTION_SET} # Specifies whether to generate animation of execution # Names of Compositions and Mechanisms: -NBACK_MODEL = "N-Back Model" +NBACK_MODEL = "N-back Model" FFN_COMPOSITION = "WORKING MEMORY (fnn)" FFN_STIMULUS_INPUT = "CURRENT STIMULUS" FFN_CONTEXT_INPUT = "CURRENT CONTEXT" @@ -129,6 +125,8 @@ def construct_model(stim_size = STIM_SIZE, decision_softmax_temp = DECISION_SOFTMAX_TEMP): """Construct nback_model""" + print(f'constructing {FFN_COMPOSITION}...') + # FEED FORWARD NETWORK ----------------------------------------- # inputs: encoding of current stimulus and context, retrieved stimulus and retrieved context, @@ -161,7 +159,7 @@ def construct_model(stim_size = STIM_SIZE, input_retrieved_context, input_task}, hidden, decision], - RANDOM_WEIGHTS_INITIALIZATION, + RANDOM_WEIGHTS_INITIALIZATION, ), name=FFN_COMPOSITION, learning_rate=LEARNING_RATE @@ -169,15 +167,17 @@ def construct_model(stim_size = STIM_SIZE, # FULL MODEL (Outer Composition, including input, EM and control Mechanisms) ------------------------ + print(f'constructing {NBACK_MODEL}...') + # Stimulus Encoding: takes STIM_SIZE vector as input - stim = TransferMechanism(name=MODEL_STIMULUS_INPUT, size=STIM_SIZE) + stim = TransferMechanism(name=MODEL_STIMULUS_INPUT, size=stim_size) # Context Encoding: takes scalar as drift step for current trial context = ProcessingMechanism(name=MODEL_CONTEXT_INPUT, function=DriftOnASphereIntegrator( - initializer=np.random.random(CONTEXT_SIZE-1), + initializer=np.random.random(context_size-1), noise=context_drift_noise, - dimension=CONTEXT_SIZE)) + dimension=context_size)) # Task: task one-hot indicating n-back (1, 2, 3 etc.) - must correspond to what ffn has been trained to do task = ProcessingMechanism(name=MODEL_TASK_INPUT, @@ -188,11 +188,11 @@ def construct_model(stim_size = STIM_SIZE, # - uses Softmax to retrieve best matching input, subject to weighting of stimulus and context by STIM_WEIGHT em = EpisodicMemoryMechanism(name=EM, input_ports=[{NAME:"STIMULUS_FIELD", - SIZE:STIM_SIZE}, + SIZE:stim_size}, {NAME:"CONTEXT_FIELD", - SIZE:CONTEXT_SIZE}], + SIZE:context_size}], function=ContentAddressableMemory( - initializer=[[[0]*STIM_SIZE, [0]*CONTEXT_SIZE]], + initializer=[[[0]*stim_size, [0]*context_size]], distance_field_weights=[retrieval_stimulus_weight, retrieval_context_weight], # equidistant_entries_select=NEWEST, @@ -211,7 +211,8 @@ def construct_model(stim_size = STIM_SIZE, # - continue trial control = ControlMechanism(name=CONTROLLER, default_variable=[[1]], # Ensure EM[store_prob]=1 at beginning of first trial - # # VERSION *WITH* ObjectiveMechanism: + # --------- + # VERSION *WITH* ObjectiveMechanism: objective_mechanism=ObjectiveMechanism(name="OBJECTIVE MECHANISM", monitor=decision, # Outcome=1 if match, else 0 @@ -219,20 +220,21 @@ def construct_model(stim_size = STIM_SIZE, # Set ControlSignal for EM[store_prob] function=lambda outcome: int(bool(outcome) or (np.random.random() > retrieval_hazard_rate)), + # --------- # # VERSION *WITHOUT* ObjectiveMechanism: # monitor_for_control=decision, # # Set Evaluate outcome and set ControlSignal for EM[store_prob] # # - outcome is received from decision as one hot in the form: [[match, no-match]] # function=lambda outcome: int(int(outcome[0][1]>outcome[0][0]) - # or (np.random.random() > HAZARD_RATE)), + # or (np.random.random() > retrieval_hazard_rate)), + # --------- control=(STORAGE_PROB, em)) nback_model = Composition(name=NBACK_MODEL, nodes=[stim, context, task, ffn, em, control], - # # # Terminate trial if value of control is still 1 after first pass through execution - # # FIX: STOPS AFTER ~ NUMBER OF TRIALS (?90+); SHOULD BE: NUM_TRIALS*NUM_NBACK_LEVELS + 1 - # termination_processing={TimeScale.TRIAL: And(Condition(lambda: control.value), - # AfterPass(0, TimeScale.TRIAL))}, + # Terminate trial if value of control is still 1 after first pass through execution + termination_processing={TimeScale.TRIAL: And(Condition(lambda: control.value), + AfterPass(0, TimeScale.TRIAL))}, ) # # Terminate trial if value of control is still 1 after first pass through execution # # FIX: ALL OF THE FOLLOWING STOP AFTER ~ NUMBER OF TRIALS (?90+); SHOULD BE: NUM_TRIALS*NUM_NBACK_LEVELS + 1 @@ -256,6 +258,7 @@ def construct_model(stim_size = STIM_SIZE, # show_dimensions=True ) + print(f'full model constructed') return nback_model # ==========================================STIMULUS GENERATION ======================================================= @@ -445,18 +448,41 @@ def get_training_inputs(network, num_epochs, nback_levels): TARGETS: {network.nodes[FFN_OUTPUT]: target}, EPOCHS: num_epochs} - return training_set + batch_size = len(target) + print(f'num trials (batch_size): {len(target)}') + return training_set, batch_size # ======================================== MODEL EXECUTION ============================================================ def train_network(network, learning_rate=LEARNING_RATE, num_epochs=NUM_EPOCHS): - training_set = get_training_inputs(network=network, num_epochs=num_epochs, nback_levels=NBACK_LEVELS) + print(f"constructing training_set for '{network.name}'...") + training_set, batch_size = get_training_inputs(network=network, + num_epochs=num_epochs, + nback_levels=NBACK_LEVELS) + print(f'training_set constructed: {len(training_set)}') + print(f"\ntraining '{network.name}'...") + import timeit + start_time = timeit.default_timer() network.learn(inputs=training_set, - minibatch_size=NUM_TRIALS, + minibatch_size=batch_size, + report_progress=REPORT_PROGRESS, # report_learning=REPORT_LEARNING, + learning_rate=learning_rate, execution_mode=ExecutionMode.LLVMRun) + stop_time = timeit.default_timer() + print(f"'{network.name}' trained") + training_time = stop_time-start_time + if training_time <= 60: + training_time_str = f'{int(training_time)} seconds' + else: + training_time_str = f'{int(training_time/60)} minutes' + print(f'training time: {training_time_str} for {num_epochs} epochs') + # path = network.save() + # print(f'saved weights sample: {network.nodes[FFN_HIDDEN].path_afferents[0].matrix.base[0][:3]}...') + # network.load(path) + # print(f'loaded weights sample: {network.nodes[FFN_HIDDEN].path_afferents[0].matrix.base[0][:3]}...') def run_model(model, context_drift_rate=CONTEXT_DRIFT_RATE, @@ -465,12 +491,9 @@ def run_model(model, report_progress=REPORT_PROGRESS, animate=ANIMATE ): + print('nback_model executing...') for nback_level in NBACK_LEVELS: model.run(inputs=get_run_inputs(model, nback_level, context_drift_rate, num_trials), - # FIX: MOVE THIS TO MODEL CONSTRUCTION ONCE THAT WORKS - # Terminate trial if value of control is still 1 after first pass through execution - termination_processing={TimeScale.TRIAL: And(Condition(lambda: model.nodes[CONTROLLER].value), - AfterPass(0, TimeScale.TRIAL))}, # function arg report_output=report_output, report_progress=report_progress, animate=animate @@ -478,52 +501,7 @@ def run_model(model, # FIX: RESET MEMORY HERE? # print("Number of entries in EM: ", len(model.nodes[EM].memory)) assert len(model.nodes[EM].memory) == NUM_TRIALS*NUM_NBACK_LEVELS + 1 - - -nback_model = construct_model() -print('nback_model constructed') -if TRAIN: - print('nback_model training...') - train_network(nback_model.nodes[FFN_COMPOSITION]) - print('nback_model trained') -if RUN: - print('nback_model executing...') - run_model(nback_model) if REPORT_PROGRESS == ReportProgress.ON: print('\n') print(f'nback_model done: {len(nback_model.results)} trials executed') - -# =========================================================================== - -# TEST OF SPHERICAL DRIFT: -# stims = np.array([x[0] for x in em.memory]) -# contexts = np.array([x[1] for x in em.memory]) -# cos = Distance(metric=COSINE) -# dist = Distance(metric=EUCLIDEAN) -# diffs = [np.sum([contexts[i+1] - contexts[1]]) for i in range(NUM_TRIALS)] -# diffs_1 = [np.sum([contexts[i+1] - contexts[i]]) for i in range(NUM_TRIALS)] -# diffs_2 = [np.sum([contexts[i+2] - contexts[i]]) for i in range(NUM_TRIALS-1)] -# dots = [[contexts[i+1] @ contexts[1]] for i in range(NUM_TRIALS)] -# dot_diffs_1 = [[contexts[i+1] @ contexts[i]] for i in range(NUM_TRIALS)] -# dot_diffs_2 = [[contexts[i+2] @ contexts[i]] for i in range(NUM_TRIALS-1)] -# angle = [cos([contexts[i+1], contexts[1]]) for i in range(NUM_TRIALS)] -# angle_1 = [cos([contexts[i+1], contexts[i]]) for i in range(NUM_TRIALS)] -# angle_2 = [cos([contexts[i+2], contexts[i]]) for i in range(NUM_TRIALS-1)] -# euclidean = [dist([contexts[i+1], contexts[1]]) for i in range(NUM_TRIALS)] -# euclidean_1 = [dist([contexts[i+1], contexts[i]]) for i in range(NUM_TRIALS)] -# euclidean_2 = [dist([contexts[i+2], contexts[i]]) for i in range(NUM_TRIALS-1)] -# print("STIMS:", stims, "\n") -# print("DIFFS:", diffs, "\n") -# print("DIFFS 1:", diffs_1, "\n") -# print("DIFFS 2:", diffs_2, "\n") -# print("DOT PRODUCTS:", dots, "\n") -# print("DOT DIFFS 1:", dot_diffs_1, "\n") -# print("DOT DIFFS 2:", dot_diffs_2, "\n") -# print("ANGLE: ", angle, "\n") -# print("ANGLE_1: ", angle_1, "\n") -# print("ANGLE_2: ", angle_2, "\n") -# print("EUCILDEAN: ", euclidean, "\n") -# print("EUCILDEAN 1: ", euclidean_1, "\n") -# print("EUCILDEAN 2: ", euclidean_2, "\n") - -# n_back_model() + print(f'results: \n{model.results}') diff --git a/Scripts/Models (Under Development)/N-back/SphericalDrift Tests.py b/Scripts/Models (Under Development)/N-back/SphericalDrift Tests.py new file mode 100644 index 00000000000..3fb2cbed191 --- /dev/null +++ b/Scripts/Models (Under Development)/N-back/SphericalDrift Tests.py @@ -0,0 +1,34 @@ +import numpy as np +from psyneulink import * + +NUM_TRIALS = 48 + +stims = np.array([x[0] for x in em.memory]) +contexts = np.array([x[1] for x in em.memory]) +cos = Distance(metric=COSINE) +dist = Distance(metric=EUCLIDEAN) +diffs = [np.sum([contexts[i+1] - contexts[1]]) for i in range(NUM_TRIALS)] +diffs_1 = [np.sum([contexts[i+1] - contexts[i]]) for i in range(NUM_TRIALS)] +diffs_2 = [np.sum([contexts[i+2] - contexts[i]]) for i in range(NUM_TRIALS-1)] +dots = [[contexts[i+1] @ contexts[1]] for i in range(NUM_TRIALS)] +dot_diffs_1 = [[contexts[i+1] @ contexts[i]] for i in range(NUM_TRIALS)] +dot_diffs_2 = [[contexts[i+2] @ contexts[i]] for i in range(NUM_TRIALS-1)] +angle = [cos([contexts[i+1], contexts[1]]) for i in range(NUM_TRIALS)] +angle_1 = [cos([contexts[i+1], contexts[i]]) for i in range(NUM_TRIALS)] +angle_2 = [cos([contexts[i+2], contexts[i]]) for i in range(NUM_TRIALS-1)] +euclidean = [dist([contexts[i+1], contexts[1]]) for i in range(NUM_TRIALS)] +euclidean_1 = [dist([contexts[i+1], contexts[i]]) for i in range(NUM_TRIALS)] +euclidean_2 = [dist([contexts[i+2], contexts[i]]) for i in range(NUM_TRIALS-1)] +print("STIMS:", stims, "\n") +print("DIFFS:", diffs, "\n") +print("DIFFS 1:", diffs_1, "\n") +print("DIFFS 2:", diffs_2, "\n") +print("DOT PRODUCTS:", dots, "\n") +print("DOT DIFFS 1:", dot_diffs_1, "\n") +print("DOT DIFFS 2:", dot_diffs_2, "\n") +print("ANGLE: ", angle, "\n") +print("ANGLE_1: ", angle_1, "\n") +print("ANGLE_2: ", angle_2, "\n") +print("EUCILDEAN: ", euclidean, "\n") +print("EUCILDEAN 1: ", euclidean_1, "\n") +print("EUCILDEAN 2: ", euclidean_2, "\n") diff --git a/Scripts/Models (Under Development)/N-back/WORKING MEMORY (fnn)_matrix_wts_20stim_2500ep.pnl b/Scripts/Models (Under Development)/N-back/WORKING MEMORY (fnn)_matrix_wts_20stim_2500ep.pnl new file mode 100644 index 0000000000000000000000000000000000000000..2a46665f63c7e0009915b46e01909fb893627213 GIT binary patch literal 92975 zcma&O_fu8N7X1q<226;G0TC5NOsJ@+$FM-aj3_3=KoHC#Q4vtdNJfH^ljNMEu&d_$ z!@lp+-1p0?SFh@wTlXB;ti4wE3}cKryL;WewS0x0-RjkL|F3_X?40eQ5?{V~`#JV_ zOjL4|cl`SgSFdca3;%!olez3$f}K~`b-o@P6(9dL?)BZoPw!%1Ccpg@x93&jr;mFA z9)^Yn-FdL*f&cwmPeX(5KfL|m`JUuYd#>LI3=F!n=eGZopwOo`;^I@1?|w>5K6c`H zg8j?5l#lUWy&hOOkkAEJ`q#+^)9m7lOWG+|9af@5gH zTCZF7x0c3C_`lZydtPSJ~NZO4%ceIw&nNX~{2{*5A^4C}^Xf_DI3WRQ-Kgy2K_e zmv!ikmcP(WOY1dX$6jBN{=I&eU9??lUul#45t}wXS)r3sa9`W3@DyFRs{NYf=yyhY zU8F`v)~U_X;^!yMXkC&_Sdph>^qHd!Xrm=vN5iz_y1n%Cr2h3fuI+cVT`R4fnkD6z z9b`gAEbVyY%RqZ$rH@AyYsp0|)aErhYwc%5xp5n{U)uQoxE4O~VHnd+evxjWE16eO z6klkP+557f)&9~fws0Mhd@X#&lPWDs>ZCx*;_gbN*fO+L+oV@2-pXIG#Y-9ED0rv6 zvS>-UbV`@DM`Y-zwd<)?NQ3;P{Izc>3zx0G7aW?hxP&o*oR_WgcR_eKgXOWUq( zelK-=r;ch-vbj21Yoz>i>N%aVZuthv&rPA`$=alKk}vfaxo+ChF|Bi4<+)1!TK5m@ z_?D+nq{5OOt(Org^bFN5ODf#2S=+Qm^DUi;mTv!(-6E~KEAo6+;YZFCp^AMk&ao?$JBF{Vb*D#x0Y#3vX(ln+U)0) zELBn_12Uu4785Srp}APDPU^+>e_Gu?hY9dH(Nbb2aSEQ03rO7K%i@&YY-fi0yuv}*>Z6t7ht|U$-w9itT47`^b&Hk>+KuoTVB`coHg zJR}@OO`MKvm(D(marV=HS{|c?xU62vtV^WPzwj9DU1RN)vUB?}Z{cP?VtMYVW_`H& zm7%v2z2jK%uQqG16~3I?Pac+XPd^>fS;^7CXqlFumjC*fCkVqbo_Fb@bnKT3?R4_v zd$YkSB`nmi!3DL(B(&BbwT>CN;Q9&C*R7_v_R^%`lW4p@czE< zH$MHNMJY17m*>~=Tkj*u+P(LrwptgpYSpShTxv3@BV^BN=AJ2aB;*5+!%4u;u=QO= zpKBGDOp;$j%1@cn-jBzKY@1GfysTvh7~~%->STEy;`~qcIx?MoF8bYIFv$XE@ zk>7a6rd`{fKi1i`x?ugsBt}ET=Ab_0q2d!9FDbda?M0BOKUh1?jUMgQmjirSyagLhx#$aO!GaU-^d}{B93md;3 z(o%ARIFH+MfS{f_#^eEkCas9qaw3tbx(y9xWZ3ux9%_7BO^{^g1wFhGRBGBe)lhHs}$}CbM})4oz=cO zIB;OQpS_fS!JHGjbO3Bl(5@rcTD#$pXc-9rpgXkPz*r>~p9|S#@8>SRUI*!v%wt3w zPN+T`M%q=c@Z;YFlBttqL#3NvNX9V_8M<$R1K7z4yDp1sh#{H0tv}D{FE=yyB;q#6 zF$N2Qp7^;)M#EQuH{7K_<{9qDK0h$q1j)o6%w$Qwt9DDl@?FM0zb~XYf^>gm+>*r* zx7QoAM;EQwM7syyK-e*BzpKv0#!D9CkKd<_Iuyd488g6`Cw~%T;4yc$>EJoZSxdZ4 zn3$9N|2%K*I%bfp2A1l13`PyE_Df?7{y(y|V4i zvNucOWBVHv#g5Q~okmgYFp9$UjgwIYcO&fV_B-EPb>yWCT3)(v@36Gq#V)=!0;n%&%ND{6n%mr3rWgYetizrBSnV#7YM$ zC#B|$cAEg{)js6JB9QW%IIx|Q9v?hBF11=>$t>|Rd(9xM{I>S)*LiJT;VcE_TeJEr zN!xeF;D|Bl^U$eJz#}04IoK&C@(YP`nPiw;bws)zaY^GLZN33qRx6Rzaeud+4oS-o zE8Os&EXwEwDBlLu&VMD}rtORr>edPQ$N!rNl~#u0B?Cty8DbUAuGK7G&5@Znbcu8T zN~KZ^H4OnD%VaS5KMd0+?bc&0^M6lN7n;jzwQi-BWxQVPmna=ywyY(-{#g%X^vP{4 zdIUDZS(7o^7KDwOETC!Nj1=?DsU&N8QuxJvng_4=5yX`;5X>-0y3WTk3pUB5fp?pa zi;HBwgwnPAYMOLjI4FHq^gCII@n-^tyM}I>fjYQowzVnLNVBj2Z6RH?&hxmIJEdZP zGA)9zSEDf&tU&Biv^g1Us>j&_fJxlUP}`*1MG%HPc=4}x z90AhXt#zO&aQN`Eb`T-0IKNA?j^h7I(Qh9;#A*xN$stUNHu}Rgcb_=u*zS8q`IPxc z;lopJw7^Pzx(=)veW6*Fwxz9(gW(n}E!`l42OykU3?hH;qC)x*X}^pK?FgJw`J=_J zV0mqS{QZ%XZZaYwR|m9sBQ~sCQU&??@8NxIN|y$UuQy5ajc=O!90L@=Cd&+qTQJ`I z+UrDM0pll`TLSdoXC1Zo+oLnPw#v9CdMxbJ85zE}63z1USqk`7!cvbT`EHi0bbOW z_hO6o5Qw#XrOrUf3jlqW1)ly&1h;eG1Gr1-Uu(0L&V%b7Vxd zi~>kDnLFo@;Ktm!F&@v1k?d<9AR}!%PDqt4Lji!dp<18HL)wsDrPj-%l6ge?_sG2E zE`!Sm>7L_Ivdup66bWl1i499*h4= z_c1v6fFak&!q;SktyBeXdhs8*gKlnk9H3d!ym<**CZC=AEWbaX+H5exn6(^1GG=gJ z2mXUxZEnyvT5wz{bs{M9N0-kmPqW-)@S;v`y(JYqyN4*x)6C6U8zHvmAXy1c>is0$ zRz|26W$0A8%vg6ah{eBWV^b{2x+CW)?f_BUjBpwI^{`x{&1x$;q+y-5 zp3&iZ@2t-<;HR@;Hw~MtN!P6RuZi?QP(O+{sBo|9!p}$&uQ~zcF zyeHl9VDo-#K#ElCx(G4O80_W09g@9U+R~Zjqq7^Mw36ZA2y}b%{T~p=6#1aP*U6;Q zO0pDZv~AN8P&3;b?Gt%hdP24Ne{L@TqqDEA!QABEHz%2QJFtr}QEr1I_d~$@^j@)hB&A*%khoo{YZbNqhSV4 zy(spw7)HyvC3>vDd841?cO;J^Y1*ZF!}Z(JWW9#-Glqh zSH&48$4mA8LlM#uEw)SArX)gxw}~?R*zlBpH4*yg=Y|X%Jd8853{*fiEvvOhdNtP! zMn-<oQpY*5O-GfVy9oMpLr*;t22Bvd&iS4A6 zTr!z?%tvRx$L%91Z0^uxm6jjbZ$!q3bZ-V)OAla;5cJDnIAPyN_8#-ziBM@eYrtVn za;{5p1o)O|X&+6Kx8HT_8`a@GpJ&lDK}`0R!+66eiCU>hv-sVneRkS)kqSzFaw8%v z^lO9_4`@ytO`wn^U<6$LGU@|Stke0UKH3v#n26Y{SaqEa3M0|F zGcZ!w<%bw0(INRPg1X5t#Z?6v(^D(_)fNwzqmkC_%~HQ{ zj|l zIyYK&&LD;s9EW_{H17yGg&HbZ@y&4=CLw^9Jzf3oi`cI1-sUB>u_ng-qC(JtxN5{Y zg92!bSpLd@sLY@?n`to?#->}L()7YlYOXRITQZeay$&2S=;8yl7PyguK!O#M`c=pM zm`JS!bPhf{e0cR|I#|t-&yVPkvGidXdGwJm{B3SS4E67J)ygELGH5u06>9GD+;MB4 zHmuo>ZI-$}Ph>EdWS`P@dI7tUd{TBJ)tGme4$I$Fkyl`Z6aL;0V53a(Ayns(u|rs3 zU@6~gpBqL#M%ORXd=^L<7@(O(G?yexkMD6v1Ppm}>>jnrq*lIRSh+x5Zd#ff=Fs`; zCL1z$NDF^h@RGvIQU|VeS&m8aD{}L7DY3NaOPcNF$g-yGbFW(1h8A{cu+fw8p6)$ljvLbKw&3gzT6Sa>ZvyQ_Ev_b;_f4pI{M2-uHf$w8Rl> zNn}l-^emU61fDXp&TZGZGty!Ot&`b}DS_xjt=-LY7Nvh5AFItNY?*YWlD1O0iK*12 zGVm(vgLXRw8*sxg3_C9Xl&^XRY@gc(UUj`ZD>f@2N}Inwg`2o%`3Hingyi~Xn2?bUm57Y5p? zrd8Q{?2-A!AF_I+xd%+LD#-Kc!wCjVeLrq*r_eZvH)3salBt;E2=xkUjb;G@`SFku zfn9eb5YouZpv=0TDxHba5~)R&W`=7EKI}Y=c!6$at?yrO(unm{ZQs621A$_sU=g9D zbsp{}DgJ2FCLMfx@;1>wlJ?+C>NLZ_MBsZRHW@AIwO#9iq(Tu>E>oYWq>4cZh-`p9(R!F`(PLf%Gb8+l0Y zDV3&3>a_zuaATvQgBBx?kx}K*_~dbj%-Yj%z*F(sk*vcXkK_12vE9b4^!G_w3=c}c&{7x&6O>?f1J0gLZU^il5f&`_H?N2`uSX+5aw^sUpXH3_m;Yp0&F$CyVVtING-v&8`61}%T*;7O%nT+pqZS-QK%G0dbR9dvTwBUkv#!8oUQq5xCJZW5LreO!GX39*$C9H81 zpOra0JV(K~>cd)Ze1UyxE_W}Ncu7;4onGj1oYDNA9dtqW^KhIdb zimqyl%ot2o?Vcbh{R zqnL9fnW(;=LO1JSJV^151Atz@;$gA=I*ibxj`YUz#gqkg&S((r+xBDov3G#BEruK@ zdcT%-0k<-%o&&peCe}ChFaD!m){f#kz9QI697L7O}cJmRy0RWkJnrf(u?o@>!g z_p{Vu+O>PFwyg)$z@<9U_czO`#1)Ytc`WEXww>ZSv<^*vQ1@+ur!D=t=Xnl zs?l=(UkB5dF9rQHpfD;UA3rU*vy)PEN{3}C70g}fxavLBIb#0um8bjxr~05&1M(F* z93X9e#8W5a)xFl)POHuQsA7UKvX-TQ2eawik9h*e2AOW0L(@ zhs>JHj7;7lD<>(nO;W7%=;S7WVBe(EWsc#>s^TwT0FFwN=+Hn1^@8tz$Z z$onF{j$hW!5GgwM&O!_XZDQ~vCRa0mq^B`D^XOo0f34%!@tM*8-7(y^gs`tPBdm2h z=>U;Un!rs@QWlKvW|;Uny!UOKwq1o;G*711;PI(lOS5CuJ21oPhw*YJW@`)<6l@ob zLkfVx{o#NMwtpZK$C4Rxjgf`Zx_I(AVB2TD)28_p1?yxk-oh3$nvXpi{md*`vs-_9 zJ^L>#ba<~W>|aOt7o)MipM2_ytBel^z@f_P2bsrD>rl`;d-A%|e3xh>f@cot-&E@n zq}Zv&-vKc+Mdi6G(skJbVW2$;!T4=7)>LY_Iuf6-4JT6rIc$y7z65|CEoP!tQJQ=p_pqb?7@B+KsYwjes*o&Qo5yFvEqAls&1|_QVrm zW;f3-D^`An;kE`4-m}>glnVrgo1Hh)>EqberdR~ zjjY~qX#HpH{-0E&zq(&DU14M!>;Hp+Sihb~2Hdk``ekS^Dutmo0Y&4%pdU@t+9Y(` zs}pb;YNPHdj$)?|k0Yx}b?(b~x21UKb|O<}NSZ%%1El5c(seF~?tlFf^tZ{};blL#KXKp2`o7Gry2Fa}Aw=x&x6)*< zbkU|By;F?E|Y7>JKHW~dz=0Yg9qxYZE!8^=tvpE^vNvQywZwD za03f;N$a!SMv9fQMM66^y)t=2v{tyEIc%zDRMt%wEi^S(zvbh!LJC}unZdToLIff& zGhita0d#+@U6yWAf{s?IOoxFl{nj%Y!1Dl#?K{iN;Mk%p9$kky5z#WV&WoJL+J!mq zhsGX$ssm=+&!eQ09X&o&j?ZO?ZB=P|jqx8#o)6TR^TUL<|CKc;;}M)>7E8g-a(?N# zNAPIx>xVi;R6(SsSYEsb$Furj)Z6S89?GVid<#;Ij6i>wd)@TdB2^y1Fw5XH0MOYz zr-|51>-uM&QE-zx&sq*Hk`n{z6Vad2=8Jc^o zEs6}ye5#$k7&#M%+vtTYO8fDX&rK{B9XCcJ=i5Nq88%^|uGErteUY2qn7gKtc zyttzyr@&m$qS%Gvg3iH7+;Uoby!Fp*rL8EoeH*ps5JBBwLXi4qCt=d`l-N9|zfP>x z!IN5HIl<~kE2;ZxwEEOZ9rJiYe9`l)dL&ix)CLWfOAJo$+~#01@}T}bB;7$g0BYRx z^fHRD;rO0Ua)V*s=DBC2lEIZq-7_MPY|56b)7P&W!A0Z9{YgL& zKb~6&z2x`s0Dq%6elT;g5HInS?)}S$Ab28lW4qz8u%C zxYyuA0iV00{qyb;DJ9mSFdaU0o0wRVu@2W?Ay~!?pj1ciVl6b>@I(H8o9;?}0)ZU8 zl&`Z#p?C80#?)7vgJt|S2OHRYefv}*4laRk?9$f62#TVQGX0odLv}+8VlkDrV!5g62H^?X$Bh( z(6OVk@Q9Q%O@b+ibFr z6CH>+oxjh++BDOw8Owa^3n}&pOJS2h4U{u|;+5$Svu)<1WJhlU;dD0W@;Y}AtHSz# zaH~mYy9cpn^D@KUphc{+a}iOqnc|{~p)-MvZ%$DawZhF={B2YRz4wM9L-H*Gf8dxl zc%_j)Y4XQy8Ah-%V8>OjptnjqMTx4zA2jNoAzZsUZfY&BSWk(@NQv`OWy zZy0kCNkAKuO;FMw3Q*)*k=syawK~6oyIxOV3p8od4nsVVrfne4nAzE7b7gnNzz3d3 zO9(p_%lHN@@v>94;7rM2T4gfdpl>m1HL~ug-RYri8S@5^f9;lGZN8)v7+&+G z@sPtVjKb2%?*#pqNH?O%@XWu%!8#{1-{|bc`8|2H21fmBr8sZ)+Yh5uS&3l(h_;{> zrZcQfo*-w=;ai3Zv!za3j@$!C=|z{HxZow#t5BbNW%TuS3{r!K)9<}MNq!HT!Q|li zwxCokWC%o^cQBy$PfJV)wh&(vn75lN&Blotq*~D)7t$TVP&NwIYYWQ5)cB>^wvi@#yW#=F;~s z$>de@{a7-M19)L`ph)GfEUm|?g zh5lmUb;7HOdZ=1^X$uaAoBbc%hqv#14U$@tbsn#b9bkksm^BN~n!Oup+{kHwn^OGR z$z+kVMxKn8f2W9n_zWp~rKRbSeA=mF8OP$6w#a1J-<9&Ow^8>Fp}U~b#qAivLCd3< zRBeDEomVs!dU5j*G01+GtjAlCB$J@sqP0@${LfzxAR{nK1=74y%Om)H;ML9YOBXVD zEEP?|=?uR951PYd#oJ@uJV^G{Yt|)b_6mG`N(SG&uzn!3-O(F)TH^VGzkk)i3^3)mTX5yfTf3i232A-Eq^OSB6)HpUpvRu>Vtneym8g5 z=qHkU-rbnO)N+~ZMIq4$ogLgCGTNC6sDyZHd7Lw!qcj>AP??l(Bg+TCyMuuJ=(*=K z@ZVq5KF5G018o^h`nUGrvH@a+<*VuYgcj#=mY|`@;R}KA?;I%SqxsyOH8TE_%;Y9t zB}Ev#*Y3c;!>6MSp}aCJ{ObmcSsD&&??H1lar-iQ)v3wuTJeSf(sp7` ziCgd~lmoGQ=Mkc>j~E#>5rF}!=!-Uh(vwRFMmFuMecisMtIGPUyQp4V?fv#(%B#diDump=XAWAVYappKqtI~pKquOppq|ib7W%{u; z|BNUeJ;@gutk`{ke!Fl=13e}kFE02{jb~Hf5hu-0PF;Cu@UhMH&Ay9l2gm!;!`2-& zO2cB0fnN|2)@PEx;@%zkx0)yYVQXKq=BG)?DyZBQ( zeqKi9HJDBr>nPVk53M^5h_ZVaXH9G84U%;~306nI&SF%6XOR%EWoKD2bC;2wMnjJr8@RRkz0!Z zP#&6~?cys7%ho*6`5pKv(~8|ehIhY>Vj1!W#mrk8r_#W3E}RO7IDTlAX2?N#iXXxrs?QgdQga1OA7Z% zvCsXpI%ci8F9n+ou$lbtVC#xztS=Ls28FOCqE(HxQ)q4({1ZeZD zRa!#>q+5DZ%!GI9pBKb=_bI~TQ<(gVKEV*$s9hJO>;RjrOlUQ~HI6Ldj|x0!j=szn7(H93%<&UrpPLbrjPSBgm9lf zzm0wknzS%T8h0_Kdh70%P15zy)Vh|ma601&V;;tnw&tt9eNq zLvKsfaZ_mN&-hzDLA0O0D@h2|Pe|uL)CcNt8K{uf%>gGDQ0A&fWM* zmSD9i8T31q3_B9O6?fETl=XsjHQdr^AKG#ixC35k^hQ$kS#jb1Ufhnxx)nkZGpgD9 z;)!ZD5p!T^?zg`^qc0aV{BKzs1}__I>H_8-zD8xMeGi7bZAnBgEj91>~DK3 zWdYxS+$p?c(*_@CBJ&}`H2YGznOELw*zGSad!xhKsF@1w=~ukx*~5+*AD19taE&&E z+`Y+m0cqNyeG#BPhZimBvXh~7@_taLw`J&#{M{j)U#VB4P5&U7JmwG`5Rkm?>=r12 zi?lO+t=2o}M55(fEJJ)((e64WSLg|u-?7` zuKC%+aE!b5KUptX?q+w>pR4B|`RKG+;u*Ea#pP7swCAm-Ctu%nNF}8F+?SMRULc~& zc3K+`%G6%t2pjfXe81r-)M@glZ<7zZ54eU!Pa=YD;_$csC?t=3CE{(;cn4RdUVIZx z@yYJ?>_<}cJ`!)0!h}AF`YQlCWm<2!vI}DDGN;IF2$z!cYu}wj@LoLpOlEg6tu9t~ z^PintO;sZ{cC)UwyR*S?cESOOwJ|1r38QIhrgGUT? zf@a!tF>Wik)oGoULZabsz;dHT=lrmM4aJv@kEK}ayq+X{GLLwBA!AAC^!y-F|43H^ zuKQ`^(V|X(z6}6Kmg!P`^4#9St z{f|_Wh$v<~D7o*X&eceExO>qBr7{<3G>hbHea22zQj~^75<|}oA#Rz$l~rtz16(?j zSu}i3a92tFm!%l!m9h_E%0ApWznWLvPqTSFt^a`~F2Ga*Xc}=|Eu2ScjziEYR2sha>5tUwrtuCEncA zgzs?-F~%Qg?U=m^V~|EWNz8RPxkc$0$2=Zt;To2P-%}}2M|Xx@Kf+W@amP0A zYtqg37NS1>sLpQzcEY>_pAl0pvUBlD)H1pS7gipYDt|4& z&*WN%+Aa}kqd38~GL}9La@*OAaQ@3#U2yvk0+V6)*p1ra6$F@vV}_x%=IF3((>lCY zXOug%IsHy5PCuZsz_xo{e|FaDXgI=1wyAXyGM#L&VYPH7X}>c^ajmx((HiVI1vs$m z*h6UH@K@Bl4u2v72>k0NJ8JZ20BJvnnb?tB$7EU)iHN@pZ{t&~&G01YTP*hjpZ{YA z(+E5Dh$v6?)fs$pcuGlUHqB9o99)S%CuJtc1YIQ7-V{zqWim!{dipNR8VZ|(aOeuz<{#uiU&O$=$#;L{kHb4Bsgg%5PLcAKBJIe0r|TuW;MReu^vn5 z+XErM6(HsC+&kr({ES7BAJ*Xnpt~8+xPAH!^J|t#vu&fs_Xc6O@ZL~*%yaX;hB29^ z%ChlkCn7vQLwjH;8M*Ub2I%AcG|!77I(}0BT`|&x7VEss?^jzW0P^XumbykhHvQ9H z$^U$da*6F@VsqMh{smHpt`$v@hg$9l7Udg{z)aOKYFod_TooP&Aarl$OWjr&QQo$H z{}=PQ6uMrJsTV{xDv7rIpjJEK)IoBowbnzPppaZGB|AQ}mr`Et(- zEC+HWLQOchfm@g`EFnPh3CldQ6W$gJI?w-z;3hPZ2Ht;&()@MCawjy~)98FSfcrV> zKNr}#`qdIWv}*54*ZZdohRMJginM&1RZIm_<+7wUzdFvyjb2^w-ghB3itTanQuTv) zX)+O4O(0UQ-94jnO_KrvGU_J0v9=+zX?2M)ZAt=|gy#CfTfx z-qx&SOWL=t*lRX=L%ofF%ZM@jc{6n#kg;G=pB;r3(xDfZwc?i|iXl5Kh)TIeOoKI9 z(sRZi5M=wpycA#2ngrzir;?#SOIaILTS5dArD{*E+L@2E>@mP@?!~?%F6_ z{7%Vn;Oj@+l=m9(A$8Z{Z}EIi=fEqDIvR->S+aPJwmEu4#1M%ep>&f_6`Xluaj*W= zwNSX5PMZ!wxlRoQCY;qU+orAUUrO+hed)e2cmCE*Xi z*`=3d{1SP^3`ekUv*TA&rdM1m;q#^;@hd!LyVf1!*3!s%`n@NSl;k~KszuDn%Sk94&_W}AI=-5q zq@T?WOPl(|cF)uT*Edj5Fu-5?r2fHU%tstckeIkwxBYu%vi{qNfPHA0E? z0)K|peblrnAgWiMD|Xuv12MO8ZGNB>>$G)MCswT3z1EPI`vGaa>JbQ;AJ**fC~5v- z%()Pw)mL@FT@eDMYr~G}`2SYGXMO(Ldigj|L(8>@w(){JX)0~+BjcoGIVQ7d@h32D z5_OvkF5BX@G(tOFu}GWsX73>v2i{s^?UHFJNRM&;$ayeksJ;LF&<>Pv?EViV5k;I) z_h#c@g?4S=i#i`CKRxB=d%CWtPU>v6U6q^*%2DoKEwesHYQ>r_Yxs$&cF4b=y@= zeYs<`6z|~-6-g&^5489eAW5D5SAYAaoZu-V)?+_XWmXqA;ftfvPVuYdG(h{T_a0KV zd&ei7?kj^2((Xd1)G(v^%QHp$?*aYL8xHQ_W~H=$WO~=Dw^yKM(r>SJor(Zea{2sN z+8O7>hxA_~uV_(5s<@h37b)tT=mmp6+?GZ8=U>}N<)^f*JlqkI1$y#+p>UCr%*18eUH+@WwL0&f` z0a}m5T-k6Y1?EYD^+%m=;#`P$K-ix{m1`0W6~+GhG1N{~GQZV7g<`Y(mykgmg^)NfBu1%haSL=fR~+>rf1AoWu)A@Qz>?~p`k+nw+ne%?%S znU{#n?BM#hI*dkX-uT8y)63D?{0OhL8R9RJfe4*kAGm~UY_B+SY->2sc4ilPEhm+^ zFIcbXkblw^02Sv9n+4!T>AcM<`1#{6A*eoIL#!{32-{cM$qg>5NztKuNU%l&*|K=% zON7>b#;^}$ko^w~y%n>4p;T;PX@u-$gS`~^G9fxL$i*#MaCnWjA2j3pNe6=dg%|(n zwYDDd()KS{9_dYQXZXq!$-ZP@B44xp_XV^Z$do**+ru>LUxcB#A&L&y+f;JyGzk;~=!ijRpxrwwPE&JYIaOIsF4CV(# zu-VDe_yhu3FCA;}(cDJhRJyeA^mU}$((8lpAg}L`9QpZxs?WUWjU7rlB~`D9js}Eu z$Nn^mcTB_1u2?{|hx>rZ92D7tL#w_5YpkgLmY=IN$3$kdjP2aOTc9w_P3}~pwvANU zdbKxAW>Ol$GbXr-G;7wCbX=b@4>kcpbIbA!^(`kNXnk&x_DxEbwm z%rJ>%XuMKe!eZaAGb6)aW7qCVD@X4z4aV=gz793EV7WL+@%4x>qs_at&Nl|kEJoq{ zc%)e?!9JS_Mjv)XYmFD)BmjE|%RVi=B4tKmmq^Q|l&|4@gO0=;B#_7FzwzEh53R7j zq2rvcqo$sf<~5si%v^?8isd2sM?tXb6nm#&(sH^ipIkF!Dh{x1w|H6B(}!LVH`rRt z5E_)0TCf3=AKI^l+r97}9SIgWe_yjZ2{e)rIaaO{QDAuxPUS&| z4lq~S?dXJe_bjDeoC-?kndaSFm6rV#88PonsUUBdVb@2%uI)Bv+H^--5Q7b#V)}O3 zG1ts4=t}03LD1Li)hSjK*kuF)oA|l9d3M4q4vUNrKVCFR6SSy-(hSt=I%3wvRlP*HgwG#LU}{tOeWqwcjYk zB56GzYahVn#9odgA!+tsm01}w2s)PNqCJO!_hp2KJ5rEQ|Z&P z_4qU&1DDfMGRpYzFGT7pL%2>VtR*Uu6?QK<3eU#qc!aDBmLjp(E9(bIGCK_ zMM1lv7WyRCffqr%sfVG^N*|H>Q>>c#ne=2wDu+VP7z-H>pb}G!}1zcB-HVb%<(--L8QG5}Tj$Z_c$-Ah^4&jVb~Q-~iP-V7MF`rYEY8!$GgqB`lW_&d>~CuMW}*?# zul-2>AfL+`u#F2Lk}sLNWB>zdA+HXU%57Tk&APH|M~qA(?@d2+#z1@JZFJQ+FA5)a zsINA0ifd13mG<%xhQQ0A~?^y0HP7GIt`zZku6_i?=0;eE^_3Q`gM0UU0l&@mJVyK zF`qn3`_DWCbm}ELgGtX)6SY3QK!=-_K&~!4{B#0#WBHM`a=K1RZh#J0F;evD+hsBq z3Mh&U#m=0=GH5rTH3zm58D?{N{7N^c!_IV;@D9A^wQi{zLZ6f|*LkM-7iQD>b$ed2 z3Bbf3T^l=IRFDe8J_drTKi&Gs1D0PqDjizC$Ig$JM>&9uQ%_l4c6yN_WdSR}@g&YW ztkgf==t3!2kJ?_lk3e0$Y%{K@q_I+Axx*#@&^mpIG?oHn-iu*YZUb~;P^;qr)krGl zHEEc1gx^aLy>h%MA)LX5!f6J>kt9f)LYhaCTz+j`WVrgx8 zbqSgn{SR$?;-mb{&|Yto5+`Kg_>ai-iOji_*A+*c=1on}XKy=S21F`| zkMokZf`sfOm_3s{rE1@2&0=Q>oS5mF!riN_Zx2vQ?e7UX$gK#-Wh>u%R2Z)D{JL)Y z{qN|CZ$NOB{QmeKIHr9M%>IZnv?l|xL+1~?edud4EcZUIInw4AnDXwjYc7tM*38a` zfkiqXBV=u_<$XF$S%CvUSwLvGt3~!#U#4RSub{BY)9S&Wj)=^G#Wg= zf3bX@bJqJdo?o=0E}egPYAM5H&h=ulD|VounAhIjl}HT668C;}ev3 zdfV3X>4$-fuM~KlGT`e%0NS)L1>Y3|E$nu&T(9e5ik2+HUJL8Jh$-cWU_xloU>$kf9BPN%=K{G9yfxGv#--7}4C2K&lqV zPgEDU#>?W4p9i-W43F{JgcI&M`{5=pJ)-aOMXQrYm~>Rx{A$VP?2#1&u-3(rl$`vv zV#ud_Kl?*VC{YnbuSzF2nlz?}ZPJlf`YX-UZ`IDEM6LdPY1ph(%2+yz{0NFiu3N*n zriJR%x{$yUa1Gw)O?kZNqgzJSe_@bll_Ab)Cdoh`mUMXSm3OyJ@9~^#+ zlR;X&j}&U-1uanEfGcWffldQpG%r&po*l*BWPOqSN7lUQUYmM>v(H-SjcABCfSCqK z+ojh_f+JM~FNB>3+`U@%UMFsFH`9&df<>5UhJRzytYEFv#`W)wo+lSN{=5H;+uE$% zKhoG9!~J_un)rZS^<03AXdy~-&fjzggoEO~t1nH^ZN6Sb}Tbdu!z|>Ag}qC%#+x!`zdV= zaoK9UlA2qGu`A2DG-qJiDXP9m+M_PlLNIb0I=h&qNRN0#MeLiAMRE zq`Z>N{*L~$-kaBS_wfD)V`k2TOXUGO8q(HHoemC1l;Tirz5G#rClSC6){#rvvt25# z08qZ$K!rA@N&AA9>mpz|!9=>DRn2BcybO8lL%i6L4&x8$Fo9H5k6M^6&8X}N17yH0A5VnIE&}aH$nSh~e>}B@dwEw(8`sq#K{6;6h zJ{_gp!a5zHCG{um_v(zZ)Q2qB(|86oex%`+TkW-V6iQb3FfIo*rF|I9?9Z)cbO%&wcF^*vfLD=l!)G*SbD+`y zXxJdFq0)FC)_j)i?&AdCS6Klb?6!KRzn%P6IF+id_Tk@z{YkHPA%?sexH0dZ_tGc zEPiTP`WpHB(iWE@(a~o2fxc_)^JqL zNAd}1BHec5waE$V(e>pz8m{a#1#h&MzZ>;HsIPWhewOJ}U8EPnF{gXp`^gO!N?Gyc z<^AlZr6;gaTld91!-OtzaV`$JbksCfI{mPGM-t%p#B0M=ELo0ma1RarG0amB71^jT z=40KR(Z6glt~F7tGtYQ3us(CpMgM#bWW{WwAWJ)VADO-1a%Bg>O`x;O;775OG=4yY z6U8<%h|Vsn#Z=Kl7K(ioxBOp}oo7>3TM(u(q9|rDB0&_zfFg=u*oXlIF`^g|5l}&r z0Tcs4a?Vl7S#lIaPgVEdnh*0V?#z6fs+qcV?-e{}?-jb=e!Ew%?ko-i$gsn28ll|Y zZfb2;Z#TD?C!_Jk0KM!8ZFsX{X`oE5T`7~2<@%p{RC*i%FQj?~u|KVKSgrq@G58o3 zF3mNhyf$`nOuNnHl3G(?3#%}+hUy8K%c=<81t3veT!^q#$p zC?Aeb-INaNlEWP%8&xRiGb0z*(pMV0H<0`aYj_vJze8&-5i2bt7L5H&d`>kS>3WTn z6fa|MU{?-+GZmbU?+iF)OvHY4vfJ>x+br+d?bJ0p0|qe~VnQ_c1*ZA)4qQ%LNg75);-g{~wkz;)@Ug zXl3G6)&~#Ha`qxv(^dd_lpgX(~ z0u0*j_0>J`qY>fGI4_LUALT%3{rh;F#r80DYnU(G_Lp`!pC&-)$H^H?@j8I#+ zP9+&E&>!I}evirf!N6g>%Rguy&|7OxVk$oDirID$D8rljaasC2@0%T?T&ose4PdM` z$Xg*qBS1+;m>9GAZ*wqiu7K`ZR9g&M{YCbg-M8p^~Fv6R7t6N!#-Ow;iuh}ng0 zG3C?Fz?yy?#J+TxZ>*68kHZGe1WK5?9ATf6p9zM3_cEC}FOEzLxEyiG18zKy^#H!P zJ;i~D%*Eq#d+Mkk%nPq)+=|kn2fNqM&A|GyS$a*e(HbbV?F=ZH=gH&QVaTv6i#>w3 zmqqQi$>kro@d7wV)u&s#-jajSWSu;D3HNedXkA!_n$E`+FD|PnG3Teo|3|(g+ zbJVS%j3a0k1VNL`6cqmQvH9{cp{NaFy-tfHg#+rN@F5G33BblpvgKd*p4 zqVr?ZA~GD%AQ^Tbz+<^TW$md}csKOiv73fYj1(#bCD(eQ1&L*pJIhp-X;dzt#13{oi%{-ArA7_dBV3OZi!jY)YySWQfr9IQ+tHk9WqSqpZyQYDRsACuTMQ=hxe1RZi##%T8TAR(Nf#Xeu2 zY5f)RD32i1`(-5jopDKe9k%Fr&=*rcsQt-7X!@8jMy>(!7f*u60sXmb3E3X}O?E8i zA?yy99DN{{Po?RiCy&PAwr=c=xTC|q23{X2TBQwdHyI4s_(2Mfdt!MASlrU7HB0TG zAH;`EVP$FtQbms0Xwz4zSf)QU-*O<7xXavQuO&J^L6v3Hg3NCtX-cF+U%dZR?-NK|#9oCKu*9|f-oN~?O#s>KQ3vULV zR~5Azv9ILLMYhakayxDDOAz}FI=poeHuSF>1rQS68_$hYDXO`;tp|?p$Hf}0V+Fgd zEd-#-*4neuq{wiO0rs?@H5cG0wv91srGMKMAi@`khkG;F5#mM1<@XBOe}IkyfCqqU zl09qoV%0L@Psd!2A@hk}DWSM8Gs|;yf#WNmokt3x6gd zO$lC)1Fzk%HRWjz}W*aDaO*jl7Vv6=q79bMS;qQV@8W5;`1}>+o7P`jhxcIWe*gH$TDKWX`&D z< zU|pnLWDgmAo(God_Tuf2fdrf#_WG>CyZFoC;KOWRHo8&h!`5m$7fSh&OFV4*srZD? zGXKP&IFyGf9OfcH0aLdj>BJMcv&TYX(`k$p^5D}p=(8>43Zm1hJx{Qb+V)DSc9VE# zHWdRo_f)fibrT8>j-n3wN=o^W^NrKPDF=Q-Eh2bQ%~c^@mYkh~E}Q;q2HVD}55%jvNK#8)U~xu^+!7!6@|b7ysSH%F)BhU`MV=d^q) zM-;x;lkw2gWdB(&cYpeL`#Eau9Amt2Y8}C|KYN0dUEV;8m;LKZ-zN+<(D~};bi@oe zyFpP@?Kvpi)`0{dNrRs#SY#N|BW<`$+-V4jN3^ldE#zX~-N0d!P?{a%bOUD>1o^o} zxagCI8j}DtzvGy53K57Y>NSmihW|EY@Vkh+@$Fa&dV5PjRdyN;52d%l-CgVqsp5Yc$#q}*LO+B{phD`ZBg6|j)0QvB`CIv@$ zoK5pGSscfk@BwKLV(or9D7Q1LeUD`!=*ljrDr+$O;Y=V9j6ii8wm!G@9gd0}rg($o zh8Tp|gUl7+<<*{n%=u>O)Jry~Qh z;7PeyoN~~o#@ZP~pkppgtD^*Ox($^pL9Qo_q`{1P1J3!;$%?(t0K9wNUNCN4f&5n6 zR#dAL+H@Nalj2-9wmipry&(qn-p&_088D|nD`YIpg}=HjFf+BLFtDWBL zpg8~=OGklb2rqna)4A7S%MrD`Amz{H6l_6Qm+rjmAiv1wAR|@_mJ(bWAz;qYR?hx6 zJh_wE{IE`I-~NsFl^gXYk%@GL``YFHO>EI;)fVuK-GxCj9PM~!k8L;ZZ+*yV^Ya-p zWk-n~`9EUeF{-<@P=?PlDsHT@p500bw#<_+4KNpP&sB1NI4BH-&$|n)QdJ6KMQ!!ut%bF~NeI)mnQZTnjTG z{o6BE>mYlUWBGEgPa6XFzOiWU0t!*gXh|%?E#lPk*&!WL_Tu*z{$l7ke}_SuUKzbb zTj_ql8AsO|89orox$9ePNJ(Xra|wIeUAnKO!i~}rYhAF%Jt;VD%zV98yw>6OFAXBm z_2UTEYsTFc1Jd}N@Y5d}5H%S;}1wY4(sJmjM@1>f^N0@$o`=fZfAFC*xb zzq?%w<>K|eURp$53hPDWLH=f#&43wmyGI9~Q056gKaFEH!m!FAjO7*!F)jyfe-Px3 z0uk#&@O$ClXpie-$#JErqNR0C+Gn3mnJ=U_jXWD**+de6N+*AvykZy>Vka*5j0+HJRxg@G;y0a;!?xsjj;~JLNK2sDW$Jwb~e!9g?#}=Y<1X zyzzbnye|Ek2QxbGStd_!+-MGhMwgtGS}xlfR3+d312vrU7A1BnZig-iVZSu$la*=b zr*&`HSIl1tdPyl3H>1Jb?<19j(GTc5TK+a^qYH+!03#A?@(l9uWP7Zo6eD=m%5NAST9?g|X@QdL29`Tn$6uw#qNahjay_yy=cFrGzsH zqjFI`pQS7mra5n2*x-$*6ktCP#qT;LpCAtvq@&;`-MtlqBFTb{L%kE;yrJ`B|%^-MX($7gNH5Wz34- zk2JRKhd7m49$ql?Je`a{Fg5E|On7P}@kfVXZb-|*TSF?EwIVf&Gb5uN#3>~!mPQq1#L%a`RR8X7x5XpIfppBr*F`&=j5~(Bs$MU%c2eqd@5nd%# zgj_x%+&o3gpB;9(1c9yO!mGIiNaEXfBfkwx4-*GNRaONu%6c@n$Sm2;SHzfVq~Z>Y z6zRDENR8<*fjn;Hn54SSJwcfubov9WdX8{3Suy*T+EYS_P*%Ja-;ve;W|?Q5y6}Dv z$CZ5afSF3u*O3G|YPBDZ}pE+cMM~Y00}O3(rHq z`YqR-BDLZLy8OV|;W4DkX5zUvT32e;)(!tl`X`uSwNB$U7IVzfL?C8XrnV7HWJt{$ zbovy}EP)6A`6`7UZZXDNU`Pv{7ocSb@gOt&?XjE)_%P<{KJ;f^J3a#ap6h6%F>4RG zdCIRkwKXveeTNzxBh>KK$UKQW=!+1YDv){>b(>Jer5sdPcn zogAadByQ-a2d7;oBHzBB#=vCH_tBGd%3W)h0%qfckKJ9>W_A+SyOe%>g%{8hPL=Zc zZgr@R9$`J4-hD#Ga(1R#TDcO(ms?WF3^97OOuO7)T<0@7XBfOa+x`O=&`!p^AChuU z&d}Hy4vhCNF8}Da`&t#ps>AzgpKoGg=4cLvkvOblz}9oTLYd!xb{$aa7>S zV&++%DG1`fU5xCf^}vl4o(n9j^WY=J}D7$_p3SP|k?B`GNOJg_LVpkLyGe zrey9K5(sR^?K{3oN0%DIb6@jTOW&~-Flik>hqtJ7bmZu!U3$Y<)d9sL#Spq_?zREtTHfT)~P^aomcM`4HI6??*7kC_i(sLG+b~WEsDlM{V;{j7^@yDo;Xy zs+`zzp7VFL!x2(6CM0v(i|xKel#DVRWwlS6`+&7anuv<-hzRe2Vu+I^~(~wNOq(Vyfa@0n;<&{GSNJ|RBMIsCr zF*kou3O7Jz>NWcShJa&m1BC+^pud=oE=m>-n8hf zv#y<8{!E83V_f~xF>_yqUXtcvU(H!!(OFj`0r=@Mb_CPLUy*g+cVf8|(M-hmY^R0D*5i2eAh;3r^94>&)69f)q48#HO-3#88MWtK>SQ{da5P=nDXdH zX+3DZ!J~(dK4v};8q)H}p-w<(s}QO^sYg!AY!cb%S?fXpk6ZIT?kDfmyh)*qDm*yqM!vAHB-Svx_@ z4tBwXf=gR>o}|N%u9;oJ;&FHp zv%~|rka-tfxTu9IjF-W}SB5Jcj*b~5Xj&d(F5OOjK(X|X6W&6zmoc(nMqFwSU0|g`OZt3g=+`|4K4o&z}p%iY@nlr3WAN@9(gHxsH1b$bl za{!VE`$BiJ=ntC4z^fCXF1COd*MW!$>#Yaj2h(n(!OG+#Gx%krAHYJ-J2(~D1n7QA zzZK(|9;NJo(Nhoq04*}vBml8YbE`j!YY3QR;A9TK;xKQ&Sw8Xu^21>uoD5vQTTT!jJYZO%L;rh zMqPTA0=_^RWlF5~`bQ=*S;%oibq4IXD0O5bs;PyU;9R+z7YlCN?4niYHlp^%Ma;dT zy$2x&%{+;MpOn1TpMFxu&9*;jo9M|lWxXQyvI|(8#VQ}uDqri<{(A-r%d}1Yu1qon zs{#$2U*_b(yizwljFq`J(#sgLutwe9yofxH<^*@-toU>s)QK{|bkNH(O!}(O-FokFM;zy5z4Xg1pr@xuo z)o90g|A#E=$Py&Nlyt-W(;p{5>MfGH1o$lrFaQD%tsTxkC^jkE`4gvKc za#%)%L%JhbtoQuA7t|9(VuQ6|=YBtsr2>2+Q-Tedf~+qP^P#G^d+Js5x&of`I}PIZ z(|}-T(4GutNy0E09%n;<&-x(rDK0?9*0R_GP!1iwk8ayGIbvB}`64RyO5t$`LD}c! zEJL}}rC2WVcPHNXf+KtODDbq*9>u}O)Ah&fm22A-77=Rkog%Y@U;eyc5nVaNw&_@u zen*v~?4(Wq9K>F=S~B|*GFKwMj+vhMQz}o*c5PsCbQ?GoE zWu2xz@l_t#5bDKVaeEzo=T+@4+UCiuKe)q`S+!E5WK|KcvVI?gxR>q*)tFine$3%= zu8DQD9(h3c<=LtDO^;XeBKaNdZpz;G;h{%qkIQb&r`iR}NGOK*2>3T>>~k>;CdVEl zU^Z^-@@1uy_X2F)-|NuRrGRD|`^<$iNd3?gJ|FZkkt!{T#(I({{MFl&VIcY>EAnsi zQPO@ayRFi42XOd)%V1RJGggBud+k<$6QQ*+DSFFTZBHY0Hce_yflJXy`ouF+DX`Ak z5p?)5mZSpVusu#N*@SaS%XiBW)uLL)Ym=94(_ZgTT#H%5X;|py1C&wL(M=|Lg3H*q zgduga5<2lf=B`HH$5z|c`ax|<<)@3WfMckviBgjDi-I(x8m1QFc#Kv9-8P`LfxD-e zTATa~3<>3bQ%HCa*8cNRWFBkTb1QbKT>wbO!Q32YQCjefU&A3rxnwQ)2yFR7Ceh({ z39D*FIe{qoGz>x}la8)C6J~5YBp5b08H7fcn&>y$ek~d^2XKQz2c&Zcuci6}yd7V> z$mC+r8NuWvJxa4UL18n1;sP`vFFe?g`f}|*d>EarGdtPlVa)j@EiyRvHHio3+`Me$ z@(vL`_*5(c=*oq~4UoTo*0*<;{G|67EOwpRR_ee9q_>bdb3ZOSLA^M0CKr4>@ng6y zm{g}~gr!Au4xI)^=$D~??{kq4|E|JWhmn~4cO38BUCuf-$*-uzEo!9kYN-C8FPG9Sq#vc6fq7iBu>gLGUaRd)COw**Hmx{~E( za8g7?1UwL_^5Y!?lx`#yGTT>kUA+uz*r|CJj%kT=EV&YI?wXCLAlM<@FVE{ZVHft2 zkAbE6&oD47#6SuKqh6gz<1_y*@xbAyE~u#v(w?SXTkLVnkkT^r0;;L~Mh9ZFIK@u; zNZzwDk0T(nTJx0L{tQ|8mhhPD7VBe7g12_?l}@hFO|5uv?ksK+VPq)}glrjf|E*)& zqti}dOFZ~kUMO9j*{nxiGIxp)4$AfPG~!=~K9m3|NHPjIeBg32!jk|>wr{zSX`mwS zy7pXGqx0R*+gMHJS-Ma%OcjWL!C!HGSR#%TC8)4 ztac?=|Lb;2Q=G8{Hu=6{0}ApBeM4#YYD>f;R^UInM9Z*Oci{=9U&t*TjTfMit%hTC zdOAo~hW6o(ltm)keNwT;ys=#iA79>`V$8kWHCktCN37AbK4hYUf4-9fj5Fh^M3|&| ztD>8K{TOjwu)pDLcvGWCp=yNyXqNRYkUY;-I>E$jc9Qq}tCW8-Fkj8yGDVGiAL@??LqNUH`?WGc^i4 z^TQV&YK+2P4wo69_j^*N0bW=H4PKY!1KQVI;ReT6?dpbtOFXz*lDNl?JfHK07}`R0{2|F_=6q zZDY}>9E8cQ#2z$*JGG~xK__3S-=ws*;eNg)6VJ^FOq}DQ1*@03sCVm9jT?XygpHeg z&U1RmzA6WNN6r8Oxt7z#&~x(FWVMpv9wyvbKIykTMge?`<2e4Q0qzt8l{sL?rJTC zx5!)noj{|3I}pXK+og>>^f*T8sXw@STXcoplgjJTuJix#YY41+SX?lVhk z-27pbFHbSq_*Za?q9pXU7A&`|TIqXUb5TU&6r+9hNr)t=0tVG8uWCN`1dQ}F>W@=8 zg}o)fabuaxT!Y>W5qXip;61BfVC`6T3a3l?vUF!9QZf%^p-%id8BtuxYBz?Ei~osg zYW)FU-ee-#xEzSiw8@I+X0>&WzJ^~;;96ko&-+IbaARDfX6Rzcd=PrhZ%;yzXpR!TfX@-zG8DXKGvvgiT;Va+@4L+k4kse$Tr@j=I10f@ zlZq33eVXqfMb(e@9skHUn=QKxAm+uAL`y3VhJq-Z*O)gq@Y$*_Owd%ki#Zk>60XGv zg)}F_{F;)sr&~@d+_XDX{=Kp;g*rXH{K@dB(uma@TO!LujLQNZVB#7W0wCpjTw7dZ z?mgbhXDPh_OqWsu18aKVDbpzX57y?fv<=(jH<2Q1$yTZOs?{lm8EMv1t>44E{#lzZ z+|lMO7hzufIDb~#w9StpLJGH8kU1zXYIH?M`BQNkvXV_!+W@?_4Zwps6QI#*|2^w2 zD(tI1CCnUm5d^iKuQ_2N*O+)BhPzG52Bw)kKlzZkw7_D1peAcIhsv(6ffNh4D3!U_ z6tXwRBt_nLm8{-jI4lBCOq^^~zh^Q%a@Dp+N>R8O?i`g)I5If{`?j&jmD2s-^FH>) z+|6yo!EiR)h=Is{>zuLZ#}Dql$9B{po;+xhfs^#%$N?ONsvwSr*w^>5;?nn&3ymfh zmGjK#e*eL#@6s=&Zwj{*~Hqgy;|*gL_nw#PW3w;GugshuT9QJ|EfUVV;J|e z);l00V;{893R$&0&fzO4ZBq_IwCK3$iUV-PNu!gS{(Q(slmbh+TD;Adjgu^Ht)A17 zRWJr^rv&apUAWIVYM>0*g@nk!2T(C1uYqyZD#9${`-9M*e zj|qRS)UGhc<-Bv4Xl^m$NZA3zmApU7sb@*aGbE?eNK7YD-xHf6lFiz;1|PKsLHKP29S8%D z9055tSnHxpQ)e8lnd3{LFjF^SdD3)iv*vhkgvo$Dp86U{y&IEj`PAgm)k23t5=kX3 z^fF(|JMclPHe;r+z$|{r=L0^hS`$AM`-2;GrXLDre&-=|d0+6v1{m^tQ0Mo0#G--I zAKz$d&GX4Fvj=uEdusVR9po!qF}AT>z1QCvug4S{ZiYw#Xy4*YU0<>Sj2GXt>~o-t zCB3+t=ZIR=Z^D;Br?7(?_|v~HeAOlmD-OI(RpZt2>|svI0HPwXE!UyrcR`L>Xo7TZ z1D&_ML_}78&{KDr4OOLv%#TaRw7Mfe9z6{OP}Wn}AG@CFN+6Yh7$ zSnWn~>u5godVdBx4Qf=P(D4{BOdhaWu?^!$Oi^3DI%}^?bKQ$oyGmIjnAQw?ntmVw zM5kt<(Hihp%f00HcFgCaEo7>=La6hRhls1t-M`2Oxryze_^5<k%erI7p&_k+MdZ1rnMJS>>GPPT9=u3 z2O(R1#|eXGo00b)+b7L_2wSa1C$4@epyelAQAV8Em0W<9bXLpx!lKVA{a?5VBJ(>* zA^pgkFW=0N-w(oZ)Kg{X4ZvRbltCmN<6EV~K@lS5Sc-|ZQgX9|>#Fm+Imw~EEThUz z2~Ln~@`edoP>^Y}m${A;zvGMcMkN`I`FGz5g3PLPQnQT&2rXLYi_XLNGq$|8f6kIW zi^OS|->aO14(iZjR`VBN#3C@|-GoWoJ!h7%F_84DcgSOh6TQ)zDc~32iJR~(*@QL**#pxJz<|wBtS;!!( ze0q-*90TXuUvYsgnGj`><@gRz=@0fczm}%V=X+1yGE6%Udp)ExP6Tqm_*~yB5556C z!`6o7OVs8{9YpKctBpFf3m+R#49-8DZ1Q3_Afa-s*1~#|uq->mUWacwf&1_#;KmlR zy5iX9c_`={Y>eDC=;juwy{cVwv#7wQpE~U75p7@(%hD~AD?D^0-MaLFaWvwwOzBL) z)I)NswTSDZdBO+;t#<;mCvdT}{=pq`v@ z;Y#*>HC$sV9Dtm-mX028km6MjPa1*1WqAHjeo>d=GSV?h9m=l{4=kcemWj)1uP-`S zNAh6j@rP^HBJ*A;Pj8~yw44W!X5z?KL~Ve2I8DLxquTfhi3`wQUtF<1b$Dwep1(hf zl9>wvhNMoeB3p(pwDVJ?P7+oGaJxWwoiw*y$4}TTqDVVH`|Ic_S5|;<{xC75tX97L8l^NO(>T- zWE%YCZ|Fj;omYaTc7+gmbcN2 zd>NY4i~zR+bAVH?l9gS-wSEXoAhxbQT4s#(wfTMob2T>#wsto=zq<)S^ zd-F8g$IDvbuU-2IR;*zs=FpqjH;Zp$&Ev11?#wiX$AxJu*8Tk6U3nX;6dITJ2MQr7P0ZXs4)&xU<7~5hZ^TKOWecq%5mhBWY9@5RVEPJJKx3|%h4|uNl)?1NA zU~b{9v&W6cix$$xLw-vmJ{>0sjH*ROSDe|4yaP9ncDqTvSA;=?&V9R1vM@vx%$2*E z5M(ab)x1Tf?YOwZm&r&l>fg2wqi1t_%im1y+9eI9b7?iDAxf~LvtEps=SC$Yenrox z4Vyxo%MP80W%oEjfEI!BJu-LLj2V?3l?6b53_^*Pypo)?Y)St17fgCc{sidwDJIwe zUMaWE&?Yy8h2Lbyt)vivfZ)}4B5?3ZB0%$C{d4}@EjT%|nbl#((-BC^q?LT1mry@O zfd$KoqdogN-djl16Tou-=^kdweP)me75rj( zTI^F8Fbk;@(T4jSb6s3zO6+MVv#7x^BY+uLFi^ZJ0U$U1{=Rkn2-+HQ@yVrP-{K9G(&vFj;qe-S;bL&0h zq5P5G?2ToI#%Aev^tU6Ffl;MGie1ngL*EOu=qqyA{}t{Q1y&%m1T?T^Sx|^STQDDE zoI#upb#|1$hrvtMeIEZ^n{=6+x}eVQSF~W42H$JcjTzH7;|b1 zy{A78crd*&s=3U(Pr#h!W6rdWPQ)t^x@Ihe^PNb(O&*=`bUT-6f8Ya1`!Be6pMH-G zar|nLi}TY*hF{|-V0qKAwEu9W7D*X)j@usCz;>L1U-WglqO{%N$|za8|mja~vgyh``FzUuu{Eh&UvTqXb$XJaUI>P`e zsm?_Gr8ZXUzv|D({SpN3=A*Q1|JBRHY;|J z)6tvc-^SPA)zY}rpB5>HO{qoRt-msXU~WB0d+;hel?LmH@vIl#$e$ph4IW#y`_NY7 z{S@xwV6-(EXhVZx->7{%*wk!r3AX1&^K}zo$v8q{QJJ=+n&0=OeUrMNl}4QovAp#O z(m@qK{H;|PjLcMI0b9l~`g2G3XCq4TH$((^LmT!U0QsCHnk?mI~a(cOX?@$i<+>QRaeFLd<+4<-)+8-MP>JRC0GBkLd;%o@s}|vUUFSw>XR>Iz;D-OxVGfj5p5V? z+lzRt{JEz!o-s!elkNot9Wv$1DnibDtl zVUrT*uf*ccT+t@_CR*v9UVMzg3HsZLaN$Jt&B;Yz?W059Q(W;p0~hsdtAQzl;uwQH zT1CvxTvw>GuCIffjIe3unCv>g>|T2G0aLjqqmQ&J=3#_EgfX@!3;ZYBi}?=6NSCQQ zhK%Lt@7p?i_=TWUjoe@iJ8H(HUE5s{iP7teOor~f+q8YRH@@{w?Y_e-%-QlNI-bDv znluZYMt4qL#s1La82friZJUJhJC*v2U1EB_u=_uFJ1e2px1})&ZE7|Zwe1Ato=~)ly+lqplkP*D@}D?L&pW=F_Tn?* zOLB-6e%u7mb;z07?Pk{HK03H8fw=A1owq)aZOCZf(TJe1B$EeN1IpE{(uQR4pNMLj z{L+aJuvWNS@s?9B0nA~1{pSIg5Yy&_U1c^UO6z56v+zTn4iezxE;lM|AC7A=fx&1r z56B?f3&};0`%vVQ&od7)duY0Gg{Y?$3NryW(qH=N9PU-C@w5Jd6_o)p0@yO{eC-V1 zs$<~#AUiRUM0rLOTLBcr;+3GO zWT0|tvj^*D+n_m{p6d6McqVw_gbq=}7pr}Q?+-ov>;)Tel-^A3VB3a-UWmN85E3wn z0!&*pHF^ORs=?@@t%alrUA_LyaJgu01Dty4AUkKJBx*KXu=$!nV)~$C!@VDMQCkq^snu&W-W5OvW2XZio?aQ-{tbhLG;aU?%!?Q!>{%ZSXZ;l~a zyrj`nf4=1f)M1U(1-6Ji!JRr2PKPWaRmJ)<@VbtDWk89{p2}ZEtTKKVJM}B+GaQBb zJJ#AIRA0JEc|7a2V_X0gCM1_Cs^w1rVXlRP^6^ox*PxGiXr7iu%ltDUR&C%h{~P2< z1=mbUR;mF?t@%XJdn`vBu4nNhxTbcuyNa3o0@_7dmN z!b>SccMXqDxvD+(#!vi11S9tY?j7ZOjlOL-dHvv?+gRs|I(Yvk%WYCJY^y(dukgZc z?;;C=tceF`5Tuq})(V{H)My0jw*NXZV$B8G6EaH$5E=JabUi8~l2$DP18`i}#Ywnd z9b^}yx#_l#f$CQC{BWOxI_|}i;h${4taDlWIi7>@nhSTB#sU?1xg6q`ZLk2w<{hb^ zswX?twUyg@=8Uy1el-AAZ`7A=XY9}*=^4No!qomgev3&kSsvwoH{zj*@sGG;&nnYp zDxE&#PAl+|x~^ye8p8q4i%8AG=l=HWu@;|XxNR0Uif7rOe~*!T?GJI$!qcDeP+?`KWirCGa{2Smo3jyv_152kCk%SyD<6izN#^5G%7#*|vNgPwV*&shfaR9oy~6Oc8X z+YXJSMjtRyX4GNw_4OEp7CSdWTIxH)?t86GhHwo#+n zdv?waF$6zyF&x~S0bZbleK#n`?t+CUsw2~2QwRP=5i7|4XQ}N~S`Zk(6e)(8vU=qi z!Uw+2+Oiq?oo6(6(0?n9Wsx0X)$V6?a5KbE@EVETZ^lB600*_!^=~ohCqENqTXiS= zu~Fo&CO|`yC;nc~DfWg_3fm?uQJe7%^QCr`PW#>`^cbm~I(sJ$UuM)IscgjynWNy9 zbuKvL@lM!oCq%BEH&8>n*GTt{O}ir^Q%t%a#Wb`kTn4mF*eP>IrO#ZKt_J3nYvgEb zId+D?HTz{6D1zUiC~}VVNk*Qf@&<%e%>Z-NJrI~&t z)*S$*h&$wN%=eqNJITK#TQjX2Z>1Vr3XV$2wp}O83InH<*m|So@j`rM9r`;T zBRz1i9I&P#%w4N?!lG;`r-R|T*>4R+Yija9@t9#*I|G?lj0I}2B;q@q8eno-> zOrYopIiUyvsOHN5eo9y3S#!=-CRuCUrE8xtygHN4!2p-C$*Y;%y+_I@j%doI*rAGJ z^6Mc+LWT(~vz&jiJ0>te86$?*c0uw1F{cj?spYXwr!9B4ZI8&A7@#g2nOPU2*~tC6 za3EU$J%g%A`->FBEz1oTTzi9IH7(K?WH?+``ntmr@0iM6x3}#0Zxp!PdiMpP?Td#M zELcu1@8kAhY(a*(1iL*kY)}u!I{(ng8X%3l5Gy#4rLG~!gFq|eOE15)@F%93P-TQ@m5-N8-_9{`` zGPIj`wfPd*^?0q~3>YwKfFVv3TkC8!0*3)&vF1)0+7 zWchhf>z+cV?2kF2|Ha@nf-l{#V|Cyjzkm$M|kB zY$9KOoA6);x*C4xi@6MgeH^N>2|0<9j{NEMGyi5hMLw8aS|DOtENB*lo z=E6I%Gu=AxdUvHk5n$`@b`FQ006?pi07ECuF9>dWZPMQaG*-)m_3~C&g49v>fc7iJ z$!nzb*`l!JY)M;t83{De4s!o=`U8nzCWJ$Euv;4m*X1FGX$dl)BJ(gTj>nYJMGbGb z@E|ts6zJ%oZ%@&d13EzftO`~-Aoe_83jOl#QN^}4`1%ar3aAA*> zp1wmT5bM*{zITqjJC*GYki;xM8Ain`!NeonqMu7-E>RKVi!fO!j5Ym zE|5YhI?41Eo&RWOhQ;QMZ`an7A7PDJ6HQ^<9edeMlhV?R-c#NNJKq>8Maw?|83dnK z={PqhC=XJb>p!lvMNNDUe09xT6fh>Oy zG@aXG6*@&9-A0?B~#WS{r|VU|_iQ z>WdISkESEmNiBRR&8C2h!Hz8HKmSa|BMp@UsRs^wVN~+c_!}5#{0g+Vk$tfFN>f0P zGed51N%x2(Q^&2)1 zBv+$c<7?19e8epDj>27+4|-dBytD`g7-HUdW(&J8`VIRFa;T~JKnMbXA3}+==i!TJ zY^j+J9>f&a0X|$~`#fMJ(~wcl*HLIpMg{nu6^jw=#|;}=;jMoOC6b$eNAmqq8N>Xu zWcZc03$%`iL8nZ_$DJ~jyImPVk(OL$VR6-?*qyPpmnGY>bLS*4gs2B!T(wz#e?3m> zuh$X|fKIp?;oG8rf^M1YI#PX!q)8R44ItoXO<0?ce+)7-x0>#y9>-pzKqwD zhJ*NXdF`X*zDEbEtjK+grU$(oFc}5?2{O6mtCSiHcI0lrxxH*ektLw4&r&WNv&HYz z)7*EWz(1`m%V2Aj$L|>}E?=QpNo@55dW@YI!%OIQbm5UZqmIjH$Wo+AJD-|MnF|L9 zVZ21-mzf4sI87xOz#`azNfF~!`K#FT>EI}XxLa>oHzFxcVhl3*V=QDAo7=}EO??gw zs*<^EYE{hA`Qj{!1DP2#fD;iFYw$D(PoVtTvKv??tv*Xd1p+9>AajRPa}J=>lohuQ z9YP2PwQY-eBS~pr4sxZ-cpvmQ)F0-Wzj+5Ez%X!rK;0wDVg08BEeJ*Nm&o|mquBW$ zP@fDiw;<{P5y5IJ!c~S}JfjLcr)Y_-JZywn6SkX$d1U#HX%N$v5f^C<866 zcao;7B*~g{)4j9q4y#DV=f^?cdq^~@Kyd)`S+`gsZeZK3Xnp((#zVTmlChYolU%p{e^vde^| z<{SaZYVkp)q#+4s3YVMhtj_>}P0|!ZkO=u+_AdX}%0TUMdw!q$N~Dmq`K(9|jWwu( zr&#=gEu+d9QKbGgI<(Xtv&_4&2;Y~bJeFBPTAb|k*-M@WGnR!7QoqH0bv&QweR!8Q z_d6j`QASZ+bUeyX%<7957)TGvYJBJiT@?4T9j(S5Fr*=FA^M}(?beHk^XFMQ8YIK{ z+Ysvc+kAuJwek|D-5ZYTlSY zwBrpOj)U-;{D8xo61O8W?s(lGYp4|f@w`}26R6&lQJ!183 zVhcB<$NQrLHo}V9g^c%y0{(p#D*fw*)?~(Lo%Jyo;U3h%=jPcQ;6Bqn)2+RcvpPF83_uX}0%%0o)mW!azu0s0CGzyR$u7c^6*UYGhY_VL9Wt z`hRwn)Z)HZF^wF=g%T9&UY^1Ifcvg-ps7`&P9jIppAI-%@B3HF17&1s9FwTRrsG9d z2Z6xDdyW|Ax^dYDzz+X%$a>;_P=7lvW4i&35{D@5Tlp6s=+6EFF7W?*8NC_^u3yo+Wutz|1RkKJ>-@g z&|vA8Dt`pqwtd+Ks@2^BZG@st=YqUU4xmhh?7>=8tiXyrK%H1U@7266Y}dAnIu>no z{)o0@8YE}>*5%~%^BKT@FZbghxl92*jT7QI=k3TV*^VZWnJ9U=BZ`Q%a!u~#olT^b)S@AIUQ#d0<@|TFzgGl%TCJW*S zX(Dn*e?H6%E!Ez zcr61Yxp6J$nyax@C#7hU{PuEVc6c1)7#XPiSB&FP?BeiUMqWXPE#_#bc0A*zqNPpW z$a9dc_mF($GGVcHo&hQiUFMUN-UEKS%`jy8f{qKU#h!eHVYK zGuJ78=-3j+_25SbU+7$C4-$0hDBuyK^Ea@L)Qe1^tbCc%(@&V8b}RN-7`$nUZGc>4 zYjLy=zD9m}thdXxA>JIUW5}DqyLJ@BRPGz9tE_8tXra_hUI^TRLH+AU=_pTUPiNat8iS#5T~UkZ*y^7YhGd3`=RpN(K)hOrO)J5+&QIf zSmQh%i925>O~7K-D?iYB&r+$6GUZ(W32Lg6451aL6RoE^Fv_z+5|GJ8c*x>SIc(=0 zQ98F9C%8nBDAg>B&C$pg+J!LC!>qJXJahB3%d&sLqPHWB zh6T*Gc5BUHxH{LTS+_PcJ(uS5I`j7*oH@wFKfI(CyfK+5V z$Y3~}fWB%Jz6HanaWK1aFnL&C4$u;XuM|GI(2ygCi`N-3BlgfhlS(nV90j!^t;|uZVzPho=C5k*jC$ z7uRUn*T@H%nupt=SufnR64%B$>~b7*{a`%aEIyn?Nt$(Ecxuk$8`^V&l`*cxKkR+o zN1Xx3P0A)EB9>lvW9Nx!dCS6Gq8H4gcqbuZ%{qWxm7f@Hh#6VD#1FKC=yVU?>SVn0 zR_I-6+9M_}0;8Slw(g4d`@(K-p)`nPKYp6=j+e{$Q{J1!))I8Qr1>Yjd3U1US0>wd znl)c_L#hvf%iV-K&uQb5FO>Xe4QyfPY3XB1b1?298D5*H)QK{$8apY|tCnI$o2*-2 zt7K*s10)=D|Jzz-*@;1@XaD-N?D$PKPXAm@)G|{iCC}pIqwxHZP%2lLL!A!7V>J(V zYQ9BD{=tukbQxwXOh>d$n(tCRocFd%=Y`#$yxB$E?>=+l|M!F$&>n5WB#k~`md0F5 z?Fpz$|59_X{D&lBt@M6~cC3phGurMVagD3z5;2^U=`12|tpK$<-)p{+CHb9^XreR! z{Gk6bdToz!DDr8(F#Hs_?vu`U)Nrm0(!Qt9(I5Nw2YG`D{AOpxMn&%RPgP{ixrUo}~Xc1mV2UkD)+w*JD2(Ko4tuOdz;} zcT70i%Um)7OdkL?erAO;vSU zbo+r|QCi_F3m@SRY$&=n^H~~MDPklh6i&7hXOQj)6-mpxpjC;^6G;hN$qu5xRZlmZ+VjRb zw)|+C_H0B|$&1hW1{4nOy=GCSh{^&{Ur}}2E7p+I;CNV^alc1dgM9amk{80@NVP2@ zQi1ujz8rXhIr+H`pU9p&!x8K#E%*)TKAGako-Y0^xGznZ4~mXgWC^i4ZSQ7DIGY&Iq+Iw z3bxTE*%w*=0#kxV|Gc<)Fatw~hM@ACYVw)36z(Eii`^kruMOI=wK%Pkd%-1z(@mQ1 zB%^ecL$H$KU4ga{AZ3Id~<<64IF_ICCjl6r0R9sA$Rb zrm+}O&l zWh};=<&XV>bdIK)Pu(J+Hc?uh$*^_itM)I$t`=cOkKt9)%q;$dS>8y=_hfRjY(fFS zw{tR!drU7?kP@70=7^SQ2bnR(B9=A4<627S^P>jD`EEdbXNdsDRwIW8s-oD7o_ z+5im_bcX}jy2;|of3w~WndS~hFO0$9)@)+czS6Vh?rzg?0UUqB;&X66s#4d8 zQxo(NYyeNHP(-Yvp0<8N1yy>@@Mca82xDYWvM8OIX!dn5y@qT4Y*MQ}OEH901LY%W zmi>B69?fNm^bn5Z+AWq$QJZe8NKad`eKs#Q!`gDT*F%SW2H*rdaG+x?CZ$(Fcdv_L zw9)(1`C_-B>zMY!dKb0}975Mk)#cs0S*}_>HR@$~9SA^13UjInT z!xV_3E_CAoQ}k@T92HB`FGh#srA&(^SM-BUfM{T%PzWML?PUC zFUO|5LC46exnf4{;FRKpyV0>Yoq2<8i>B?F^hdMp3vl)N?5(`B7tV!r(NlgJPTBl~ z1Ez{x8RL>R@cXovX^!(@c90M_QWwo`&qY8?SKoXoU9|cE!!G(jfp);1psN+p7x^R? z2l2?k6mC4ZlYl)TlhibOYI0=1WtktK6^Rl0)lD*Zp76z6q$mnJDV03)?;3g@`!I9k zIiRb-l3C9^GzD6xrXc1a{#=E_2WI_%>US>k&eH`8m9~Aim)2{}ax^8Y&givalf=lw z@zFH)oM&uTnPnLnh4NcD9yE5+R0Jjx;IQuOK4!RMm$WGrk6Eb-MSHsq38Vjj_(j~X z`90|2d;T;Ob}`kISSwdWNS*nc2Ad7h>vTwY@&F%Ag{K24OIf_o28icx;mS^|c^OIS z)lxxM8funQO>5}RIAz+k{)BvYJO(Mf%I>;ThaF8z5#l^jN(&_-P4`d`8{0p8H6s^E zog~+yC63VH{Cc3%{n1HR?z-n+kAS4_RIBEr%I~6cr0J$TD|A~bZkoc^&aqq)a2RWl zcY-=iMmBTh75`h$z7GJuphK&f33!=-+gwBMfZNC_zi&Y~aosk?5b?hoDgQp8(0JX} zv1r`e4k~Hb+|5Nj~WZTUg>3Z^5tYdV5*KG1GcD z=Mzt4*^8bq{mw(WQwh?r8Oyc+;6fe??py~?iBO}_NB?_le(qz}Yc=b$mO$zl%iG0Y zplTzSufi(`7O6gqn6Z+lfyV)3DLdh{>-|0AE>f}aViboe69}DIBs5?O<;P0J48G{R z(-b~iLLt@_dtC~3<1dRx%&a|uH(mSqM>{apdQl*u@Y!2PM(KMecJVPn-~Moi45Tyf z30f6M@2O2O1m<{GnH20=&pMqvna(W?{t#fYsx^aPjz3~p-6YQ&F#l`1GAya5vgwRz zpi~h-UHo90;+!J!BTRwa+6d-VsCMArOVZ0THnB3>uCMbn6S<1dY}wC9mklmy${>ax zpW-*Y901skARqZa&|-pu@-pIYlj%moGSS&swK~o5;25^{5X(sK1{r@rK2pi$hqCY< z7z2Q-oRR|gJlO6BDH-0j<A z^aKI14pV!F!X#)LPDzFQ`2@Y=@@gyorKg#7G~@oNz;kZN?8X2-b0}7e^L8_|CLJ3Q zXY!t{LbB;QkGPdf$Ae`F3}Y;5yQkwrWj=joc$1W@QHDaaI+I%2o80q>zdR_F*+J|t z>mPEI=M57Ds~{$E^m0h3E0WA2R!LFtjnH}2t9I&*$@}L23W2qyu6qw3o%<5PmHO^lpp)irgr)P<#SDaKDbw8 z`QcPnKR3m90H&O)X)>TvY$KQv9Ap}n$AL~ZnNCJ{ev*~@jhoJ!(2?eK?^G4OP~Geb zR^W>uhQTOB9nY+Xf`ulTQAfHurm`f{p$TR&MoM17_!>6LoCh?2*2;-V!$%$@c!IfT zd}qqj9S?Gbb2>29`4CyAh=5eTGLxoVyN$m$`N^g|JW!pRuItN8$6AaO9;UY18N3j^ z4~G5axIoA#+`|f_Az^=HcGpV9;A+IU3@fmck}@^8@6h z_{4KXx+T4zPel2MUL9)No1L&}g?39JZUM`w!Bde^S_QWIT>zWPu}^nGHf)oEBk(Sp zDS4hQ9bqvTr#xqwqKORb0)=ZY6AI9Q4UH)F!4F%o9>*?K9x!N!|PPJWM> zrZBcaD=`=FKY3OGdqsHddz!TDf=UuF2=?KZzlh)f{@3ZrXZkQcouUWjkFgIDvQ{=!aF9NVc}fXrK(NJ8U>T{E=-Y?xYA!60OyV z9{)*^7U0^J(eLqUZ2UDVJme!%-E2xV`dR62@|SV^+009%%Kr>5)Tso^Ep-Qu5l4+v zR_4YyDSN@h^D#>kbgJ3!fx51RE_9Ak)|TCl>rF*G zDtGb_oyy-b!T~hrfn(s-eF{=$`oFELdBa|2K%b#g_fWewP}*pKqr>dhH5s)Bx2gXBR@81T7%)jFScWX6b5c1 z{Ii+^vqI~3;u$IQBFBrHu^MH_{(wKETk0K;O8y#YLap|jK`#xKgz00FJOVFEo)p9~ z1<$Z!(3~%L2jF>062@ib9%RI6cgWlK$OH=6_Yq0`;&entu>oDn&BE#c1H4Ydjq;yv zfVKXxXzED=;EVfsQR=Hqs2TG_eJ26S2{){5D|SL^VU8tm4)jMkX9vF@H-q$4v`$BA z##i|rAP0AX!3>xt_v=g3|9e0teBYtKZ3MPD)|}cZv$~9fPfgj=<7N=bL4S^)up4f9 z_`u$hNvyEo>cp=&`YKH`;|GJEG3Ax!@8;EvvdY>NDmGsTa=DTa&Ny$pN9S;ewX?9w zG=)aKdZ5=AhbFdr`RGhq;A5@qOfg3%)@^FCIxnqb-h4KnZ***S=J*vME&&mnrSCa1 zEk$vsuOLqoOOqP&4mW+zcxfTF3e4-WI{>>O23Eqvf zWbL97{h%6Hc<`}f7!N9pIYF>YQ#J}R<4T_wT6#1~Z|M0@N~S_w zvzNZoiBmeFcyG`)&hql#B4<68(-Bv;r|l+(CYUF3ypWN$6oav9`9RQ+7MECH{?>K^ z2U^c(HFoYR)#>RxnJE+tOZxfx22#oV(DW)Z5M~-Oi2H08iVROKk@i2VHPn`rsK-31 zz{8!31UDcVNm^p1`D*lJv{vskBPZFp7MP~Rm}l2Yr|ExBW%GoewSTvI#k5q2HRHjs zgD%UP^0rz)qXyTA9v~ZyZ5t}eam?r=1V?jMIWzzQD$tV}T7VyCBiy7T)Nd`bYSCv6 zn|v+~zpU=iwEWve**<+NtSIC?4hX3dJz1akGe7l!gXQ3y>A`XAyf%&dw3w8& zs9{_4n83f=ia)zxW4}R(R!5yg6a@t3K_H zebw8dia0V{^pm2MyTOjAcr$d&&Co30+HluYJywN+Md$L*755lxr{!%Xxiq+nr`0n) zQ?UHVrj5U7J!Q1&8(0yXPI6F9So2@QaE48 zJoe$LDLN&Eq)ia1jc~ZhH3mo+xOX`?_Ry*@jLZ!-vg|Ft>;vl3F~Yx1j}(G31J=n% zsXNYsOb7{|<0d~Uf^f1-TK}zhY`*VU&o^x89Hz28^-dgWWL-Cn@hsaP^&uF-zep%_ zZ3ZhO-8ol@aed2G05C&W0JfofGQ~dNuCSqKYU_Pu-0!fH46fS)CBPJDFXchW+hp?j z&Yu@-{U~7~;vOe4W8b8QZXd!yVXYfRuU|^g3iQGZ_|vr23G5*(N%MM{OoTR|%4f}= z|A3VuY0)oLIzNB1CzAXhx3|)%GbRfO8vzB!iw*~nxDF0}9vn$ zCr#WmcNW)EYm|rRRAPlJmqw>$Cr~-t3Uqsoth2B zO&|1Qu>jWKSx7!b`9#l$2sJwcBctRtRnsG>7efJ?Icr^ti$Zq#>55L00yi k39k zkpC_x;y4XhtLgpg$rsenr0Fg{L(;#7#T;T8{yQ%l5`;nd&g` z9BL|wVw&z$6_&bHWJLQ0VRmU<#49>@TPMtBqt9BI3PUc@gTTa%PDn^I3?4Eyw%pWi zrEHWns51H55u;VwvsI|OaKy*?nNv7uj=LO$TSD=up=-2D!?f2RfTb)Ba8AVS zP$0GsMhPx0!bHfU;%gNgQQk-7{%Cp}2~VapdgFGs8BuR?{CteI(1c-bZa1|4p-nSi z$YVk@30o6^I?=9j>j{AY0V~pbZ%d9V8^+<)50;+1rYQ2jE$mAU+t~5XIv0xu__pDISS%R@r({sszUDHDE+s{w~u6)raqm5k0ir&=V;dNUY z25K`cppk;b*Ap_t!cv8=pPaqtzCDD^0C!qVPt69)1g`U<^icZW7SB8BJigcRi$9lv*}y^O5M#)AH`+_wcrMq$A4!X{prY6nwC(Y3|Vn1laj&F3<; zaq{z)^yB#8bYRKqPrFI6svuF3Yw6d|vJ$Dp515ADKT^hmFQRSZ=t15xo^OWS(4QJs zqK>EW%Fe8~X8`0BTGlONG?d2iqg)ltcTSHb8gQEIi$iNoiI!v8)-&?5SmL5oC5-a# z4lr|G!w3seM~?a)XKIh6A=K52H-qYTq%dV4=QpbDZ2ez7!n}MYQTbHE5t)7bkZ@+Q z_v@|v{Tr$i#~r|?%8^e(FNNl!BkSpt3MiU(4@@UV(!7&C{2uU_neq>r5NX_`9JmH!LiJdmeON>ZG6XTqaGJbhvk{$dCsKCDyeqnIY~1lSWG4m?3BRa)gJR z*1Pd)!N0C>W83g)HN&PTlRt26HhpqWWvDMhp=h4@mU#r3MN;*`IZnEeTk@5DrJVId zXDQJl<3$S_dZhL(T4Ljo3jy9jXO2QE$?3B%7MR{?EFuybW4^-owSBlv{)Jnq!JFR$ zWD{|ARC}w*CZO^+$o$^3-`H+=WJ)1T&Dcyh21i(smo(Kh3f8{(kn!^x_aRZn*WFS` z1OZGzKxX)bB`}a`!?iTG*l=}~JD!@q-iKHfxG_gVZtP&4?wRRMbdx`vS*sxJ-Sij< ztU4BHh2C)fczVBXuo?C;B`yK>x_jT!LviuS!A{2W$&{}6xHeM-*7t;?`L0}lfJlE= z6c$u-@c44Dsj zL!qhHOuM(I(Dw%wM#Z0{$4K*?Pxz0^pDx=F^^QhXKjK&BgtA=Z7_9L(3wrd80ihpb z=sm$~scHG+n(664N(+9~4v@L^z-@)rL2s$&R6Sky*8B>(4S*gbVz+_7TK^m8gV^}n zG{Bb9VC)yXfI#5CJ*-$Y#Be9;mG2d5f$5;c_qzyp89-HLWxNB^sR$!N%e={=d*Z>5 zT+^mLN2@eHT0v7IhT3b&_R7dr%*qr(0|X3Xjebf&EFD2Iif>ZVSC zQdk=7p1({d>BCv1Jk!M4F)L!BF+VP>r_U!^=VFTXx|=W0IWTg28c*rm$Or4w-2-Oc zJ6XTCZdo>IThffCWJ+F=_cB*3IL5m7w4p|~PQA|whE%bgVh&YAh<+z_bTB&uuDj_C zF$}7bj$j-(+^x5lk-Lz&E-OC36#K19GbMU6<=;p2^2P2X!{vTFjsAdvG%YLiqM^1= zN^COV1YO!@7C3Xm?KiVG7px!RUFJ zzK)tm=}F$QD2w}9c=_uadLrb!u9^Flc=>tK6eqFH1;ECa=eg^pn0+nD3AB3-^MLP8 z%VJ16!7#lB-nQ0piIymAax!!59{Tk|kCHlZ)=MlX$v=#xr$_uYJt_Ih@EL*T?9@T= zQheTQ0I-{}E}A(=|FtBk<*`0nXlHZDgE5@x+@6ay~NKH2f` z%Qw{AkTr>Hevz*ls^S>(x5nd8=Gt{E&ay3+r9F~ku6}EEnAFpT^_>bm#N*H^s9zq8 z6zu%7FO6>FVE7esBXUidwx`(X_xg0y9ke2t8BaWeadflAG#D^#>xA5Dm{cxrp{mA= zAALwh66B~)IM0yE96Ew%kbBOQg&}L!vKSIvym)53HbW${gNGov)AH+my5yZOGwNt8 zuqT5|jJy`hJzD0rel&joM_Qu6gv;+qPcrYYf2XENuR(i6hS2rk_t0PBZ)^ie;WtV9 zvfTG3pe&2qSPAoE_mjI)m!jr{nbO?Ibe%T~f!KVgXPuNBxu)n#oYrZN2ZE;I##+xu zQV_oKhGlm@?GT$Z&C6|+oS2cAbhQdc&hH^ydIDQg~JAm{PZEGbzHG}g() z+Y^9u;*n7LsH#v^0aJMwh17Um5W{&~#O^p#6GO)l5-X1*1QRd$)HWQOLR5*YAceh} z-PGG*HQ~R5)W-B}@B$Au371`(^0@u@6c5gUk$=qmX%*u%7wI{S6c+0bn}TILmour3 zB_LYd4(pqB*^p|z+hB%$9SMBXKdpb$THKRZ##6a>+xcU!u}wCac)Hvy>Rq!fEdA(e zllOKb@GpxS2p2Y}1FquM7iG%*r0+tm_4&p)tXDMAoM$y*0t1;LE79_%D+!bmKCg&p$t=vJ$U)lP$2u|4up3;ji8ZjS$H1&J2-nK-k&5AbF>y>U+dLEF{E67{7 zm747?v#xl<1Iz~u)kJi9`j~Pw<1~u*kKcV!@&!ZK7{p7Kcv%Ad=AXlED|LCIOdfH8 zUu&0yJPyUo;}sN`i@0?B zWL=qgy}V?;BDXDC@LeU7@0-*bffspKMRi zs%{%j(J~?B0J&}S=0rX3is=hA{m@tfZ}jgoRXGPCC~6&QS2_Y9a5>BL`kSx&H%J+- zflM&_{-ltN>J1H1q5S}8Q0s^rxcs9_;1Wxi+?0raGUCtY(fXfeC_c$W=eXBYOD_RA zaDm0Y^%&bysMSe**7XQ_0@B}8$rwu0gg{(mV$e&RvC`Qnz(xC8 zH3xY)DVk;{Y5_i2gwK)0MKoWpb8d~wntyv)yDw)_;9zU@qJ*MjD)LhK`bIdbL%tGk zqX5F6rb>TtlVpqzYN@9E4a;MbM)M^+i*3;EkTOF>T8fNbv-5ko--%I9gbF8jC3G1i ze5PfD*(;?r>?Fwi)k=UD;3gGv$how_@Lswk!iP==?R1uT(!6+Wqp3KIyEwQ*`8~2r zRwtsIFIv-%u53ysP)qJ{ADH| zFu$&Ioz@tHa)2bL#xRz)0>70u1_fJU4&U`;J-17~`>OR2P{_q#JT@@zcr@p!^k#dX z7Ii^aM*}g|R@^ZgHWm=8PdpkdV={pKa%X%qVJR9+YUlQM%}J#-9Uk;^GqM8=p z1~_HEbDQ$90sAX>ZdlSnD+;f+-cn%ZPd{uD93ibh#>R z)`ge+c1}9B2tmD(l_Z>l0*}t}!DrhudQ6KR;qQ+6eaHYfjzp zr0Dd3hT@wZP-|m5ppo8u(}NCioQ3|Ycic~41#PXDTEy&)Y1^Ojj4?z-L$g4PVs20R`)X~C->?<| z`)g+L>rC4wzNyV@5Q#EVu9q#hXMopsR=~4kHyObRvkG5s=gNNHEJDN5o~}Eh^1Mt{VjXIA#l1XPaau|twB4p{vFD_wQ8bZNNDUxqd#EFoJhLHdG$D|8- zi)!~E=2}HxgJ5%d@WqwJ4ahtX%#fy>KyV#AU^B{6p#>zkT1X@902TTfcB%F1y?k>4 zsp!Sb7`_E)d)YRhWb5|bBbx(nVe=1de5Z+z^s=`>I_6E&p1ZE>x9QAYPO4(k5*MRL zLOP|`%(6&i91y80y+y$c(nyj-siA5uf)yk28`kiTPzcQKG_6$R`&q&^r1?}k0uhxm zyoben5_I3;w5}5!_cz%=7pI|qJY@6@kLb=J+H>{kEtaDiM5DN;FE&jYF7H8mg$73_ z!TX9eL_wFErjHC{7GuFMCfE&4D5cbYfvVSWd(-#X6f3lr?_CFmOrkwZsSBgmLUs#U zQer)!eHeuFH&XuHT6V-+8Y3}XOL4*?Gn^5Qk`5v%2zaR}v=3Je%6|Tmt5$s!ba{9F z4h(hM;U%D6+?SHz_J$YU;&Z>#?x2=mEvKMlXByk-nIrwCJp8Z>?P?te%5-i`54s7! z$fW6&ajrE63Wuyc5LNd2CR}98)^hdSj%7zQi{@`c_|05Yg`-$0LB44qCpOe`+FF}nn%#~$qsumDTEB3W zBBBl^fMCUDAt8YpN6VFq4rOG6XcD6>8l@_CrX&Y5G`wT&=FB~~3hbxD3ATE~jHAju z%%S|UD`1Z9|64|E+mzFbhlh?oHqGz3El?@%jC?)&oH1x>J%O2xuA6-km09cYmgw!T z14;3*LniT1&FFD6Kp65QvZg&Erqv*GjqBI5I5>0_;!h8*gDT$-Vx3td@;Nja4>y$s zuRjZgDv(-bTUYLGSe@mDXwVK7zVrvy#0d)N=XK`STFd|kjg~2RE`#SboW{7o$BL!( zrYv3zVzinJNH^!40EMm2gUq!Eg`s~S<*H?-`#~UYEVIsS-sThypQOjxaiKi>d@N!q zU!Mo6ioIni5g*Q;Oh4CGDPY#en<`=?1wa7K441=qNuK9bwq>8UF1h>wN{nsKhd1WS zX2gXX2u^%-NQF+2u+?KaA0(j>!*IRTPt32k-j1s91Z1cFxg z*$g zWr|iZdU%eK+6&UK)l4R!{UQ;uGMbJJIw;v$z`ap7uts8&tHpej9=Beur*RHsH_SNc;1-&GMd=lHfHPHcNpe z1-U8_bspfW-(_bXG?5S|k5a+y@II3aYd?cyIk<tU=C+-9&*gA5)0WTN%2ab(6WiK4y^D?&= zfAQZj>E^Jp?M-Mkg4$%rozSVZP*$6)hSHImo5M9@Ge}MNNA&i&zm)? zheN$4!a@3K8cYiK*ge#w-_RlgSjwe^=qOgFVX#rTY74VeUV2ai$O-tdL zbnd%$MV^~)V}dk#iuR-*M5j3kgJ2!nv6TFY{2eycuigm)r`)4M=0dJOHhJH@#~>zc zVVyCZCvGtv3Jx(_8p<5WF=Zda!2vRvaB}|TN}bClzsNJ7%v`j!!cYEWNO=}!0W3;b zkMS-8BIreJ!Q%5U%)Mu~gRPRc(zL4xUh1b%cb zt@8(7-VT4knqqCiC}O-a&P|DG+qSSDbaB{x6##OjAdpnlt3pf>sHXMiIgCAy)7CUu zw1berRoBION!2n#`GEy{pk+NOL%q71`x+iKx5}HmKfiBow3XlU}aIOeZ)jBn8yf#%rueycq{ z&=q+b3WTKeN!1Rn6#}88@vPocSq&f%z+~T>P2uB1fCsm1rO4djQ)(2N47+)Tk8lSG zHvhMDY;!oY5(~-GK~u?t<`VFz%y2x4VJ>C^ZLc@W6Y@YeUakJmrsz0Ee<91?mk?=R(g^HjfeG4)6m zB=RS#ZHkw}u1H%hQ`uvCG>@+G{BebO`SrleI`NmmIF2UZU1N4Gkd}VV8XB*=?1Ovf zS?+G9UHD5o@lE-b4AB@gqIDrV z)qgz)3S??Goj>VmsE4wxf8dm5#^`ooJ)`?i@jW;d-wQv3rhL<^xK6Gf`Fm|AqB2aB z(e$n6?w!{d(*X>p7aB+d+bllBqo;ks8~}kP>)sJQKLTKJB^#XNOX|faG~JdOjO86o zzQ|V~Fe`w)ug?%_@QzDAwy!y$aTvDZ_8yImOrTU?^Sq4iTX70U@r3;KN|WjTGA#Sc zVV6#TPDCF_kZCx-{sYl^4puLqij|0i;wKe^PXU(N+t28#ehcZDvcvEIIQeGa2tB8W zYIBbu0I_&X7GIkZ%l;~$RHEshL^wWz4rSa%-|%9obdEzJ8U3Vcv=G8Hn@!W97{Y)@ zq(FP;*T1pCJVH&=b0BuiGL^|vu*!V-l+Gv$;-o2(1{&|t<&>!V=9eEE%AS8eBY)p) z%;FP^$%uZd+bdo22-AF6D%P&yGk(W3PpN9FqA5LtP2DG!nE$s))p4)=1j>+xeSYuR z_~OUP8FKx+q7l++gzSV2C^uMzF@R0FA!Xlpx*#)?+;UQbr;!zoB>Y)-J`|Dssjroi zjj|9y$XhkS_92!j_3b42-8$pBs=$mfYqso&f zcon4BLl&g}^m0#h#D?vaH~Dhw7I1!1F;^eVS^YK-^MwcjH5qd|Bg6Ru6#NmW{N3q- zDa)Zotv}-R!3>ntOs$L|+7xc!!}IS=-D$X3zp1k>u9B}2)We}R9*1S{>Rt9l34X&H zQ+aAv>Jm0^?YeqkX!I2Di6tnIf}M6bh=fgsQo>EOsoaYk*?22HWrO(@s?Pj{D?7|T z*KiQ}trN0C$nuraz+@-VdG;f$oz@l8p12yETdE3YJhkZ0^j%XQ2ZP?_nfrte-x}it zy?pQH>Nc0&Gh>ON`XbG)YyluiBBD@=uuqeV^gRMPORT*Yis8tN zlT1D?s!C#gbktwdqj_*0ThAWw-;w{0vvw^ik7t~Y`3Om|j4#j^gV=*jDP{qH3^RKF zRG=Hc;QEM$DJtSP?0+;20DnbM$N_PBW(KZCtF8_bgzk`z+iCiGGiT6OArYx{mvm+AFC`P+7y zyeuzl+K`oCYWHGm=(wjo_0(%qy~`8CL?`82&JO0;jJN=+vv`*6M3S|H7r0z5r5WLz z>@ct~f7AUi1|w36Jd#d_YOUv6whP;~p7y;!kpJ{5EL@RkS&IbQuwJ!siJ;^!WYR2L z+GYNnkWQ4m@R$^A{fwF+;-)jw*E>#n!=PtUXZPX;;@M4ZLb2GS(@do3d(Y}-oeh5i zCfh88v^>mI2CEXXp;czVmCSV5%!r@KJ3=f2R{|ca9nq*?yrXbyT6~54+wx;-80_p$ zC%EwDek)+(W45V|wP_=raZWFe`(dX5`jw3uG?k$al4q@Zu-x&Ir^+T#HkjJzEh)&f z%>5=oqjq zf>#V!pzne*ZEK~Dgng>BmWlBl4JJ`M(8e*^hkpusy|oKxZ5O7y;yE1kacHV4z- z5h8zOhSM4D!g)Y2sZudK@pYo<-zn34^_22cDtI}j##?G-YWLD!HW?Zta^}NS z$!IN?f~$AdgT?pbc&pwRNSw(s^C>vs<-r8=trR9-zgOtZ3x&ajqZ{UKI~p@ut9=qu z&A@}Dg_Ehw8xU!@9i`d$zS4Q*3}S)lYoJvx8RrVBccT-KtwAKM0tX!qUy?1~Rqh8_T@x z9eg=R8j?<*IjorNAhyI$yD&r=(yyjuTgE>TwYI{;pz;%FtHf*(y}hz)pdGMz_}_X`2*O?B>x*J|j# zK+q;l!wt3Ih)lkx8TDm`_<;ji2m0;QLF@}$gRJr-;)^I=9;KQz(LwhLKXRmX z=?*=%%|enclg;kpqi6nxpd*W0h>Nxuu^pjL&@QeE4?!W!qv>SRYGn8dqeYuGA6b3d z6w-_rI)-3WNbmlowm2{OGt-f_wQM&eiH3KLh_607XaB}Gc*1xBkM6PV_%;7?A!vhj z@ZlCDzt{4-_d&?7xUKSiAaJ{*=_kR&VnMr?-^N1Wane1aP2w9xH%w)wsm<`ZGyCB?g~)*!cnVTY1nlSOtH`?k&wSJLaGR;cY_vRM|>C`FiNH53P^+_UPcCyG;P@S%nncG{>rP;q>XE+ zaUj+?ac4VHyu?3#`QsGfFtT!{bBDUGt`3+QAaIIk<&{VjAHP!6xofW5g=)j-zs3o_Q(k8 z*nV3ri!#q{^IRrA%MuW3G(GpSE&rQBi%G70s39r}v@DuFGL%#j_izbU zr+kZh%r`1Gyg0#z;j0J8&zNk{euo{FbefK3#5%rk&}D6YVz-uxMpi8PliLo-+k--z zKMaIyy;pZ!EU)-?{7_>klDvFcJ9tbU;Stxdtz$>nwrS{+N3MMPFM_2;sLbNI~2c z%e2e$oEP+RlfStUDKJ`2Cp*q~aVVL=_>0K*ZI8Jk8J?p7mj#riGkk33!)=Z#)@xA1S{Soxh}f+?s$c zFsUSa>AWN(iE%u-oer<;;seeYlK0UdDwXZ&q=VgtkS(q?D6l9B#J? z^L6Jbf*jlTHybcnlNvgG2Rt;VwCj+$5>7Zbl`t%tTYTf9{f@=N6%*MN=AV1^@)j8A!?A0u>#4z}m%d zPox>3l^k^C^nvt~Yo01Wc_f|Yi+56_Vuc6CBPY>#>OqKyJ4r#VbZUaR&HQ`0Jt889 zt$qoihIE_n=XfFk?V@Y(ta*>+!S9Mq!3sB|&R(iRrQ^eHB>#QNS{ZwVNGOVqCBMt0 z=7NWZeTH5xvK(Z(sY^we7urQYogXzFF-qU@IIyohx+awGfh1Q)B(yo zk61JkJRjVvpJR1u>~l;BNf=hp$9QQ=N+mFe*pv>nL$pfBO7i<5B>vR|sLyKY}h&vDd!JABtPUCP-9#$0!o z>Ca|1j@QD}@-Ve+Dzr-8v|1%!TJNFi{IMFV)3rPp@xtc5mKo3eLZoxpx_szAxc_h% zv$28ZmHfASE$_fWO3^KEB!8B1^LF`(g=6YJWLURNuY4LmXDaCy%HN7to92*nkVlIN zFQ7slC`=0@p>0#kb&TQHa3YO-vsV7>XT2AgT-sR)}1PFvjnexLn>!6FDQCW6q463e`xAO zF;c%E(;GMICTTdV<84qPAG6`qojWW;)!Q zhchi{rsKj@;>e>p+{t@bdT;Dz3i!6pyIkGWu2RYEgnAD0HdcEi96QKOR#gyZGZ?h# z^hMN*`Ir_nc=;i+_nSY^XYcmP<4{v;J5irDA~H)H1xCGa1Cllcf?}6ucF45RKCXSK z*7FD{W%gg(Ko<6q%a%{Tz}uMTxfk%(Dv(4=$84GWma3MO&vZHbkoc(^$D}&Wasz_O zFUa3(x}lG*(>C<}QKZloh@cg0yYd9eWjnIU%aGC43UHW;)w`M70C(vq8En~c_tZ|+ zgR}G?no8LKXy}>nUyO9mhG!OrZX#&Kaz;*0<$!$4^bbCUZO97rIhccG1Usip)!Jhl zP{t~X;=Vh{x1%T>3zQeKRl3Y56G4_(q8_LFG}ZiKJxDyv-gerly!pC=Y)bb3>H1DC zJCL|;4Qk{?K5>zBn!%7U1=Lj1iEhU4x3YjlUe9IH{{^3f*vY50?+yvY63&I)D zH|U%m$aUA9V1Mbg{k+~;ov4yCIG8e4~@$}R2C}-KXB5zsit5wS_N_jDoNL% z0P7ycmHn>T6_W1(lG&k!tCUz=Pq0bg_R&PmaWSXh0 zP@S1R1Q(mIlEOA-o-|z7!dgR15vpHXk;Zf>ci4TMfwx$%2%=48B4*S-@{3Av`Nvb! z^GZc43ehZLzU`F6pc)K@MfuD+sS@L2jTwzJ+O#T7oi1EfcT5PHp8^}0c3456hDYX` zj^*d%J7X;YZo{&r!VBQwQ~c&HvAoNEDvZZKn%U2ELuChFJ8&&FP@b2C`CPFKCPEq^ z^1b{9!a1|wA=KD*%I_TQ(fhwR;GYNV?9!`2MRcA_n=BAx$-2w0PwZ-~X0lkS^Cj%>_7(r_?BkLT4h^@qjnQxm= z0cwhf3G8_$owArE?bau|2-CI05MQ6GMO;?L|3C zWbp3+`dM7#C8Xzbr971^1iZ*|oN$xhi8AosI(4PWX3XC zmV)zAR*SGiY&@Ahhwps^Z&Tc0Fdgh~KJj?3T`y!P9Or58CK-8cn%AHK-j5m0WaLd& z;|+V$zb1{_g ze#sDcXf1k7YY0+P_L%t?;73X~Di!_Z%8Qq=ghOtdFDMqs#3o)x&kR781jc-+a`Abq zY_BvyiUKJ*(|^N-)q8#;&OT$99M{&P!+gx_%V@ebl4g+A`4H~O#?vT47?{tuEW*I1s0M`hBFSkO8IiUu?HXSm~Qudh}|&~50Txn`@AJM(s0vl69!|-x(rke z0amSNF)Z<>j9Q2gHsPOjC4J4-hqTF6H7i}qr@y)4BIATHaxx~4OFhwrbpXKFiuD@f z{g*+wgL+PFKMf}e+Ci{@zWSUSx?jw3zON!fN4{|-^V?Q32s6G$>)flikcLAyy+F!O zJ(BgJ;9_8hmW1FINZFD_Xp$5$u8mv3?9E8YP7KStXc@_s8djfE-X@tke)p3h$(#_+ z&4Ni zi=^>1P^_%#3!=UlcM_q;-#UZ74d&2;5&wH4`cwEiUcY`Dk;pQ(nxZ%>Bmnj~<@rj{aCrjFZzyg=7;I*I#ZCxG zVOqRW+`OkkxAPe#cC=72_=#G&u0lK&;d~jhIIq#$Hh7$Aew>0nM5YHn_vN+%%iZ7Y zJeiN5aee}?V`)as>|OmDl>7RBJQ1w+e0Uy~QN!0skNKq=*-Dt1c#Y)}6EAZ&HJG4W z@aKj1cTpA+%6`5v->$gs!7jXE7>%F?fdwW5Bu<*Ly`W&F60GvIBubki=X9onnefbg zun8APrUK)4y_ep%+;a$mHf{t0m`S3U@xdI^2~W+=D@th6*wJwnU+LryDLZNoiMWQ# zoZ*Zufp?(WJ)aXD*Ar9V%sEF!qv`qdkf>7Z0jXet#+CW3HKTiFAi#MEM`oZQ!%!=G z{fjJa-y44i0S2?#v~N;*^mRDmxM?Fo(t96(D&+$0>0ozCrZkq)%|?yG5PIWuZgsaP zrOLsy!k6knT9 z80I@;WvnfI zMbC0TsB%V%BH^BzMc+N)I-l>RW{r1LfJ$4=*$X?R5S&5ar4udt?d)r6bbK47AYlD* zvFTelxiHPw;zAjq{8^oDxAlnn3hg|()huv2J85~+Qp>dM&0czO54gO9!nKf9oEoV{v!Z#ie&QP&MZj%zeIAYoRx}yD~6lg)y>7Ne_=Zi8)^uOw&X%D50 zkP+5Mx$jouH=TeV`ZyF2%SjI=9}O8GV!9XX_^=DEsH%~+rr z&WX}YLNkRC?va^Ws9>G-=`|FiPUh?sBwWJ{>k`ELPU&iNcvmUzL*eHuw$6r<= z0Wl&qn|9lRqSQICU+nRcfVtj+VYSZQe%-cCB;;eBhsW zL|W%qDKoDzRtOBaFHdC9E=Xtol<`%BY{%%c?l*Bl8tGsLHh>0BjN#|XIBj#a&Gg?u z%v+;XXX4+Pl1t`q{F1|qyKM4TtpLgfEc+N!n3Vic1$j|=-4gUbmTm#2;(Zt0IgKrm$a<|`jnrV8`>YI>cV??X8Wp1X$a z#X&CyLdiA(N?KyLO@r%Gr8dw$6Y;9HICRH(oM0(B6{hy0HjS&&xpKqGz2%o;`YGzb z5`Cbx>rIXR+SbWGq@SSWb&bOR1_FCrq~X(NUeLqg$VJhbC)L!FBq_^eN_bHG)PPRV z-a$KHw^+2*X>1B#ik5cJSmGWO=%|9k}km!j=QAdnPvLY&6n&g4r&O{eHq(%33q9N^H#04 zY`}s4s^7)uGc)!#S(*V0P5JF}3%gs&+WMY)xo6dDhPH?KVqENjn|_9%Z6g}AgbXpQ zfY>Vtjjd84xpifkOb}{=qSovJCAcZ0Uqfak4~c=5nN*x_rc{QHc)HolMV?td+7dcK4al2RNE!vWRiUUZ7qhNZ;2uG$*_MW)-fi1 z8FA--Y=cHOTQc#~hkPGls?F$zTpGqXzG-W*iidtUwJK0{x#>F_iY<4%Y(597Q zy;AG>*@u`d4o0~&UI%-ADO;uLZrv@1B_|z$D}oh$dF#frZ9?c_-T{x2P{MhB)I!=t z)~1wymqZ@iaUvR>#1G&qx9AM?k9V4e69_TgDfg;G%YrW#Pnt1SoWFepFUq$-|H+W$ zt#;YZCKx-1QU*hga@2!Yjw!F`JiOfwwwMp~?iqK_(`O>;~ zA9Bfr%O9d%RUl^v%zoG*0%8JDsK4u(2CLyqmhTh!zWtOPf^qNIRUiVM2-UF3pET33 z;^u)=A}@@xeFK7AwZs7zmTNZhIHnfl`0i^)~Y1< z^4n_;$ngJ#^6{J-7^VePf-S%Tg?*ag#~U3?S@Nkne2(#P=zM}kRv`!%#j&0&j;28= zIF~}p&0k*tku{V*kwET8(+eNXJ05FBB9||r?!wIw${9}4^;drG;w3#=euQZ^y{A}> zYCvz1b+plFlqb(+n}%@A2+B?Qf{c7-&;9W;Yd%QnT^;7zeKFprWAK3F+qJdXvC^rN zLOq(-Wg`RsG-drS@wC$82wJD~TG9XUm>IoEj&sbm>(@EueMev-E%T635F&7Htw4yl z`f$nYXt1m`+bqEIpXs2&r%&2ce&+zvV%hD+ff;in%-J7Qm^a zRm#mDbJ^6k^5Dwf~ z84mjJLbGga{^uTCcn@Jhs@J+6k|rEV8N@G>-UQh2H38+mZj!P`RBbA#Qh(<8j3ioJ z#-GgU`%CsD|E+cMu(o_C-KG9ZW$J-<*ZtIR(x@kt%qW%%6LBCro@}Vo2#nOHI6R1b ztLhoMD(MhI*GQapq?kJ|UlDaSjC-uMLx@oS-}nSFkn?F9ufslh-1qhkJ`AKdow|eG z6@XmR*2@@-($D*-^Ne-g@jL*~bWev~v*H+cOn89RC~4)IbMdSjKGo-gX7K7M2UcP) zGEt5l?~zXYW5Bd0-2ccLywT@t{o#CL$Jqhodu9(eMQjwj-MJUZNN2~HVCyWoNGB0V zwfZ&dLCGQ0=?eQ9xvIeTj|BFO<$Et6{1~qkO_C*B`E!mVbcc=Sn%6dgiv7iMRxbP$I2A2HgrZ;I0iI+rJB4g=nCQFKq+v!EoIzMVF@HDd$r7arE>qw z;LCSX$ZDXtNk>nad`A@A;>klVVZ4rohtYLodIctN3=`PtTU zl@47*!})X=&Sat92(+M#8`K!gkd9NFvhc(2lgSBTn^dZIqGVsCK6;ay+v`_Otl01* zm`0QR2nm;Lom1a)GCo{JKR#qV$*kDL=W5hr;;uT=y*v2;3zV*V-!8lS*^DfA1;SS$12sEQ*a77s7}KUe1b09>6kn5XGR*ok#1CxH49D(m zrsj4!{dH^by#8YcnKK_O#xM+HD0OaN0cjCCs6Pjh(dBovQf^7UyOxwDzh1jf_Ez6_isN`vao@AE|5F(kxEbKVmr zT1r?qrF&kdIOO0HY};vucLNyQ5s!Z`%4lb9yC(CbycgG{uvBq>~*5gE|1CFE=!8 z_nUQy7O8J9Qi*Lb@H|{Mj$Tlt2(0ekK&eYICDuM>-%Ja>zr2*3ouW_PNM^p8Ehx^sQvxP{2_5qNODvsOf++==dfYgJQ{}m=SgYZm4NUxZ&y#T<-3;rD#NKCsrfypt} zNhgX*q+o~6W`#6JfA(V^)#+bp+s*G~k`Lm#Y<`SY{K|;CtQ6B0DBmHYEUL5Ll{=Ay zSgj+GTnm7SAsY>GapeZ@5Qa92zh%lUxS8*#RsK>m<$UCxs-z; zwJqf42{WU52;S+bH15eR6(yjPhK#6vbH)-zmr-Zi3Bb+i6c?a)1MVEDTIToDM_#!A z%Mk39V4t8oPOq(VL0klCev*X5;xxs6q{LM^vsvA4zRv-kHjt^w zRRG+fV)ED-o7ceQmEC8hJJ(6yxH+B?cKMp&SA^#0Pw#b78)my5ZbptCqxr9OUJoz6 z`p~8}tf~@)HKIiGz`^ zP}6VA-OzX$T4CmnUx4Uq%Zt1$8Th;sg9{j)bO-?Rd##%k$0S3!3UMXP?2el<_?)No zk#u~6eWb~PVhSp+l$8|E8j%aCZTl@CQOS%OKOvOZtbJ$K$=`KeY=*nD=}j?(*OY%$nBOnW9}Im1h+r0R`~VZp zIE8FT(_oFk zTC1gBlucsMmu){}>MsTJucxd-9$^u@ekrgh?c2HyDyfm1v!>bJ{EPdnsxlmGs&8V0 zXp#zdZIQwApjQP#Viq}d1-!K#BP#mW)NDbfs`2UkS*aqWDp{&Y`~2*V3TQTI`4KGW zJ#N|$svCx=lb(DqjVHZ1eack4vR99#EAeQ79R2)AeClZWUgIv1Qyr1J!_0?T z(QNI&|0(Ruo~mq?HP0X@;DF$Sq9O_i0-`9&pzjg{WE2q;XGKtO1Vm&|Sj@<*3=SY5 z2skmxjKGShj5v3F(SJza_I3Y?z4wWIPIO26#TT-0t@T!AWj^_2R#sNHbi%_%wffA_ zix+B)pUi!pLZ<~7cBO$ieuZaDQUd^>-_K8x?qYmBz)h@SMsX97wT^~y2pz10j2$tzITtc;jG^3VhHp)OM z?umdr%~{M|s>w%V^oQG-Fu##>=>rj}iYwG=fHsTn0CKVGBhc-=Qk^41nV1zh&$;MG zjGuG%5CAwom(L$eW+518XEvBl9z#()Y=mzFg?<)RXyw>@T(#Vl;iMcx{kL{{LTv=g{Z(ionYs}}5KfxcH-@E2&| z@|Wz1_Lpse>UVTLu0)Me6aYYs+Vr&dV1O>H$%y;*qr_ua zHrF-Bygb0yWoNiPCqFf+^Osch+k;2L8i22)5X0h-Qex&b>O)UfUj0H-g&Olf|{HHMP?ijEn&kh zYL7_c>f}4NGg44~upEX7XC4Wa#S$mj*EkuGrss*g>EkO>@Kxmnz!kDUz%rD+@ur?0 z4j<(Cu39|2v};n0(zq#T1FB@C9|qr7A~-x@C^ngVy|Z`&AhkP4ejNZF&6qtvwOF_V z?V?>c2&o#*zSyckZMU@Cv42ts%0K3$V~MQj+jE6IB9eqFcCYkeE}ebz_{vROijAi% z(4FBm0-<0N=3-8u-L4_O!iiSqU*~AsKm5QAivIR0Z{Nn1OUY`y7$pGkTDSo64%Tjg zXr>Yifp@S4OrOPBadAinV6K<(J%7|p0h4_O`(NB~DIcP#x=pG^Ujf_<#GTts!bRT> zHN>tT-2l~cku!M12^bC(auotV^jP(8Dq_D_<~U2O`$LSAmeZLHpk+0r6#Ow8hfZTd zmP3i?b6n1*XIy}WHan}CYdY{E-HA&6k*_hbvhNH@6bz7E3i*5m0HM_S16aQhWWQPS zK{^~{JeGdddXwE5`oZw+T(L1|9Lg!s3W`GG;KwO7t$$I@rFTY8Tk=7jsH=Gwe+z7`>O z-MYk-88N1-DBvHLY0+nR0ohs~c+L+StrN{!^J-n&`1x%&;l3G|j_2U)lR>wrfeWYq6A0*fre!_+KRr^b=WRWmjI*+M zGQy;BY|!l_VntwTIKFU-RQ5H)q)t!E3l2@%$*Eu$95rWwQ9_%IG~IqX1Wh5Luq1`vP5!%A4>NJ37d6N;r)|B zsj)`!r7M8{H&BDkP;02xa2Ja1$e%RF+sUFIP4mHqKY^;UNPfS-MD>a8-ooQ}<}7Yo zz2yoKN6`CTTZO$S(e`pg^-I$+9Ag_yvOtFR!P6%5c%mf%Y3bUSHh=v-z^m80z%^1s zn7$O8ZeGXtz-PnI4#JwdxH*jWM(G2n*)O~`hEavtk=DEwN3@lB_%K*fU;LDB8Z5XB z6<9mPyYT=lVzn5gU_0W1(5koz&V(W#lkr1pC4(Uh15-b^)Xzf9O04-%JFH(@wa;N= zL}u8J)jKuPmdN-2P_t__{nKcg1)2Bp?eC1^<*G6@;h~;Wpv+P{I(?e=jvKyLvTaj* zAjem|<6g9d9%`$&4Q~X)Kdl9q4`jQ2I^-H(MU6kv)oIge#b5 z5(`0@|6c3@P-<2mZPEQEo~NcSsJBnnuv4#OH%ru@%;%ZV4}$k@nO>KwMszs~>D$m3 zFkoFmW`n#tP4uaF&1HWR(jTih6iFC6mYnP_b0|7Kw)^4caMgQ%Ee26DFQvzXdW;%J zPNg%aL3U)RTt0D4eWjXRQ#9rQj325^yKKQB$io3l&Quc-Ap(wrsA z1UL%g*+fwt{@4OHkwCh$vT|LciFroJynPC>#lf2OPX6?zvm1qLaF{30skJRiVs&~s z3vP+Ontoq%d+uQIe#W@oYqEu{HrwJ1b)OE4OvYa!olj43_8l^Ovh7tr?*T?V{H56) zmoN$RB}Juf)&^Ekh|37kH$X~ZI`^?59 zBX744+CUM6raPV3T&1h};0yR@_xV2bh%bQJZdlVas33zY=XJvdcGAnaRAzvNt(iFG zdVPsAARGnRC&>R61T>Y#tIi6myBgteKR;gKT;6-?Rg<4r_9Oac8%*SOt_|AG$>tYE z(LQAT!y^>vu#~xA9`u54gC+=Vte-j`JplIKSA(STjuTO0(6Q?0p#)sE7E^?*dS-Ad zC7xv>_(8V!X07I{>6c34_Pij|-RvLr@FJ_Y!;Q2dsx__=@|V0%y1d_-wX@thuYO)u z)%+efJ5ONBxkwqdBHV;`AhRZ)jX(#xz9jRdBT7v_is3{4)L|i+F^EA_5hJR}!$z?B zYSrterV_Tp&1G;)1S?VhTF9aPN-?SSJeeXvoJ6?>-DEf?sF5Wudj}60InD=e=k2h2 zlZ&*IGGw(}YJ7#%!9?UC8A;d!K~NuE)k3K(6R)?^x9?l%neD17f@mTUZ2m4?IuL!c z@%(ivs!-r`46|K^4>0IzBMDvX;S}^iNPq=p=~d<&7fgHAh(A-tCf@`>GG$o5)v~Kv z33pZ1hT}O|3M<)Ditn0EM$8TB?`>Ko?@&R9W?HD1*#q4U3xYrDBhyVnsQLytGgZV~Y&|raAF~RTp z;5FBwk}X((w2XOa21(R#i?=Ce2-}K zh}7y>WY9tTxe(P#f4DP=+8s@Ee**~gp1zrey}wX?VCHvPJ!!3U_Q2jy^u<}ICvQAQ z1ZrfGBZ=B|Prre~!({1O)L)!FsMkbH+b2zwugff=swrH%vw)$x1|)*`li(`Nt48lo zaeJ5)5HsBhJ$%dB{NT|bC+DCmV~8Pm^biYD&okZ=uRvG75G{G#k9-V~|9gOAK;CaC zw|T1up3?J0+ZAr2A>94EK1Z$l(=7Q@q>GjuQZpxaU9!T@-e7c1GOs!!n8MXv`{S;_ zVt(?$7O8fXYEIR$t`_x?OU8TXXi0!URsvyv$#p_#7O9uz)O9pj2lsSZp$&@|4{*>L zq&h>rF!sK6@)j%t#dRytwYp0#V<(PNo%i?;n1agX!m})VzQ%8DY1GKF3SOn@0*tQhxhxqAjb&D0_YrP}O(ttar z`xheriMXV0n-|R> z@der`L6%0ItLg->kkiK2#nIcfLmHIX!#M%$F&L$3^3Emv8I*24PoE*U81+N`yFE7M zg2k)&2l?&;dJU_;>4Kj9IwPZ`4llCuuB!g!rsPr>cj?Wo*KlF;LXNOT^#`+<-Q42$xL97FtYtvIc>;MGg_)J(i}TSy!l2V7{~x6uKe} zHeRU1iga+Cz+B`mb8IZ)dtDjA*D@%OCXKSRH{+F+Mx|zSgCl}q&?0%0jhZuZJ_x&L zz*!EJgxw&vs-?_7)mP0_=tzPa_QI^C@;pSqQc5EXwMh6pz+IpHh4OWWd`~s9Ft6@Q ziO`=W%2oSh-v7C+(5zX#-=XGJW1jW^C5}O=zv7KvpLY3XUmz=1GHiGrxJIZrrY~WG zn-xbDZabkfWBoGdYC}~&t~L^{ILYK1*Dj=CEBMwfS(NuWSg296$q!_49j1&+{fU{r zxAKDZ;*>kNPxWjPqYo-9)KRy+&3RZEW0IOvPJZQKBBId@a71C5@V1Uql(-vlz>#(6~6yI~e-V zJ|t0CeBwkQo*EG0Up!u-W#oJg%G*;ZFIiJ|MEi@>=c||sO$3N({DU56$7@lA(coWx zB${+0h)t@q9@f%*mGsJCG24)@r8k(cxd+|)&EGn%Uq&8dic?SbfNk9uvs?abpH26ZA?%y{+;R8S&W{>T5h#{;5`WK~;~;hLL$0gTauk)wPVZ z&(*gpv}ye;^ABDDQJ>ghs|rYUBdV~FlJAkv-YGK2ukmw#t~lUi*uS$#a~4QbgJJx{ zy<>bMsMRtAs{`q;-&v+B(c0fI_GeTG!hUY%{(zkf;}p zrzI^4BUJg{s9@b>Vl89 z;)z3?3x0(8yfY|6pAc&3!oUs|q>iwXierd8X~FRb^58#!)fIT{C1S418{Xj>3@=`d zLb7xP!;rJM>yU<@k+`NzRat(VLsmLO9WxbkAITnV#lzV6&~AFvtXT+E&CjLk`Fd7o z_uT^N&kxk!$bUFZPBE*mLWX%b48WnnIRRIeO?xvt`i$vGG@>JQzN(rt1^Ro(>Gvv& zJ2ewL9eywA4rM(?2GR5K4bv&~QQ9!d2*&?gKGGM9GIHtPaZ{(9Mk%84;Enahfg}Kb z4r?9I>4(A0Z3&4eKPDx=Y{8g5)tjaLLJxlIBaEXr6nWpvE=x7G{B5-CH z>ibTUH0kz_#r_=XkJldDxGQaYJk{JS0D2!Y;aUQ<@Ip_4>t0mDUHMk3)i|7c|wTv*sCurrrtD0Kg`m&>tz_Sl)fUFDZzH>3(H;< z8=6@0C$MMOh?dqA_6UaA=QL#8(dj8WYUqZ8dT-6ZzhZneHL&6wr8UMt?gm4o$b!R3 zgnR?*bK=*-VT?O#cbv5|N~YPp7l3%n-Mg>Q9j-x<+5AFQwi@cg@g!+-~ zZVjgN|G~jdI1+D687?<`k@^Cp6KH6pZawlGGyBCMsy24Dp!=0V4rb74$EM3ifh$-X zi6b!Q6UfQ#$0rVDz`6SJQKe)2di11?^gbuxOtJ@&dY?1snZ<$HmDiy;e>QMaZm@{#!rru{up3;0m1ef;rFHX zgwtB5NBqP`=FyG9QNnIe7jf~lz_WfrPdcS07tJ0m*YVa2ey1)_H^T9CNqeGtve%i| zHB#LaeZh1H$)v}>B9&AFE;m?yMQ5vlS36C|fY}!(7zA8IKBTJ=KdIWrY95vrHOB4N zGHY#;`ezJo*%v-CV;rGcCLoj1j2RJI@Y6bF)(#Q@u$8pFlHd8-U&*I%_3dhOp{fH3 zf{_OlWKbK{e3+-TJMSedwmPi3yl=CBlg9mL>aSzm0rigaRHYfxf8E}6GEXh(&6`!7 zDEcVBVx@2M+7at0zQ58w!VmdByNsh>jrYPYq{O^tNzD29S%i)2iBO$t@H}!o9dL^# zbkM>Z>eEiv_^($OWzzf3Z;`SW=h+J8P7sncw0K>Y5-laW3n7c88^J7D(0r^9rm41+p0fN`lL@il`;v)1cZf# z(uqjs93e!%iBq98pQYY&eGDd}aq!biRB2Y)ZfGXhsGGW1UxHly;i}el_5J2Sc2tO& zE~5#A1vc87EZkLzHU~;=m|BT;Hi`K_#ce>Y*Ty96p5;%Z)5g)X;rSt1 z-cCH(V^p}PnK+#P5S&3bksUk-w9y$6oW2=Ts@|tD5U%A}QFABq3b2sxFvS7P=^e&8 zYwTxeEe?sBO^A1K@B>OW zCB62tXk-W%NYq~|cQqZ0a3f~`%4WEeUN}bDnRBp;}-)AQszB@F$wqJJ3JzokR^N~>}KlZ8cU{dQ@_;iCa+;7*B$L{jy zxK^w)3}K1H*xIIwEZxZ1M7{CaJO^FX-)kG0aKDyE<4I+9+f6nhPc71gFZw0gEy7InJO`ZuAsV0rLs*ND;BdmLw zo2c0+XOAeeR6o*Co(nkGxdP64@u+O==^uOY{sAK{4WdE9QO&ZTs@X;AqfFBO8oA6L zdFGVP3@P0_yz8=9P?R|4Mdjwf1jtd4R676|o%jYGWth&p1KMqf+A~2--@@NeDoOZo zQRenu=9T9ANiXEbQNE(y;Ov`EeBMqN%6>AEi_)<5IxB|UNQZBLM?mZK;o(mwkotFC zu>!B50qtDT0iQ7ToI&d>nNNt|3Z=e4x^Lcs@mijEqK6|gjnzRM^`sirJ5Hhj>qCC% z%%I1Ghj@llx~i;~(w$|6{$Cm72l;C$YC`GzO)o^@_}g@3A39wv200^>@ zs57JmYD~XqQBw}8DdldNQDjPhca3mW6Kc5}$!8Y=AcGTB)_S_I2K`)h3P!lFMLe4!>qwuFtLYMG4dhkky_^N35wm+M2uQ0w^JyuJ}60n!cWNWAKmloD~u`zJ)FY- zK0TT2=;g{KmRmyISnxJ}#Du@f=U8dEYA@8A*fu|vpoL25cy&F+pr~E=>>G2u}>pgvUaL1>xrK)`wBPJ;?`)_TgAqvJJ5rl z*)}$eeB`(g|4kg{XGa$SI%q)p$)NZ-mI_jip|WAIUj{RzCO}^y%`V^vsb|K*%9Fh)yUQ>0ui|A?oMt~MsG@lw$^%Wjk0JSnuUM@fRS&iq% zs?W#YfF$P!!mP6~mOIh|VM#V=|$94z#M5kaz%8{Rt z1olL=%(j9km7wXoMjG8)wNA0n)S{qU z+~o(2Rb6}Es1B2MrWI7!vx;B9WX;V7kDe(%M@9`CI_!QSI8?sbiuv(PnWqiM!F9;1 zYVTHQyx><#KIJ|ZItq?;Ya>81z1B9o9wWhH!J~b@0O|W2s8u2i17%rb)Ai!{x!x~* z$PE7St_D>tJ=a}TB@dC#z4A?QWGR)$Y(z|50rGDD9{!=8omKe_)#Jr;^(7n|nkAj~ znIU2jlTw+UqsP9xz;~Y-zFJ{C;jTS3e+BMFB4+?WOqRYXSTANPj#{W%U)v30J}YnE zfC|&{k^31pVubHXcOAp_i=>AK)~qWyfIu64W#v9HujfUA46CK0?22vtehSoSr%8z0 zcHTQDjhBeJwz=~&g+ZIVR7)m5>-E|vHQPKjWU22b)KCz%+^#V~=5b2D6Y{Qt1Qru2 z+{$}C24im)kE8eQxkVxdOUfSCXX^Jhyg6~*4M)QG>8Dyqm42!-CJ^4wBL+~pE@|7$ zv(P$O3c0I>Gk8CrVbw=pC)}f5xf#dV;N&4+_v@bN;@FZKME^OkWu}jk{~wO>-(vAnsI^TA2`ecM-Gro~rdJwY*q$Up=Kq znX*>~fP&UG~p^NquC!_7k6j16f)9kAJ?_zfDQ7 Qv)8|~;XnQUf4ukq0Bct-S^xk5 literal 0 HcmV?d00001 diff --git a/Scripts/Models (Under Development)/N-back/__init__.py b/Scripts/Models (Under Development)/N-back/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/Scripts/Models (Under Development)/N-back/ffn.wts_nep_1_lr_01.pnl b/Scripts/Models (Under Development)/N-back/ffn.wts_nep_1_lr_01.pnl new file mode 100644 index 0000000000000000000000000000000000000000..4903636b03d0754409e6e546094e0c9175a3183c GIT binary patch literal 28527 zcmb5WXH!+(*Y1g60>yxeq8LyS%n`FL%pxj?fTE%#Q4vvs0Tm_Z3`)*9BS;RrYK$*& zK1ILmx81+#zn@dp)zx*LqjlfH-fOKn=Lpxh#+YNSK!1m&c6KXQ+Wmk2tg~~mi%g1s z{pQolqcM>wkw-qfjlX$yjop*~*PnFzv_w1a$M?DYUgU=lZxUVwCcTe)8J+UxeZsET zr1$T31>Fb^^1BgwhvP%W%T=m@be+2d`i?d%l{#=-M=j~p zZq3)hjj5Vr?bB&T>D+o&CoSzgCmm9lbTmb5w^r-mXK8WJ1uIn-r76t1p{3Sk$%~Vw zE7BnKmX1oL^y;uwIclEeCv|6cXr4~~u%gc1S}x6gJX=PkTZVM;txoBT75-9&kG)wf zotCRq$P9C8wtHr6OxU8m@3l?aEnNt(U#>0AGHAuF(00v}U-ptEHg6sI$XMx`ZRh)C zU5wT)T>9n>DT&F5u8vn`&NFXiqo zN!n^Xm1(V(_NUMFwJC^uLSC$vDp{ePOxkoKCEDVRdtsn z-%G0{?b5@=b2<~DnbxVx5rL?86v0ZZraLCB);>+QH2)i)Wkugb^zn~EKV8sT8L_15 z#zC1m9W1?;)?C+StvjJD2;|4Lm)f7`DDBqACl_==tL)^rbxY@^lfkM{&!Q#2wN-yP zK9XT+L;II?PRgIgqWg9Cbs<)3v{pMTE!13@x@H$EU6yMscQ&s2t{K+rz$emn%}-k_ zso!-(ro*`Iw-vq;b!DK~zly5bHcNi&Gbz(QR#<=(pVoONJ6|2=%JDPO5GP~SMct}1tQJIq}t<-virE@!?He9yTVr%O%nTU~GnUNYi!q;1> zgFRWVR5-nr#aCK~#I;#Q;_k;I86F%XRkDE5&*~IvEO%#+QEYX_N?0bDI(XbwicxuK zgp{Rfk@g~XWSFHLd|1r?Yb_K!Ep-9f!L!@NCb^n(Nqg2tA$>1tdm&X%wbgipbgONf z1A}HVo#*mb3eE;;xpAR99AzlwnYK%xm7)vh_i2kXYObY?SAtH+L>gmC|7jW5Y@SoD zmDbZQk|BjMd=u;U;nGEEf6n-SEbY9U9)@_$RI%r`)Hm;bz87q_Au5B_J zBO{nSJ~zB8(M~6pAT<_w9lrs4t{~3%U6SK1IX*`Gt;nW54O!+&x1|l=F}(pT@H19X zk?;W>%=<~Ybu7dQ(zDU3t%Px`;D3y+@5W@YD;WJtzyBtpgVF>y^2Un!_iv*5@NnWcu0~RN!!9^@#`g zQ@?dee_@@KZ}5vLOKVpoXx}k_2GHwz?5>#`o){ao?n&2m$qtq|YnRN*FRgfY4mD7|vWX=%|x7Xia7K>mq%Txd?gh5oOatBpS~*59aYBIvzV zN*#Kdv$$B=?yQwAfNTBvK-XK>fHwT43joc0j(Lt+TD}(@wac(hV&XD}@ikmN1T;=q zyO~wqP5-OJyc<7QQ;#+|W9`c{6V>NDKA~Bblu3auV44M%^fCA^ET=%b*K&1+<@eA4 zh1VG!x3>EjL)hl?$^xCHWzt1xQ5Iz*d4_?9JHjnEqX51Zllvh~AG_XP(qpS0hnU<0;Kq;;ZvOXon9z z^4p00@BRdkKmP!7uZQ7_Mz+{%c;}tJf@Cn1~e5B=( zvS2*2@g7g?eI8`}&<3CT+VaytNG-0vLI1#FaA8Gi-A920LdkL#pkOaE^836sSEWl^!NI zq`%O*_|L#`jfF?dV6}NBthZV#k<_eIy}^_)*HPSN^yWt`BShQ2LZB*E11Cfc8?yc5 zwuWgj+orpxu;Mw~y4Q+HzVP)ttf>nj=4kZ`P`CT!Qr{z6q~NNQZ#EAlcJPoY841x$ zyi^CUnP048k;dj&kIVuq>X^=zQVKK@?$={=?cQh37-;eQ(@z)`m1D*e$kiI3)y0B2|JQi5#f$jI9D9r#^TN!AW09I>x ze3YT>xL3VGyJmIxv{uRDa|}}|-a5IhJL2G*_*e$i_8Mtb8VXnohBSPK3s@ltu?(I% zhXD*&Pfp3?W$nI<5^RYZUXyxk3`*RTlIR-s_}+gfdAb&GKlsMu;QwCkwE4CD>wk&R zme*uAZb6A#O?KmEvYTzM*O?4u<32k(z_U%7qvh8r1nlaeP2MlA8ZD0M)P^nKZ4V@Qf;lH*+CNAGgTY~+} zB7Hn5TiZglFYTJZ3F@I1)7U{SCWkCbLZn?6wviXKTAotn66T?c!b}Fj&Vt~R`X?2% zc?;ll==fnRGR|7F;Xs;}JhEgYUK{Vr{A**|$iF^@%lct)7DhIiK%$eilaLtXB&8RU zH(`!P0)evM+7f_=Cdq&lfBcTW5Z-q{%BCLdy9iMSsT$F0jw}YE$gA?}kmh}7(lDl> z=PO`~1pG{kG^%Cufz*wV>7^IKwfwUq!TG*D%GuwVXKJ`bagud2t75YY4Yw_VMj4k@jGyElL`8#6Lke z{0xkAK|mXlKfkgx`_Sg?fuMDPl`Ma7@?qB#LPpT?LgvE#0skf*S$ACj5agz!QSmnJ zU6h4mOae^iZ^^;yaom|mWbBO~m}}P^JoeWyFT+!Crr!@>&w0zF(b^ejMc}fF zGGuu^m(i!6mKxQ$ror8Zb-)AN+E!@SEwu$*0n>sY+IW;NHiX2duYB_2y5BNoJ>N-6 zfKiRYzMlVF2KEfZ8q-)fw@Mn4jbg36k{y2#l0klA-TAp6DmCN8)iNvDSGD973T-Di zwi9#z$bdVhXd`L(bxR9^pPE@828bK9{hItvLj--Fbl4GeD3B_6sh{-oUDFpM9T~yP z%Oxiw0z;FwTZF0n8)&QDIuF{{J=O-Uz+%ci$G{905NV;6_DG9%UeVGdYvc2Tuyt>Z zKFP6lWXYLpl5P23kUHm`5&sxO4p=z9huBSs2Y-h=q%T#QdHSiV3FHSQ2aWiwQ?Yn# z&pv~FOHfeTN~wr4zuX7At4>JU2N)~MY|;KhhY=3Xy7cMk12{m9kOE1^3xfF!O3uUY z4gkT|k7-#D%bwBZ_0Mz&t7VjR=gvx|C%#_CEp5PLlVQa=am}r>`d)Z5@;PP?_4LWB&}XR0cNQdcaiE8LL5x zwf@OnZBO9AeXsmvh!5xBmr`;0q4a(8N64D}NLoX1u0a+MaxuhC2X1bo5V9E;Y>mNs z+pLvq0*+mZ{b+GB^m=prN~Y$|HxLoH#Z(&f@PPbTLhSexCbpQ@KF_s(E3nDzr%10C zpde<@cL*z=aP7l_Y`5=jc`w61S-Y>69g;#{nG}NR%9mRiW8no-B~8UcYxbRc2y^-5 ztJBZkN~sBUTC&{foSS5Rw3e@OPTBM7lev^^U#jh(WmFZ0V`IW(E>2plBe4&pX0My% zaLEGlY1W3l`+3~%SK6F&bTPSN^`9X z(ingfVE}2;@nN^NUzIH2N5^30z+{!liq~$C;U9OgZGuD1uJ>M-PMHaR@ztiKemDo9 zgUx0i!euAi(PF?C3KCiG&_Y7QH)K_S+na@F>cD>EBHH+qDRth`rkzsvL<|OzcI)&} z&AyATxV-o&zdvBlnPPjdWL3|(f5E4G-li)d7Gu--A@c+%_c~H z=Rv99uZ?NtI2gvV2VZiz_IBi|0X;w`JT5BTfU>%oS0zta-CmF z%0jh6T2{UP0Jh@VTIMMw+n2+2`FTpGV}bg{0E551AYJcTRFUflf_`+c96VS>{yv1h7PG8i3*XTU2kr4G!l4w708=A3|w)aY;bbwn2smzwIl z{yNN0HZAa4C8Hnwr2{gd6UTke$@mZJ+6(76$D`k&nt7Ui0L-t3T4H+?NG2g7gIEaK z)vnatdk$)k)LHOdp!wAq?d3sdaG=IDu$&?@T((R)Y2&*6#-B*=+x6FJ&2horp5fFT z(l5Cuu~$id3EmT{+Khj6=;FS|Nl5oM>GKURy+QiIj2VPqJ;u*tVF*Mr-4R_EL1ET> z!^M5cIB{{SFU zUr~LJv6pZ7`Y5W)=DEldd1Zd%@}4-O*0^1ak&aMujVG-Ay3Puv3fE|@A4*7+M)S`lekGw62h)E7s;cP5fH8=4~H_Sk>{ z8OdFyeb=9utf_}-PkK4~X})z-yH>;#>V6sd&wxP#TKzbl1kinXf>wF1`)7fL(z^*K zn;_Mxuq??)Li=qpF8$VJ<~Km7Yh{JS)?wfgcJ?U{*Y!FovuizEkg#NaK!jzt5jUi< z$5;CO@#hSZj-6MeY`YeO@*o|yyWo+4IYSO6KknI<;6WKAUUPxD3LmYIeCs+mV@vdt zqIk;~k>rK1*{{E>CCr72D=VEdO1~*3cf6jUlw4DNAE#(g!50JYXD$Cif>QQUMnfzt ztoMuKgOlWzd;8MLFw|`=Zd%q&TZmkd2OlyM_`h=|X zQP@rTDWX;pl}M?!_I$!FrZ<5VP}h8^dcKnA*u(WEhx2@^eR~WpRFgY(p0GomiB z`%{FcIsV$A{ZM@45Jy%>&UXB#9QR#;a*DC|cEj{+t3RabaKw*)F=JR7?rKJq6kyXl zEAuBa9mM^*Nhi9rIrOyrCP#R1SBt$5;Dp$E7oIqzc@O!g#d5O$bXT(0vKUODL~0+} zZKt{cRQ$0^-g%y<=txY0j{V??Ef(%uzu|Vk`43{V4|jDwhL=@Xn|32KXvYK1qK4MH zR4d)sgc!1PZW&@*{7JHvW7glH8IO<172aCt=W5WO=e6&;rLDkfsrAtR>@96R{?a-h zNqxTKE;<{sj$%X$sG|9rsf7I zPIF)D$Xom{1d<>dq)(vYOFLn!^?H_JZ2dIx-^A-@KQJD4vRxzUcI_S6&!Z@}0Zd{aU9C>#}La)7L0D zope4~N}#q5O!VK5OEY#of!7=5Ent4nv;{^@0#QV4KP#P^Qz<;k!tx_7*IE2}5MgdD z)FKNk^^z7X*x-F02jAl2$q-XASTxdtMFN8!fY&HJcRjdVH8RZUd#rqFVru2CeZYDI;u= zD4UoFKdKcBt5p`Gt{vtfg;Y>_2|ew{lQkE^#=9FsfTS|B7g(X0&jL4dSEm(vjmrSh zOzyb&3QAk~#2Y}jg+wOdZ@-=*zZAPSGU{;KEPSV?-*~$Ym+Y|~ewU2bno9*7c@(e- zI=R{*5eeQs^^L_$K^_>EVkmR@<)?o!?mgJU0CniB`-aJ$W;GdUM~Sj*OzSUq*j+OV z?=~Cel1BPP#q5|KM?>)YgU%$4WsWNNba!ub1vcul{UZ zl)+b^Xa#qf#mpYZONM0pHJ)jE?4za0n)MVzm7bIYsjv%=x7H>dB%{wI7_k%*(@gDh zgQom4@_Hm~yI*Ke>OUwX=avEIN0=;W@9JO(RH04=q*?ZXp5d;V3$0p;-KSHv%Zf73 zI*tWZ5dON*1nb|Jge3mF|7^bc0&J9T)3VJZRN6pxJgGBMc*rQPL3$p1vA!F}AhehL zaF?b%<`<&<_@!7W#KNeGxnT)y(tboI5+UHn7=7^=Dlu6I`;tD1fic$B!{4{ii$FMt zIe1ZCCg`KXJz|IuNxl!q1HSl;}{8@y{h1$iF9xKDW*} zlXtniya6a>6Wl4DckRBr?;61c%Jmn8lswScEtm&hIT-c)$4=bU4#M7XiFFsqly#Mi zEd!{@P5H?lb^02v>tO{MET=@5jDG$)H`2Psk)PBSorX=@kY0{OH%7MnIf;>_Q^?W04YzTsDML}&ZC|D1 z-|iB6UJ+pi9s+WI5!k9X9?}?@gk_Q8ecP>z@uVwD*o@cqunl267JouS#3IRq;{>4j zA|2e7%G{)X#Vhmx$EH#h2JAGuzQoqZisChEsZ2XFHqKY^X}kQrc?A57KdLzftwKNZ z+$u^ZgWfU}LcU{1kjV6wVl6@-Hmy0ZT)O?i7`E1W_5*tL+UWu0JFE%0vDTjXMP1Rq zae-lU6G>mJRP2~K+)X*_W&|f3i@wL^9L-$vU@=~+aaUJ?^*VQ5Hdz-JfJ4>GBvhd zd`s)s27JN87wC4Nw<8o=(OWHq9MqU`e5DOa3lh39{7U!WEU%Ng&v<_*eqC=3?^kExn_kn%)g3o|42>PS{rFfrllY=V2~>MHR%|0&s$F-!RJ26Bs5C$++F~Y&DPT$t74Dbl}yUa@CZz2zb`Zb^lA0b$xuRYFAPsQde^=qw%$k=#g5LQ>F9VgiIIZd!ArEroT^DhE5O2hYyc8HiwrZFu% zyYe+{E$ncQ$2~em1go`DBH>X@IN)fU75EZ29F*}hNPiIx&k33z3yVO}cd9OQp9D%b!&RtQdr0eUM<97uIHYMrTckUZ%Rsh+w=c4eQS1CNI|x{V%*w145dSJv8%|<0 zEMj7x=H9+;jCPf@?jrECV@?ZJ#Fw4tE}Z;Ic6R)qRaR@ zzN)rvCSO0KWlsUf2q3V}cL}4ETc5$+oold;a(Oxd zEkB5_n@#zp#G6cM9vB(7R>1H^sAg?Z>T%`Eubw~g)^HKTG$46Tw0tZ3x-f6bEK-sT z$A*FzX^s;s2bWVU*3tKosM`i2QwGXCfxI&eh8g9U;M*kaj)-G{`r-@ezs2A-?S(Ux z>fG`hl75zGCHbcvwf?y=7%h|5Q(NpYyi#ks+Yw!$XA0TCzUc6YryyyadT=dn#c_IO zBz?oV2Zz5I9Mob5nsvbA665(fXS{v+Z63V47wB*d)wjTAfysKEy`~N8O~B3#KdqBa zZ?IS6xH7);k;Dl}Qq2Xa-(#}xVcgzkr}bWMUp>YiH~Q?vMtc)bfYjcG$dr4W2K!gTn|_}GL2C%N zY>VAtdKs@2Te zF@-o=)|iQasFZuReRVM%3cqo;5y=*En0d z$>(4=9$M-^Edi2DJ)k5^O%-5Vr!2{}OT~sew-X+=O3P~WRi6k%MMP_ddkD0D!1CqN zkw@;~Ce{;+ChS74V&X8p-V^|-Q(Jc6cxIj7!+^KvEKll>zay&lQuR3^SE~-x$#IayYsvgc zvRf7iU9rcrYWIQ)WoYZ_mp$6%EiI(tF8aqYYF~^|iA^h<-FBt^3k1^rQzk=6BC(&E z7cRF9duK!st=N7$?ukKEJE^(23tRnX&@Dg1uIyuZEtb~a(&7)2c@Q3nU(vH9f0CcC z^~DalFsaAHP8lCbaj2qp!@` zYo)vV+NtbN%**)Ob6DR1c4^yxaHo<^ENFqT+QNee0_jXbcLn!g#D)7F8Cgy``-Ub- z#d6=(7P#i@}=A>rAyknpYdvopJ7<1Ay2>QjT=A zI?xBWOJD69^HjUceKb_SU+ z?rrW{F9*IcFx&L;BlD;s&+N307^`%MV#a(?vWenu#cn@(p{NEV%H-+$&c4wbcH_de zFJ&6w)`BHJahFz*3#983Dt_4~(2bAO_-pHhL#ErINT;tuo7%XVHbWbaHv1&C$p%^S zwDFP7?*rwhtQeg;i6S!bD{42tK7ZFXt-1z=t_KYB=Vdh z@Xp#J^J~~Ss{9Jre2%(?%&HlVeWrw0fYuOO!GNsV4r+e1wmwvls!C?h=}ZV$j#*zX z9l3XX!@sy|Lo$Ex4T2`68B@&e&|iL640vez3vG09-;HbA?AUaUcgLP)ty5mBh|m3w zh6wUEO9ni*La{O|#}&_xp&!~-ttaP$AG)2UH^+7NF|=MBq|~>1m_s$#R)HIkCj6C& z7hA|E3XSm(>yVH1f7y&nVD5n!fJEo0ZCTQO`&}AfP+9@_WK z^-eUOQ;o$hsfKThW!0r_v3T_ecGqIX1xD`M%4$cc<}*^xH66c?b=ty0wA?x93UTQ% zDr?u7Bh<1wt+R3R_s|oR-EAF8<)JO6LldVN1tgk$q019I2?vb@Un(7qjO~EDR-N30 z4zdlWXud#*@4ss*g&8`wRI(kUJjr~~2}!T##0CpGtl8gqR>Qh0v|U^#a283;68V=? z{e}nRTt~{eR?zh=GDn`5ZrwU8H9Ls+z0jju&S?}ehp&P{hu3JwEopy*9n#LZaU(EE zD<*IbLgpDblo?8nEN@scq=V=`LVR|Rxvw&O0iKY z$R+6xu=9OPgS!l`Mb{nHmCLsYDQufXO|c@r!&FCQ!q-e@sf?zEChA|H_?#4d#AV=rX4N^-r)hb?<5RU_-=<*P zg9d;cD131+$V|( z*OFtj4WxXk^UlPP=JjTtw0|wz&QR^R)Y&8{4tWY7j#zH{6C~Rk9{^1KwZiT=K_Z=? zskLwXQ?EQgXZlx)mR#5XL&-H;!CgC-`U5N-*7~*KT6Peo0%Msxel>Lu0Vc!R{3Ukd zmg@!us-C)00P03hm6p3{w#w77M|`nTTQG=U5ZN9K^WHn{-|KK5U1LSV#{&p4lm`rk zbm{!o4bNG)KX@?j0^Y;c@!jVk4m!T)ADYuJ!_udsHrx5y0Sv`ggUG4|t<{e_rN@!5 zP8?W#bTCMVy%BvXk)m!r5-5l>ra#A8=u#)gCd_qWM~kkWZq%l{b@0RbM5w6&%5-UB zKXfZAe+!09XyIjVQ#tzmnuu<@=fmf&y0{W*^LHtaxrkgitP^K4Fql^-0boQkcfN#C z@00xGYdeh57dc58K_BV?Zj&6eeTojhY4|NAgi*~3LC1N>UkV+u>_r(N>HYWJWPq0x zx*%Z}S48*c=~naz;14mAzwR{Th%Hzv?rL=?8qT%eUw|IBJ%aec^oKY2X@?uO_1pTu zsetD}F<|}H&v2~nrAU30j+>4s3}u971KQLfMX4kh-NFF*^sa1@KYqr{er^A$UGE9t z6g>2g{C2+&Q7N!4_?_3XQ+Vd0$>5qiz~lMlX zGVMWFp`28zMaGwkHyF?uhTjxw#$oA$V4XhtbZm*h)&1g}b;9oFE9loIHQ~tBG7w#DcCg{k@SG zaZHfo&aUI+b z)0UVkI^YI7k$$J7WXzo+pV+n@jgGdgFO!ZU$6cIL;>S6nfKbH3%5<}fZt*D`Gi(2SSB?V@$& zJdMHO%f4A#?rjUzW?Y!S*0mm&;sl3>_E>~~%E-f)KN!&j14z?1zN1bm(RHW%5?(U+ zfOS^LY{zx*A%@}^se@Z_LUyAq{5wm2JK<*}ip;Da@rsKJxC3TqMeCx6WTQZu2lDh+ z><%CI6n3|S(u9r+q~@@(U=-~!FL@FBIHbPW?}|lD@)M6 zw+MRo-OJ&m+hRK*a~DDwou+h)8P4jT6Hc%k;3rmEuWG}tbGW%pYnDk5eLIW^i64@I zG-~)WiBfKP_`KRoN->UrgHN)CjGJ7;qjIF-3f)dG4fWA-`;Qo|aTL6-`A(>eEa$2k z>)E-qO`7+V$a^|4P^K=ra_6HRC~iTT)&mmpI=t?dUj$<&O6zjzjND8WO0%WmSdiy) zlg|!exP$lbI7lI;d5@@(M{KUZWoy-Tz&nDTdnbkmNvWgpUN+Jg2%Dbdv;N{aQCbflfacB`K=kB{22U1oM+ zHE4K#sjCityqHQ$d9KbzlLilgQ_}WUGo)_QYGU|~6H>D!hJzEF+9^dN<{ zLQVDChhPOL)DNkj!wO4C;%~bw*?82K4SG(DrpljJ4tBm?>vw#X>FtI%aonQGL#y5} zsY_>|&E1$I`w-TJOWF?On1a;Ul2;K3^K|?Xn7@N7Mn1sXe-Rok=>$MNviT#DPYeK} zI10NLz;O>m_MOm!M!@*0X|gKPj_Y>5X^x=w;Nv)GgEnj<`ET}M` z2}0xSB^BM3u7lbaYwdQxEt__0fxR!TQpeFsZY+>8EBT(}xTT?MT+3Q)AdQ%7SMAJa6F>WhB&pEl{GZNl6bR<0(FKj%20t&}LXGLm{k3rfTvNVAUSApOFtAFc_erGYNmCj$DxWmKb+7#1WR6{Z!x6sh+yHkWVU6O3nUx05 z?e4B7q78Vu`(8e=|NM^_V>ge}Fu@V%OdE64&K)vhs*Al#d=q5W_ni*tpA;)lvtLQ^ z{(s8WnE-OtxyO%9OlXsZr;*7%xP|1~A&V(g+;5x#oR?hfIH09#ed8T&GX3n0+kN3L z4e`vi65cpweZNirMjfa}`;P72!}Hip^D&?$afrXPNj z>2<#Hm(%h4aFRa;a3-ZP4=RK^XPJ5q7t@d+3uLnQPko+rKeZ95& zI!l;51=o?0^*Z(XmD1)ycC6#^I-4A6@4Nf=4jw}<2P&a>VJ4094B}nZq z-&C3Tbm4lqApp2#x~KMTVC-ItZcKzVbvkjWuQ%fTbJT^eDWgwF{oUQ#xEFM!lY9BD zjWT5C%gIuh&uAFMjY;d){!?W2R4qu;nxrE#h-l~HOCGwwLfDvZ+U4ta{*qLLyHFun zi`eEbJ&eM1wS~>?{;fwHeIxPHOcQ0mMpkpg_}$T<#A7Lm$Dg=oJ}AMy@-Yh>pP_w%^U zp9~K%mBjjlJ*dAN@=Y$AtQp%wU^JWrBFdiB_RWxtd@PPjrnW5C?#=%!uI()jQF_-Bv&&YdY*esvOi^cx?P3L{^CTuB~1sHoR!5eK9#Ox-%(bx zc9{@2uI8p?YAg?{wlW0N`Yx zl%>RbQj#;AkoLv~RMiA`rD1#cqc?$;pg~eXwys4>i3Yo%XQo}Va}_*$y>x}@7|jex z);C%R?`rm8Ze6dfu`>D=4`{c%Nr`C!{%Vr{dFlYgNm81SOx;5V@5FpXa3F0NZdQHc zNfc}mm)ZTpfffjigI^nJr5E;)6#^Vxx@Vl&%2R`UwiOj31Fd z&wrlkz@y=*idqd>U8qUs3Bb$zlG=9MiHKFgTK|1!}8T{G+LPpr`M_Q0XH;oRP zQ)rpQyWCi7%{uO3j}=An&8+Ju*wHv>kqry#Hglgw!gMY8@k67`Veqp!%t}X6nreF< zW0o?ZEuMRq#z_%fAQ@a<=LTMZE-Rr)B;3?~g%=;_+&1E8fgyO@Gb0^usHjkj7|^noSa(Kmpxd0B*GXios?U;p{HI27fXZsjCS z0|=6`NqbGs)Wqi;*>_=?!|RRpfIBSVzpsNn)CR00&mE_(zCv>>C$DaQdO;Lw!mPHZYF>&hjajkPY2srF0XORlf7qE?-MZSP1sj!En!%QF}r+045-#-z@h zkW^vC+9FN+uO|}ph&yA_x^z2^7xvKwh;4;TP=6kbm8@0Vh*?1( z_0KIrz?4@0V`B{mG4|<|%$15GXj*=2BcSlxNK?o5OD;7CfWZ(XDZIf|*bQdBaUDu8 z);g_+kgVX}#!b6Isbokx2NvW{BxjyzmnvFs{ArNs=-)LwqwT~ae9m@VMsy<0feF!f zcYh<URt71rv#t^u!Q=&`|g;_;j=?lqyhTY2wK_id%Q>tDz}M;jIk z0W-myzOBh$H4i9?(jpmmkxs%jApw<5%F+1PBj+WCSc+d3LS(K96utGQr|@s^v+yy3oVDW_=0$poCpXa7~r+jrZrm#hAW zCdSZyw&|$_MNDq3X|L;GI7w^a29N_ zTw1>%aR!ZAXG%DP=L&15R$hJ=w(%>nDUpKhmrV~0>nzngj(uX}gl3zIQiK{Jyj~56 zSi|G;kE2VaMoQ|A)+CtUio38^m}o*i+yw2HY*t?n`{a?HK;An_X!|T@&6%W6pAy!^8%7l4hNe4zI%u-eU#fs9&@` z4Rd|^LH^PdX;T;O%5s*Y<9;0N&TjDR=fYRR(+)vhV(az zQNamW^#^mUx!Jo?Z&aA!Xsk*SjMjIKF&5^o)c@t+NRESH46fb5zK4fg?q8|AZpz~jK3(!VAZeC4LctAzO%e^Dn#-*xyIaWK5I1l z)<*&kUiwh;K7l}yvB06E$IkKkghi_Jm;EB?SFKhr=#Ynz(ZPo`5ij3>FD7HQoB_<;ogq;&LVGA zx`RDJiI)pF(IRE9(Fus3e)XWInOq>-K~p~9ROL5b2Lut|1qSC?Z@oz*GW7SWi`WtZ zDx{B1yS;JcL2H$^;V(5C41||`hFxHq!^UGfHgC0;sV#_&=v~Yk7BG23PV+P~awq-! zmAEJc`K6Q;P*K=6sW^Zt+sySgtxb{96>GE@YgqsJJjb4}XBl~_89xzN2XzhLf8fY{ zZ;Yec&_|or1Yq=wShwx2qkT#uy@Al09D5%rad}GX0dwj`DCLf1g?9)=oa(^NbFJv7 zJ060=XmJum>|VduMNDDOhI;$q1LQ3xje*U}f|Q)WxX^;OZSi>1mKjIEPtsO) z{~qdyR79HyTTNi*eJQkrAo76LIF6w^D{Yu{oF*=7%7{*|fw zQVgwnm*%SdK42O7(V`1Q*e$p?_$16#i}&Hoe_nFzIAqmpdk_Ffkeclm$Zeq0C@VYe z@*$YO1Yt*pUP#TQBnqq6A>N5F%p-HYbJhi58$Nh{E2aeZT)27G`6j8m&OGTEqW=1Yr#n9UZI_|^LA0wxoF#QSaZ@|kiunI zn{U2n>K*qDE*&=s5hVBT_J2N6et!E)X>vC8?rGGS^%Lw{1P2`6zI`kGC)se$W)oBL zPWd8ztq6a>nNmj#1#&MjYHgh&2eG#PO5oAlWoWc=m|x{+}r4D{BfGpvS_UR_o~k}kT@u7*=y+tkl+|6c!c zte!$VXC>*cNO%=Y*t+ZTRoWehXFZ$?&Elcv5Pt!tVnbSkaKn!MEFL)g<@WCE2|jPO zLO&+QEnz?`X;S*=J`H(hO#eOcvMw>r9d*OCO$$$p4#%6O^iG0NhSqs<=EB-2Ip5B| zc(4ZszFrsnZtgdZ|lK5 zAifNR6SJ5XX1I|08vo5D8RIO~R_FjRwK+1C`DYV?cs1Rx#{k-U6V*$JEv1+4rG#6& zqCjV#os)k@0oXawsT-TNkl;82VvRe2o*E-mRMm1+CkzQDgbwJK8#hdl$dPT{1fYQx zY5wRTwx5z?uLj6V$wr>NNBbH2H_cG)Y}i=$P0awIILB*Rcxvrw;IrH|xM|0SPb^M* zBUkPT`%l24F;6l-um70<6YF9_ycsj>qBb&kc8QL%N}73jD@nk8N$2$kmg)bWdncJ%HS6!M;7xd;~zZw(auV`b4zc8pGhE_`1}bUFIZt0B9KglKl$w$rfAa4oAmp zAVIHnQ_G{-NBT+<>&G|KUfQ_=tr>*W>0{202E->F-lKdxFOQ$0gTuT%*w8Ho9@pP- zZ0(q2hs-EL+aTX5yyA#+q$A#1u>smbGJEjLYnED1ZR`C39S(;0mooGRnY|>_rG!En189U(v(;sLKU5b})uR(j0bz05X6t-%mN@`}Gq#_tak-FRnlg=~l?ktFO^> zw=8gWco?^AdaYw%I8({LDDBSL_so*kwaVL4a;eF%u;R@!b>XlM!`hL>*w4+5SHSL{ zepeCi^eba@9U(U(0mNC@gzfTccJVqfO>>B!#17c1e7sBPs3_i>w#DHynpq!mNvBSH zBse3qtli*1@o}ZAU~;W??M6Glk%02DjPB)9_2r+?T6YWW`@A5__$=@{sRa*c#X@%#y0C*IXXkKAo3!838Qw|rnRtvv zF2ZgrFFs&6$qbc}tB{Uy3c@kkk)Q*gac4q^OnvZ%QB0H9{kGt7QFwR-?mcP!5AN;zuy>%dAO}jS>g&^+6x46=u0ctka)m(VNw%#VCzpLP%J%fr6A=U zsv>4KUjQX}<&fs$k2fhNIoUBMn^dPMl`GAUFmz)UL2W69)AN!v4|nb3Ug47N1F3Zb7&wtF^o=y%;%E-2>h%f z4x3c1?iR-4i&cNcD_V|UC0Ia+3ZkfFW8;}b9KgI;%I_oKEF>`=x6}>V-wYe&nLXM@9f!t5#yf_g z{UtYmoi0+5*h56?(w2X(jxhT_v){NX4-J|ouX!lq5Hbcg2cJptPcXd`T60TkTw?;xps1w59DZ+$rn5jJ)&IF{($CDDt;dzlGVi19i2?iub@&8)h`a>mndx zqxRf=152%aX+@-vXl1Y^@tUQSQBf+?B2Vcz?;>JKLu7{K?-;w;vi}}ow#E&JpF5F8 z0#kx-85eAqvAgbc`e}}}>&y3qhYSvGN4*Q8^@1prN1c}@m!&B@tlJ7mroul*q?$F_ z9N_%aJMx_jet`o}Lu;14By2ZQ>UcTM3c_rw*m%2ataNRD2V@cMa-|_y^8-;EVyFam z2Y9t5m~`NZG^S#>quhQ?c%wv49ABDh7-6O2o~fjCT6aRVDOg)VNZZftqXyEY126vh z{t+qmiUWfTk<#Iv+L{JHlkw#QY<5ReMM{NnlE71a6|wT3b6VBQbSg=ETrnjWMd3xj zs@8O09Mz&{l*Om*`NADQ|DAM!{5(Cw<-#0SZ>i`15I*e(g@ zlJjYA00ioYTKz^Vpmbqx}o}K7~ zzMZL`U_`MDZZzwzy07_P?0I7w=2d$_^LI)fr*W;#-{~%W06ycCi{TvT7-k2r5nr%#J9(IW=sl6rdSd%6 zVA#nX#4V4AR8pA|FUo}sRgVx4YkMR$b%+1Rc^BQ6Vay4>T(yq_RG`Sev#FBrtRqi; zq=02GU=HzmCqt+~&FI3&IuWm;>0oJwYY^aSugi(d#w2HDr7A@Hb_TuAx9`#STlsIDZAHy{up2uKi=M52HT ziYSOIf)4~4NP~7@$N&Q>5SB55v@D`BnT8NVKx7qzMi7+4&h#K1_CDvA=l z61K(sq^%UOv1r#PZ3Q&Y3oA6Zf(=jE;9@kp^J2hKiBqO#=@FTcrf}!<*08fqT zh~|vL^>=A#Yca_)ES|p_tqAtxbd2Z?aTs^;us)NG0}3zlTMs%X6?4yrMh&fTBgGk@ za|rrrfRxKM^Y|yV)Gyt<=?YN;k>9CocY6B}7E2*f3R1t#*Xs~xh3Y>lb`<4bd@smY zDLgQX-?ZWa?VpD0#8kRT2F9Xa7m}H6QWRs74x}J&vUC+oa!2g>u&)pwA$(g}r;*1C z1{`GVD1cNZz>qn5EM2j(0kkiu2Q+xR!Su^&9be1o)=qf}Q8Gl8V}BF~`YYz!7BlvAVWxB;C714LwXc zXSm;kIDJBeq_(WLj_T2~TxB;}0$`?VYLVzfq#DUYWNCXNKhex#S>{^(8KEu}?Nghn zU^qGL&Qoq`sYd3_uv0kJ!K5E!ASuy6A5FmFT@>}U8Lr;o#dCjU*GgwGA~6(vwITNa zDk}oum|gc=RBvv;-euZ!LG}Nni|fg=&|Iy?4(5hOrE43#LoHuY8v{dD9RwYTspi-tIhWdG&P+^8?p3AF0`^|Kt*}! z6;BXu6|)z{hYP-9vlE1x(f4G6*Zmi)h){~qgY7%D8jM`4#X5PsU-iVJG2}&5qv|~o z$9S7$6-7JIEOu?x^A$+I-lI0I5-U(on$Db1%b#nN^-4C%k)qdcql|rOeP3|^#wQ-o zE7e&!k!77K$8S%cG*VWs>4Zt55Z5(SwRtg!HA?KT(6{xNJwsXdcklYh2%55DW{xU~Z48 zwp+y+02m>%;gD#xeJD>2Yr?z%2Sz_D)#)ZZi9QhoTH2-CU})1xDa3Dw4L4*jPpBrVY#|%a)-ps6|dsCx3NQehdo~+b4zY zRkG^MQ9eZOggPTcRlujpl9DSd?dikDWB3ds$Hf^UHf29~U%fe}Rya1RwPr8KHil+% zqRHtIn19VuP1CRE7iGNZz+tiZTfo$vARi8yibq~&aEJQAHK*DF9R!(R+iYx7#1E&O>SaSbb&td;YMRJAKHwUMPx>GmsP9z-IxeR zP!7}bES)d{p4W|V*$f5_AL)}IDj`YDP;CT9++Fhc*OqGMSVl8t{a_6+s3p9)(#zD0h@I!r%Ck~ri|B{812M)ijD3&r=r|t+e2){P)_D_H_|N# zw@=1BAuiNWqXhf)Iva-1K!eQOwh}tsB@TBC&q+)W9F2CF_Eqc>pwD^()PtZx z)CHFNgiL0iln%N&GQ=fWFIVk{(1{TKDD)cqqlJgsdF%W9*z1J9D3N2TDiD^@5)Ybw z3pteb6t(;@I7}Mn6pqo7?y2p7+ylmawJl>2xD(FMdewGbJqQF^o%9PStv?py&eaZc z!X=c#&@Qn>dZN`|LYgJ0?xdp@9MYb3RiA*2iCHef;fsIQ#YW{x9zgatr;#b)Ndw=a&9!GU=`6EM|AT(;U~h Ln)K&?{_KAMy$<&% literal 0 HcmV?d00001 diff --git a/Scripts/Models (Under Development)/N-back/ffn.wts_nep_6250_lr_01.pnl b/Scripts/Models (Under Development)/N-back/ffn.wts_nep_6250_lr_01.pnl new file mode 100644 index 0000000000000000000000000000000000000000..cb1e3a49a5ef2d1fce3a283eb8c8dbaca97c3655 GIT binary patch literal 28463 zcmb5Whf-Eu)2<7c!HhX#Mo|oyP;`P>Km|d?1R_CD$ta4V&&zxg)-+lMpJ$lXnza>lU>{hO{`(OUI-fp8^cydHc z?8m5+k>RP~?uqZ--@3Ne?%DtJzf6aWBsWH8%cq(u(|~l@F6v6(l)6 zOmcGdv-flGbIDIy{r_DuGWByJ-&_;oz&D~VCaoj$7XAr72Q>2d?yvkwP7@ z_mv8MDn0U7yR_xb9jUT(DpDG?^KsA`ZzpNd3TeKp|5kZRZjkn$*9je#3QLFdhqQ5p z?o&Es9c5X)OPysx7c6a*Oc{LqkhT7`Km(jm5hbnWs?mjRiyUP-n5OqYTn zEw&=0`pR}U?pS0={)-28C+|p;*jzPN$D}ky#@Bo6?8AMR?4(@ltk5fgS{N42*R%qQ z4C(xlJ;n%95+-??qq8gI7k|rk+0GqDK1rU~4ziNzdqF|#z3*y`Oh)mqKFPCA>!|jv z2=Q6zE!DR!JyaW8ov@@xXTCj;+2!gT;`vmHj%dM4vAvZMt>T|9DU)d{L9(@f;|eK~ ze2ib{cTor5dGGetf|FAF^^(kMfhA2}Tx9m;x!vAtw@5>n_Utvbs9LW*QBQT;(K}Iw zuf3L0b~I&a(HE(R)&+YRvvl-@-)gD4xWW6jv`BBfW=rN8Z>dPp8ToT(smxd|TCY?5 zKmW3hSk4DdYkrEOy|*-LiT1CNaw*XnOMhuqtj;HBtCU#Uzek&O;l7L1h%HWwt}VeC z1zKrIL-2X2lAltl0ZPnHU4#n@%euKPdil}@p3i29JOfX)rN1NTelbmA+sZ9@)Hm4?Sc^z{9zo&zTH-jJEz4`d& z4VR$cyZ0XXKMMNaBZaLo4l#=nzAc8Z-Tp9XhY8!2CTw@ctT&;zd$k>q-9oh3bT(R= ztczOgv}BzOePOR7((^zj*?5+8Svqk0Xs`}w0eg_TG_78{BtpimO^3E>&pB=4r_17F zwJp>ibFcN%!i!JuXup-XTU+GMJ+5ADMVx-(C6)K3(wI@}w?>|J*I`QwWssZX9Js3k zmS%$F?iY#6r4hIde)5pfYf@{1rHlVe$I1{<&}?b>3F-cv=E)yrLb_jNoF4->hBiV_?6*%@sSi+z7GhD+6_7?nHGKpoNwhXQ1QHk{){a|%q2mb{ z`o|I?AL;Y~DVAAFvi*|0x4tmmmR#xj#9BMFS30fNTCT+eNIBT}DYlhgbwKi6v{FZ{ zt(tRcnJyfIFl^@ov?xUj+%bgIhXSzz7-dpxEFHOa0|IHizsh^5=14=53?AaLI>Kja zW%T_68MfYp5V~z|9-G1G0y=(*1y-Wcw*F!AsXVmIr6 zR#<*FJ#J}xB=?Qc%!G3?yd3*W{jTHMhY1^{!vcb>ceF_Jp~OL)pcS!jO09J3Z)*oZ zpMPcleksN4GA;A#ud_8swp`owAJozOTH7tQKdHHRvl~*bv99O>#F&ZWen6Se^p72` zTDXZ@WQcnXY7e`sWs@(Qy+Y&%f0?v&0_%-RyR`0QJGM~$c`lj|Eyg9<@lgk*@qspo z&AJg6=C9rQQ>v|0KG+e!Z>509wby30oUaw%u~DtgKE81cI^s%4b?Vz|)-`Oc;3kD{ z_RBEwZ{z>}K53SRj%cs7OS(@Tf=@GzLw+cXD6`3{R(dCW()#0{K|~?WY}WpFPB2og zmElQF8jRy2(7XQBzaO4&eg*@{j7OlB_~|bg>ygwZ%O7bui$h*X_MYWZcUNnz5ElDO zY65je`>dBTkhTV^cWIUNLhAjb*yXyRp+g=wrADU>#4UU2#|}LK*x!;V*JJR19;Dr8 zJ(GHvIA_T<#Wy42%d}r+9~)WbOE_OFaBoJE<64_ z@bZoPwUSq^Bt-f_*q9~7S9Bx+b}Nw<>sv_7vb*6pfaw2aM?`dnX^y2;TRbnmcn0FM zMrRyW>o9~yy3xNLr=|0l{Fa|EF;^_o=6$Iqhii_DR>f$wOc`h9dM07$sqObH{kKta zzk9wgA<;ohw#$?Z-h(p;#P$!%W&H3;Z$MfowIQp&@LQWU?eMw9btX&{`ai)fb-3fl zVoVO5NpY73qUx`uWq0t&U@UkfDKuYRqb+9-DdAibAN9%~16pgfPV$ik6;h={PG5SG zqc(muanbW#bNw)DLxOc1%J03zMw;=$bDj4!Ij&QCtc%Bpm`dr_GD7ppL#f>yCj%rC zfSt9E^j(vH}au%K-u*_PpTiIZNjqZ|i)pwrPW-s(!J?ln3AGaPz(M`-)$HK6E!G(1?dg_tU3`5#||Adx?8q~*pOZViP^MokzT3$9+nL2g6o0L%c6Bo8!hfQd0lJo zYmt+;jNOt3Hd~F>wN^_})Q)%DV2n-wi@WWB8-H-G1u0!2O=<2rYw6rEKs+peKgyhi z()iyW(jOB!B+N8`kO6$oN36XrR1rgY9?OyxN*fk6PZ*$d&vk0J8;9*;OLZ?q@ zwmY71xTF!ft2>QYt_TcBbw&o0;^% zXeII!a_d7GuRn3>BJ6^Yb%agEAUs2R6V|qKk{=|~+`G<#AAf1j=QAfwoF4+fQzYOa z07?+#gSkx23MPfvtjAJsxB3pLqR)D>De)6odBGPe&0dp=BQo~Yfd+2CB;lE4-Xw?$ zF^XGI%0Xa|i#bUclPTK{65}=2J$_lPd26vxpLIC&Y2syKXw>q5Aw7pcbgKyigv=|4 zw?wgF2eIA0p2}yaWU{2vS+g)rD`D^|8A=X1N(`Lc6Ln`TxviM3*16;EikFzM(b}dB z2UdA4H?)!uyY{k{`e){vg&uBX-CtAy&~J*nm*k#4K>+BGAnfq45A2Z>+ZZiifx+^Sa8Z9>b(|=w10vqWi;uHKlCzYJ&KZ_-*oB_vhtO$ zP9+53b?CrJXDK`e9zzEYvFC*EbzN}s-Xw$3k3S|znFYZ&OQ&N7MUM4G7xtawVzd6% z9_jX67q-UETf4#8Z|77%jg&0>vS+=PP%YRjpN-l{jvAE_6yu)F88Jpk{;~XCBs_tE z{&K(LJ0&OZ6@~x-WeB9)=Lb=`CR*#TOucscnlv`&wM>f->NqxIJGn>0d;(yfNE!U- zN!Zo`d&ETF>Ra4)LR*kGI<0wWi*x|cVr2e~Hk>44o2(rdQ;4A6E$k%tV#qb^3x8%( z$@n=)e?aFb*^VJTuQ{>R7J-Taa6?L51C745^A-Wn=SGyBVe^GIu`C3w%k13BXz-ooJewef;HEHkb5dZ(aNcpdx#s67};__c5%05$~?0=Ya;D3}T2me!| zY&0dxkk<;BvP5!>&@26LPa8e1V5C$0unY|TAT?K5+eiovS{a8RoUt;lYV*{_imr8H}NJ# zR+N^m!k;$!54K}(b^M(ygc%kDmHo%GO}chSD_P*4&V}kXY$bU zcT4=VhEm81`TADN4+nm<&h3sM*gIs{WT0}XUVh^%psfODQgk9(E8ea0PCkRy-7EdU zVvA4xsuL*=#t9QwrB2E+AbnG+wQhHNd6!L3TZeBvr8;6?P1eppDTO2TC*=qX0)GZEdZvwp045`}h*sLXqFiVZ=%E!q^jSfALS2FjFQPf~Q=lU07( zd|8XGfCZEH5XmlAxXN$7{+*lF_RR9im~veWwG zGxp0wFV)Ds_d}?$-Q$rVw)E zwz-(h#+Q155)LMwN@-v5>x9~WB`uBp?q_ZDCO=b3Wtpjmr84WuXccNoNU!ByjUAvaae7*k52avo)CcN*nr)g8B)JHhVz$0)xWCdvF z8Go(6O3mn@b%z%HIr$ku`AZ7E-ybZ!yq_?}@!@j+yW~tjuf2sxtQo&mBSEIS{ls$qx@`?UUgxcqD!nVWO zv{9#GVQWIB`}7jOZHRy+iN`_A$Pr>l#@9*hBN=gDr2d*vZ-Sn_99_f5Nv_iT?gR0F zQH`LY+(IBUT8>cj z`imW6vK+5n`|lDETb@d@Z`?&0wN5XChAU*|C5C^+HGiK*Jv3wb6OscVje2}twCTX} zcMqh~fhg1BHOTCR^sC@$4;Ss$%oxu=>vV+ofm6uAKIHv^X3Ln2UBU)By-LP52k%

wkJd*v?Jp)l-ZFa9z-lf1O2K;tD7dP$rBAJ$pEBEVk z#leP*j>bNeHjGhYdD~$XU|x;csWlq3G$R^EQ<@+3_r`QcltVhS_o-&xBL^Ip@#_g5 z2_%4yKE77xy)hVS3{oV+{)}T)89DUfWC_m_Eq<8npZ>k+9kMKCUIOoaUPM5d zj0YQ1)ZD%L$KC0wfztqcDxpGaI)Hf`sMuP-4%(8sd`?TgV!?6X1nUiM-zJlnAgU?r z$g=Q^buShdk?HO#)5jl~im>q*(Eh=9Cn%Yu#d$B-`URcLS!eET{Fr`?q)4Wik-zKK zu7TtY2xQPhnwJ`msE_pAyanuZnzWRAW2Fx*DE6C=RR$vXU=;xTqP6EkWjGZE-7W*4 z-bcbjmDc0+Qm|?rN(Y$bNyqt%I^tt$!5P%izl-u`HCA`Uv;&ZBp=92`a?{`M!8Rpl z@j=vPDL?*LCqXjUrM)xgW(Rv~=NsusJ4X>SZUrsZ$#fr-Shl6zQCjO|ze*aR&}{9E zA}LlHYDw4m+dBCWL>60D*z@l_7f>9umh(05t7h$jhdGIuI;ej_9qo+^PyuBw@!Emd z8??@?(B3Or9UO+&d?rU_Xo~g*nzFueQ>}OmQGvfeBYz$_hEiv#*IUz{!702+e zVM9LuUf{MofL_Rc>||y;DdXp%w-&}R&Azf=SEL*XrkvJTy3Cut<{$0RI{zC%0xTv_ z%6U4Sv56FCBL*5S5deS5o@3TV?K&0adkQHe6A}K`@2o{nze?Z!4Bz|`D$?<@KHC0d zGclTys-qdwb{3S1s!)E;JQyH4ch zZPQ6>?H%n6d=P44;g@#rGHLx0D-B(t)sf`33dnwq_CF>rS_yzs>-j_MJ{}xJ#2hAK zilzGs;)Kei^U_9XO6HeHVl8rmXUe{5n}Jz}{NDLBhD_aTZCn4uPyTMBE;(fClEV*^ zj-)2Jz7Dzf-`6Go|KZ8?2ls<-h6Mlbot+&0ufpV*DNNiRCLK40$#!#ga^gRQ$^UhT zW6Of4Qj+d!0@3pFqY94qHTH=-Lg+^ToooudB z_KH(N>GScs_G~}su}OUlp*|1oHeio!@{`k@0PI9psZ5G@p2vip>%A$4rnvj0-%fL} zms&zS(0iCoc5_nG_+2NL_#q4)kgDpS2$S92wfe+YS$O-xf{1%zyFc1}m!Dtao9&3y zQI0{Oq}6txedy=qTK@1_`*&#z#UO$5dsFnXqZBDxeHw0Vmxj-%t&7ladIt4$uI2sq z41t*y%1s|%{{VhD(Ih5JmTs5QB_Mplx~BE758jHCIctZPe>}!7F?N@h7rxNkKPrQ6 zD>*|)4(R-5nn=%eB24o_(YbWp?NXqUF0DJsXEHg?(=mimfc(49DR!La?OQ<`t-%<`zPC@FzN zY#XERypzr-&9lOi){+oQxZ#Qm=M4!` zxOds+Xt8-fcAY2AqfNW4WXTCR_h|7;%lE-JjR=}4>9L$Ndu!Ov)W@c!kt5AZWX#VH z$w&XsP8V3fkhSU>*8i!7sx6hj(FQPHp?Q?A2ChTs(yq zuj0O$TQ%<$!R|t+{2@yYfk0`zCAG_mpb`V>AJRZCnLP{{f!TOO22QTf;#Xojm&|Mh z3JQBj>jh+MyUcy()){9pPYLlmC+NJ%B(y@Q1DB^2wOg{*YU{1fT=(D(wY}S=b#dt5 z_Ll?{zsD7`4`_HTB!7xDV-SPV9PLgeg26fv5mO-`v%`?ZCmqECE$>cY<_}LK^R|@I z_Op&9N`<5Tc_Isz=Xt_5TS^nS)fbt)fN-s&2q6Lf2m>_Iaa0PdSFxeKT6Em3Ofr3? z@e+UIieGfng#xtdD1VkY|K0J?$fi2$n;Y%Y$M?aA4t{|=TGO{Bi_JkBc3kicCgQzh zKK)yO=6s>T9M+cg(P#=zgJWa?Mx5pU6<@5S()%IeH2JyN;yC3xjxd^v22%fFIsj(A zq>f|^eOJ8NEw#=jjgIXhS@wnF?+RqFjCo7e3MK66@ao80ozYt7#qe2>JRco;gq?nH z&;jvGKgTtsO%%Ys8!P#A8wiN>m69VsnM+z;OQiZGJ%A#@c{e7Y8PbnwaN5t+C`WpY zGW>QCtX=x+)bZ4B9APUBN3Ai99P|#9JnE(GQR$%G_TULrkJjNA%&@%MBOUm(oZ1tK zHHeHIL7^hmF9M8cZoXW0z!iH@+t3OwVNRtO~Y75i) z8*pYd9b|fHJ<`05ki10AW{bV9S?N3IOi>AyJdp~A&xmwwI7WJ^z6S&bEZ-+O@+nB$ zK&B4FP`^*%(-j72{_5EJz9Po=ddNheo$ABL*0QC+pj z+%2$YjtKIOzr5g!AJE^PQvK>7H~NU(tF+&p)QhaI1gkwj6Yot~ z?h&*puu8k>b$|+x{Zs#)r=DLTrEV-KQ=1J-{yP+*eV(UG-WiFP+O0%qws342wHJxF zaD~ia+k%xQuZJ8YxL3y^wr0C= zEKYikePIdd@tWf&*)hh-;B{;bekxOlkiQO+NVBy0iK{W)0%2SiD3$4k6Y?C>eKk8( zdcaoP^QH7)=)YM0z`f#xHnYES4*eF^%jjwnQvp>@fXq2ul(8r(HYXk1sNG&hLyjPi zhGZ=Kn{_2ADJ&?F^Cns`_CetgdXHtlGc;IRHltWhll(udBIJ-C&CWV_06H8Yu}W>= zyS2OT8e*Gz7bc_bDM0Fgw-h*n%wY(TuM}T+qBW;s1TB7aVB_hq#bD>abJr!ExI*HU zoS4^A6Y8cE4OQ9@_w_F0OEjB;NS8WKpqZ8|3hBuWDcUH_)-C<*Ex+QmH_pj!xY(2; z?1F>kYs-O4s=Gp7i#|j8=_lCpzBSL4+1IlC4Wbbe; zn8odk(l#xlqlXca{nktCQW8*uTkdFMXLI(Q!lRvX%o6-zYj#U@R8O?H4dNEnVnLTZg_Y6x-_|J-K|Te zWUjvFv0G=!OQ7kIW=hds;2CG;E@p7%5axLJgv^o|AI?sF7bQm-)oC3?KI$)imF6d) zxK3?9>c0!Ku*7!gvJ)FkWq_2K9p3tPtLqH}pib_+EtNMQst^~j(x8P7WU0#uQg@Z~ zJw`D?@aCRG753A{HiIv{8OMXppT&Xe^!KJTqGgsPFbVPn=$aXf9;5=i^^X9eA!R6I z8<}ay-rkXwe7~f{PZ6daTxgI5)gOKZ7@@Pg6e;;Ezu!n1y^Ijl>6qlbL}orlk(TPn zYhZ80!mn){cl`iH;CKtiva|^UCCljGI6u^d>`+j-kmNZn@Lx_8K0GWw8ItMLMO0_(No~gB>wn}}vJ$*`Y)EX@#fQGev z^=GlUoQd8Y;i6zCM;H}Q7RPqUFBQlkZC-&L(Vg`&efJQWqy*nR3#1m&_E=(rDI0p6 zzs_s|;ODt?N#L^oG&RT1xZ~Vvu0>$>(m1MpD+P#Z%l$5Zn%kRfG)PtY2X8H3V*WU{ z;$^t}*dfzsKZIWedEy)qk#Wtmgmmmau8>}ere5-H9==0MHxM($w-Vl)cn;V|5nkd< z`tS{QblC*TNVi#<R(h$AGVO^E5x-d^mPnECUr*(zYw8xG{QQ)=|T|B%Bax=M(QF|CZm^WGHAa_ZwQz z7V<6MfU62NVWfO(mDn~ZWm%mu8JS^{|2Av!5()tz(*x!HL-RM9MADNU!MZZRXF0Su zYiy}S;f8LGZr9#$EJEU%I1?_#_YrnPR+qMUCSDG`ZsgNIqGKAH%whpq+@=h_&3=Wu zN!>N4J_W~vp>y)@l0ANyc+ZD8b%KwwHbfY5d$3jVxBEQhH0{$ysSSW(=z&?!rIk;& zxU(0Ido%ZhfB8aa;-!zLwUs{U7IHfr$Zv_(QkZy#0w2x(ity`3OWxJ|B~Ml+!zea; znbNjpQpNN$_wJ62!S`lrhJxg4vi`}yQ$r|D`E`25h7YEwVYqqvNDx1)#>@Ke8&;as z={VCBs`iu8r#iJC&I#Hi0}l?SqquBA*t;8ij^9}%hY=#YE$IZ}fD@Zeub3A)Zklr* zUK#X&Vv;P>&jNs~MO2PO5*1;Qo-*iQ1n9y8Eb~Wam@OfS4(yO$mo@h_zH5b}2U&GR z5?Lupi~Ke#6l3txy`K82h<*SeOIFq1t;=Ger4D}8>wAYUl^m6M3n6%xz^!yN27zAb z?z$V!GfH%JhYoCY!{+ct$4<0n$rl*Qw)7%C>Xyo&cUG9zIr`H?>0uXJ?6k~DsuK*P zpp;rINr^XSfs()Fk#>CDjQ0q^Td`{;e=EW!23?ftvyG0eS?o$~&ct*P^m(LWFQZ@Gy4WXR~PGVQbZ+Ry;C(+Ww?uE|X5fW1jIxtAEL+h>M z$E2UB!|Pubk$Q)D8S;WfQB!|Vd+aDDp^tLy+eJ>Q_^kQNA7rtzn$v6qPW`9Fo3!kd zn_(4BS~_v$Pbz-gI8Uc@+JjiCB1tUOxfj~Us7*S0Hi|${n*%#W@-&?#PKJ1vmNxHra9OI|FJ%5m`YVD1@|s*C^^5b)OEfcViT>S5Rfo z4pXI+(wPu!-WUaGg*FGsqG;Mqv>9ao^d0Whd>F!qQxJ;jXAma(v zrX74a7u(G)GDIcj{VRW>xqX-QHbQ>x`ucv6*-G9d5=^E~&j{CpePR6Z7b~!KoO}BE zrOe=68C$PehhFc&bk_ti#Iicn^wnf$>rVNb_>gc!Ppv<7`H|6hSV0G`1#G_q$=Rg; z6}ZmU->1!LUrCXp)W6#cD*&{RyYGpdI%;iXTvh;yso=HY0cl)z4OYozyq#+uCWkt( z{SGVUgkh90Zk}sJKz99UTVxK_FHgMc03Y=l^!}Fo^k>1Ofov=3*m1b472%d^9a*VN z2kNM!Vb{P7iAUW1`Sdd6+_IZ{2mx?ZB4y8{_c%N!O{knEAPWg}+s;!eqI|lbM-?Ig~)BtI^jX>mBa_l`|8;iP(f|Sa4cFYUCU_tv2c)Sl@ zwwPCg{WNQb<|0G@R-23;G(wETafZR13+J`Xkj?y6pA9mbBAEaH5k0?dEh|L|cU}$? zW?zx9V5WeKP)~o3<&uIrwZ$CO4bx<=6x$W;UPnS?911|RgM~j|Sg0`)eBJ3uq|W#m zR_p^;`Cz_dD~qx5`??hwxT0SRt>bBDSKr3RJ=TdculDQAYvc3+>3fc8e=7;aTE1S} zX-06Cg#e$ps#CAI5;(oF6?vS0?~BQXWhZo0r`O*RhD4$WtBHM-U6~{`$zDcuPky0- zLlyr*`!}s)L$e$bA3p;%kcj{hi#o4^^w$~8VWGCr7us=B|E48d={u!mr`jNJNMu+W z)2>La*8}$ON7@q3$jobg&PC<|c-SHkJZn`D3`fTazqMHp4sJuqcZC~uUZg+ozeA53 zr4evhGImmE?l&)$p&%CPp7pr<3>JFGb!s_vxE1~l z_Svrs7qsvVOQ8ntKnCP}*TOIc$0R2_S<2SD2KwZZxi!+aCk>otcx1@m?V6U za9T!4>BONB8c!4VO{U#=B0(!sH4|PsqRf|n*#OY{t=pfxw_%bwB7B9myx6OMfgC|i zpZ<~eS`-Si9Q1nrydleU^0#n*>9PlSMRX~;wC36?t13poJT0VLT$Z>JMvSFqF9xPG}9d0<39` zPO+fDTkO5To3Hc|$9;r!mm7t%kIqEVh0gXgCLlpYTW@YFuy+h16TM;tlI*?Qu{wf9PG&`Eo9U^Vh)h=U8xA}`t9x(#@JCC^Q!uQQEJ zBlc-poOGSXZ<{$`2mu5+IN`ICmVeT&Ao?_#?*< zlfN77`Rgg9#pLk*6n-(o0Su)=1IVTfnD^7hN78r{!riF@R|uv#J9=#JOlI7DYA@V( zOll5);Z(zb?(_?(`1D{;ymdPmhV6+8Fc|H6bwuZwWv;Lu@5dzNRGAD)qI+~YfWWCH zI~Gx~b)uYG*w}BfPym%rKdevg?zMQ-utxq~#qy?1mtu!K;gAJtS}T?lYCi!vG;AbF z-M5vl{CLdTr?npl{kC}Cw$DM+c`BTXwHJRN*$4}i_n zEX2wnT7HBAXBdrx8}eA~461ed5HYO(Quavs6#$ILn01owyLMMeOyQvYFA!z`1qZ!b zu~pl5u!U-ixt}7`)DbkV^hTJe>1v(*Zm`8uTfV?Q+hoX2t^5q&%b2FJUVCiH*uNYK z0H?X%J((_M$|?j&3}7iaH7={GRMv-5{2bOv5xhOJ8- zPe}iJdv6`tOC#$AL}@ZqNpk!XuiXgc)0?l`$7ziL)`y5wr=F9dJFU1^2c>r>zUj9j zqqOMlPFHNUexr}JUS@XET0%27523UYb8Lcp=e70QbrX^;N27IU1%9frJU4_r(5a6c zXn(sAC3Byi5Pv*upylo!NFI!pBcpJrlq}~aH_y{xtMWMToPi{Y;JQYd3oSeRv1tD%K2@Vr3{eYD_0VKO*w7j{9G}cm z=$nFpQCZk&qAPdh{m{cYPPp!1BC%8FUh)-;j>jUT7A;@-%1uW7OdjMZ$+VZ)Aj_y0o?h_# zqBZFR2%Tl5TZum6E2-+jy^G6 zR6LabOgqvcbz_J%eoL_DFc^+cmA_nfSSL1n%g`YxsK#y`UD!Ku(tmB2yXh|Sm`WJ@ z*3G)nEX#G&M~1gT&;4YnFGlpBv`q&&&JEs}JZMygF>e8F(JE>Fgg)V_d|Z3nzG1*` zp3<@(g*-!#Y43Uc;cfV##&Yq~p_@D{i(KNXf2C``=TjpWa;`r7V#U!G+W7!m|8jQ& zGOnMr&EM-Y^=GUUFNeZ7bU}0-+pkTDuV8D+o!wMM*+(%|w=tIt-UpM@Vp<<#me+Om z+w%d-ckmP_1G@4%KA1lDy_J&xn7o^9@X>&lNPCDV>$WyX){RY?Eu)Vp8a?Q(6faq_ z*itB6OCD}g8T}+BDT;Y!SbcT4e@2B+8i(DMdj9PhJTDi69E96L5 zBKfj#DQk~kb@|OYGb*35?}+4uJYOt0=9sQ+dH0wS@2hs5(22VQC7puF^_RFILu{mr zXZq_UBD6V_a?k$!W+Kf@%k>gfFnVcBB9(;ATJzji$#Wzqv^6U|219aVW!K@K1@3IKc>sh|7W_amFcoK zJIoeuvwPCc6>Y;8<;zc*aOOyu>Vm-lwn(S8$a0Z}m72YTBzYf-9+X~3vV!DqBTw<{ z_?smRk4p9aH;`5OgmQjCdKjRF=^vbe95{_b$JS4E$o%XCqlQV49^Yd5DkH92AE}b^H)dhx}`H# zswc9z)s#+f0L;i-N*7GgW8FI@-RGqElHsSPR}hP>t6rJ=!Ernv37u3tlq_SyG5N8b zx`A<0D^TaSPd#1P(ZyKnl6+^$+sDTdd}XJdrQq`cW2$aKZDyM`M{V?$;ZScg4_dm} zdgbnS$S;leCBc%2OF$I~yvtMr-jcW{wHprIo z-N-*ub;n)WW>fxQ$fF|}d}m&Mtf$_+y>%UC8zc-GtgjK|hlN)p@CI6+ zAq44gEVN~2?aJlfM$HU=FQ$<_w&psLZYnL>y1noEd)86_k86vzMM8-=3;`m9mL|jd zXhSLF+v5+l`58HgXH)pw&#$}_X@D^xmB)7|A>$b>nnY9v1yT__IA zc-O#@o4XK^b|19imczMse6JW0fc_mhYc?{tQ-55>1e3c9J^WPbMr zkZaRFOx|2j8ybmbaUt$+ry6d`>& z7@8 z5S=CryY%P!up^6SM1FAgZ<$-BRpI=id7F!LJA!xcy`klS?T6+Cy zX9Vl?qs^G3%X*!*ijpZ4bI_9fb(A4#R7Jk@itTPZv{xIr2ys&Z3?3o_-^2l$>vQzY z;?qe*QLiK`>^j|uUAhqEe+~=uTB(kli&Rp`ms*fl3CXkMtYHZ&`ri3_X9x_21_C^t zc1FT9+(sweL0>5ut-vUwT3VLBj?f>g#8=NQz0^+E`?&DoQkixM=BrFO9$C(7c;2H4 zIN0V9BU)xMyvqC{Re4CDP70loHm6shRo}ibXuhFU`#BFbgDpIvr-jR&`QWn%?Ml(c z8-z4{NFToqL`XI<^_#rRnj z^WZI+4%Uek&(}Zal}$TiwfQFRNU*MJ|0x-NiX;B$#~xYf0Y^W|xIb%HyXE|m zO=}4&QzI`#uH$N*Xe>(B)mrx8v=wD z{E0VY`iI73FM?GR67q%{@AG&5Uu>eA^lsb32)27zBCbyvFu4M&ErxzReuQ zV^!8Nf-~~i9s~T1PAT+Qnwg%){VB&TzP@2jEvjVt41{|EzcrC*pC;osgj3>0 zQ86m{R?J72_l$bF-PZi`gnuK)X&sv$q3Nr+>=hZ6pBEDV3Yo0Q`;n9`wNIes6R5v- z?IxcZOXnaQ3gyqc<;3f)+h7TVwHbX(1h?A5SM6W%t<(j25|lM1Ft1gFjwr+f_Jt!JgB!h8^2A z_X{GFYYup7wiE14GlZAMl*E(&J1$X(+j}H2T6c#A(u~B-h(fa1*x)`=;^<7 z(My(gwdhUVVP%gDbAO2%0}5gtK|n)5Mn*}*j}qX?u;ah z+HRdXMv~5@!OK#J=wS*(-dV$W!PUSO*u@Mtvx-=l@y9!vzHBxSEmMgf9D$Chka+{l z8jL*aX*Aa}DG!tp*Y^e_T!w=UC&GBqet)#W?;2XitaHyY8GChx_@_iT2n=UPYqN`w zM_`!}&05J>FIaTo;V)ufWRD?NojJbiC2a@8Oxm^ktu`M3I>=4)CNj=<2ZHV*g0F7( z(`>?^&RW4k^8nL6wWm2%U)X5;+k}TGn@A?LCLsUrCYpX|rtk=^B_(@KX;z39 zM_7#0d4w)zcb*9;gg*+?q`)Q4y6PZ3ckIs_(KJrwlXqSI^0bNA+_fM%<;*rye`S7> zt}`hcULiY!&g#HxQ_|Dnjf#;Gs;cQPr08eKcCJy-stjGAs;XKFZP9%Bv4u}CB~QMg z!!+$f{XVXLOn1zM3v{x`AGn6G&jIFs6XS7wuXWtP^^%jxN@pCILJNDu#K2mudU{(r zcPF9oG>7K5QEkx8CDu8%^z&0gFeIslb2y`dzBGTs%Ap&b(L>hCbf!!b*OL}aR-y}{ zjgJ={Sg(DIbi1vA-M;GdU3j5#Z>-qTSbyo23`A3hbuCpZ+`d3?eN3I^oDe#-^(^u7 z8rR!!KR&hHV;yZWeMz%Fi0yzNFL)f8XyLIF89$P6@j4W5J14zw!->#d=;c828_nJ% zQ|ub>%mp8arL8e$txUMZt7l*b&Qf!|P6H(~U@~bP-!EmR=x45o8LN-LeR3Z#tfmz^ z438Bsq8`oh(VjChzVw6XhR#IeoI&M0*_;XgTeIZuV+T?ZChAHfvsU7;31h}`{rSNR zB2HT}X8$fw7YAD(;Ek^Ja1Tvhd%jdIGqs9vY5!507PwXpb_fB7Jf4be0iG-(V=-4Zyj>G&Fp2% zYZ-Tx(j=~$6}xqx>1UA&Cae%%d=divv~cwh z0)@_u))|xr*xg~zb$B%2vfq1k^EFqc+-?q__82{{LlKsa`Dj6m@Ngif9NKw$vmcE1 z&$z8R}jA3;|Kp9Hx6aoP#YdxzGYb{tgl$|OdC1Zijv|uwp>cy9I{@- z^3C7c&1(p(z=Wfcy$Lx-ns?L=$D7W?JwVq|&3c!V#*qqtF7Qox#S1yPKB3HH=HPu^ z9b`pZmcFyv^~gAfiPfQ(GP{W;0BxmlXQ+NK#H67qv|=qK5!p1ta=0u_>t!Uw5EKuY zEcvow3!jPA9@=cASi&RIN`Fh2)#z!$T<7ePe6Rp}9`IRlO%3)oU0vsPb?A^gbsLH zUy-hPiIMc2(vILyM&e1ff0~r<=hVeoeYKn$BVwC@UN3%Pr>) zQpx(0V%pxhcpJcQ*?+Hfem!eL*l>8-elncX8ak}5((RP86JF!hfSY9E8R~+L7ww#mTfMcE~pJYdbW$Ahr}uatXcxgBSpk6m>k z?Yp+cS_if<$ArLJ^i$(WnWGRp%VDSXCiwBheIINjbDDQaW}Z5-O`}w`bc1HGre@?{BZxeW6!kQ7e@zdi(VOj-;~)uDYB6( zIfqPi^T^0S?Y!c=C{HpIsYIrAfER*Tcb6kU2aoO~q=KE}bn3+tllvUmL>-Y}%1;^< z{j_oZkbkrdYeUk!*ojvePzp?1t2Hm`_3qP#V5QD$owxn6#XxQ0X@M}>^9>lAK8Cho zBu-5V4W>=K)21DV_bo1_?0zD_lJAE9Q?%MgC*N$MaV6C&H(`iQqLbx#p-Xu&3zP5W zHe*M5o!D}~4a;4Kj|SDJke46DZQ;Nhe;Y!>Dx?h(a!XjCrLuQ$dHYGkk)7hWI*1kxQ!=1k|xdJ zV!zCsps*^SNNZ%&;5}^IeFkfx&`P&!?Gw`AUj*`d`FlZ1UP8n_ncDe{Rc&`L43uu> zda88X` zTU-b|S?yeV`1~SU=H5Ih<)|&wfox6Tza<-W;F~#Rsy&aWXn@yE=ySdkrzJZM8x}6t zkqGL%UShtVuvvLp|J*dLmtlY2toZ&lam(=lR-PitN1&)^x9>WF<>fzr-zdu@9rI3>C`oj z4wIpqTx+WXzZ)Y0DzVv452#cAEy~egZ42BclMnD#n%25wk6PhX7uLPd)Ym#|XRMLT zF;?mB2$L2$7wbE5Psg7FAXE`=mYOF!n5kEr(<+_wNIk-7j5H_c&$rqUN3uSxbqN~@ z-Wuw_$dwnjv6Ti;6@AoM#uM)uLI%@Wnsa}H4yH3AqWKY8kQix7Xc;;F^hr2$p&i?v zJazeIpyp3|cJ&T6!M3kHgO)m7jc~H~UGsXnCbhr|U&z2#Is~O3b%b(I=MVB+prbRm zU3ndU73|XAXrQa|_r>~^1T-6Id;z$!`j?xan_NC&cmB5Dy>C#G)L)U_ zizgQk^l{cC&vTtZp8~6am$yNx4KgB*2ke*Qvq67Dt*x8RN#kMdGgOhFy=mID{St(p zsEuo3y?lJaI!^$thV$!PjQdw5t==FDj(oP&p$XZl** zR#<}MpCmhhP;_bzQP*>ne2g$z&s*zu>EEY@(d^k;iIzIY8b$qs*Zi%Pu@4+@uim@| zEA*1q(66K*QrgeU?0p$$+~bJz5ga_=(or}48Hp08;3XwBUdM0UBwe4=UYfUB z_0pJ?qM66mTX(;)UOdn_v|5p;k{G7|bI~@wIKxtemr3r|y!X=b1taZ)K1)uXRHChc zV-tdB4?L?(8G1SBpn1cNhqP{mO{*e}QIP`Uh^=fb^$FvodHuy3+K?JeGoyIqX4tO8Szi#_-ZT|?=;?0L|r{az^yc%!p7%vN}y zNXMeJ*O_;Mw;i{3o5xCxmaUR(j}KbEdJ*b0jY3XoE^i#P9b=IHRXD&au(WN**>g$z z49qn1QKGiofKLjoB-am8^c+eUWx}xb>i!_Y5>GX28}chtIzxyC^CFn$WoxwEaD~@* z?Mc&yM;u38y8}4Nv~(ZE3Z~0V-Ft)-!OUZ|*TFOht?v}~Y7cSd8AzQb%ufRPPAxVX zbqE34o4QhMq4eo&G3)vW_qX+%OWu@Zy)`8oQl-YMW!cD zsXFgbYM$BqlKpAf8k%!jx12mNyo@Z-W2{!R*VXevqP1z)Vm*G=co`z@1Cm*O8K#Ld zDRr)w-vEJZj)mz9K{h{Mm|Q;P&f=#-m>)N-$5Gzjo)JaH=Ft#WP*r$KIdbj3DD4|Q zd^7-uo&OoG&wgk!{3HFdJlgNTqMgwZ@dvqaBf>-fcxdyxK;D{3)Gl0)b393xb4wBh zIUVCqB#R3Dw-q3_JhFnHua}8y<_)5uuEz_ZhYZQ*8L>^Qk-Py1#?&&Q~Z$GufU7G>JONUaSB~q!k*XgGu+cP2(pMQQ3JSw>;j@h0g%P52{ z=3S*f%@m(-Jl1{JzKv0NwugOcv`ic}Jig2hjo@m~@VO{UCzzNSd154&mk|4W2-TwE zC>yvrqw-?Nt*Oa8AZi8W7K4pA%S4FUj`E7-H>3vj{>S)=jKXBKP!gza)*B8o&-nJn zU|r(rS)_BeAFmT6WJKIRx%9lGCEQkbYLUklUaWu!S9Sowr0Jo$$n&3MeTOVylQFaR zv+F&Z(}-~?mM1Z1-8WEzy5*KyFY<@{)P1X3j}m*W#+^sakVLn-nGcP*f|5{!mPb6@bSslkAVR)1T%H!0%6wm&9!#20Fn! zTdVzy>w=mdR~F?-XOk?(9j$2;yvH^TbG^QBM$h>I>ddG=441M8HNi-cjM?Y~61nh# zWCg0BlUJafNjx2D_6zpc+1;eplQ!Uvf6q@3B*LE>N<}|IyxC46h zoQz_(8?~N0j!BCJJr|FJn%c$pY%4-GAYBvDV69PyQiIVMtmJ4u=vLF76^;z1c6`}XcL6NN zBD9v~HyN3p1T~TkDM=p*?`~h^YL(y>HqnAlh^%SB<+LwdEu$8J()F9s->$j(E}x@Rz}0sE18dFNCU%U$c#{CIlyRc3^C&j zSmG-zs7>rP6MSxzowizl!+s52FqQ>k8TCWdK*Xf~xbBO9TKDF#1moesG@k#ikO7#9 zp|)_DN(h%B2Xr>Onj^VZTA>21xva;-HKT5r+4R4}A9*^Kt@I9eZX4v+FRv0dSfn)| z)7=DVYGwD!ZMw;SU`9Ka<6ZlsROT-k=ZMsnr7a|D<4aca)oQ>Qfr+T#KY47i&!_Jj z9uW*#N7NTolYoMCOfBzMWN^77L8tL%O~bO8aPKTWN32DX5eSw{34r;3H>}Y*MC_X! z>#%{LYp%$`H= zeg`E+KHEJ2ZFil=Cidoo5Mw{;8swo#2H1UCp1TNz24mafET(hDNX+@Nk& ztb71HQsI|%3EzF1nhZ(M$*yxsi{^>)G!ErMwu_naTPCAm=hZi+1`cJygJ;K7n!?JsBkkK6D!UqKkU7U9oOBeV zQ=Z^YnAr*h>7}D*5m6PGfP_m^7=Sb9okBV9$oR&ZWrj^2LDMpH8UR^okRD%})?3TS z^p!{(g}4;_Y`U#BR4h9a^WccG)fMUjL-7REz7>^CAMs5k5w^jo4C(Jwu_NgbSvG6h zsy1J5Zg$10l##M*d6df21IEsvuVpkYteG{n984mydFye72Ir*jI~2DWGWB#%2>Fzb zW*PzvuZ+)HL=!eDXueTWx8zn z_u5)aYlX?Af3yDbL^n@U!p9Tcu9aS`;fS`B#MsXcbo=CVxBp*=bkpDe=WzKMeO8+O z&B}Dpe?Fcm_vf~G=^LN?e15{*=WqYHX3tOF{1<)w@b>%OWMw)#hrNF&Xw$)uk8iYx n(ucnd@9*SPzPsQ(7k%}Q_ho%xTvq00eO?^zIGXhH@1Fe+OhT^V literal 0 HcmV?d00001 diff --git a/Scripts/Models (Under Development)/N-back/nback.results_nep_1_lr_01.pnl.npy b/Scripts/Models (Under Development)/N-back/nback.results_nep_1_lr_01.pnl.npy new file mode 100644 index 0000000000000000000000000000000000000000..dc1b2a210746ffe22289588e2ff38ba0a4396ed8 GIT binary patch literal 1664 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1ZlV+i=qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= zXCxM+0{I%2W;zOnItoUbItsN4WC1P)6!5_wh08F?9}N%W5ExAl$f3b7nm@03=6^NdN!< literal 0 HcmV?d00001 diff --git a/Scripts/Models (Under Development)/WORKING MEMORY (fnn)_matrix_wts.pnl b/Scripts/Models (Under Development)/WORKING MEMORY (fnn)_matrix_wts.pnl new file mode 100644 index 0000000000000000000000000000000000000000..f94b91cb0287746a9479354275b382dc06760d30 GIT binary patch literal 92719 zcma&OS65bB^X*Gg5EU^k6%%5>EM|-ovjR#G6fq!RKm{a;h>GM40)ph6b5sz8r?sjt zweR-%4gTlmv~#idyV|?f1I%a6F{*m?>Q!Tm864o~WM}8%V)y_1pS#^UyXchIxHsu9 zkG+Uai$0q4F8=0KH@nFH&;PO=z9!r4j|k-JccYV%-Xy*bPI>?KWo+7;_lcgbQr;(c zUJDNmy>Tzh^M2T!yWxT1_aBGexE~%E_S7@&z31&a*RS8W=NWkA(T&i@cM_96qy@iE zNju{6G}%5j@k2t=r=zj&Q(hiT_z<7=CORc0`jcx^%FEdIiK%HRA7ay7lO6u&D%Zpp z|MS;m#{j#6WT)7u*!M49#-_TyOf(-{lb1!{e1-l)NT1*?oV2$OCa=AB z@1Fnv=6_573b0E{UKdjmw=5zvz|QWvvleZVe(jTA)_JMF`A&+oO$#jTkTI#gbX2Oe z-qI=^)3V4r2X~82bJAthHO5zZEy<77a?Lp(B(;{#Z&>^M!dA_*be!+xZD03Vio_Nd zs(*B9rM60!*seR@@Y587*ec^je)tKWUF5_QB!e@p&GDX?T#>fZ;XYL3*1?W*KR&rQ2TETa7% zbyzyC?TxXn%ii?2dTFbr#R*z3 zO^I4#W-r4}UP-x5ZP04#qIP1JX`Kp`-_{EGds169=Zm>Y+cz=bd}-i{YU{ELJk;7P zVe;3K8C{UPdn+*IPqA%~Ir$+I*Q8p18oQJRIcW7h85G;*=NZ!GbX9uA=B4wvqE4$^ znYT>JtoG^Hb#2#t>n@}Hqm%MW+N92scAWN4$4(rO9?P5AlRtp(H$15E*2G-iWL0D?!qtsZw^7o14yLoC3D0-we{rfH+Puui+ z)SgJ~+Ns^vkq_}v9$b%`x^P~Ze3#apT4$wlw<@jCe~qmfLB*%v(~xb@ejM z9}A_)^1pmVd$b2@a)SkFl%L5b`9q`F+@)MfE=M0t6x%Lrlc&uwk=kTw(|xJcg3XU~ zlFv`1^<>C$X|bg0iI%>VQT-Xtsfelsjv8*d6()HCEh-v^&!K*4#7ovUa_e z!dUIFJoVSf5ZqB@OfZ>oR~v2}c)%B%fm6YL`5ngmwJK}-OZ}yb29zFg%k>8zr_7Bu z2j0<|E&HXKZ%kj)?^^uDFGXxOEh&5Lj#ug}51j!p9d^5yhGWCi6<@c=gqB;fC{>#u zYQHXm9R6Ci(N9L1RiW7alluKJ^4rm1=fK;$+O5sI&Evq6m8I=iFvm)BStHHb6iLWq zH(S*H+sCBa+P@xO6y1~@ow2n2>1Tc#*Gj;2>zGzc`K8_XT#HWL?WxmUkrVuo2MKPXLF8fjQ-lP)@c^U!?lG(%fl zwMz>=zZ2V99@0#tJ#qS1{#aU}l{&&CY|+|p>5Ptj zI7Xax65g7t|B|FH76Uh0-f(NI^AdCQ~=1!8*7O82t-iww;#P_B+0a z3+A`Dya1Y-dja5@`90>Z`J|H@^oONC554k;_5w&Z{PBH`&uSin?b>Bt26r%x`!Yym z{rQ-@1n!wz2TnQQkVz}@>_$R+<}o*12j7v7wDOEjTd(~5bqJ%58itW3e`$OoCEm}i zZJwK7;QTVeLR(hDW<8x%advaE(mh zi$ZHV4AyWNG%RWb&mP;8fXUmW-7@j0JqM)ym_c{74n5OiohInvy;&YH33WAd$3DaG z3@%p(Ut)q=?w10km0ZDdeEC=D-uw4pO&QrHzx2oI{d|5djhkEvw>rKnRR@9efQ(rN z6)hezALSS#Q&xhMXuex=xJik50H;Z-cSGqA{rFv=p8b#~GOWo0ax;P%vW69K^`lSpVkHKe7+%0`iNWk8+h^gNivuT$e<0{pbJ=}#nKY(l%^0O zy3Y_r7e4ND#6~4%o=p$kUhE74KF^jBc-h*pQh)PxrdhNMHX|OGQTetj#B7h73|L3A zwADRT|HztQPaTTFdL9mSU@gkH$w$(jBZd zxA$Q_Tly|-e0?g8^dQwP-a5VdJhzp+C>m`B^C z2rmErR{N}d{=wfgf3Jz^KQeafq2`{D38wsJt2S!?8XdA;fx3cFGVc)RkumKJhV8Sr zX{kl}s9Ry3)Eq@~%*)jKIPJd)R-PS|*|So!9lP6P>azX~mP#G`hOzYDW0c0EwuYO? z&-GoCz?Z8mBCeA`Re9c%1uKNKr;U&Ikr8aCWAvYy10=A>1iV&C?unC()$YGrIwYgZ zq!R7o;qys4oQbLE|_6bDwCOKm-RSai`}I@ki;8vS<5a;#dE`3UHZcdD>vw( zy`OYkVOH6DWhm78Bm?0xoOm8H%fw3=g)d!FEwo>+d|#J_#pbOC()Uu!pY4Z~;Dc^A zOWQ8m`MuT2XhNw~elI}=*`Rs1A?hMy2LNR|@ABU(ZL<=tJPDQDeTIv)DD{2lTM)g| zKqg9iQuu8uIcjLfbFE9V7=N2TJWxwg?XlE$K-)9@LlGtb^DnBa8Ld$UVzx#}?P)FY zCw4Mc8&(XzBh8i_BbnEqB(+h4wMRaeu4(gQc=zqzRIK<%hP1)j20?dRxOLf>WI^kV zcKf?Ssu=N1Jn`Jlg_X?H^(+xO3bWXZ+~LN>SbKO^7QWYcFPZQ>qH{}VDbjYG*dcYE zCYy&!%@ap%Tj}JNq2(uKVCOX>f657x-iWZ*c$|q0Ytx6}{R|k^YTwCJrZ7&*%|9k1XF%+Ov#vI zg;}II_?-M+iS7L}weA8LuFvQrnLV>szrXl|4>xOf$PQ^tKt<#3G?|Ul@<)X50gyYd z)6b9>1WF6#Nj7@2(~%66mgpRUbhEyLa<^q|PP4J}yZ|QLLThR~*x! z!ybsV-XtxyB2a%#38bWYrgTtqCps|1iGe58y%=w zis6Yti(k#5jQ`1$qu;8nOn%XssQ+h}tl+s!#bnA43v!` z*0sz~K@otXkz%bw~XxfqsxZ$hhk(@2Uil6gw}fLIw6fNGiwRQWYXtBg#!{f=;p72o@IEKNRW8}*iCV9NuLzEsJxd~LDtO(K zOJ6A>i2M9+Zfh98oVA-vCvR-P7}PgZfZA~5+(BfJZJi7~jCgb#h@=4J8mUWr2JEy} zsqr&TBPb{o=I=W&|L@+2(JvoYh3Vu)Y_d!%*GsW4$=}q5T|Qw?qyTy#Yc!EN(tMB> z)HgcxWT%&OhDo7y-_v=MeJI~Vn3XVuT@R(y^3}OzkZ)bQ(Vh8PmSi8c#sogx-Isz_ zYxk1Fw9WOyS@~lxRhFxZevdzck8AMO_6P9lh}xiQTrtguabPn{JU<6X9n$w!XJaj? zO!3}@An6L_wqxEh^npA+11OYXAASwo79rCIk36;ML4>hvODdi&-Jb>0nNrWDv?zem z2;~qJ{nC7ZG(cW$hRYF#Ae{>#$xty)8cVB0D+ z$F=yu2mE5j(RbSGBC=R}-oJ@k_XgdLXC}Pm*M^9XaPYnhQBFkM6jbYT+TDo;JD!s+n_NW&DJ%%vgSFvFUGZzNrP4cFE$gM47*_bWNW8IVU}&j%+C-pnZ3o zJZR{?vKdDQ99*o` z3A=6^9o4^H{$!Hx5lK)-c|=w!U+A>9Ny!DWPn)y(IwovVgw)g)Gn%4(q?8}Wf+gSD zVK4d7+POpe2#LG;_r4BZG1%`wLM$FZmQadXQkWsto^K;SIjVB@@)qZJ_}^Je_M6Vc zOA{06T7SoSNhESfjWnOU#ekZnSZo{lv6I{NVowI*l@x{y{)5i3BJoTm-=$4p{-O)% zD3l51 zlzA6VOVgdF#FIDpY1bKVM*|(_l~VTksusu)KI&j{)zZF=Hq=4FX4Kp5Qx^8#wL{8o z9j8c>;w&jUsw3;ZV4-qq!CD#JB2$uQIk?|}W@^ua-^kAqE)Y&GV#^U@*E}TFh6KSf z6MudMz#7CP8?v-bsZxUPckYSU;FNN*X5Ln zqVFd^KRQT_TxXpid5=8x*w5ItV@0Y~$L|AbtI#w3j)0A>ixqxS{~g!9NMp(`?;mOJ zmaW{w!C9Ju&zl%P$?uR~m&vFV^pu{!Zpm>lelaEBCUW6~ixhsazG~4K8BW4Trn}Zm z$uOjSrnp_8T|)RvPTUIvnlvJ7&SUS#8>VQ@-WdjBTERs+so7@uA&*AVlY>t-u7YUD zDyg!tCI01JqjC?Y>(sW0EYmZTrT}QL&$L9SMf&k_i?%yBAZWqu9}m)69&ub^MQJnt z6!?cN>C!fPZ9GT7&67CYfM&L~>-=qs&g3yXOtGFi9Rf^Wg&>7>`KyFhnQ@apd zb;L34h~+wXNs2%mV#- z6^oS2drcqW+J4RUq8WsYNP-*Yo+QP8;91$4qeF*$jwV}oGzKg@(&cQ)p@7guC5H@DLxnu*6JWr=shJ9w5F`^l{&ENIg0XW-=`!aBw8%~=amU)IlMZbUmZRGaTkEP%~ z(LyU|>1 zOAK+K3rS^;z@K;V)YcDZ@1K_YmX)VrlM$w>HTQu*{^ua3`gmoK4jqOOVVtT5abY?S z$=E;wxB4Cp%@mqck$pt}NadRqrhPpe-dFMmB~tmt!zXooCD}{H1GM-ylk7(S$9;Z= zF6=yE6l>c?X>+{6lR8kW>+VbQx$Rs5-;8EGdc+*4@v{lFCLb9JG9#sbwp(loyCByr zoz&75&Vi83^G|9!roEYH0xw;V9BJ4A!-B@4$5EQ&m5lNNg}sNQ@Z4J;v3=CWtPkFV zaSaI{5}Z65CBuwz^_oxgvKv=m*?n{wOQet==ecSvc&Bn~lr8-sGnZdj( zl-m1-!O?v#)tTt!C*SaK`MRXL+#%1nbwcXD>7e}EVD!jaMv{O35Onw3`oPpWwL68J zLIbc=n;b6cztx6Z%K$>(hX?YVDQRxn3AH6+H31~GXSLc{+s+_zHyxI?MCs&< z-_dYwl%M6dy-=q65W+2>JbRB~tVMvDcg=&C?qvP;A6RIYybP>@V9r16uosQfX}$6$ zDEmOgr~#7{ebx@|kO$yl|K(^M{P+M8_mH3I^7r)~LlT&v;5oVB*EceU%u2z^1gLBz z)e8Tr6;3jg0DNuBW%{WWURrO=KdlrA`C55$DM8JIuG9XkS2)PVbaS=IciBFneZ8oE zVjW%{G_~>8RogBPVKO!G{l zaF+Ia`ZHc8CC9pt-?pZIp*pAa)2zJ*@10-@UvyT+&!?Vv4IJJ)TYF17KGPAFyj5r9 z?@B4K?7pDWY7eAC5F2Q867yAnq6lST`kgGYZUS^vr2)qwrbRMagOzmllKkB8TB(9Y z4?q6Am!gP8y;5xnN+stl5QPr6v`F&PwNu8ey)sFs*VBuMxUUb^s${H7|99%^Y5nUC z`cSbW(wQg?yR?eJ(a3_@yPAI~ zla|ZdLy6i1Bh^?(WjMrp>nh2%TrzZ+u6NTlkhT|jTYW|Qzj04exHNj}d=}}q#IifC ze>RBi92vt+W|CcYC8Lhk>5!+iP-UBPl9Hc$(~GVovbB+vfhzq7%iVl@TmG#M!Mj?t zH;G8=@La-fpZs1P<$zbZt*`@|wA6Ej=2?fR)&`eLTY$D&_jWqImmzODdk3{=hg9#> zu}{oT)-^`46d9Cv#ad3Gyq0wdi_m!nfJ`bKdbo?>?PJMv{ z%+rBPa}ym&B^wkS*1TvgY71p?491(HI=sFt_8mf=fJu5n-G1`Dsiy``U6%6GGLt~! zcRO>>X{r0yBlG@i+#q`T9mf~ySc@Zf7*>k zk+!YUMLYgFp(QOuw@qs@Kn11g=+(@x+8WPzUaytzqmma1a`$G$5hhLE$d3Eo(tdCQ zVa5viO399VwZ>3(1#8tQ5W^H$g;;3+ui6b!(M0~Of2H=R7JcChcH7SA-*Ej&kL&2V zueunpO|z{uDexpn$Ft1x@4yEgKV?cnIDKCl!up>SZ`FKjm5kGMK8{J)j{^Tz6Ju>M zvl7sF#L&RcqQ~&VO22#8o<9{q2i`g)1#h(Cl1a)lH}FKa)P9S%G;d3=)?ImG=zi-3 z`FFsRyo!t-0S^5-uokKUL1WsO;irpR$gmG}{uY#+_r??ewK{v=0~DK$6KN#e87;nt zVNj>^q2^^iQA>`3-OI0}CGgx!0B_5D4}0`88rUaIat>ceFuP7p%3t}u#U8)>utFZQ zh-_|v=j5MPrc}q>CVHelR2#O2%bzdSy5wVFgjgRVxGm*LQob2=RfN1-9&?o;u{>t! zjP@QQu@=5GF_wYZDxDk;NFTRJakz^bEH=p`_|X%YlL6|ECoq`144);S%bsR3>nX&4)uvT@Ory(#;h$RO zjZA>va zg1#Xl^WV3(Ecme=Q-Xz-0FU9b zX+S><1QcHIU6tdtPg+PKn9(0M{ZFxcETxwM$h?2t&D8@=@Lj*uq!<_#kaS^myiydxT3ZnoZQ>DC|}c&W{VOGVa8e4YdCt+bwb#GW8HZKRgp zCwyK?FTFFTbjhbu@_9~Hc&nB74ql=h->AbOTDh7pwh>P{uZ4bqs5ga1(P4YC_T)D+ zn4L0wTITdmn&w#2yPL#QdmE|VYS|+tzi3-Je)V@@ks$>P0P>-G^6NGAiCJe0;lh$c zJ2Y3e6?A@g(&>lL!581aSebX`S{BSMX=Q4V@osO-=CHl^m*6{cnb>O}U6+~XUnXqp zWMJieM9*5LSU`WI;(+#3Gv<-+#t;BACiDFKx5{|Y#KU>4mf@fogQ);*gx zyTKZ!;W2TDWLxo-L`8$?$T9K*OaeuXSvvY<8Tp|43T2P&2vi-#hhN%mXzdUGG~$X% zY_?u{eY9f7D{lQp7M{OlgAvBrl&;l0G?b+D-!timw!eo9d_J=3k(08E)Qw;JvzBAl zMglV91WI0p;*lfTmG0w(hBT8VN=kR*y6>i_R+9dR`J!=CA zTN(+L7JETyLq2Ac$x0~>-+$VyuGzw*BQzex&_wLwpNYGCy9y^(S+5Y=lXgq%NyAn; zPHGBfMV?xn-Df(V0>d`qn_Iq_GVYJZ=#k0b4LF&&E;WYu;lSrglkt#0j!)B0p9?h; zDA3~2fXugAXc35Jt3u-g#y4eM1M?#;ECZLA4MAgWAg^MxH?Di=E~WI^$T+m*q-PuV zt4>M)h&%bz@h&OKr0Z&?FeT+5$;8x!J9x&M zj93xtMc^813{!~>OW}C}G3&?;=@L-EsocvyOz_OSQ zCeUAxePF8`Zf=&d{t_fT+^CpLRy)EF$v{XvOx;I(s?J=#?T8vJt!`5c@RSL&^^TrSYW|PU@h#U`E`MT8={eHY+Eqm_P3UAf0p_d zt#O~V(@y!G$;Cu+M~aqYz4U;#p!S7JD^G-zh@GV2toE+~on6-D3&cY&8&SZh`_&E7 zw2}ZLvCbsi(|UW9$9CZShb8lJY9-Xa-KKL?Edcbl46aYsLPxU{(DQUDS2u#CHp}NJ zCYYnc(Uoo;Uk^6(Wb&~NLwdKj$lS9~GQg|lY%a_|;^K&4pC@wIyODG->l)78G#> zN7b_Q&>WTEVLG8Uua(C#k0HLOYtAe3<~ubNd)aXSjiE<7DOU2$+=l}9EsN1u=8@)Jo0(5MM-6n&pm(p*sCrdXv2~XBpPEWMv(CuBY zxYXake&4q~B%wgvLu9=Xt)L0O*5)DBQrQB)|L2J$8%H4Oeb6>U?+wKNu-d#%ENOvk zzE-~KD)i93v_oMI8_~2hu@|5@QfoG1Etu4hpY9f}W>X8Bt@;Ba1DC#CWW9Dh>`xF^ zX`dCU!$*T&zm!~SBk8{5(wF4`PA9J(#HYPcTp%oFUQI(1vZ4<1uIu<2GAAOMM0r|s zPQQk>=tY~&Ve`>%7{r8i_hrxvvljq9icQjncx^=*S@CH^&J-idM`}Bp^9_Be1LwaSiP3t~qA}K|#)YbexUJV3 zDRnfP-}=}i6QguYiLMcv3AfaC7UOn?zXBThFWs;O+tkQS_fLJ2=0|rx#t{^bj(vWi zMZlUArqx#im~{G?RIR_JeejM}#!Jnz{oVjm|3qLlcC#lkPVF~qG6SawcL?FSwmL}J zV}L-*wTF!{nY)=MVqxaqx=*CLF*3wTLcVo{cccMa?{2Bd{Cd;KhLUIYI{JAfviJC^ z-K%+~&A4woMXM=k-jk>NNq@8Zo}D!)YE{W5SYp2)Igp)uJKQua)7;?euzFl!?ZKPA;Ww96HWnXtkL(w3~V zkBxxWGH~|i&N>g14x30CHl1I_xNno*XJpRpIOMZZDh|_bNF{o&$`AeLqs4IqN90Qn zf`1U|!u^BSHs7EYSPC(0^GBPt_~l!%(ezjihU!TvJs`LyUaPieA>yta@sb}eagk|0 zv+a=$+A@yX=-|is@H6L+=`X;O0ri)Vlye`GBy?&|sxlaE+5x+y)a(&>D(dn_ivL$ym)ZkqhYVA2?!!WgPcjyn6;1RvasBh26C{@evuZeOC#LEPPSdQAgocGwC(FQWj&XpH#X_b z)Ry>{IE-fUi8uQ$TwF&l>k)O_FcjMWzs_2Z?*>ch0bGZ#b=+IqHpu8LBcK>xDc+oS z27IF|M6<-ewnye<#@+Tnem@P~PyP!|dx-SN;r9p-e}4?ze)|j--5hZRA}pYc&9~Sf z1zU7_dmD<%qY9cQs=FBm!PzSuyTR~f$c|!b+gSwb9LXS$0opJwI|$ZjQ7CHd&Ep69 zckeS)s*ap_=e{!pB*{GYwe4a<1lIA=s)QKh&hOUsH#mPFifI^C)D!(io1U$DPsTbc zpW$1(4DAd21cyw6z!6%w)#Q}n^TweA2alnSYR_XE$vwP= z*&$?5U2NX%ls~t(5wFjcok9oBkhNF@=|H~ZK$x}Bq$FRh*RIWPHl|Yn;=Hm*K-Hve z)`Mq3h~jF+&Zu#3M8FjODS>_LbmqLDcG#PM(D6N7-@lCqfyL{`$t@L*FUh^SG5ifFukTlvyuOg z6&QGAf6Q%6fY*P>OynshW37%fBk0%-N87ln(h%7s_G-2^#`>M6@w2qp*tN?aMEr0e zBW}28HjOutjR#@SO!aN^921V98v${PJ zORKC3udP^)P?+6;;hZ*Y(V4)vBq@+KDdiaS*JHRTFmw4K={T&}))t3g>bh`{X44rm zar?KYV6Ya(YH~mzEI8@7b?G>f(Bdt1(biMwpctc#qo&%ei_QV}wTA-Ga@xI~B%Vim zfhATBW}VQv7at6LRY+Nev^wLKMeFt*8QK!%V9My7Lwep1PNQ}tTl%&2S`~0TN&PZX%5!SMFpX6 zPRaL=x^Q+`CUH@XPr6B)&1C%tG7u8Jg=nTf_0#>K&fPwQJ0kr_zEe`{%_{h@m*>Ad zL2Z|^^4ceh(t3$3%u}l_Ld12*wwFuJ9Cn5u{$4QlHd2|5s#v923t0>uPSw63HQ9PU zDW%|s{@?_KCB@|b)o3Cr0aq!BILo;g3W4`B_AMdVNcqBjcaYelcfjH|kGnflbq-=j zM^d#jIEVvsPCI*6DkH%MpVadLE ztAP6t`g>+CYV+pX@5u!N*2mk|uC3iDa%J!f?k z$%pqWZF;&*dR<{Firy0&wC9y}*%<`VF|Co>%`WsL4E^or^sqPE{;bPywB-R<9L5m= z4{&S?V5^1}wa!xu?~x{HlkxQdlV#eq_!}8+wX%b)jpQ=?ne#?DQg%>>LKM2Db1C!d zb{93zc z_Q1#2S_EfcgW7{wwR+_(8jIfod`-3T5Nmu~rgLusQkN)e;^ZTo*g$i@#L83YOYtT7 z^jLU5KGJNfBk%@}OTRA&nHIV(U7EjMyP}u@e(cuY8Pc?chCw6I&Tjn8`fESg4GU6m zva76nHA8w04>R9}C#Eq_rvuV_i_}?a%7AQ_>(38uWVvJO9T^Lkwrfl~0vZ_ggl~S3 zTT@SNx_VFtUoYXI@1WVh-HA@()}_0tR3APRh+Wbg%pv$XGaTMAQqHcLYQD=5c0mHtr?NJ0;ismJu9@um7{Fki zQm~G6zvk&CU{onX2#8N-<2LI!qSjh{d+l3UWXY3hmLMB)9KXP?EMBqAXO!QylfTu* z<}-$slVvi;#J(G8r=GCzoM#*bEgR*Bd(V=X>XKGsBTsxiEpz+vSg9464Df%*oDte? zQt)Z5wEHqATD{LGSZ8r^i4_`qRO^qr5mmJMs!=+&$6yi{kzIFa{>fyn>irLxR7>Mk zL)#N)+c&ejJX$h50M1XB3Sx+;0pUg>rhj6 zQM;lKo;YGXk)GJ~^uuwl?bP-V8KW1BqEB+L_t(Lvi+W(G%z5t+gS`j*Rf{AoM{S*K?B)?pj*aqsD zlkylhsoH1m;C5cej&5RWvK0=m7&7i)R1h*Rz%Y3^gR!<)XJCZ(hg$HFw6Gdl|LZB$ z4s4O$do%0cDkOiEb^M(J@ zh1MNrm%jqJ!NE_oK(W?Iou7j-} zedpk&(NI4cp4zih5EvhJQIg5%UN0$&4mMW<*ijdqVfn3&_&fu9{fk%~N48{PwDoYp zR&x1U7E$%@4o*^#80ya*(%vUb@`bft8y^YIqs#g`QCqckAIVosZ*h+X{#hCf25)vE zhReuz*_OwX7#&`*k2!yIkYBeBu2eeAP0m^xnj%%fV!K2pEca%Tt<=9k1SadO@#K;* z>%=4eQ+m4m?3bzXN1{sHI3G-R$NaT%+xTB#MocJ^_RNK26fMhyWPZ@xfhS zrgIxkuGMB8X6HnrR;*sD#ZCebgUw&Ns4qHB0e;)MXjXnA=uY&Jv~U&8(Pva&)odNJ z#n9__XMu|1t0Up6%zyi2N}MhoanlLvAs~?>obdNdKBSe_ejQlnMn3;Z8}*^|J?2gw zi0lr_kIj8#|1m7IXdPzRP^x{S4YpcNzc?)uDHH%C2%{^i!lWs~9OJ0ms0#;HBr;aB z32TPrH^M(1ESxmU_=<}(%lK0pZ<^3Px691Owr4$X{EJKH5hM59AhSWjp$EsaE)t%) zmHDyjw0qSPZ3jb!7G8v>O1=(nG1W@pt#4=Jyi7Qb%0Q-P2$aH6gu(arPOMLxe)PeU zdm?mvry#DjZT5$Wd07N{ujZUeBKKX_<6fLWP$| z5m&^@RG|N%Gp4D?Y4+#Lqyj|r(%UbW`V(IcQY+nBZzr|Bl)nSk*SGR3^CgL#Igq`~ z$)~>1bLOB)HI7@*>Db+9tb2@@x)qi;N0T|Y#}T{Ll1p)}y%(;7)9=gq3L0ki{$rwh z7RV-R1x1|eU1mliEhJ~!@Ciet41AIdsrN*I#;^K4=7#*+j*brG-6WBB!6~cg2W2n_ znf7~0cs<wT<3S9mI&@)S~@mkDdb4TR2&j-Mv8X}QkR^4*G9 zvpmnSIS;R!MJSIO+od;5=VL7GbCCv4%WQ;|W$G3?8*UmXqJXh>wNXyGv~o74bYxNy z>OlB`45q!cBke3$X=f$-(PMaZ*m7Q$=_KV*P?0N=cjSB^&Z6`5LdLhBjdhYf`nytb z1#153O?w+8vw9l)#LGl4`}I!+?0h0KRzk+%H%{7TPZW#G1;a#hq$G;jBSqN)nQvglReCH?RNtU(f99 z0Ae0WQySiqm~L)NdZ7PK!-bmrQ5FuJryQl9ShP79;>@AsVj)P{GW-L#LHt!#I>#pW z$;fT=AZ;dIZQzbR=ryT)ocA0p$AuJ6SYW^<{RV3|Ce!LL+a9FnNyY}hd+XU#+qBeQ zz zyZf11$eh$E{9-~Eb(NbMFO8|N{Q5{KwO@~GCM~}UQky}3>VOZ(_kX_yOYv?;KR)JM zW~x7ME~^aXD*##l>l@d&PjB%0b?zqnSQjGLS-|!Y{#FRfXuHf^TYYcIF*VmB?J#Ju zk>2S6I`zR9Go#~XUPH+!wizmTqO*+Ho6AzO z5#3=c@PrMi@$bY}giD9X_q`BfGry`zFo|L2{#6+XLPHqqI6a|al7EO)?LI`qs#Gi1 zL1VTs4_W-ky??;;Z&P-?dI>L6)nMO7PTE7^ob|C-g4We?sDrZCov)I6nk}SSLz?dT~OtdWwl(h1|pH%y)Ou1WW z#q?Sa(s1@7S+s}V+ik$uxDDno3-`WPGLGojMutP5=s2?+UcuM1#kNK3?%hDt{#sH{ zJ^J?xk6~YwaBh!z_g>$fZbadGWct82`SHyBpoN?)mc}&bZ^FvRSSiy7QES-yEdndo z%n?u_=N7JcrfS0>BU}qkAAx{!m?e8PAXpz)VzL^w&GteX`O&-{jnl8VZeEMmI?>Z} zjpDrmyCF=1f!$1XSN&kgLGn7Y;}uB}ghY!Kc}8nDpZDRKLwn%8a(>zE!T5%^%gk3= z9v|q8&=?H|zPLbUE^5yTt)l?vnn8d(qeI&1v;^h0n9$ceVbQEw4Mx4NOOH3}EPX6T zdIRKgV=AUK8*Y7k@Wpx_!9FAo*_#{jGPCG!&rnKGZ$sj;HS&GweGI&LL`$5$1iYrJ zRLOgwzTh%$!TQJ*f0;YA1dx0-Hy&rJrCH$bJFYC1*_kErm@m#>bnuReYh|x?^|QVB zoi&N!fU74jfaE@lZ!ZRE1sJIS5NNnR@Mw*R7y$8ojlT>Y=Mq|Se36&VZDgY;1GHuM z!rbMFF6OF#$tb)U$z=zv^+{U~sZHjCfxP_KAvLe)(gIHip*7h*SKtop6`_V?ymsYQ~J&v)2_!LMe4G=_Dj(+!z5wf zyu;D~sLvpa zB4`E(vgT#c-}ru`6u#?xbN_jIF$~i4!@g#;G2SIe0vv`Zmx5G}-;z^gjzVTc`1U>y0dNiMmrh#g6o zXq;8O7&HTm>a1KAkn`U@$BH_WN|3cH?P2DvLkHq`^=7z`uwjT^?lgTR1)Dv&!Hnvn zHiA8RSUSBCZUG#O@PJnuhV4Z0X|9ypz3y`Fhcla{q>4gC2ie>j zC0F)h4PQcL?YN{rZ)BQ^h7^zJ?a&hKx?rrWExhG$4#jJ3muT;F?jUbdB|mA^4G-Zp zA5207XP2}esYfFsfzwlX3^zw3~p!%~0Eq@9wh{C&_I5e8Q$r1f;Vr&2l??N>~U&cYKQ z%DVF6ab!5*ObAQcRg6@EDgST-X#*119~^=IuX}EkqN^E8zG#05;r`kRtjG(~D7FPH z`_CNIv5~JvGN7#aj$5 zWysIFYt1`J$R4cN@iN3C`w4-&4`d|p4P}Ux9=JkAudO!-APv_uhXGd}1e+h+jI+w= zS5_)1sU>I^`>S*;L~N@BB1XIQKoKld6_Q?u6Quev6ug%`m)A)@ZJ(`(hx~r_%nhTs z@;=g)$ho9TL0YFjH~YKVqe5@#SeDf9buv?)*l<^VIZG^J1o4P?bf@_aRHn8)uAs>x`A!6^|pB7_SN(=RBpa z8R`OQWn8Ou;rx<1n$w@x_|OBZkE3JFPG^GW&j(a+I@i`<`ur50TByT!|GP{elezKsQIrg9BL{FrP;>edx-Oi=&1mUXvR++4m_umh0;E50Crf&(JAc zre136W?i^)i<{f*QGlV~7_~Qy0HAbFbB30r^WHs|%UewlRC<0*)pju5Z^gw+QHIc~YT8N0E%yBzAC3Nf)8OgbBs2~%!Svj1RrsrFq! zbeVK9q*F=Szn^Vq5PRurDY*`R=YHldU58IOKGQ++4jNTw4nN{94JK3ak{JX_O^DfL zv{nWmy5HoQ0xQxne9JnVPRe&kxoI!;>lpMW!q(_-T6Y6h$oULiaK+pO)=QaqqyzC0 zrohZT9fIR3KWnPu+o0#0o;SBn=K!Jk~`XoEp)Y|)62B`24*yuz2XGLjRcUN%~oX&*65qn^v}UzTA{4)&Qkg6xz+S=G(lRy(W-=9V4U`hcr_x=FWn=B*dX zK|yWYNS5ODO&(;A{IhQ{R(|3G4u+CtyR7KVK2pN_BCD+TGVCr@Ypx+@KD%UokoheB z%yf#(R_VVo_0S66yb+eDyeNIvPU*WFe!>-%fp%`t$``u$**JRkj8yD{y-0`Objaz2 zmOsJE=e5fHTZr1eSdn{<`+FRO`EmGPf_gmemKIprzLJ+7`e3eZEAZU17iXCi%gzIc zr3HsOYRLU$h+916262fdgv5o!mmebOaM{HVRM6^`ArF)otU=3vB&3^LR z8INGPb5FP4!{+62)|QnIK4uVlMAhyG=RM3Ts?E2Gq{>Sgc^`;Lm2gxc)2*g4S4BF% zvQcV}llo2dc|Y72{7^g@!{jC(iv{=XaQiMD^Zk#vD*`N;@$yx6H@h3|^O0U&x(AGB zK}I4iPYw%4uQPL8^ZLI2J>)_RuT~oM*QN3m?T#=0H&$rVT9e40#Y+E?uvlOgPVZ-) zSxxLRD@vR*d83@(fR9?)(Lel*tz%^Id{*9n>o0QO6x1|p9bYX)SNMGe_Lolmu60MW z`50@OB@o8(`W;=n{A5A6n%KtruD!xySDp zE3_R1Uxd}hSFOF4NfCMe&TIYdgH3@#y*b-eB6;azyW^t+A-uV>#L~Krylpm-hu8=o ztl7_^1Sn_WE(d_}7JEnfbR0UHKg?v?c5|y!j#7`}&^hKH6Q}p{(!Tpf`D})Op^O1v zt+dSfsqSYn}a%J+JM%o*$pJo*(Xeh3oo! z&e>=0_kQnv_St8UZXP^aL_X$cNA6=$sr5BcLIy|_hKqI|zl15GZewK;;RSnuo z96ki{LK7|X=NSvsEW5VN?A@xzcXB>Lb=74Z9sSj86P&#x(=c;t=@s~1)RzQ89ou~0+dmN zcgc(N|$-)^bY)AU-3VWU0!8X3)T zktRK1pKD5^DOI-IL5EHm@Rlxq&(hT8Nyo9^v*$c{Hmf3k zBYb4U1(m3ye;ucf0OLX8&_Nu%l(?>z()HN(vnwZ1f;K0%p-*0vx$x(Bu^Jq23q6vO zuc^xfT1KTPK7%RrT5f6bWB*Gm7^LZ>n;8DQ`zSd!DSWI9SiU?!;Me&37Yty0E68*3 zR?sVdbn81Uu9`8saUc9OivksRDnFEDC;%7Ax08Es?!c(BT&WxNc?PI>T~CEul)0;O zY6v!e4?rj#!K&FJCH_KJ}&H)OKmbMY;{gl+*$3>rb$hbM2_a>7?CIE+xe-}}y1 z4T^R?)-G3NOa9}z6FN6(2J+ZB$zHK4!+gr7^kgJ`zsx)Y20SCYGZL8kf_0bY{#yxz z6|2a>OqyUx(N?@K0(Lf)nXaG%*Mr-n8tJ5@J7&cGF;cz{gachzAnZKBs_UZp*V=k}a}uoGoES3C2wY z`o1y^@tEOx^N+GefK1E%P(qq^$lrr}tKKp#H}&n!4%p{tf0|v<1uDOH6qTk$$J*crJ=7J-m3*kq^6IJWCxo@m%K5dDDL7 zC~w=aDd7x4>^`Wi5aE#DDJ~!_E%WixoXPxZ(GZ)Nc)9|Hpq(OJ&Y5~WN7+e#6lu+8 z^*E>!$xM+qDgJoQEArxxxQ@abW-?mmHga7GU&p3`K{gpa8K1N~oAE5;j4K}Sr#HSQ zL;3^aq%Ux3;VR6`6_R+m&rokg$L5iR`y$=EStdQ;C5yZ7MXM5(%Fug5-x-3;KY@X! zI5}4h@4OlDFyDbTEXd;nbV)P&{QJY%;}4+;4Ayd&xlFI%=cbIJD}eto6tYr>7U52w zz&|z)Oti`8fE5Q2`I87=GvRs=J{o7*wo2bafnae4zApK$ENBh!ZjTA%psRxf6LvP_GRNLt7d0)O^{A$?gDfvjB~{f7d5`44k3 z#hZ3YW!`H<;KuX{P8&Av3apnLwLQRD2Nrmu%G$B4G^Y5+kT`>Vc( ztPeCEvyqPMGIgP*>aGf4BgM1qM4R8L4E>I&xWs}T$21Msepvd?t{_i!=9YA$%_Gp5 zLG1z%=Y|5evO24tr-gplsc!Y>a+;97rD2f<0=4($_wGap%SAX;KXnr3FKd=7)$4;u zcIy^q-AaBnz1_?Ml5Y^(q+ifdg$bk>u>3?0MV044th8@~Y8B!4Y?~m_LoY5T=>%Hc z51T(1vJ6?b+#H8!$229K(aRznKIQ{B@h@2YW@!V5acnjh&b`Q&_*5kG=N(GL);{TO zOWW0suQ4^NbF9=2{H?~1f^7KDB`FFAhOIkV%3eKb80+}+ToWa;}r z3V|5WdtVN@oe@22=OSNDfq5OKJp-3Lwe}7t-&my+ehj9Z-XQF9_W&WqAq3GQ$nWFX zc*m>E6<>rI{<{1cse;VgWHc5(P|2uRx3bI7xpv1`&=CZ~fV)jY6i<~2C#l>=8mQ(a zG^L05qS^Blat<<;Tjgioj@PUR@cM_7_^c`P?~AW=tTz;imskgPZQrob5A|d}Tq8!> z*YiN4+IP%p6!NihjTxg|*E!Ym{GfbOvwR13;kqkx6DBc$+W-`b-0y8aj5X~9Y&r7N z3!eVq8k0{-Y-wXeR6+Z0)~L}<+h4tU1QD?$*+U z8h%^q@6dZhS@2&pwcQbr$Ho4`#C~01eM2QZk`hW8kC%ZRdr?ukg2MkABO(6mO7w6RoDpImbf~}^e!Nc zvpsy}mo#T5u0M@s*|MM-4Q6u9*?5aO>4P^>*uNj>Jzildo@nY(3LvK$xvT*PM^!J{ z^0uL13tV3Siy&uJ^W^Ayc`spJkTO}NSQ*}>FrrZYFF~(MVIKF*c#t;0DJ%%8!A{pO9hY!jj zKFMI*5wN-5I_gP_7aP8pSXMQx`MxE_5m?) zH-3W1mXVfe1{1YVZbS5!_(1CT)^gMQ4!te-h-I1HJPk#flXEm*vUu|y%iNbio4n6v zD%_0bDPrE0=5T3GOVQwMBJ)rP&^fP|Y<^veaDd!4D0nS|`$;v$HtH)HWg+=6{zf6_ zH`Y2O+%v}#43JZiF6Y4QfeLY^=VF?7E1nfOyu7EiVa?XH7_65XUVDvpMm=)!@ACEmi+Fj4 zYGP@nyxk@Zk7!ta=n{eBl`V;64llB(Irx?Ddz|0jW@v30nHwXMHEgTji~85Ync24@ z=D&kMO_|!AJUkn@cYVC%L2kqK8P25dIm0LIRP5=hd5Yn8wDrwO;eZRi8+t*)ih|_t-l=idl?c$aYu$S)lb>94RuwH-lKqp2S`0rU?>7d-u24$ERZ61!nfBpujqJdU&eLBxq2m4Pz9 zZ}M&sd2Yo?bYkF`wCZ|)HIed5#57u!7<+s_Kxq{{Mf(A~x|hzK5J zR;uGog`GDH_Z$&v^{VI$XdcAQ3`I)8X--{Ohc1#NnBB>_b=HOr{8+bhI3Wac_5p;g zQGPlTsYIa^6ELv>V135gdeiosG96VEnu>i(x`G(*c8@P!_;SI;fCswHi2GOHsRMIAqhii2$i~6_Q?^6|Joy_ z@ilAn$+~&)*bQxiT4G&T0YmS;t&GUp{9v7YUqxnJ4;O~h8;6Lu$U1%zfi@{*D}L)~ zr8a7bg|ZfYoHD=SO+m~WP|_7l7~lNp22!=dHGGW;!O}EeFdH46b7hY%ZQ>^4<~r^Q zRC}Ph|B@+Br(-iDCz#A?ymS-R4w+i-04iKc`DR}h-tRB7oWopf@|UK!rpn6$3tT3R z2h8Y+yJ|KaGAX5dD2v1Kgl7_$^l?s8H!9l0cay}L&~Xby4p=IYh7Z`q6L<3|;VvEm z>-`~%FT(J3-0X&?W*~^q+^qwhdK()Ez_T%g7dkCv&xAW^+SCMaqY#>^%Qtw^1*{%U zcnjV%MdUGZ-E?ndAUGPIfV2FAqE*$bUS=@8GG_zx-V%m3aNt7*Uo$Lf@q!Ot+Kg@- z;?ZSN`;769T9%o=8@uX|n$l-nSdViIoEU5yx|1s8^5(Ff%lUv<8TX48L*CwN6M;gs zittLtu6wSM(wB_RJ9npP4d3vJFS?!Ixm@}lLlriMzCtqxG%Tx$Vsh;-aVr?iqBQI| zmuuP{S=X*6J#;*$_$hT9W`R4=Ws0!yEUl%bPIsF;kxY%pF`m88>uST`3ERGnm$?hNIm8s;C!@B1>#XVUR^jnA~tZ@YE+0Wfs- zvIaYk{3A6@F~VYjOj!928PbDD0VJg#?!RqH2$QxPHxtKr2m6GD@TIx^3<8v3J|$+r z8`+e+#9j|bVN%Fi2eL*-BA67lSQ~j251sgM(JmJ#HnsOo?~?M(2jLzLF)wl<&y*Pp z(JbzFK0lP_ecf?93Cg35K2{Mp?>ois=B!I=61+`&4q@!B@Av; zdpr(3{bO4Ezy*peFJUmI!wvr9wd#C_TJ}Pt0tnC4rUQco^6|}H<_-;1EPCz-fGzQ$Y9JfQgcJjPJDMOcg zQGVNL4>z_kFo8aD?nSXTFF(VIxx13V*ONmaBien5sWdZ1T2NZJ6>f&iUKgUVW${jS zUbstv$p<(G`}%HC_YafC!`JJD8~}9owbka={>UZe{JTZk9^vPyPA6h~7F z_GFu~6!I^ZxQ$7^(qM)S&v7#&6bf@2iF&s`W-KRKrkU*7^Du{QGkq6ykL34a7P*!6Rh&GwFH%N6D&wPU{ zSICtYYT0S=AzT`^bSbKHhG|>(j?+;(j`AzVmq~RZ(_`~nT2=uj>>i;$Zc>$Iy2u6a zw*ENMvgzO|s&xtXNV6Ngw`m<_#-|rf@f-^N=G3N<&nN(o_DtV2#fP+lD&+>tTzX45 zBuubkj~*5Bi+T*nfif0T=RoK-X!$;(km#5h17K#*={Y;C1j9=oS&0j&`;VvhTT*dJ zn{PF&=HCy(?x!KP37wpo4l@O)S1lEUFzWCF+$70Uep+d_Jf$=Q8EO`MI0)_5J1lxG zSq|ZOH|Q#t#=M6Bv=*6p^bz-9^LZsdz0ABcMNun1Cxgde6I2;PT}^)kPNo2T1WPN; zG&kBvF~?0|Jia^g4Nx^;d7pbOh51i(q})At+fjlgdigg1j;$dKk2>>KycI0ik|9fOt=+qPaK{V`yLoW6hm79d(}WhPzP z{N6za{f;27HXbc%McP-1SDRSO8P1R{QL`9Xq(TFG*kFEsgca%;U+lr{(P|={4o`+* zyM$os6VtBiHwuMg-%Pz>-)Xca&`(MOfOj24mG36`cVwB`UgcRPIAR)+tbz)${-POp zy&dt8I2^luM*eN(V~}Pk*~%r&WchT`ra1Aa>UaEh~?5C#A?bvJbStUkA>J_O8-p#M!|dJLhq5o7M?i` zDUOhK@A$28_>^-fTh*f#hGZ}zJ^b{*1_wpDpBEuAkSitzllikdA(vm)J+n@|50Z*p zEcKH`i>9dnPm6>u70 z89T0sR0nu)Ez;d&)@zC{kdka6)M_{1cB50UkGK4f3;DP`iS4LXE`8`ay&pO1`p#b6 z`v8Eudh$wu6ra$A)XTP#ZDUc~MW~Rs*$Gjn=F~4u4kKDIp85>oOGWsBw6$p*> zm624GRhg&;t*Yr#pJUmX;@dL&Xw}lPcEB>D%xrWHr_6egW&c?c$QEl&9tN^o@CE?& zzxpU42zw~9T=Dh?OwZE3H{8FEWwp5?J`{Ezy=SE49?1tPFX3M@7As-Y9gYYK`q^VWX2=FnyoT_ z{+2UKiGRN!v#Gr63k$g6JkV33zHN{c6`rlfO};%*ZE8%QX4nv&y|tdkOErh0cEdT) z_9i#B*2o>UbGvb&DZ6Wa?LbL@)~0_z_Ptu;DdoMrc)+!N580;gVsU@B;K} z_)I5qki|1HvS}HEN5tFL93d|L#u5UZ3kS?F=_|0O+;Z`e#-k3HbB$7F^DX^6?iR(0 zXRO0^?gXQFG>WU&n7#-{e0!Mr;U9rXw=t%mB(HQA5E)-pO3`XKuDcd(v90&SrO4cN zU36Ll+`~C{{)gV;h`Ex0>ohk$(QWv7v*7j|ITkSMw1;Qq*ODzhrao|(4#Efni<@{t z-|Rq^cjIf%gHnR*Sv(MvUSxH6Gwu}8hWR%41Qz=`P}qLVnYAJ*anj4n)WmJ)6NsdY zbRS2?fD7&jktCYsRBDl4 z)IZ=wOjRJXVbY3b>}nxJFJRg=1foQ{v1{pg_6G>QnKTXJy}0%V-!_U z@<%Dy#2VaSGhU)mimdbRO!;N5js!P({NQEjPX^h`EV1DyYFDcbk;yo(bP zu=!Tv8<1&w7cfT1%Ro$D<}ve=yI)BQL^05Msv{uIBebS$PVfY-iq$c!F(vs}UlnHX z+loDc?G$?wX=^xNu#6jT&oFbe;6GEf>!x#;=#6k>mc`(>!3nhQ`g||>WG(c{xXI^e+61=@I!9dwA`RKqz37Osd_ofJOg6! z?z}z1Kbn~-bG;jU{Uoy{D`2tnQ1#z~eNw7wvqYnU9-u{74V!6PtZ_Y`$q5x?-;>O@ z4Z8AN3K?aZcC+LdLBP|4Z;95MdAY9Y+{IJ)TM7tx`$aRJE_y{E+_N#|K6X2g$FCuC zoqJWj?O=|$dztg{qbUQ+!HkCSxg2VdpMbgf6UI|3kwfv4QXbqTQ=I)^2|{+7#;wqy zex;TDm$OXelfb3%n(E`P9uqr|O#NmjjY$-7j9FP8GUxCr1I5uODFT@JvmNRf#ZQx_ z^&xQd2eVj+%a!qWp#QF8hjyHh?>f(FKyJcJ?pTMUY|EZOm^otnmXJ35v>p0TarFbk zZ~kDNN5fvt&gb9ET;fI?DBMR{X=a zXCinVl&14EEcFLF>kHOde}vE|{qdSP95h|~0#C8TMHU6nSC~KtJQpu&Ch}h4)ta$u zQ*j>XoM9oxRrh0Wd*RmS&_<&C&Vx;t_Ei*Q8MbD8QBP?C$CITroPj-CMls)5g9oS*NF)&y5cpsC#$Vh=xlfhH81 z$yc{@)r;{Mr*l|A^#1Wi({w-J6?+&Rv_LUVM9ZWLcG}BXF6rEi-4!D?NQQgXJE>g5@oypr52MY5Yx5M^0n(*K z9bnhYy0JIGRIK6)AcUFO&YQlkeV%VU3tw(J^sxREEzi4vm}2wk6mI!!78I)_n#2Yl z|NTHl(hlrmbI+7LdkMNDa6ESbLoPv+#vT&#Y~{S9`K+9TAjIk2vU6r3o59))7bbHV zTH|FU7N^fY;mZqAyv_dNQK^cH(qM@LC9Le{crdZYa`cGWdmcb6qN_x|DP6to3^r)K zVqIqJsRQfs5q>mFOnHF3m?=w_nf#C8OG`ZSY};P5a08XSAR2D>Cxa(EUUM5ghuv^x z6d>agy7&nWRbXAn*Jb|9DC;f%!%}(l?1OA;xxdWYd2eAPdrxHEh(aDn7{k(ZC#CaF zx{NgM8QNed@zi`D5i8{p;Y*FCLgm+C)Ay0G0~nXJTE2R1QFZvfOTL{l-!?0cl}Xok zrZp}OU!A?x)JoZbAa>ApOGO|%MYKsmsafcWtvZ$Ejd-E%SU57&ceUQuJLgTif zkdF@N_16AC6$cle1fE=n`?N*zg737Xyv6|zf^f! z@5cMjz%htci_;M>0{9)YZc1hB`V55jix{nPD+l4dmegQ;z0wg9IV2N}Ex#WTW`Um+1cHi4E-cN6=bcfqAMJxu!1SLtyd$4;=7w%m2_CG~i=WNrm4;`lyrNvs)jgt_%XX)D4I)^i+>h>u;;n94EzZJjphD2U5 zZRN7A@jxTzGs_UB=+DNCj4Y5)BLGJFBYcxuNJ$j)30PQ)zi zq2A9ul6URb4C8N)N2H$K`EOtU=q^efY7gzGkn*cbT2$%obLZ49mVd3(`>zKJ7KtA2 zn;&l+HrlI$euSs|k%@zz##h)(Yc|(0$UweoNf5m1b4Kb+g*0+ZME(?S#_*c*KPDXpKL7lT%uT zaG&3>y-&^Y8HK>z{J_;=_Ic_fiZ5fVt*I&4k*Na}J73Gce4zJM+U1W@5~@ZG427VI zEy>tIu&Mrt_0j%ulH}6o?J^&PA1xsr}fu1 znTFMb<^7<+1e|)C&g2HCw9#6hMB;5uOGT@Rk#_4fKX=I=cgADe6?HHDeGI^FTCeW2 zm*2Z}?54==jQq^WMQ@nwQ+C5Dwbb?%M9uZq4aIbl*z)VYw;OaFK5t85=H%r{tj_)nmaG>ch-03fqD58zbu0YF8! z6~Br&bm9e9UNM>(41evLQ1rldZj)(0erpLTmg5IF&40!{oz(4Pp%fMGS0unp?Mb4d zK9~$;0bIKAH7*oJI0j25wNqo-AKvnOXZgLC?(5m$Uae*CMp>H-IjgC+aDJ26WjI$A zLuzj1CzF+DrBQ%bQ*_gZ$Zgni3;=|Bj(BDQ*=FV}1Fi?_S|Ko$);%zv4=dnf+cnZ6 z-_G&gVOs0W(dPt~p-i~=v}-O6wGjF(p(OG1Nird9>y0?DoRv|yvmwj#rpIP8nxaNx z=9X{WEA#s(aKE(+n*~2iM-n*5&8u_JOjSHtt{UO662!QHOanZ|tuIIDTa*zXe zHh?{(?P&&h*Lgcm4>$kXM}CAsdwZ>8ju+65Zay@GFa1m8`15Flz?o*2W$HeMkaw}7 z)&RFfT&pM*R`N?%$e&(D&5rM%-N%EZ;lCvpw+IUS@jpu3z`o=Q^_cQY4VX3~0~e47 zmEV?UfEcKz*%elN_3Z~6Vch$e2GNnmoU^RB(hIlxY z(g1gYmP+TjCmHhXjB3brAbem4Y=|}i>^)V}^hHWHqlE0IXXB;X@f75VH*EyC=rQ+N zzfao6a6R)&{xE6UV!^Ywsf-1QO>u6qiv{=SI}#1OX*k#d+%Fyk5u*cdl0m>3FIhpA>> zmV!I3mlT*@pynOi&xPaxxld*=+kC9`^~8Z=sP^QFNj#vXVnq@+oOL8!MC$1EF`! zmlq%m4GOr$r<;C)$4&nwDwt>F53Yj5y29h1{ifz9_XenKneXqUmk#cC6hY_~$DvU9 znvJJ3btHk;!);PEX%`q%n!UNJ#2wy9~pbHPjv%#H-qf_kRz zEeH5GRX3!Mw@9=1eUxj8&g(U|&Enfw=CfU<4ii1m83q_DmziCcoX(={c&+)PJKp32 zVh-~4#O*s85L#wxui#iXdZ6QPPP>M0?MF_Z%7km z5mHUTQp-t-c~7!rcNx+r?&+D;nMXKXcz*aAcI3a|$XjW9#{g|K-9&a%-tMZI9CDKa znF)uM_F2yo&tHQEU?8tGq1$2>mnhtc3=Vu}JX?atPkzWmPeOA#&`Mo>FZ4tL-ix5B z7Gc(W5J5)DE*@mgdsM64%G<1g6m{hbP?izFvsg z$a-kDjh#M2dn+q#Ichbx=NcAfXnCf6Q^xHWAx80AYVuK?D%h9M=&VpGc0}L_ZJ1%{ zY18P4vn;nT;9BnXx_tKoQHA*&Dx;@7h@S9Obl^dddh0VkPCL8v+VA}86EqX7l))S{ zxz!51=CtgVH@ePZeR(PD7MJoq^Gi>+aoxu^ec|zVUjBocAJla_#x-OnX;jyQR7h-U zf`HUA&Q*$1qfNm@1)JB9qQ)Ft3dfZQ9^y@(D^z+&J;pF?YM13~QC2*UuxQ(%smprn zJ$!i7Eat@EV&(I7UuipdgtgC-5%1$Nmd^^W-s*`kW<*b=!0(()1+`#V$l$F#wlsp4 z7lMCgQF%=VH^Zp^m@$1p>uG*9ukqwx+*sk0^E#!Ko6WH6Bfiwbqq_E*&S-bUF+dhX z`nWZ>{=DvJcj7<_>~T6%hy?fJ@mk7mVkI$$H-WwL2=Nn}Nq@mt;>j5x9_Pni1)J1~ zVQjG^M{0YHB6!Y*yBOQQ1Rzes!zw9q28K#JQMmjhKx!<^jg=r(QoQycgKOd&F1Vyt zxCd~!e~49?3KP{EHe_panuA~wA(_M6}Nv6KXOeNW4M%PG)^jq8%Y^H}sOGR)FNlbNP1_hr%_Ogmc52 zOy{kqs*gY83As%X_Ss>jn=aPn6%=A z4HXa(Q?SvD<{r*Sw|3qB4=d@TAb^;pQ2Mqziw!;f@44x}=yH)k#veQ`)4RBd20|-E zp;8Ri3^9+;GfsQ>;RhlJbTTDZKJe)8mZLMc&@M9}z{*9AhJi8*=wJuS48y88%f+`y zBlq1s84tXye3pg^cb`Jn7I3{ZHM!@933Rb+uT2AC)#=v?Jf_csHw{xOhv;MDqgD&& zB~&CpnC^joHX}k@&cCXpz*)K(Qrr{Nv~t;25Y{oB`9r!GfmRi@$>b{<6CRgI(whf4 z&1}zF{KA{Knr>L@SKR586?x(AnJ8`KW;0_krf>Z#7fmOx2+H--y-YTg=JO&UF&!c5 zZ`%KZLsg;yg9v~s0jFqb0y~B+Ux*rQ)EV;YuF*pHv%8fi9_0~)uzH5kf`v(Ol_}T8xHt26Ka3LIZs5sJOO(2S#LR? zj#+;Dh|r^C}bSd zyVsZjaC{N|FMS8OGC+I!{H71Ym!lj%zo3$i2wZ*zt^TItSY(W!!W{5MER5}={UtUE zl+FH#x`VM)stJ_1$i(w4+$O8rS|w)C*TFqok3rOMbkBT`;3+uXmR;u8iIa(l^rsz3 z2b!{Cqh2TSEB77BlI4opv1rYw!Lm+hhK_tldbiUMGzELH;R^?(@WD=& z9Egzb`9w_Rrw?kd13ku&5Ox4F+{wyxCGd?(rIp>%u^f4QrF4}h=OvG!^JVhm746{r z+3wNiRqQl2hHRqJylKf%RlyDlxsWXjU0jFT?sJ#HgUroLCT}}pejYR=3aPxlZD!v> zAz)e6F;eB@&%f&}rzBICx!qsK=lj6rB~44*UjsAD{r=xdPziBi#Zm9Fyujo?KK96s>^j74GvzgD~&EJhNRYgLpU? zGR22!$1UYfIPAQ{;N(|`c!Y7az62-cX~@9-2d=4xP~|hymHs?xg@#A3EapO$GhvL-U4PgKa?1HN3FAK85HpkJ0>($Yw^CL3;@V z0e!P@nF2m3vx1*X?>njAKv;sH+b-?hj_1YInBUtR*Q54Cl#WLqd!~n|B@+CH(A)#l zqhZ*ObQEtIIzr99674QXTOIzm$9ntlOaR~qM%J4%sb}VO)`Q1e0wyUoWE% z{nooXvGAR*tJOqTn%`#5BTu^dkg3AG5|A!#L~z>TdWd*N@6?$@k%!=ep|I>B?>V&lNBF4xFX0*J6ti`*yk-@vUwRx zd4G^2Y258z;Fs=ixDeKo5yB1(G%`Ux!x4Xzf;XSQy%ro$mGRD87Ha#094t(GF_)G*l}iKx5xEo1met^Q#VYu<&3-hv_hp|2T?>b+KVdJyd?u# zL=1sn8DGzlU+FO=a793UiVRV~D@aX0+3_zYc*rNZ?pj>0IVwNyy$~|{HQ_$;HCm%H zxa-JS{J?Y?1};smM(^JHzC1{DSF_Sor=yEnS9cz-)5E37h{I+kR&CR~TdOMdw>NiE zQcx(YfBd<8p~UGT;er`GYYH`qsArN9;@>1Q^XLI)i=DNY1{XZQcmBfNq zw^IP;^jp(yx}Afh-|68~G>=<%=%F$3lqSj%w;AJ*sgZi;Q)?LdGd&gid#h4HEVJ1; zG1Ji@2!h@hc74rWlD$`#Nk}j9=|GZW_QE(QKQ1S{-HvmIYBz4mQ&sL*sc&e;u16{IitI3QoWt zt^en+JP40jZAv}m-y@d%#K|j6_Y>wvf2sUHGkqjZC82db3!Hmo!MN9fzkUIXI7Pd} zOsi8KymCr`AmfuuXr1_y^K~dNdN6(U{3;TDUYmlKlmC+i>?>Wn&v9sscXBjm{>ld- zXw$%~^ctXADnAcnZsdHrZu?792rz;u8weaIa*jauF{b+&sg09r6>Z=)vX%u8K9=c% zMSWp>-+|Zb2Tk=(*WKzrTf?O$IpH;sZl=zjlu=&Nuq^2ClcUR5ZQ>!0Y-X6YV6+O% z1j4n$Q6mdP%_8yP0Z+CWBOzz5(w-*^IobG+*qbZ%WKb}UvImFnnSxqy zJEQK`_?SOi&~kdpTdo(f-?GjHI9^*m!9%$;zS?FVlzjK1w$`-u_%rBhO&UBo31F|=;+;u?KpbK*+4^(>7{0cO` zoq~X(5(ZEhh8cdgI1%_)T2lko1;x@5!`{#)NU7(vwEIi_HV3@A&CT?rh%Grps6?#} z@OKxo&gjbtUNdcez?+8bg4E~TH2sgzL$bdeaXt>Oli=3Uv%`)C@rN*bczYWx!;RoY zGnEK~XE`gs`4mq~?nWi1MJGMBvM+Rbg3&u9a0@ovy7W4BJJOwvX zmceTmF+!Nd+}fjgAA~T|LSpoZ*YY~5DcS}+}bEwX~u6U zyJ-eHK1Cg5lraC%n05BAA29CN-NcsKe1c{dxIr?MfS{w26~FZ>%yg)~)Ns11BI6kj zts2ku5Gq>XcpeiUwyt`a>Wd&$H=l8N?@PA2KV9{V;kTH&ET==zQu8^}RENCA)|;$= z2Yc_KQj4kqMHJPM^O?Db(23!<2Cl^U7hu}9Y^gj$6xpTpl6CyC2}V`+LcyiSM`*a$ z?4YXA6XtCXOK4Fupipy}`SMC?Gx_(F^rQ!RAj?IzJuW1K@43n<);pVY&({`h(iy*5GiG1!DQ@DWuiPEPc9QKs`<9GSpRS`zqX=s ztwem+ISKOiBX~rhHSckVts{s)%H0HEMo(%J*B!GKXa74W|q} z$dqy%>X8S``rDFm6u}QaqnM@I9MR|Sog+dvWz2F~hJnwlb;Y%B-IpHacL6PUzn$9j z#K)|pY4!?|$%p6)QmZ$MepqPdIsV{$-;BXLf6$fRiaG+QOrHu|ite=sP92sxVh=Sx z0DyAo!bUbs2qCv?^Zgx>uRSwgdp(K?`+qmzkGy&}GW2TH)hj#i+zq{T@8-Qb|9}1W zcmCi0`@6LN{_e=vd$j+)oA%%L{$Kz7L0bN=MAEz}OcyPm%c$q~# zp`lw^lz1PoL%JBn^!dybPk*JXAz|JtHYKTQZbPhJ*p!CTm5_@2`!tTZ_Tg~#p7~MK zs(2%oZ{viHR2-#G__UR>!zV{-@v&?duH3SpLdS2`{yQ?VGMtmxmKiu9Q#ps-VZ8Zf z`n;*xk%1c7??Ewb?tx0bwGy@&?$H1VA#*mh+rg3ACJ^+hnda236kaBFdu@s?YTWgW zl;+2v+ngscweP3{bY%Y)W%*4ySXy1R)RCfVHVPPHhXZ3QI%7sG>*-ZfN8AgGKmUMz zLL3(N1M#5+9P-zQo9QZoMALUA@*0PRv8h7Nuy1BiU4-ZNAfW9)iiQO?^P3)ypj6Ec z#mwIK;bsOBRsc?*KwnM6z99HFzTgo0+J6S5*Q?`~N6POb@(IMDv;27iA8J3*Jl%Gn zb;e=7+5ApRYnYfAUBT``7}g3@=KJ$F}rJP|fwE7+os zma6wj)T`ix+O~Womf|CersxnqH{VnrhC$`}tS6EYdo0_}ZJ(4w@GlIkplQ8eL&}8U7sAj+zo-%gERqNZ$;_ zOW8$C7&I%}$9yNA%c%9{fcCp>=QkaR;j92{hHZxwlVS!Q1;Or*YyI)|6eVXXO3qu>8 zGtUyZCGPG^z+>%u$eWmjX}u|bHZZnb?xx2c6)X|9Rob@jtoF=2wd}Ge+K;bDk^!<5 zUcfW4eKJq2dH&(y9uWH*^@{CVj^(H|SSOR7=F5P)>XTV~DC1tBg_*QGu1M9{!?B99 zq-UlgV#qmFu$ZRqc$$22_lDT7vy1f&$XVK33}+pFnDQ7M;W}$`+zlLzmXrK4pEvAb z#9d6oOy(XW7^7@ft^PEi(<{8NS`C^CRT_Uu|49hgLq=M6<%OD-<-UeuR^j%FFLYsP zkg7G(f#DPR{d@W_W4n|pG@QQa^wAK6wOdEI4tqDwqU(`mrqik9GF7=61OK=gekGk( z4_HZ@qGyduUhs@F&f%->JR$pf^riW+TBbDow&if`FgID@XVHYj)0a$9Tr?)X8at_o zl)@+^LyXdPc#k%mW*R5>tYS*HP*lst8Tn_9Q`zC&q|MokbM6^1KpRitRBKq^8UP+O z9E<%(SD_OhESWp(bAjbmCC=W=WB;{OVAY24y;;2wmnx};rs)N57_)rs7+PO| z%mBw5WOlpMU&2{q@O9E{$J4)JmON=3np_NP;XDoxUJHV8sRKvl@fz#QzbrFwHCiWV zX7a*9(oMO-QUVpE@ksP9j^6d58Go0+g&x3O%mH$@e3Tun-Y~95HI&@9mVg^HzWrwlmAXlf2``<))- zV|tv(EHR@W=5O3pgja5Dv3FEWr!T5?@WF;F4%hVZL;vy9j=6?vpxMM~$ zBTel!g38VBktvs`Ac#h5y~{mZ@s}{fX*xepRYPQ=i+L154X2y>W$Y>3m6j)-oSA0C z=kRWW=h43sh{Yc8W!gS2T<1(%ywNu+LP6V0@fW1dH0lwJn6PK7fRt`#WT;KgW}IKm ztmb&mnV#je`(D9oaexg{<|XAgK z@()YzrX@c(qPQwuFO!i2JJ?}TjD7!ATPy_cI?e2fd(7Y_uZP`Bse{IA<&L-4$pq*1 zX`S;!CUTMcBor@3Y!8_i{aKihYqBTAJ58YtgC6Gs)@At$yubJ;vA$g_Q61;|O)ti( zK5WXm8RYc*eki}fdN`UgCnHz!dS}eQz89V{as|>YU%A)h3RP${3@rQ@DH8!ak#@IQ zJy`Z0#~gYjZS%I5>LJ-cHGiD~kOKg5JNU>N$DL3q%BwM~DPCE&6l?82X3AgkGxpX@ z2;b{&zLQ6S{$5R|XYE~bQkc*$GjiTkaj?mfZZxvsD87x&CAM2kdrC6>pfvh}-uD~} z##6Mt1I5hL0X9!e!5t>wW}4m-r}b`5wvKWj!u)nbLNj!6nnnG~-R@(^h`C#O>C2D* zOqb6XK3pjAAuS5hxx+|IRUH2;=@Ba zwmBTkv?5R3^Z-f*G(9|WcGHzZe(xDg=?a)d@y2_swoG2J<8JD+?5N7W;O+0tymqt6 zF+sOz_eg)4hVZAQdZ#eq>9G<0r9p6sabE z=}bqRQ@G3eS1KHi;)(Iwc*Mplgq>hH($ngIA(`<+iBEAhun9@DE?IUJ%@~2?s#vB7 ze3nrNX1<~{7;qrzCVu{48v^c@k`15Y7$nI7_Ol#R_62 zdoSmiw(Ha(w6PaAd~ln7gy!6yYl1nKfm}(t0K6KVL4qMpQ|GZU>{UHw!*e^&7wZ&Q z*4{>pLp#@HS&^n9nQOFVnz5L#5g_uUj1v)Cxq*%_tO7dYvzXlTJA5b(4YaLcn$?W4 z&t8Q-iI7K7-gcb+fR$>!e7Y=6*$lrPriUJEi%+vf3pi`@j{@m?55#5K1%U0pBaF~sj(vUY~Y{tQl-9211TpOJVwC)T8bJEP9$GAijZ*I2*Rv?hZ($cXSf`aR zBzx|ZjiRhn2e?nc6`LwXBL6rqSxGNGw=AU`#Dr@T_u`y@@2zM-75=Wmjznn2%4pLX zpH7Y{`=FG*lyV=)=m{Fy$vkebo^KHSn#yCmwlbDPkNmor^O&0-Q4|DIsh!}pVtc|m zL|Zt3tY5TxiPN+;Q~RI!l!WI$PHFyLXk-PR-A+2Pg5SS=LWdr3Cl%h%!jcB%hM3OW zH!O-YhC&*2`?iq4R{C?))qnq}4r~Z5E3r}!GZMCVvF=Hu+vBwgQ-95T&^ptzj3p-9 zVA@v+w_9LBo6>@o-H*mRP1DBgkoQ<`H}z2DDj_2+%&*8w&(3tsxQOrCf;u96a)V2@ zSH6`^IBrdF((RBTJ?Gi$m#C6Qy0^{un_)!Hk*Cc+`RkV;-qE0|y4^+Dsi{C}IB_-=)%Sz+Z4; zA`Dka9XP0G{$?&wRqhZpqnAOH&k9TD6K`CKk)Pn08Q-%#)Ubo1huFu_=pOJ%8)`)C zrKn2^FG?A{lNK5B)fBF0yxcbnpP2XFo=Ses4A9xsbOnL)GT=>eN=hA-{Yz6eJ9*Wo zH<1)MwnK(by$=H@51Ww}STVPJvAQ-xGYs>6qp3X(*)iIQw7`^I3e;@el#8MKf$7v! zw(f)frXB>s(hfJ5vLl&SHcE%COS2iLz42qmH&hj!l?kOj^tB}Elm-$E;Xb)^*;<(; zluW;e(CwGSDAOpDZ~3ktk1KXxja9>JI3RtRu@s{}jNKEt4_^s43wYlPQkt43-`1)N zf=PWQ(L*{PQ|Y3>a(-)))V~q7OKF!tw)Cl#=dzbPcHRB~Nsa-oTR>#kBUw#A))r=H`R)QO1`_>8B;dFY;VY z!g%Vf6F|*u@aDS;@njltrO7A;q8q>Mo^n*%_1~s)`SmH50U~M|*WvJs_UbHRrT-Z> z*glpo)|(WW4v*w*i&or=49C#JSnBh0%X6eMk<3IO*?j5A12bwZKPRHHw(Ux=XN99} zp_o!Jo5@T>Yx^rbbjWzN_gvS#J(*$C3NPf&VOij0TmXw-^^mU20%VN``G=k4;s)+T z1b)90BEZhTJ2SU?Qm>+&d-Lw# zeQ;yVnP%}|C>akRy8spFHzR2YSt#8g^ai`{&^WySKyEbikt{r8E0B>#cY^UeomQG@ zeI}epnG~j0nIf&UCp@eJt-hr-I9kF;_OJ+D;&^6t4$F$UR^8i-rzumUEqiC`UF}cc z;_qF>0D9%C4~lv>_c|L?f3M3gYX|lIGP@te8MW?42|?-1I(7WJ%}>YB%P>(kqicEA zFmLN67=Da1A|62_%Xw%hK>HIY{zB!Km6whOo4q4?a^7aB-7Cfhv?dT8aHcueXb3+y zSTFp*IyR0;eOot=$Zs}_hdArWSXaM>Rt;gPMV8wxnSQp13?u>{-i`h=0HvR$#adw& zT#;inZ)vx@@!?$9!oBKUz-#F46_A5oHL;!M3!^MZi@^mh+qWa>7=#;Cx{G$u8iwm7 z0P!(=b^{~or)LSOu{TDl-=ZdeEcd`wuJXeJFJ(JQ9i3ao^;gX27moAaOKfvd^}$X? z&r8b{SG3;F19PHU7UPCro(Mmh%9?W5y0JaBGt{Xfx@y8BnOJG2v*iz0Nnn$v89 z)1$q?CR5;|%%I=>zNy|s)|XuYdsysOtqDRNLLG`srRA7^py=zs;G=Z>!QEzR?y1!L zW>NlF4#}!p{Ge$i+tUMxtl~5pL|{+d7XV5p6qII)_-7*g2C&$yY^Khav>W^V571iH zxAf8@V({OOElBEs*kYvNDT8?=Wmrs+DLkNZsYNnPUO2i^#U}*bgffr_yP7~7FzG*% zgHNfV%DzPVwNIVZw-oJ_mY5I>7Fj$Gm*!Pcy&G}KH#aW;s6LCa0K;)Iv={CX%Pyj6 zK4BL2t5YuWz8DY3>rjX+H-GHiF$fI;Y%p9Ks?>zr;Yct}pg7v^aJ0VREiHFE&F4+% zvozDR8Dpq5)sIy5CK&822GYlp>XCv!Z>2XxMKpvZ(mPd)!lB!rS8ZeW0S zhd1oUUw7HOKxvRpu|UP$eke`WvP`EjLZ0 zw||BlveQ`|dC=tpfzo*kuw#RpSjg8&>MZ*W@?+Ny)S;4hvd=-V;fH9Jo9J;rv8~WoZRJE)kv}$sG117wiN38ww`Ky@@Z`vLM5Rrh0mniQRT?_JTtcn!dS|& z2Pa~QX-7x?k5IpXeouFT#WU8r?P4SKZYJJw=$2svb6-EG$jrov;|VZC>9VZUXR9H* zbGim~ui_C$Wt!&ChtH&#FPBfl0fJ@9?}bws&!biPu_$1VgGFp*NYj>#cbhPkk9*Ks z_O{~nd*4^Y9M~I;-P(3Co)LN%Cnqa-nr+^C zR6p{}9})%XxfjoPq4b=mb zEvQPjtypPly$neoig)!aOFxKO#yF+NK0Pb_oasfp^N#_kGV>)Jr|{&NwD_3vX#BsK z^0*ZRq`Wq^OF^IvRmsQHGGh+MAezEM@I1je9qPMFjm*54H2`zl)gZ7E&P%V6nFJFSjC& zyT(YTKiVNJ_oeujnWw@JM(UX;KYh+EDMO^pXG`@fh5PF+>+#KXrXmgCrJq}h`Evl_ z*c`D^ZUSH>s3)}SLblqmsC^op{dP7#5<$X0I6YW3r|=GjBBeiUnfs>6U`#sVKVj;8 zpWq)mm+-kO`c(J}v|t#%&(<38K@pLa65s%IvQ_@xSASTvYlqB*NXtV8{t2w^Hot&w zb#-9)z!kOO5)ykmX5JaNn zoTKENB%>giT{ZoW-OupWTlc41b?ekQ2Vw8EW_r4xe!8cpM~{>cG%4d;oqu}AsY9E_F)?2*=ubhtxEtxN}x z_AMO*Du&~3@u}KCyg~)`P609c@{cQPr2jD2DEN^}o znF=dtWyfe)jl|Q55_GV_MyZHV4S6i4DG#-*Q$yb91>Uxn$m<2^O1!8VJbN?J=@OYQ zHGVb}UeV4}+cDpIQucVns~Ronyf2m4`4yXcUb$Xi}8c}Fm$_loSVZGJ-pO;#RalWsO znk8LxIEg13A>w$kl}e&3KBDBlLK!^a4sEH`G#$posJWu%p#IiF5V!6koT*qrw^LZ- zI$k~@eH_V9o#$oY!7P>OiZcc~lzf`*|X4z?Ezwvyn=J#lm-SSkrz?;J?7(tj+oQjskr*H!DIx-xV zLjnK+{JF*<2zyIy%E@ZhQ3lURAMA)7{cX|{$OqV|n8QqTFT@IW{V^GPaGv&!^81UC z@+F2}>+ZpmMqG~iZF$=r=5yLcnw`}A4^UlJ!*TI_nYNnOF5XrH(QLdU-h080oGl^A zXoOyOyc%^gn#VTSS}>79Tzxs4RCidj@dGsApf-F}EpR22{6*HLr^3B5QP1;;8Fbh0 z715Qr8#*mp`o=fyRd?G=W<+*2$5I0CxI z>|{CgKL5N{4jO!T^Y{@nz~nGUF`8rk4$%b)nVvjYeT81!zuQz*=EYAs;rAGYsQoVv zQoCfh>5Gv5dVSi0!A`zh1o3b365^Brrp0~Qj;Fwkq|bIznnxy%BA;Im-_t`8ufif$ z(0Ea8{KpYm!zM$m`@0}E%y#(a@o2PdjK|3IHVdll->Lx^H*xtd8P7k8)<)3$44D>@ z?E@<|KR%ZF+t8d*;t(=huW>64hy*Ir{XxcjuKhrdXfDyF8eeGxNMrR@RA~tMYh3hx zBIWy-*k2>bPI`9YgxhdF+6n%BNtZoo_M<#XXgJ1daRa5{l=4ve90M>!WO@`;YbNgG zUG)%2yAZbQi6_s3E2R2K%=a4!(PymB>`EmHmL+>xze&7NW1PKWt8=Nkh0T65Ux?1z zaI~&N{<#J~A}bSEW@lMU<``VEA!A3;x1M-37f#ppME>Oy)YfTgK=2&`c8s6bsZY{{ zvi3PM9k2Z4q%x%^BOYi9F6AIO`r>e%@gw{_u72e+;t~4H^gTItme=v1PLG>vp1A5K zqbQ)dC)zII8y?}0s>oNH^cXaxkM&%z$HH6qhUvBMZVtW38i^c5;V8bZ)!vg)nzA6- znBMpaQk^XL#AR7>cEu(VZb+9}&*O*`VpF|e+*rQ?zHQ}5S(cj3c|}4NptqQ{exErwXiN&=?)8&I! z&tr_qgbuTPBebp@)Cs5t_4jaGs4EIBr5D36ExqvtA*o^y(=(Zvrjs@d#X?siyk9ev zSwpC5y8Iv+!-Hnm<|3S7+OLu~w6MMm94xZHZ>0wKSF@S$Vb%6B<@kTR&BPScdZPTf zl8Bj-`1{X(+X+pWH!sCVo0kn5qf;aEZ&kZ5K>@yG-bDeJRj6c~Ufw%WAAyB2xMO5^ zQD-7Evy-6f!(<)jPNIohJi8KNNURpjzd*Yn&Ls_gk zt-X_E%#hI}&MAL{HZWycqj}wY21A|Hu8xC-DB_4((i-BQX!kbO`IaG`Xm{6=-DVW?hIlK7-uM`4}Fl0)!tQg-!E_- z%Z%Z+(;T~-G}peO2*3RsO>MzK81s$85&D-XBxLZ5^H;T;a!6Bp**uZx@QIy&XlsM& zc7c-3vINOc`U2TF1?sdKR$d@`^G&s=SDOd_I-=$epotKyqJ3&EBqIzm(8q8w5ZKqoYoi%Wo+iHMsBUoGq;uJ6bP1Q+4@ ze>~rzh?833YSmzPddluR3^r9d9lj7c-~&It`Wyrj(1X>uYfG7|lrhl(1N$9adm&_09w}+>lP5`|LLaEACZwNd;MHa8cCOt6-dM}`>lMfU-?8bXnRS+L` z56pga`XmribQJ`;?0bwQYe#PQm09TzmjQ~QArqL`AFpIj*w-k@)@3zC1ssoINt;=q z*~27d8+HC{r-$S6&Q5##l{(s4>!;g=qH~Vwu*JtEL1)X3LnVfEtXd_dt}^L+6agfoee|oGj7O%s@4)e^ zbov+sL1asqxNnxg1^-0n$)ctah8;}fudghAg*WF(LwvX{*G1vT#5bwxvr?C1?0lvE zy}gP*?WYI&8&#XZsTUo0kP8?L4Y9^9C}{9Hza6*VMhaOkv_9R%9vP}Qxrgut=LVx# z;w!2sjmtpqsbS;9u_NeT1F`d(;kx4o^BF=CgbC8)=(S4?cJ{kZI%RqcvX^Awul}B}$!C2qs;P9Pw_)}E^ad?W))i?mS2s2mWI$RX_eweE z8H^J$gfke{#1Fy$tT0UU{7gKQR^?M_AtfKlQV)K0pY9e1tm1*RIMf|@f! z+V}2AR|UFF+pV3}XHe^TmX9kS93ueh55#3snhL7?bs|i@d>z@8n;#;mM+ZwXV5Ve8 za2Vc+jvO}&1*aaw#7N^!ghh(ARyPV5lNkrB!o-{D!bLUYVy+HLoJ zJH3%n>vCVMm|eqna*unk?GEsi!;{^*!7jOMeYk<-VR6(&&eKaXjd;p#aFEkadxyT~ zsN0Z~KIgq@EDl!)LS*%1Wd7|jTQ4Y?9z=!Iyu5z{x?Wh%BQ`W;51^w z%*Et)lZY0$-T}E?wExJ*!N+NRbYlET3MxUAweP4J@#Xj&D9-r2%~8H}!mT`I zz<8R`Ju(qYc7?!n$w?}t@g>VPYW#?yQ)u3C5Z93#-?tz9rX6_K3&K9xcuM%~B*p(d z1EM(LM>1c^t^9+@d9ZsU+@esOCQ}~FoF>TI$u$&WIm1-Z%wJQ>vKWmVp0fe<$Uai1 zDJHL|vYXs`i;Ktu^4ZJqA~J3yCDTHuK~5I4P1Az&Y&ef2Y1MS40cKfc`yM5nH!QP+ zCdq0!K|{B6CtSi?E+JJ&zi!Cbc{U4NYavXg+Xa_(LCETk-zC>N@&M1G2EWThQbshH z>qOO;E9GuMd;F86)PhxLazh9payJ`%DABM`H__*?(+xIJtRcRQ7gcf~B1rq)KhV56 zCdti+3r4EH8oV9(QhS(9F00$7#uCl^UvuC`QgNoT)z1L4MU5XWSO$+ZVE=veNaP0CIZ~r@P`qUguaDr z_BNb+;uyLpr&Yuv2wMpTB~@?ft{BhGzals6P1ICUD13m^r+^BORTuoV&$-rHgmuCLRYT&!9IKBc$EXB0l2Pnq%)7L{P^%Y@# z*(@4NVp^B@Oh4#`#N)8k~NY!C92p}t?NDp$aslWoV#&@;j1#U5BdD7 zL~jqas;bB%`6yE>B~dLew@ZCC{Dhk_>q4YCm6=pY+jE{;Br88CMW}au9e#^h?~(4; zvP$D6q*0)X!yvC!bieUGi)oL2^8{$wHxRm%uFyQ1b@0&S{+4ba4zN~pOd|*;QL@9I zNwy@vSJR#>rmXE{J!UeWaY`B*@2?NlbTg}?P>3l!`ST=9ph8W3xV{rNK>|JU4S6}x z8e=H2Ju7R@d(q_sXjhXb!tUGX$i^yVttCTq_#e<)LI>WlMIzE`Ic*|2qbcqmoM zjMKLJ=)!>sH5;U9`$!$8#t>y&`(?<`btvh**PrRNwG%M7o%<=<74Hvg+Smuk<5!?@ zBSuIJw$}M#A8To{-F0I-6JDgWB16r3I^Zv>5dyqsg}yuy?gR+9oK>lHbjm@ebP{b% zb){iv#X5wX|IAwJ1~zLz8}p(_#VUl}3q+c}QMjIYWXQ%@Kkxl_dsnR9Vqkyx@+GzL zUF#>rq3Np7;%FVs4Sn$(_dK{ylhp1kJX8DQAmEUVZ4Waa^9Za@{dJ>O&zVVz?t@jG zeHdK@-~(eIMqhB{)pYvlOnPH}1#k0BTJlt>oi!YbqE6Q>K$^qIebWG z+^kNX#4I_82^=rdG*NZw)(tctsw5?7FR(RgI(ksKSrC_$mf6m$O8Z)_0hP+*=NNmJ zk(Ir}M58K8^wsa9L*x~q38`mo0-p%V`^PKGm) z1J~hwBbt~rxXeV2FM3V*6LFrD;P*r{i_P(09tEQtqvH=#nU$th* znjIFeM%+F~=MzGta=0j}3xGhc8Hv4&{CLYdy72}kbdid@b5BFT4j);(!)FGsh4TA? zLwlv=Jn&hH71?}v%?Nw6;)d#Nw@#>Dord;*z%aSZqmJ6W*7Csi+#5L=7@-tIis{Eb zyyrz(Jbw-yFUPuHTad6Q%FNf}qooHidkN;&N~mPvk$Df>W@C{H#qQ%}Gm*6n*g=v; z`29HZOu4Mu5@YGmt}_Rl2!AqtKOZ9b7!=O1>!4y^xY*|bph)FRs9{(9yh%E*a?9>F zKY)T67KW-x*Fep(@A9u%cHzIWXy=YyQWfkAqATuSyzLpCE?n8}fX2+>Q`?vzlI&&) zC;hVu(Cy`gQlOG4K6U>0qLmw`Pd#YoWgZMz|WnBs2FP6SR=uB;n->td@97-!Y;U)nta4n_0kiC^(t$&};Vi zj8R_3YE`NFue297T*B0f{W11g8o1)OSG78Ri@+q+TBr>Ba`z7c*o~X+%JU@gtCwNo zHgg|@!r-j}0h1wE(j92!rOP2#KM*<*1nXN!W_^FKCX@$M3DxS=(qi`2kiuXu;JWlM zf_(1|KrWn-X{eCdb1@~I$dFCHz)TrAD#T_Oh?mS=j-Zt{vORw9d%BwN)M%6~G*GMv z`+=_5Vs3ML3Hr$b7!3W76#fVfW|V3^E-sccB)sN{VIj5lw@@9EnhZ!|-OG+Z8s44M z<_OMjbp{^ba50shYpV3ys#U$t1FADsDuYlX z9Dfzhsga(Snm%eOQWxqaGu-KGqA91lnvdTqi|&zSS^lb4)o%gN zdYm%jotl#|wHXMBB(q;=FOwM#2%4P$*y((*;w&n%zZe*IH6w2IQfG7_BF1*kWEiEGccGc)ul^?R?#5*`iK)Os}@Y*s3B41 zd3!aXrkDqN2hX|65Lc&DgpjTNcdCdGZ(*DEu#l@iku=D;4_`(EbBt;e4b)hG!Z-n- z74a<2qsO~7S8qaK82@m*&R%TjN!9L>neRBSeyy2gV<|yl03%=)tLuT6<`OB{Jg=oB zWlw;_@%Jbn7=~r{@yQ82^ku%*DwDY%bdIy+lAm-qyW_P?4-Q@2o$y@;X6?R^#-S80 zgP|!*jTD*D+Kwz&jK0LB`y@+9YVznMBt?-=DhQ8VuB(ka%qF>kv= zuhptb|gw$plLXHV-}^rHFk zob)8+lRW!_nUh46@+efhn@*CZvk%nRCyrcUBOLOkFhlW?53$<; zod7|gK~`!Km~2Z8#8Bx+Tm>LDU_*0;wS&wgpJ(~B%IgG6WhjGTYe$DC0~z}fZ;8&d zESUM8a{ZxKG9c=a6rEvd=P5eVUScZVP5(YUM;0zpR)uB1j-v)7R&D4op0uL3l@nUW z&cJ+-s`QXe;JL$i@EsIg@w<=j%6cmecEH>O2>2joTdXV|(}9G=5iqa#z$;#_&m(WD z=>vRZ*tqJ2@-(_VMVC!?UF^ungM!9cI~hR-e`_tWKO=?zT_L6)p3y8IEjhv7R80XtpMpM;R!` z(Pd-baPk*u)1FL?>OvyqqK*Iu&jZTBdV0i>fx zWM#am)}<&_Z5*%;%*Mka$3o}mZC+SwdhpH+8TrZ65}Ja5$j#?+N6TaY*s5TdJ6gHds-UT8~IWRsZpK=rg@@#Psg$ zMTN)X96#GfQXS56VE5%mB6kCGZAKIyZ21pnu(gMp#+%lzV$enWhY*Sj`bp zJ(O3a;c61w@{DO3eQkNbgCUI6nVus*mVrL`Khf>A+uij8P?t6}m#O9+>$fhL1$=S6 zlSh$B8I=z97!h!JTn&AV^L>#HhCjQGdS#WE_F;)afhzRHt)38>hTLEBCLLsvP;|YpaKs&87Ld=n@3FBS{}T zNhiRGZ8!dVc;D*-D2UVs^Lc2UtdO^$56>Q%JC?zUn54v-P(*0|sy$)*AI7{g? zSFP_azW#2!v$eKjJ$uw3U(;J7)d!!mDBRs5)ixMwGfs5Hh@j#m`97;U!V~{|yB&y= z)a=rv)~r8S!7vx~?57=6H5=(}P`-G^T+)w)n!b4T{c?^>+V=Y}Xj(G;^lD0+To4>HhYz1CV(b0JG1TF!u zK1!PfO!zqMXN|@3JE2zoCI9@=j8M-9hcPRlhqT zoPPQ0jDPNB;?;q-W@tYN+)1in4ythn|6JCz>zFiy`m2}`IbL{c1Zid{L$G(WvCE$a z&>JdmAALY|WXoBE9QhM)&_6-vD9fTqSri&CVq@;dXyr7BM5gv|_U53!b&4u`kxrPW zNn#J=b3EyNrj(I2e6?Lo9mAp2!3k-(@X$i@6jiVtZ&VGC6sk$!`p*8R02()9^KBE% zr`n0M1~NGF^L$WZ00QB{u<_kT_XO?rf9)$Z7w=x*(%*r7S^r7p3v+flxb^Tmu^i63 zl^Z4qF-LI!L^8iWq0_XLF17GK6(9e?K3Dz`19tdx6seEUp5hq z$z=9_wj9<`tb92pk*Ew9`}bKDzlIr(ORMqp{`WvcC~2oLLZK=(r)d~ zh5;Qme#z`MXa6|0`Q#(U+)d^i0oTa{<`zPtB<#BsT#>og@G=vd!Cnrr$Bz5&RJ6Ew zm(O8ug90y3=QCzk*2(8)qIks_;BwvZyyVVrJBdxtqSDAaRxmAe$ZxAxh?MO(ZnjsosnWzc-hN^!baH`% z;5@nPAkrRx!$EW8J}b8LtX+m_sBp~I-wZt`e{sTs9W&L-1{zCLhdGgWV{@LeRv%e9cSB>`=G`(G7y!J{XtB1?=luAan$OvXoV+-?FYUMErs{n+t7Pt*# z0NC}R{}LHLfRZ;G5XyzO4s6BpcD}jkF6}-(fCa)_Q01=^(F2%KqV$H#Aldx={^nel zauAZo4m&;K06)lC& z391oyyNz|Wl$lxh^S*xOTK``VB29CDn-5EyQYdM;Y;Kpy)MrqDH&UCX+O!|3B4zML zO$C2^#ww*=)L!2+IQ75uRIiwbC;xA+|9_i=-g)*k^j<{h|DWr>^uMm(Tf2Ur;MB|7 z^&io$|H}Wk{(vYmGrGI>SgOJ_jg{TH+taVG^H@cajBxEJ0nc9@X04(?rmOdGcz>e&Y~e`^ogd-%P> zp$ReGpofCe>wi?_eUC}tif1F}K5h86zXN!Q)|P~`*6oc%=P7{6a6J zUe|RA-QK1zoRtc=poJB$`y&NMgk;_yP~^>9)jjr{-WhJvsYcqWzlJKy z)Y(@-d1??EK?jM~kt_y>e~nK&K#qoscQr7s-n5Qp{{34i6ML z=RV+~r7z3fD&`D3hl+L(fe$V)can|$GV8@`$M7f{Mph;_O@HKf+-k0B(?3Wh_o}j! zOvSp(g%2O#*Cz>9NaF(ps|GtF0brD-25r~5oA>Q8? zC>xQ-(snpeWg0Rpq)kmFJb`G%#mxVjTFph~k%V347uE_Y8sMvzwO}>@_WnqM51uG} zMC?f+X(*wnLe3|FcQ~Ki8xMn}7nM;nSCJwDI^TTaR1kQHJAK(?v z!i8`;+hF~TQ4j8l*H89R6>qVaYWyu%Q0M%R=}0F4A}` z1JG5jhr{ug-&HFIXo9Zy#kvW%Rz){`LLfEXw`2~%(pC`%^CJ1f7d3uUnP@iSjX+7; zH5!-v2^=ZBbJoT^tJt-2ldhv$oG(5T(2WR^fuV6LvzKZkS;`h(5=pbJ1FG~VgSsRO zdpP}_fR>yO;?=!@zURwR6qK7*IAFOZ|6Wnh zj6G!&fv>9N3+J{JCi|#N1*5yabyo2YH3az~aJo(NcX(V`Mhz(x&r?+w)bf)vP({^a zdt~niV38YX+;sa1?Ma4>J*l$vQ`X({nOTEuGJ#nx!^Z7z9@292kqy?)bT~_v->0xV zO?r!RHr~=9>75H1vE@-XB9V2VEQHua}|i`2-zzB;3SO=L+avXd=4*C zc_T>`feP;^y8f(7Avb8wXhLO+UVuj^-n|#9;C{FpGKE?eaY4&*b{1(;Ld*p3{kwxC z3vuP*DUeC?Z=6Y8S{fd$P5SWe7l7qb_8=~jTwzFkp0e-;S>e%;`lxgtO^oC@W#I_{ z4(6&KcY0T??!L#As5-jFjjX3HsZqEYvi!4EPK@gClB%Q|@${u;-krBtd15BwuTJCh ziy&3(gyH~VAMCvJ;!<&BNbge~$Z1l}&Kk)X+l2dBhZ$QQW2>UgiEh@19oNHC7TC@% zCkMtm=*NXfXAlzQSev*F+ z-kQK_)Dw7Ai`Nj)5DWcJ?~@Nm?=u(QEP_>4|7CyTwPy(Ms_Hp)pYnX>nwjo@Ck~JR zVu&67<#Au-4XnomRBeWkO6k&_$S}vkLm@V?c~sQ1z7-mH;$cCi6gFUFDS59ZJGE{I zd5v|@1(r#41n+(K2uO$86wAL1=?=^Xggs-3+2%fImsWgEQ@Y=xt7N6^zD(YG;ieV6 zMh(VF@!1!sCPEyg_AwJREHUi9s=tm$5f>;YWyHmkchqu>{?~qQ$25!gSTk1Ki5R(R z>6Bn4|JZ~NKCCGd-&i%1&69!K@fcn=xicu1Ikg#vdBV{hoSny)EpowgBEe zdYV)tO8Gq*w$K0*4}KQByB(&TY6}`X_x9WY5>laj6u%5^1}RhS+hD%39vy1D-bcp* zp%#i_gaUUnep?)gUvv2wm6VOX=+xzm9hzjy0SWNNH$otu+*2ddTY$m-Oo%G*#lp@3 zlkEpz?nTQ#T|{;*s=`V8kvUjfE6#(h4CF!sXtfzG9iPc{Mk%&7Pk6uM!6sRY0wWis z{Cb8irsw*;m;ADyKrGVBT?*`Jcwk5szLT?@`|b3cOSm%Q`CAl3M`Wd(n-`NKqyy{r z#WFfl_hf?9uac83;utEL+@R(>5mX(62ZK0#Z+KUW)uE+$!GA zdAEiEP=_KCl`HU{(m(;nS{qret`(rX~b{|-lZC%7>M>a6r^xU3P)9i1)EgM zs|)rn;4Q|B2mOuMa1}500-~^8YSK!zJkc)(_&S);YAh{m4e2Md1n~4c!$Du)BO4b{ zuSvj4=J#Hn|1=F& zJ5X>sm~#0^tt^{m;8B^cDiD*~sY5x8poWO0{V;T}5^~A}+hG7pT*ils(R91%E~{0K zBIs?1u4fEyMEY2{ek13ai`i2e)B$ud&WA=3?jJlCZC$1Fg8G%ONAMWUYmYs}4WG)C z;LF45_mZs_YxiwV7^{XDZQXs;8U{;gf)-tVE>vuNX={mrkMGdY7q2)kdr3 ziwKo8YX$zQ+&z)WcN;SKmUaT^E{dpd-GOEQ%i1;Z2bz?!DHjM}mC$mbK=tORx!syX zb=zcXqTdy&!Mj$K+574q8yTuT_Rt$NmPz#7zQefw(!fBOxWr9|Ja=R&4W~|);^iY$ zP3L53iO3p3j)A5c9p^8H2ZU~&FS{a?7_(MYdvui0&&@1SNTX`)gFkM^hp(hELQeA} z;J-(0V*sSs&#H8v+Tad6L>Q=Mx8(*hQJLBP+))Phh8n)y=~AXD(>3C}J(<8Tn-6K} zO{n_&!JeMrvMFJwzTRRPtpYzUstV^5pDutz6xf1ZYd4WVHC>z;4Hoevu^I*hOA5uLC0XL{yP6VoA>mRE>}exdHnb-?Y&BM zggKgu<^eQ^Sz6%X-nZ=Myt&xO4bjYL>8(yM_I%e79pBX_RSx4DHx64o6pC?Mr6iiz zv0of@&zTT}2p zX+5SWRl;~{KeOt4WbH)6dpHq=M??p|zy#90>{q+-;W;RO2YZ7SS3ZDUeHXKgI}ehG z4<|{;G;@5i292!I~(8)^9%v9m-n$K?WlxOh9(c_l<@wIVh%|Xve3EqovDF zS`F}{IPs==9t;uj5ntOypj6n*u>3vDmJG#b^d^T@9F|fxz_=N>?X_w(W6FTVV}Nz# z+-_Haey{3xob|AO z!#AZP*Hu*{Z2edW(hPqkdT7(rFY?`g1{@sN!cN+6XsjD`K?NcmW&Uh{J)o`eX4@lu z>Kii-$`3r04kR+-F@Fv|z6(%vcUL3fT5sUJV2?L9d~mreZnehPhALEJ_gD;y=tU5D z$P6RKSD&~F_!bN4-)M|-dLMwbNmpaI+hf$U07EW9naE78VPp+2xVxMx|M}D-a6nvS z52UP66$FA%DE>{}!bZ=V6A1XbDrcL7->>SwsG1B__ZB5s+j72Jr!?D11t4Zwd3Rb( zhp=4j^zhgq6OT8fZqx{K@W4(>xz1M~bJxUTScahX{S>2gKwW=-3)28ubR^qc{(g}H zs>T1QqClx3YH2l&uyp#yb2<&_{K7K-9l>JjWDV(Lva93+Fe-_;3&27Tl< z=UNa5RCEI+_d53Uoo=_*Y-BtEP?of}S5=0yG#@xGh2eNA)e{u+06{E(ow_sae4aDw z+d~QAua9X!4mIwCVV^;kT9#l2UvoO^Y_1wRuH)R}jBLp&2{v7C7;%Rw`l5-SW{qq6 zRb8F~7Gv`W8VEU}n}MOPI!334PlF#nz0KnjSUPM?iX8kHKm+U%hghhJaMh|QTC3}d zR2-Mu+#Pt6r+Z_3_q>GE=Nyn>1X=Nr^e^BwtB`%O(BTP!6)wwrv|*r&VoLLMPZ8bq z*>vQ<8L*}43Cn_U{QXQ^RDpzk8OjtRP5V#1AZMgH&Fxjyi#*oxxis84CLO!A&aZ%R z1FGb}J^TgpI2^1>Z~W)YfF6)rt>uGV1ip9&4r`IkYj}g)F8^Si=9%F# zej^dc=SIN>Y0cJ}T7fF)hEh}L>fKLL>TwvX9W~U{S5}C6MQj2tJ#b3eE=l1|&6#@e z8#E?RRGyPSPjEejP)pJXL8l-d`_3_;0$w#IeQLp56~$tjSlBFHU|jwCu8oCit7{nA z^e%Gz=yfy46|si_-g??ot$vGSIf&|eog=009L8Q{JW-R(=GC4Q1SPFeF8}R^4wz*6 z-QEu}ny%fm>FbSivf#4if+}wLy_7$$e0YeLk|?mUWp4i9i>|_IX4Gig2tYgcRC?HD zLQg1Qu#~A6*fe@Q%-MKORUpLg_f+*)MmojEO=|k2QxwEoqg35#=B!;4?HKvX{C`*W zNH1No__=^%sxlII1dGl;@f}5Nx{^=WY%Udd&{NHeHj+=@SJQjHlEjjp$I^V5j0w}- z_KK~@L2hI<;`hiB9jEFc0XP#n5jvQiz%~Eet?f^Lh)`|hO+@w%1_=am8heMUn!X9N zC(0N_-Lj?+p;Fu+gYr8W0ymw`zv!Rg1$0s8ud@mr`1l?=5%F)L&>4WF9=I97f0pYs z2*r8Od!{~Q+>gR+I+atwvk2a*y7H_<6`9Oy>c%*f7 zb6Fzlk7;GNvv-{!q0(n_Q2lz!94CSQ*-ufT_xP~lc0g#u zB@HLtY1})K@|w7-3>kg?$!qHq$-4C#`){PZgn-TD8oMlBy;Eq?FdNoUz(s{y2ocu1 zofd4Np!IOSnrFMrfkbrr1v$8k!)a>4u>K_Vmsy%Bz{S!P;4C`54+LVv=Mh`F9$Qh! zYJ?Mg?r}hrOf|-(N!N~#gi+KOZtlZVPG}|)g5B!tb9zIJTlKdM@ z6#->gCUu3{HB-GLe>0_BaiFPuOMT{}Wia*X`z?E{Opy|o(`Rr27apqLH&rVQa|~ba z>lecEw6Y!~QX~S|WKIo6zU`T`mt=j7_ZCt>toez|tSWrVOG<-&ASgGf*>lk>P>b@H z+uQeMNA%ldoZCR%)KD-Z+)ZX;7P)f@RSSrqp@|+Xk6C6jzx&=lSn474Zv3*CxHj>$ zHYkc{Wa;K3@_JA)xN?om)9yIWus-&|BNwISes|~)CzNCg1tw1Zs)|k@Q?u!Ma!|e= z7ITK;c;=^OKYe?m)}Df+2xjsuDHD#5=%4`xT;$08bEqtS?xN4j1Qel_XNRGw1>bR+ zTFb1V0_(vbN26YIE{-n(KiU9#9Uij({_++}rmeG6H9<3OkaW?1(8OvS%WID@Zbdq! zD-=l&t>T!KY(`$x;~GV?I66+GKu2oMXY{kMWkEiW8rsu0v*AKGPKk}7x-Q5sJ2t3E z#om(MT#h)uhVw*4V83;CcojVT>pXO?j)ygndwX^4h3|gU9-7&7{4S2EcPB`H@dXMz zpf-F^q{C{~f}N$S$mIQzY_%FA1)z~9_ST=G#*b@9IgBII2mfjiBHL8u?^KC*CN2)Q zKX4OVY_Y>R{h`J5>}?Vpe{!(u-Sw8{V#OP>SJ8?^)oQQ^2Ar9gM02S%=lw56?Xa zmTX{)UMSlGI!D0`MQZ`)-SbF91j)*y;})7l6K9b&%U4y@VWEY0VMG;Iw#p`T)pg&M zeNq<9sQ?VQ%mFV-`~*4e)^N*RoGeme{!9E9sINVF2&`e{CsJw9T2z0>_pA)XfNz&n z-99g=@npttS%*IkZ&jbh65Uk17dJy`$QDGvxK7=SWp~2xjgwmHM8QaKKf%v3XWvOF z)j5UwC>c{-`G{AQCCQ@dvqVz~>XB`iIb#^`(6Z$IbQ}^LiC<{T=9~KFdSK#MqIG%1(SiNAQY7b+Q7z)|UgIk5kf|C#4_Y%mmM+PrXkZ0q5Y~ ztGlK5u@vvc<-JmWK&z_v(#I-am7?qHH>$QYBi{U{S|%&0LA*cmL*ngIT8$pENinF} zA98t`q55{@&`JG(hN$am^3y|(UZ_%?fum%Y`8C+09Y1!RzD`!53n;+imh_#{OPQ!( zxSFP~hBTg(0(|kD)I9sdS`HW&-(U|^iNoEvOKX{Okv_l)QFGM_7Dtu z1(bwCt@<9xI@r@oT-ymehhWT%e6?5sTMgIK!BHhLX7pZd!KrH_nHp^7tuUb@ZwFC1v{` zu`q|E^p5-sJ&ieYKoC#H*^Zk~A%rx3RuykMt|6d`a>Hv0djRtgM$(wY(cnJLo%TML zRTOSo$AG^>;aP*ILNjdU^@|IX4xz+EfOU4+(sf=afi+ydY%#{^X9W3QRlReT8vCL# z3B2IGI2Hng{bR)9x;UhQtnKu5Jk5TXi>f0Js%`i58B zi)Xk{f7+{LI5H-#ZgQl;=^ki`>T!OHRKn0f4^l*DG9F3gc9;?enyzyHg8E0V0OAkr z#&bMg>S*d;{Cg?54lSL>b^Xpfzx@s|;MpuSeNakawktG5=u|aQ^ohDO#@?C7LN80N z_B-BJ0+{H+E=KP9A(tZIQ+i#yfUkNs7U_q^=jrjQmcH;mn*i7o_Ov*;0MyK}NyPyH zaE`qf01V)-(CO#mbj=i0GcG7l%_PHJ9}8IkXVv@otC9UqxbLlXpP;;pQhyWsqupo) zK4Q!z0dhbBr$1O`!s1ow17&i48pN{y<-aWN(!R6qpO7VvzR)}t7@qY&r#XgIdXrS; z0iJj}glJr~I}3TTio=IxJd5$q7$3gU#^2Q$^CP1Ez;qhPUhC|Vl-h8+8%RfjhP0gJ z=a6{Jp-Fotdc_c3{gVRDl~1IaXgH! zj%e|fq`_$F0hdkT>Z^BB^wb^~hYv3HSH;_{FmhsnYAFa))x_P6@NHx<5dev4e6D%b zCbccnW&H#1^&Jl|CG$Gvsv%?StmpY);0_DVzM-bwkA?wzQvCF_8jriIvEh&!esjtV zup&WKCwi__5QPHIZNLuPHh zdjRa~iXw5~^50F{cs_9jC%pDqXt~kJZ7Hh{)bC4QHJ<0nYV`2{^s`$7!jB7hdW;%8 zgKKxG8lQ&@@(DIp8-=S_PX!T5Q{uke0Lv*Ez%OuW&E~n51=cVDGG}E)A({S+H^G+vJwRdArZ}T!Pi~=gX7uc=pw#y8U(VKQd zukNb>H#G;xH@#GYfzRGDRCt&4?UthBH`F?EduXBlx#aMuF(Y>`%hSV4r%}s%hv36% z@duh8hkQ7YqN_Pa4K#zop}WMDx>;&};jIND)8A#onK?*CpAdb1SCVMrTTc!3tOZ}y3 zxZxkgox7mE?%#{95Nt}5j|Y6;){+BjPb{^?!j4EyoOeR%fI{(QhE-`Epcz(_vy98g z&OqW4DM|W}au8)@ABv0jRLeCfIlT4la*v<#&lXjMD0R?mi7JtpU2HrZz#ut3Cp;Xt5l zxP>GddEncix6}n>vym#@KV>uHJ!FUdv1L1!bMPYqQ&W%Cgqix4p_5rAH5{&r9Po^g zIjQDK1z*)jZGjP}IasROCkQQk<&+dlUub*{u-*dLjby7&AAUpuB|e?Idmx2U>=)VGS^l ziawKAhJ+_^#i$OqT`;v-2HS%2GWmEs*`y4yr^oP@-yoe*BvKO39llRbB&!++tykG6 zdG-1wnaE``jP$BnyM`v>VB7EKa}JRwffOEl!O)(R3pwy}Dc`0hJeXMzm%#RIeWht9 zj0FgfQj3>^vXsdtIy*e^F(J-XU}suwT*i*HOSOHhSs&e%(Klu>_buY|HwIeZPQ-$_ z_T%qw>66u5fD+AGyhm{h7&V;+G>e`$qY172JGZ5k-Jlb}4Yp{it>DKUuL<3z5Y5K-tWbqw?##CuVG76zxzF z4pcqT(s#mBosRpW!@Y8Q{`&VaLLD_#|vmbFbs1gI3 zz+G-3spJ@jjZdE6rRac%<|HdJZSLT4PuF{Gs2TL23fe}SGh)zZ6!=(i?>t2E0c2?T zw)Ibg!_54+c9F-xu>z#j6Jbyb&Zg4pf9l^MW><{7a^s#xTy)kRMr}T0>Av*=E{qp*99+zSI;T0=&pUL8KD+Wc` zN1|0R&L-6ht1mdkYL6KS&RSqxGl2TslA@nb&Tg_dyrD)pPd=pxe z;fu>Ds^|8u3?tye%ShcfA3+^g%N9Xb`9v*%R*bjYq~ueBf`{tQ2QfNxGWF)f@3VF? zwq2*b+MkEVzU394;Y!@KF7T%Q$4Enzw;`L*Qe)-Zn6gVxt0lLm^^a6 z?%a3yg&xlC?dTEOfGniPTm~5IqEtH)`m6`}<4yXBUDWJt{@N#F!CLRW6BRJ#k$1#- z*vv>eD^&&gK*nMB-`}=55^Y4>6aWKKqAMErY{w%keZ^B_G#lI1M9y)5xXzGix;`IP z+~!dZs{mm}Ua5hPylyvFSliJj6=;fq-3=w3KMuB;Tn;0PWu{iG!J+Yy>I)%=ZBs>z zZTjZv;B!Feyb&wa)L!I2)-Y;RxtR7OQaP#~S zWiqRD^;`3^RRh7AS(YHWK~|HzGv7C5S_Voh?xs%5qw3gY7%(YM);AecSh}u$aV-}K zO($+1cHg>u<>~fs*zpqMZH74)g=S&_E2Kbq7@IraWafj;3XaRKOq+a1b`bzYPFwTB zR^Cxf5k~q)9D$}il*eM`b26ENMdU=q(n$T3W<9B@@id3Zxrf|%<}IsVuCcbp`;UM4oDiD0#YSsrGt63Akvz*Bz@n&zD$02s2S-^ z!lK9vG~+kvh_AFdU_nd5p0H(_J_~MA{Z9F6{h(RuL`M zG5>$DGM%8;kUBdDE_RD253yHPuPb8a0#Kd$l_+aY)u9?DSxnF4?ikr58JakETI`<(f+{40<4l{0cNZ~CT!dZ4W#r|#d z+93AeFTqx7**B2X8lH7jt&!7g=Koo_2D{zD&9MxLMBjMsJ;-LU-e)KGnXSe=(2XIQ zV-3C1;A-Ezy%D8w_As&7>RZFB_Bs7=CKR zIzsiGgVSABv#RGlU2nFQuZt$cGJ#MyAcy85fl*Ydi^WBFlZa!6=FcoB!sBv!OD^M- zj*~{lw)b`?@d(OvPKM%Tzy^a_H-5^%3oL2fVoTJ0&Z;Yzz7Av+;?*RZM^}A+l2?@l zJIao}XhGb*viqHM9@fCLa5`Ex%%$c#Poz=oY$O~8$1Mv9_>fW_GK+$M{$*-@w^u|c zJUcoG!|QSaa#Bw`w#&zQ3k^G6RROhWdrbJCR`Y^+#uAVPYqUh+mT}`<`Nos5TWZ~g zfA_JCOAn^=UT`2(aXBa16BfLzM}H(s*5ee-^y-Z}Uvb~dS6MKkOMKRXF4|+D{l;yW z+0^X}4{!=!!p?iJEI(ox*RWu{Z7|>mC=qW^v?DVYQZB}iEM;I9kElc#PoBEsmRB*j2KJO^A11qhNrzpj z*;B5$X8N0I@RVvvwfTfam}un?_7u61zQ7@zkb)hswJMyIi3$F4etA*XXJ#@GVjv3( zo-U=%s`F(kAJ@1%g{!3}0)C2{qzs^^Iu|JH!nN}_V&p44sf6cYsdJ%po@Ob^SytKt za@1cIj>h3Ew-w(gierRmZcIREmKzp?M74bPd>@{&V+^2op z&LdJx#%Sh(EvUQBx}pw$&^QWAi!|H9@pVdD(DM^IPN)@AoEkcSxUpv@toIgmRVX<6 zR0usMF!^$#)z#Gt0Z9 zmOdIjx1T$!mK)lcAiIC^+l>~<1WY%9@R1z{t$5{8nf{JwmbZ5GuYC%ZKOy#8YuR+< zo$5IUWZ=50WYt=C6Ij&Y(jt@4rD(tJXp4Ug!7IwAxO*L1mnmI0(vjt$5kOV5nh8II z_|(8IHyhxdo|49H8dCGa6ga4(hx(W+$yTf@d;Ue|{b-yjbzeEx%26Nvh8J6(DNzfj zR9A9R-iEFJgu{GZa!tEStJ z6$M35QH-ddf*&BFfRYS=3JOLrqKG;L$w`8Ms6;W4lVl{0)_Pr^i~Tds<@pEaDb`xM zt-V{@S9{Jjoemgdd_7dX^;Y$&>gQ9`1Y4VK@xM<%#54w!o3mJF4=E>L9;w}=zMn8$ z9%x1DNn$U;*1b6GA<|Qy;Y2jCHr_hr(1SE!XZ`dVDctjLKV|c5=D2Bc+;lcWPZ^0d zK*q*}Af~?jW_~`Disb}-AiKw8fXj@Ttm7^j4GqHGz~%&E3l>&ohN}4|v+lK2ohCJo zudP9fJFYOCF_c;6HzY(LAHA$P#jK>aY^LzkGHs;f#qyjv6#9Z6cj&!<0ihP4SlJzq zoJh7pT9>UthDzuu&CoxtXLWk)r9h33NBAD^<@ocIF56g`Ci6>AX9G_9+WRvN*i6gI zmtoT6uIrubZ1Qu-T2)tpd+y7*AQ=r*KY2>}mK|IUDWV{a-X~80H!Z2pTK6#b$OiU5 z&70`x7@nbeAhtKotj{o7u;@7vZ7qAW7L091rD&|wx-8TD_A)R~FJuUE(q{(M7*9TW zVn*)eW?3?K>#6DUMja@r!C-yZZRUNHS%|$(KB7+p3t^Bi&BYdiE_o3A#A9>J*oq=-%DT z5L*~g4^xVNp&HWZ4bbwG`zwGYPAU;BVg}>VXIi6>*UuU{YBaS(`*Jw{vO=zq&FYIV zVU=Db-KUl{GLlU55=QWADKN%#A0x;%*Vo`fgXNyKiOJ-%@P(`n+wl>*Q}hIv*L6eM~Nx~!AF1Ri9&@t5hR5qXY2AlY<;WCZ{ilav!-1nW%YVFjT%gJTU$4DeKZ zfOeK(n#O5fjC?y-`cW0aN6LH-gec~slx_%prCR2X1r7A}QqOf4?CYDKE%AK25LrmP98j z#Lq!G32#FwNK+_szZJh;h>VgN-zQCur_+cOaHDR57j1y}t0~{-bndl=EAfMVIPX6% z+9As()jOyb3};nXbT!`6x#EO}@*b*+Fae0oDHR~2)yCzn7^JU?qZ>$b^V~&NC*Lq4 z`5Sq6SuiNWQkbSxsejCg=LJ)c7|ucrtH1^mn*AoXqp@+X?|rQ4FPp*-`Oj6@R~!yWtz+ZmBg4u|3E zrYs$~Dx^N#!tT-!!SS_<^{m2lis>N6js%$n`JK3hk4|!+A;0Y9Ly(8CM3e03hdl^k z8;0hpx{e_A^grT#Lzewtv}NXS^xzBX{ADgpt$)p~_vWiB3xq;T<85qKUz{0Jamq1mUMN(TH~c19csRUMB0ET3f-1^crV(MKc;RkjD?oP789aTv?Dq? zAoW(vITyTgsZ>wnyhI7D1U@Q43>OnB{+D*pFNur0~ zuQ&U9G0z{#ilLw49{(M9L-#fBkPO+$AB?#Tk^cC`Ps>30zV<6_3Jgwg2!=BAo-b;} zEZD-$uK-}QMB1B7iVW_p>4{RG+ET!mg=$9Ld_vkpDy+5B>Dx2BwA z8f_tGeKE6|wQjupQaT<>6@3%B&bygb)~!?#sNftzu2^NidV7NJ5>%BtPiJUCY&;I}wNt4f2(!KkkGSMqJXp<%r$?*30J&h!I}#*ahBAOIBbjJlJ&HvkxWiNiS+&ncnr7 zVeh|WMqZk6|2HhOKb)s`Wv1X3eqdM=Z(Y!jdh22Yv(Ar@hAfM(Z_U2^)DC;Ydj|mW ziJh=8G~_pm*duLp;HFy*m5#4u!H?-n!EUUi9@uDhqTmg&Mh!5kA-KJ1+z$iN28kW2 zVQ)~N&Rnpk4b+t4jix8ud|2#R%O{j?edP(i)rzJdz~30u*BBxQLHDGI40(RbqR{2C zgN&OEx}rp5KFgfHRH@3pms09Rvh<|ReA@=7j$K0n(fE(&&49aU&C|mud;=_hPvGuF zke30cl$Sw@)nQk6m|uBWxj35+fWY3n$ZLk@vIH!g=JZNyu4#I;BScB^__8c$)lYSI z*t}m-W@b#^5rBrIVhyDidibMs4@xm)ir3L}B0`TRY?X}_iUp_%o~BW^?D z!b?JytSXt81m-&QJZ{9&@80{DAKo=okbf`A5$yxy0)n&MUd_L@lEif;tPyl3pz$T;T5&II<=;MrPOC#Kt!EfuI5q6 z%v-p7=@jOp2b{-T(!cZ$Bb~+7B^nmDZg>>wq~-6=R=w#$EfJhk1^J zNOna_-v#`x-OpYpE{4N@S-~{kM@k!Iu_6BN9DISc%ak-hw64qWBhw$oX9XJ>B)#il zCb%=iv|jCRQ(- zHx%CiU13RIz=2`&do$bTOwzJ*GA42yn1)PM$7v5yBKV%B)SZgI(=53+Kd)o%8~{Pm z%-Ak7o0!dGyOED14-ZtNM#d~Z?q@o)l!<|T{5{14bQ!eXTfaN&=L=ezYqx_K5s2^i%v?+;q>iBYrkygC z&E61ns9Af7#z&nzDds^~IDDe~o)x$f&NuGs1!IW;#7V)eNdxxnWZ(|$xRL}+)!9D* zqTRCUmDkvKw#^hK#j>+zc*klyf@t;LenjYJ7AbkWK@hnTfelLSsb#ueJ!|~Dq?N`t7=CTv z+?agxHs4iYYM$=fqa{aNjZ2tD@>eJ_xuRC%CuL?Y^-7Sn{MV>LD;5c)C_oQdqqw|W zMQkR(>+pK`RgIPT4rCf#w-Q#uAoW14fDZ!0Rz!&D%e@7eL0)K>E5*mQZf6*$H=DW) znX*Ud_b*xPn1H%5olaAkaTK-y4681bKL@;uLV#oWv2xnzwK z(|~iW3W435ipI}iGq596V7=Uvutt4uDZ!^deUlp_$a44)=kCjjlclkoX8fa?Rp$Qp zGvD=_*^`5LGNtEeR8KAKlLXw|SY8wvyuBesst?mCgYg`A z1TgW}=iE=wu~1C~T%rrE{?&ZbHg&$7Fu6#LHp+8|UqNRzuG?VQGtF0zRS~fISmyAH z5cdJzYZhY6@R3(5VXx92;VWZD!gRoYr1kka?RQ2UsTs<=VCr{cGfpa@2102iM(BrW z56T$lYxs3rBVbHK+tYM3$E2JdODPmmFfu)3YzTObVS(fUj`}B8kT_r-5y;jRmzH8+ zvDw$u<{((ZUf?84{O9;~L#}W$RI>E3`4uBJo@bNs*GiQpX%vOa^$cnMv#8ejwCpQ+ z_qX)D($RBv6|vHEGg$R2iKWQ57cXTRX^PF>YO1op#xfw6(55Ubqc;;26@y;6GY2I3 zN`7VQxg(gI1x#Tu;8nvjzHk8}N(D0@5*r+%OfVt-M~dG+j$1GsJ2V6%*d^K zQRXN0pA-wo?DAVzT-e!YJ9y*xdvIvTN~9DV!LPdqF?3B}C=;wnn>w0bx2@s?Ek&Ik zaIaCScLB~LYU3-U)zu6=I<+Xw)wDOD{X&=mHDm;o$iJ}~;g<@}HxS2O5V(axdQ;QD z0~rc606W_8UWQg{`=kk!Kb-cBjH+PSE99M4gszso38d3}UnVu;}y43B+}# z)@6~ zJSR>RK84NPZkhUbmgaFlCCLrqu=>Z!GdGg&;yW^H?j|%pQl%xGSX`L>)?{o}vz53G ztKFZQ`_9s1n=htC10F-xd(*U=9cnxaIN4@fjF{3EP8egAyH2Gf@=<3ZAVg9YbjBhvCW8b$JgWg60Q@Uzrja&$?u zM7gvd+XfH*VWpDyge%Z`by2i>O!vu68)Y7Zh>eLd&2Ks6VTxaN`>irnoFG_<*OEL^ zvzgJAEFs~mo1JDD1wdYr%$nH{=u7Y|j8m;&0(01U8y+<(V<)88I+?qjLdFdM;Ob;k ze+n)1F0?q$3f65-UwcS8Lh(u;Nvkl)C#zT}D~-w$LV-CiJ0@x`Wf4+bKRu z^{Kd_{Chd=u!95P#Efy4wSc9V#x?$VKJdOd!ihiQR*1T}qaaS0e3Q>N@39se-eL%A zH>ydXdjYqtEo?U=OK}GBB<|>1T6oHgJsSokdyK4d@pUu}w-D=Jq*{W{MLM z^&rS9r43yg6L?I@8iG2(PF16geEou%I&P|1m9juYl4dpT%I$C+Sv2nU2;hgdWceK( zxP>Jd&>b+<7gxy_ZFvk2nmM{T95&l-?T@=Ay>}T%vGo!bz!@-t^pTN70YnSuF0IAh zOWhLA-naNjn?)7PE*aCyRlZnm8>rh|ibnJ(|OtH_uy6qN_` zyKH{J0&!4Qco0LlU>arAIe7C{FmIB7JEg$7osh^bmZ9B*Cx#B0&(64ZC3$QmnP4x> zp!v)xPw;6okm@e|v;e|rXhJq+%om-4cgwsTF;G0I3M+!=jo!i7|28lTTV#fr&mr5w ziaH<}7GX&m`2DE# za#}xOQL5AgME~4}yUhp7K@ORTBL}kIlH3fk4ieQk722fH9G;3PfQIxh@sxJfCf2?R1 z`5Y;QZ-_6VKx1fMRg|kjIc-bt-`#^oaNDk_Op*^9BLUk}QhU;0DnbMX${8yo%84_* zR%GmvaQ5;W#u}-Ei3*LnfYCYY(H&FadmgbRs6n|0d@PSXM;13~$;3*xORNL1GM>fq zf-<~9fNL1tZwGc7{aq>xZjnO6PRP3-%xWH`?7^#?08BXmIDZ=ypxDB+Eze+`@Czrc zOw8-|5H6G97$wnE9Y}bp8&2436`NUf`3P2<+{{tj{st`7f`;{Kf#>#z@WX!YhtUk< z?xVyld z=CczqK_EgC0z+t^y$-2V*J2{edK<)M167G^SY`Ar#x-DVPPyoO{D`Vm#T!?dB#W6Z z_Xt}BpVv+faGc4VX852?Z9}Y{gv)rOe-?U7m~bIJkvV96s?AL95xre{5b*dzzt&b3 zvxhW1h?-3KqQX6dk{LcqRD&qf3LnSc+RAFohuZH3=cu(ir|vnSJnLB=YWQs{6Gjnb zI_;54d#bAOq!eVDIZyU~W5$zkN310fFnoD+1iYf%Dfn;dVv$!Kn2(A1Ny1L151MQNH}G?e<0<>DnDEww)r$kE!-FUI#+sbKuid3(tmF} zDHA^qwJ;-_%=pzOOuo}fxh2E<&#e~xmiqmBLTD}^Q`X8Q@Y6|lY)pUNhfaL-JGmZG z!rxk<+6zZ~nfTQd8K*XAqi*Bu3Mq4YM{*WP5_3|0#`iXlvYEMe-fk4m)e=mPH*MaJ z4yq-WzKC6$`IbJQkG|&H$o`YtA$)%mI7$3Vkh3mkwI}FykK)mvc_x?Q~feIMqdId)cve%hgN<1NE11T}T!9 zg~*7A^24sY-lzfPyY1R&=aEwfl%L0DTk3SpsnL&f?0` z{qO{x4=H+pG)U(`d_Qf^3%zPF_1m7ZX!&UT+kJRN5sUc%UZ! zOFEPNutDh2XG68gRXKPanXk3hNtcWCkryiUu=gyKW*B~jKWQbVo8Exr2P^<-d{PQI z?TprC`)Gb&S&^9s3ozBb9P~o#Xn#s+Ox>=P((I$~@I6d=UdhN}z@AMDG3Cz;MU+fb z3OTi>4=GDS1_HIW{$?Zw`2&SA{k4yOKz!!OtjtqsiIM+vAO8yxVPWT@cU%lfIv4-% zE&RjIosGC05%aI#L_B@|Dl7Kci>w2OlQU2MtBHT0Ht`RN&pe||{9~h*mS$T2XyX6> zxWC6tym&|U9|8g1V2TREu3**)Uc=zQD)MK6U1*{WXP0{-lo@y0NxqkqA6&QAcQyRG zJ2*5lL*~~=J}ewVKSnk>5tsPoXVYbx-LlLVKj{n%J1wXrc9BFPC_wEYeIT7SHoBf! zjMh9feJ?^X_uzOESJTr9P z6Ar`SFqgya5e@=cnI5Q;MJ`NwiC+*FO&^cb* zE)tJS6`}4BU|9?fp_kwy1-{rps$_6>X*{Htwzi zlhdF2Bhe^V3L}I2Gl_EfdGzw)PL^Bl_m}W}>?#3T3_z7>*Z%v^0O4xpLirrb8gmRZ z<_JOWXTZO#J()>IG@THJ_w_v2pgEu7uMd;XK1Nq-%?~TmnWxSq@+8#5OL@gT`#@<& zn*5pG7bKw9e5XwaPs7nBgLR0KlY7j-&H{t{%U&f^6E71owg576MVbQ9&|d&l-90n;s<#`0p}q?62C zHwC)Z`<;B;?itMyx7qKI5-yT5g{$~Hsa#0uyy;qjGSK85VCnL{FLQjn+0~54?aER` z+H#eBxGwY?P^pKgFjavOCNdaMWD^b*$64x1he*l1Qo^CVN73|J7M4wt!&dl`L| zwSFfSrH#{cGfd2tTD14b84t9vRNW4ck$iX-s(a%4g;-iB%<^fj0I#Q^zNfIWi9S9^ zPYuh-#$pD(ZHMvDNPI161(SAZ^%EbO%76z8>$ z_X8aV)9;umhKm?2q#ff84SQ84f}XuF75)IDDUVJ^lRs}+lqe1~T5vrom?p+{2RBOi zuk|%*1irbSdoDFMA=KEy*HV`jdBhMH9+vjYhZ*7k>MX@Bs{)Rk^pyPfp1x)z?Zz&? zzSLA66ynQ}K&>9Aa^hl!8_2iq1Uc)-_qh4w&1u>*G=DH>>mEMQf=qojW4Cbscy146 zp@%&XlmZH2`&ZzW^qc>HZFqe?Xf-(HYG&>%Ne*LZTde zVgRZ@U2`>zz?QL3UtD=|{#1{kp&&d2IDfL4bWZ}ITIC>G>gob6BACF~_+awW5 zRTHG!d`BId;wX}#KmB2ubk$gG&subIX34Xak{>6t;fT*48FS{#YH>(gGC;(N2i0= zLjG^T!zFh5Uu=~9GCMmbC&uceE#V1fB*EqfAgCE^2z^T zvj6(@Kc^EL<>27^&o{Z-ANbF=FOG*-|N8&CfBq&ueaYhA_#9Nq&hG#G^R@moHfp(p O{%k4#>ihrk-v19IvBu*7 literal 0 HcmV?d00001 diff --git a/Scripts/Models (Under Development)/ffn.wts.pnl b/Scripts/Models (Under Development)/ffn.wts.pnl new file mode 100644 index 0000000000000000000000000000000000000000..072920a24fedef8ad5a8d334bf0aa9d58f63f415 GIT binary patch literal 28527 zcmb5XcT-l~((a3Z0Tcr$3YfE~m@q376|)!+5fxNGL@^M=tmK??M#(v65V)(lKg;>& ze1}u#{03{kyXw@bUFY5Fsr4-2p7)$%^yt2>t9$gA>;7#AM?1R}EA0M1|5RBnqvmzCfc^&%?nRdm!R=je~-2j_&Pp-UKCWWuu0W&H2^h@`h4k0mV6PFN9? z;FOc#9F(wf|LrBW?Qd_-PFVFnZuu(ddpzHC3AN`Nug@f`&dbe7SQC`6cK>aM#k*Zd zSaK0Z^^`x2&vZgGcsgxZM!r`?jr{~|DCS>pFA&b z@IRLv|=J9I&&tWTP^_3m*UlTj;j>p3gxyp~≻u9)W_(-guN~Uw z5vKK8xZPh$lePKI$zV;jq|Qf&e@I)Bl!$G`Z7-dZ9Ie($O9t*-_V-SfF^lh%t}%nY zv&`R5sspsx!(NKS79qA!8Qgbrz5mLX=kgB&luDY|q)EhI%!HSOCCsTSLMEtBym$x3qdUoWHQSfB=}Snn_MQl;G+uFH^4SW^Fk z+jbq5d2P0|P*Z<8e$q}IwKVsx*1GVxIUTcnb@si^%B=J-GTUz{)^;u8leA_2GWqi5 zhD4XG{}bZv{?ymZ-}JzK?xg@65Jz01OL}35Og&- z=;qass{z}u+`AimH8l9Y2L&77*uPnf>rHP6R*#?rPZO*jCRn}RtTh4mf9h#`Nykn_Ws!dD2F^gJjS+Znd=aG-vW<8U7EsK?Y zY|u_XjINh@nT!a+KX)&EZYO`SH8I&tv{g!l6z!H-OUuL0Xob$-mR7NO-<5QkTAreF z)^f?yekt2`R|l=94uoyFue4~6wMDuK>Jg&5Z!zdfWSZ5@)@B!s$anrZ`dUd0#eQF$ zRtI0_r-ao~`Qz$RO}Av+UAwjNyo1cM_)>dkTj*m=vz|QGOd02{9p;)A5NCXk7TlIn z>)0kA$&j@5T5ZXn?|TkQbtECGEk~CA6q}n?SU73h3MN#??f4+iPUk)b-{KxN9gC5S zy^l9x$3w@pTMEuU0Bjh11?HMg*-8BRzPx{2>O5rF`n^x5WaP;j$;HlIM}BJCV_;6^ zB=xMO|B!MlXSq&*3^=K#-vO+k0lv8_>@I6@l$E}uuL|m>_?+a6;-g@Q}90Aj>W8qQf$1Pk?;oW+gI% z>9eHL(qim24urHzv6*D6RL1R*MtkP<@d_c|fOn)xo%M0g279eoj_VyF(yNG`VP-R7 z=|J@TQ+`ij1SymJ4PrY%%y;8en!!JXZ-JFAu?4?)dh~Ocv0F&Aq^-4=rnlH&ItrgD z-m_IEt*En7bt6feh{E_VcPT!|3No6uwc%c3`?+Yoe{L6LQW_a`k97ukD|om?3kmY& z*BR!_MJt6-R^E822XIwePXEq900nijR$jICdWwdc&vlPS1(w3Pjl zsbwZWmRM5j?;0%))@rF+hB;HeBw54Xkg&Tfot>iN7TUw@#0h0ax6Sr%WQv!y%no0hv7M1@KhE`HRWA11m!O8pLCRbcx9&$VUHogn3}Ynloi1p;{ToX&j^VV$lK##N)o#7@L?%3?(+UlDllljH z7AIG7@3(gB)OmA-3`$PGRauB;$Uy6mq#u&}n3pW%zsRy0afFrEfyqHBg5~*xlIFpt1Zt?;C0i|Wd&)= z*IkG8?^8=EbWD3SJ0wBCl4ATP?~RPT-o@(O){@W5UrLqOei3`AG9V={v8$$sFsb^; zaPx$YSfC>9`gyYwi7Rz%S>Tns)^e?0dgUIlT+h6v@Oz4MIN)Cwr0X&sSiywLFhZ5I zt&-XA7_pq+3S>3_BXwvge(#Y2LX^U^RmTA@5ba;wO{O+r1xnw2gXQ*@`FpaYJ(sR; zzr=RSO;YbmuA71ViPeGoHQSv@`ibp0>VyRYsh5JTI_-jF$+W%%Y}tAE?&3mrZ9|)# z+kyC4elARhpFfuzYZGkPEKLz8s{%`>Ps@~+eUt{xvNUZYTv7Ovu?>P`Oe2u@s1Hxs)E<2@$}c0LEr%-mjU)o2)gcx}2l35dX}2l>GbHoj|F! z&fX;cJN6Nx0KkUBVBnCBr^H!HKQM+_nb?ouJNM|6c3wRb%H+ElcZN=0OUU;pQiStZ`L5xaGptiet$O0jQvGOI= z?IHtue^DxbM`}6ftECaYrP&u4sb_GLD|u+GP8lDx$$-oDMAmZ3tU|7izkm%|eh91I zAw}{zKj_mH)y15>8}l?7S&z~rJdH*urRkTQ^3Xq&X}kdE6trpwZX zI1eA9i@>uHj`LcyQCd$JFIgj{+L(e{mEW?~@7RB68>^OWZI_%+Pb70aAZR-V`)6qW zrsquOy5@Q#y0zNe2cAFh65G}we@S=y1vxzqc@?1ziP+_2 z$U#kC#`Wcv_9p7glY{^`Y7G%Ken|&eE5jJm^5@DHZ!fHZZL=RnUPDCX6T?SzZ1)vb zp~1QhJk7h~S?yB6^n0~=+0uLPhU7j5rY3gi7<}!4EsG^p()6!1GJ9toIWB|iKrUZ< znGX)Q3699f3CRdi+cz)_hbW8$hW44C7Z3*JShwM(p<*rf6o9%4jO|2>?Zm$dy-t}2 z_S}2&FwO(TrJd4x5P3(=gfrXZ?^%R%9xRh>AWevd$ZQR)(m1~|N4X{&_T0xd;b?#Kp+hr z_$o5!+y$HuQb;#+*YEJyXtn7CAfQFEJ{l6QL-6$8^;u3pXz3Y{ub8{kXr6UMs+|Hi ztupCb6XvqrCxGX;={;}op(WM{H^yeOzhv|pE)QS))Sj18Ll&dRs$T6pM(Q|@6 z;ysr%YvmV9YainZ!|u$gh1B<{j(!o_72~3SSv$xbXm7%wv1VSDzdy0`qL2zcAhtc< zEo}on%aF8)c%<^z%}Byh9+m&rI{DR|PYT_Bi|vzi#c!5zUo*hseXMW~Qn8e|>fkwd zDGlQ<4PrZOhgKT=`4Ra^npvcsTV!Dwq5Wg4j<3exe~gYe5``N6Cqr(m_V$mj_v@?| zxU5+}wEX3YWHaeCNYp}nzx|z25l3bGw)C!+`j1R#;uLu97fwoXR!T1DO{b7jMKc&t6>xL z+}ys+GW?R^Fs6+2n)B;9-yEGzMeOt9wXKxyI zc7_2O2<%os2(Q@7o%1!%>5ArCH}T$Ctbk6}Kh~+hyObLSq&&&mjcuhXRN4$#td*Yi zENq*$qSqXsZ+-@Rm6_bM{ml^_e&ImEAqyK$$H5$_pBOqhSjlff`7bEQ_*=u4M1vyQ z9k32drk*t@TA;Ou&NH}F?7CVT*6Lr)cwmG`vDEDK)0(r4)&_F+oz`isJxtBxmyA61 zlY%4KX}LmoQodz3ICWU_q&V?A5|Hb8R*swgd4O`WCB!Xpk)i95pyZrzz%!_qXcPJj z*evyxg;%71k7G|ek`~n%P?=ti+cc7G*P2mOYS#m8xU<6W@(5r*Clf*7Ocl~$*|sNq zqXa!Dja#+iDejo59afTz1hJCj9*PfFtjEN~I=CpY+SpAFx2{!mn3J0v|q8l zJHua{C_>CMX@6EnT=@R5bwkpYc`QkgJWI#ENS{o-d`j9BAHnbjl0DG@HmP2@4p$mG zVSqz=UT#W|k>ddH67N;LoyYe>@IZM*4-#ULzvK&4y-Y1I~ zfzEb2;&8XMGU~$?f_a2IKxPh|;L*18*3(t;*Ht@@5({tk?UVd?QlL_6zqagn@CgW} z0Qv38seLb5XUI!4*KLeU!Dnoe|3V5*f+*=k!hKjhFG0pHVwpQS?T@?n>#X6}9LbjH zxCG6G`|Li8PY>xlA~Oxd`6x{+D^v)~Xq8K(=Dg+;o;|b=`pv$v(=6981=T?o zxeR^Ge$=y}eE}lxxY+FV<;X9gXF(8JBWi z8YGpzFeP!f6hvz!MJWI?GOZgMm%~8yfd+Idwarf&53B@f$!nga0J8nm5;Dli4VTnv zt}DZp`t!t!w(c@3s+9-5Jra*7(q|6UlNRLsmU$;!K~&C0#Tt{%>*y1mJiz1Nuk)I} ziVDNzOX;*;$e`AJcmueAL`~Z79_@=ZfM^D^*>5%3KG*E~x?O9I={V@U7W39@;x>(l z^&nQZl=08AMt!6ZtHXkTq5j1YWl3ECKvPCAWLP@;fXU|T!WC0PN+o6*5Ge%@!0KC0 zEZ3k{5{sh~VNbN&l$T(b3s$5S$*6s(brhCf2$J$0EXIZ9XxW8i!}7$DjBCMFQ)X+S z0OE95hn_DtWT18LWZc^*BAo5}Nl;_z#}n4mrY-JLbt-N@33UqdpVq9)-mKOqY4ki7 z8-STU{?efhn|7G&fBb_i{Kg=?Uo70Ka?iGlRBHRJqnjS-;013JxkN`^;-0Gxtl9G& zQn!PL&{;2y17CFK`Ee0!ol(^Y%?8H4SdOeH}T>3~Wa4A&<$#~xrfKEdZ&e?F3b zDNlA!Ln$zKNWltJlPfmd15?1TBpWtOvqA23rDXwo3VQPj`*BjpBg0y_B-y=T47 zyfn(Qlxh#ibK_~Ev;DO*H9?(=bOhY{ zaP@fu_X{ET!E6e~eRBV_ltv@Jn9xMLbUo9CU{hGMX&DJr&)q2F#xj@4a|_6O34=Hn zt&PxpPvg8J1V^*vY=d~xt#{$CjRh&@^jefV+O+@VDeV6X&r8=j7CPJLlSUo;aXLIn zsi8FJ;CB* z1%V%KfRQqCk8up^T)4m-%MLxcb_nv$GSHQ~Kro!1OsSobT>`P$5hG)*=HJyl-CR0W(SdHS&mj0C{`=q+~^B$&clM?Gg z{6X!y_zp_4K~?GDHxClVl&f%{T1^dI&;OE758wTD^+%w|If|tt@}o3vCd>$!<%e#Z zx{$&ym=yic-m9lD+$o)YbtH*ftZ^5H(-Zd?+@#e0oDix8QIYz+I#xF`s3_0#Gfc;zeW;VsSd3y6ocB|HWi<9z0U-w(AZl+{8N#|!n6k7d-kyT22f|;;)B$S?}e@y}io`M^r z()i0P!Vfo@c(auO&RK_!Km}!5=keU~#fON(=(EN^x1W9R)J!((5K{?6Bz2W&69X9s zcRygnIpkIPCrW3$aj9R}^5NVMvH>0l8Ky z`NsoOo@~YKyb0(5Z8~%sEv|pRf04XXXBgEcs&V~?4>M}1c@n6Y!lpxGa!?zGO` z_`s&!5^Os8O*%gUi`Th({j00eX}!`kz&mx{A-?Ia1z$Gnv^|A8vY}JwwdI7VQ?>t+ ziJj7Um?-=ph0h&`{2~;jO)^)&{9}uXV}ZQ0=_l%OY#)y0tqU8L%D^Rqb-~)c8A+DT zJ|qzkrel0L^(JZbw=)siOhmVE-{rbkz{mF0hLmGHS82iX1Vf&ubw2!~9jgx={BZ-c zryjt8#nz$CCf_T-Mh({ElYv_J%+i5yFO*fW^*M6a+F!@nd6M=YW(%d_5#u`o+!Qj_sq>OyB_ntXpW;Kg&;qJd zhjERL6m9%qU6N^37=AsD$K9e@z4M4t=C|S=O72a@R0}-1pCO8=VSR!5l5yeb{gOe* z63o-gpcPmjfpFL-%4Mjn5vLOj^<^uWD!ImK!-*p@Z!i5zP)BvvlemZ4aSj!qWBrh! z3yE(8X0yJApV}|gx3DMkBiWU{^NikzhTf$_Cz%jBjP1KZ?O*98>86l@*=ru`V_=Qu z+0$#K-jiuf@?V?x^_!njJ~JePu2?e**J~!{PZOJ;G{x@Tz{CvHRbBFx(iLoOkjzT) zFHM!fJK!&$^y$b$tqh~e<(#11TTGmyO5#4k4Lu=XZ=1FEO^EzmuIV4B`rLVhsO(Qd zeXNx%cRZ5KF9W^}GL5%3AA0qJE#7lFy4DST@4~5MLPoysyHR{V=g?ppKw{w|C`b-w<=!*d&-8r<4;TWgb}{50E& zI>nkaF@^;uzk2g|spGF^Z)Zps99PgtfJ)WWNbNq%qAs8ZS zeMo6Mjuk1gX#H)wP)02UKZ92)#R`nL|FF+;Qn79WroAu|b?F-X5BaApU@v?2I)_U< zB>+YUhgtmxJNP-FtJ5*!5Ug`R#`t61(jT~^bZ%Rg2xH^-md<>a^4%A?!5?H`ug`CG z7I15w_@r%XwPTqH1UzFwM_xv)1WT8Hk&2!6PYj1#0Jtg8N#!vtxB;h5|E#HxC|OG9 z!y8PkS6dk`gwPQ2{erd!Q{&{3-CD5S!9?aEU;T4!^-*K7)Q`Hb0!AoZbmTdLyNiD& zKO5(B4UTYp&%Mg5Gg=~9Zjb09FcH)y^-m6bdCTIiz3j1rk0>ipAbDsC$g>G-mT2Mg zSRJ_p7f9hx`KQe&V!lcdLu%f=`oQ&DG-p|OD3S0N@SOjp)i)XSu(jlp47@nJ216x* zPPq=tJjt2&Ksa4w)_HgXxq;dFoOL5-LY!04&z6c4()^8zKK37jH(13K*7*{bEy#e+ z83Nw6P4iCc=x6xG2(IGDjjU4_yh7TOy3y})GV-h3I(7;G9J(HCaBW7@o{;3QAe~n1 zD;>qrYb2FSKx+4T>F}zH#^cGwqpEHD3R;M@fU2xVB&*}v zyPr~1*`h?%s-!&+Zs@r6oVfi)Vux0-B;U=K$Je97*oI5dEN6I(%y&`b`~+|^cRwR8 z-Vl8~TH^MTE}-aui2Z9Gs13`n7dn0NSt!QZ9BVJ-UkHBqHuL!|L&KMx1bv;!XbW$ehkf!)a}B^7py-%H{TM zZMU9o0?jfs?Wlwc+Jb(%u&9=bq1(q~(?((gsgt}w%`#}{YW9mTV|@5q1OJTLYp&VZ@%i|C6P9c>C{CLXsRD_VlGR(a~RS`UG`S=YBZ{ocB8l^sn8qYRN? zG4pcia=~S(=B$!D2jT$_z(m*I`Lfo%7znY`B6A<&wZqY52(>pN^-uVIk|@b%ACaB? zaGB<3qPxxEgH$Av6W=*?{5XYyIcw+l-8MD%$QUwANQQq| zt%E;wCV*Cg4^Lu_?8BbqiPfg=`VURR^Xnh%dwKvTBv4cMK>*0)LvNi}2ZLt_yE$1q?xlQyI061j@}exs*ur=sH>0U)q~oR$)8*@T zkwce7fM~W|QXg^6_Z>5`ZS`jirCQ<0!`5o57r)gz5}Z2Y^HhucaIGneR+8-?@8~HO zOvXKQ)LUntQv2Mr4+v`GiT%j$F0uwVORI@U;&;-?WWZ9i);#;q2`_$Yd-aA~@D0UH z`?W^CHHOBVSi|@MKM&1(7q0)}kbabt=DgpYz_umUtM#-uY4$er=Y9`T)twg%A&=SP zFKudDM`7`Tzmtyc0xX)X`}?GBUpOH54;h>k5af+7wOL=8T}|v6$+-)>7loX;PA+Sm zyZ8I#cQ1ugbEN(Q8&iM2n1yea=1`1OqVpGdidNp$o;Q3r!;ak$Le1=T<;sHfG+3L? zenq4@dF*{k%3k5AwuH^52W?AG z!q%jOZIPjO|2sWs4})(8KDzVh;eW3QZTo*Uq3tkDXgh-vc9|x$6{ZPo_kWtumanq2 zyW&TP56Hg+OoXEiUwE#)?7z~%GW0~V{BGk%PT1_I_R(FEtY!YFZZvX#sHV{#NpO+y=D>|P8(tF$kZtn=8FTa0RQfmD zpG0Hx3U?wg44`kLRPztiETjLdLEBx9E`qKOx=|$rq3r$@Z0jhaV zWiA8=3s~>2U14`{Jr=tDIG2t(%ZT+j?5m8OU(Q{ZT_;aifKSSR$HzLbROdeVCE;k3 zPqu7)>PRkeT&vbliAM*ppOd)*I-izKSKL$DAJV1ZiU7+)%Aptg?3`HrR%?|Ggh-`N zIQh++)n_)cnaIXK__cNSU6Anpm!Xbw&Gba!XWU6I!?`^T*gi30HZ>1UE zrY>M@00n1{>x9}svE$P4LMMFIp>SfPYQuJ(J`iaof4YcE<|0c*EPKGa_2=S7jg@E~ zx$4UopdVCgz~?fx+ibzwz}fDXGQ9=%pc+=fZP0VUAB;Pf;+~TPT_*WCd)8f1l!Q`u zZ{s3%ysBRKYTusyV!Nn$>$zv^EfZ+$kB8zdliIFbmW*nKR2|z2eUnUeT+-RdO_zYw z?Nayif@vEgf26-gDQ=9O=a+Lah^j7aOfpjan>M}hlNnd!R)FRtJIG zs2fJ58AmdcNrP+#$@YP8m8Ki20*y!%k$}}|rQccr<-N{h)FM_r*9b%!^M2}h(xRU^ zf7OYd`+$!o%T8PO{4$BmRqG;sJPyf$u+MgO_NgiG^$Ic(ZM+xs{RF<;WVy4G&p6Aq z^SRi3$kY+)lRvrr%2zrYBF${Hv$2!o8G*vV8}&Me1b z8I~s#ZNQN2cx0yagVeKxZ;?zgW63(~oDc}ajafUBaD{H2IbehW?YLn$dO82?v^)=K z_F0{jKICD5SM)b2KSrW<ALko27cW)saFQHXCrWwWT!XgW zQp(ubH@9K<)>nYg7W(E=z9RA{fH6k)GU-nNOnRUYZ~5m=?S_gwOrz~DKS{wRCeqs8#V-K@l^OF8dNKfucW6A|u4M}k2pANlSCxxUP z*5xOdFpniMMSnPW>j~mO(}M8vIfy!+3fJ~a5F$?b!)$xZz41$6+&z1NZz2-;WCBWE4c40ms>qp!m<{rs8y zeSu80&icJ_D^fXc1v1+f@@1LX1xX}En;=t6*z^-nC^(6P@MDSd~yn$+T*W(s1v6C?RfOD+>g`-u2T z?%%e=Sa@a&mt|=Qo<7T_e@3i!J^O(3+N3B!Dq?iuI8#36p}FUx?O3U031QE)*8x&l zj%i_p*INv`>iAcZs|st08^TE{`KO4ayNsc>Ny8r3)4oQ0R2}9Ebyp}7T7E77U}F%g z2R4?IlFMpMRhH4dju0Ht$-BdTQFU7y-x7V#RPyDxYk9 zO`GgFqN>r-ya&s|wP7o_W$w+Wi+skF#=^>ItX8ahdeqS$2Q9yOU)pvVrm2>;2rnIf zV=6K&dya3Gfe+LdFh`5dt&FlGM93*7*GoB@sD|en6Yank*R}rMQsNkV6y9I3F2_qr zEc;!?YQc{Wp7Dwz6Axe^Q?K09zb5cqJ*C4H#Y1J_g--4c(II;b!vUgp0$9opGrs%r zEMct&PLBNUMX(h@n9bG_V1@*6Vz)7%>3sOJTWl*ek2-u~tSga~@zT=C4Kn;D+(M{n z#)&acf z|Lb(*zYfOz?rHp$4rpoByJGZ{JjpJe3!SZQas(fWz|w@~2yW1Zh|UH@v% z1LID8Ckeqh;IfUtXVDh!Uv%3+Iufil8otR%3h+hEw}U3SPvGQ3=&oWdyp7j?SLEhYWu70IR6jRBPjffJFN5tizInAniS~hXE<& z`5v7PIRRl2;xk(335YgGskJ#uN^esTX@@SP*Ex2VG@3fyeuRIn%zX62A8-3Of7tD+ z^VU`wc}^)8bDF)+W+^x>72n7oFvHdplII4yHnP>=o-8%@W%91Hq0Y)~YtxriP{=DZ zYejMduAgqvGL?2*XV(~04N2$IP5yL-5E7$rE`6{kPbQO?Luc#Ub|9q93dfcdY`(e? z5Z{i-w3FRSo^I33bMW-=!!T$6wU2S%rkzk0nx8h)(>n4BAS<^#UmTTLU#vyFq5g$t z?jZE5nKnAD^oled{to3I+rsynVZ(fIey_6ScW^I#2G;9y??}3;=v`-&`yb1A$`b}a z<*W|g$=8XkT4kO5`R%Kg*cm|CE1ApgL0F`n*1DTd0kJ_vk%17`F9mU<0#M|zi;e}~ z;-HV|UmMiEWs>H^s!7>}OOebV1Cx;lltXnRo`5V`piH}CwCd5VW&RIkfn77E-44r1 z*Tcl+KHC<#Vo1xcAH2e!p{(ZfF;ad3WHj4V?aw6h)UFP1%{3i?qfV}@nrL6$Y&NdrH!$3o-6x@4+?~^Gp<}MvMrP;vS51ECJkP>XA zjK6s37b%$>@nkE4zAINKiDU@C10<^RYo}$0T z_ztH}v|*1H-p2E#VilU%bVBDb6hB5aK`$vm$CiKJ1@>;`-&3;$`ge8Q8y}J!vr{`v z9-DVO4lW{TY?7g;QW9tim)XQ*{m(B6tR$3|8N*=;)WV3RIi?9~Gz5iB59|K-P;Ljg z+n}{7)IC7@QFZVh2lEiL8OyJOy%+Y&%;AvJc%tTnTxJD|y|jn(4*kUJ7|8*<&X)h_ zNL-22srM|HGRs#c5!&GSg$+^C;LeXZRtSt*E(KSELG7=J%U4Txu)RMdk`|G25v(4y zR_xH!7t+LC^6~kzcxt(ayNl)heg74m*|{hfrsHmX(ey7E-2L!%?v|%H{EQZ`ebJ_^ zYW7C4M~}|#zv~Yky2-$svv0rfSL*!UfjW-s=vswObAYx}$f$5j&5IcSu(cQ+5`22a6 zt^R_M_g##H9KN1KyEi=4S((BPus!9AA7rc@pdx;S zPo&{fRQCGKua9V-dUE~|3YW8#LC4S0qYpM8+{+oSujXM>QoSFkkcWoRGL{_U4B>wj zH1`*4CS|9N4}gElQ|E0vu7d)nz&|RKtubg_9e*uZ=8y4>)FxB{~ zuLc_CLo{clmXN4ppJtcsFjG21Y@xrU$XjYQtY!%);gtYI>7eZ#p?*_3c4+Q>A0!kE zJ_I?<@Ag>k??~_rZo+@qbzqBXC<=``O8*?i$hF|_3~|Nz3=kjO?dqoL0$ z$s%kBFZt`|k*vKbjO+U5*i~D9V(aaG>}+W67M^?ACF08Fcobi#*DKfaAab#F=bJQb zX8>8R82!v$R&{Rg4pa19I3(Sl6DhH=cFo(5LKGA?%tQv3{EmFG=svB^*X}VQHm0oy z!m-XfOR0Mx&`LD_&I>%Y*iiK(r)50h`fS2F_tqufg9&74SfWaL&q>{LV>QkVE@}eG zbKsRfNxS^}tv%Zb6e>9_xO>+{eC)W-W@$?LV(Kibwt#B&xC?bu$U_G0mMmE>rU=s` z|2$Yt+YP3hoFrY*56~_s_^dOC?!Tt{g#Fk|3{6YJ(&)uCn|&|C(GM@W`LB1<@*Q86 zIr_Vt4wWWfKo{{?zy2-0U%i`u*<3MC&ziM6*+5R*9DmH{RGqfs@AHUM(1ZOUEB-p% zZkmAxNFknTuh$_hL~#~buQzOjq{hMUYHC5H%eBUSqmi%;8+~`j#^VVHioepv78^JW z*3xnG%~^9+JNmn8)(ekydx11*+$~*8fG0yvl_ak0c_T!movrN=>0BR5!ZvJerWFs* z`OEli9L7(@K7~@gZms_}Bvg(T{U#T;D9*_ICbPVV%HubGtkg~KAL6~#pkmM>{&227(FtNqtW$wnD?L%mHb<#PB^-oyr69NO%UWY_C4x>`STEcl zv-hX|lW82^YiB36*Rb`%Q4p34x=&`0%4EPf7~l5jmgar=g#%;BB7D7k9blO!l!UkI zgy&}9kNl0g<=nzYLYscX2G6D2cYe9UXs-+C3&%>gdaCV(2sJd+Vu)& z?}9eQfBCp1gOF_S7N0s80oXy56Es< z)vl+P8HDGty+~GkrNDam)-LMTe$ITdoC_OhPKOtV*?lS|3aT{K%}Aoz%`%$IIOq69 zvhII52Cwo^>zvHk>-1_;DK;)ac(6Rf_X-|`>zB)dCmE>VX{FNlfwLDk7~sBy`-f%b zHOU9VrMOthb7*OZ<{5Ux3gs zrZBwD9Y1rMcq1bN0y@1lYflLGAsMN1CJaPomqDo-iS4N!Q`eR=CW^HA9G$q=_L~QX zeg6G{x}kK^Esx=uA=WgpqJIg(QcTtDY_8iDdU~T2aUnjn{0ixK`3Vf&LG1Kt&0cf- z$?QxF=opv<_vGS#HmPWbv5V%PUnJ+meG@GoW<0q)_zh3A#o`T_+TT?zz{0ZR4axQiVlzAOwHTyY8&KFwK55}PShIs9 zL5DfTMALCf0^GH9319xI(?8(&{6of!r*uvVUD+c~q6$1EevTl?-` z-TvmaG@{L_PBXQ^%jO!+^u>Cd-eM{z<)0Iz&kTq<3_!I%eYsk5B*!@5 ze5AC#kczYT66I4Frr~xCp^lKO-}p7cz*fECew5r$5=+!>@k*0%Esg+r!4DlfbY13m zy)lVzjZc`)f5D=i*1C=1*v@8@JJ;kAM3Zmu?d%8|F|_`f)V$pa;eOreuZ@8wcx+np zhNcD*O>!=qq^>q6BX5J97_lU)jt{|EYvwDe4fj_QLcH)M6*tnh zJCwlrQkog~)B&W)sQx>}uWY4k@};_)xyPUYw`s-6Q?&%xYB9%6haxcQG!f5&25M^j zVT$_s@yL=J93>9X0fIp)>13&ROed!Suoh|k?M=pONnJ8c13GlUc$te1O4dDTxlS6k zUQ^HMe6mzUitUE{qjCH_a1F}s_er|MJ1X!WTX5`EtY&&n5Y@AqOA({Rg&r7d`znB- z#PscR73?*k-!Wnm zbe(pL8n~H4)?wd35U-ObfR;7yVC98XLcvLL--Mjoa&hs;(l@cvg9L&k-{QhI1d>iB4uS)`CzbWcnnWTw`ME1~Fgz>0k`_?2ojpd5##LfRL#o&^amt z&&+b+oZVM&4(Q7`vh;g#8Z!MM9!j=5fA^LANi-8Vcyds`()(xwH4e z9jU|28Tj;-p--2M@GP*d1W{h0e)k85`r%2)4SI6?*K#;xl8Yuz=dhh7rOPf#ksHp> z=R@v?pq^RE(Awv8?S`Ev2zYEAe$skMbE0?;PO`R#9C-Si(T%bw9+kR6GEO|XS{>b7KW4?TO={+0l;IG zl<#N#ePz@uhS%p1bqBOQK$t(iW2r<98>YL}5nriY22Qy1e|ppYmgln5XK?jxE;-Nd4=HM5@!Ky)XjJvo$*T zQ1iDL3LOj7$`lz2G+!tH>{`!q&lW@9Rj8-5jlAIGi*;ZH&`i~XMs$kGfL;2RotnYG zeYWc8igg5SxwXmh`A+uq-V>$ls%yGAbp)tff2|XcVC7NCV%!DnaYZ2vO3iOxuC#Y^ z8(vRi!m*|v0+Tizx#p4AQ!o?f;gC{4(bMa{$i|rtzANCO78B2= zFTXle#y1mKnty#82?rgDq1!X|p$i5T+DD_Rk4K=ii$5 z7E@h!(*;go7%$}M=Fmg74oOkuGxMT?f?ra30a!zB%*gz13V&x_;_Knuvi~8#+`$_@ z1~qj(2^!9H*7Kmp`4EHGyfouF4=#VM1th5hSUCGG!ETnTe6vn`@cvEi#oVUwq}DYl zXy_%<>J&*w5^QzK&Ubt4627_Z`%RC~gJiC|=r4;+h&KFKdx1IBd__1ST(UlWj@`bW z8v91+IH*H$ve^GUaO5jxi~;=KMtGYSTNFP;GQoT0!J4*`pdTSi3JQ*dtmZ)RQi~?g zXf1VO9eF}IR%_>=*GFTVn3b02Rbpl-{)Y*sS>ZnFPx&Np9k_+*vbCL%!Z>zR8#UG9 ziP%6>$n494uoACPxN`L?YtdodbcHmgzFamH3bU}Oj+Y)& z_xdh>0vSuiRqeD^eLHf}#XiZ@!iHS6=>=}lZN)LfoX6hKhXsT5Z`mO|w+%GClFmb# z2J^R5Y-4W~4s5!i#X@y-VDoz&T!B`ur{yK@&C{oQjk8L6;I<^+pC)2*?`hW!89a(P zwB1?KcY16x;gUmcabGHLp`AnCkeHSn2LmWXP98bB?OHsA3aSqE(sFu_Gm6E|-xqsN z>wn%_*SNR_9DH8%xovjqH2oGFQDfb6l1=&1n8(AyS)vpr0Uc&{4W8=<|fHa^Dr2d=Mn&{eOxEFZphz`+Ck1*4kHA2&aq)8Ep z5)|Hs9)^zb_;xGhBv;mG=WF=OCVy7~uZ`O9(*%bTDtD6a)coXM;Cig~IdjCu{BjJD zMsU0q&V1Uy@2MS&h1p_YEQX<{;_c?el5N&U4)kLdl!}QK4vS6x!1b)1Yxl59cZ2xy zbWoH`#}cMSzE>p1NcSG5-*4RqYE0dt4dDkz-ae?sfy+TBgGwVhz2aDsW_`AT?_H3I z7;IyD8KezpPnyC zKS70|>j(zBz%_;N$lw+<4S94dlb_XU<4HMQlKUKG!j3XpB9Q{s;9U=hd9$W#-x+&t zHg$Vi3fYqNrG`YgWs+?M{Di$yM00MKq}gANFxH8B!l;`W`=IFXvs4!+aBq&g3d9b`t}PQr2(Hy*@W+X?&*Q+Lx5xup)S(&1QUXg?b{r;kv8`l z!rrkOxx)Sc$xy!aBU*YK{EibN_7AsSzr*|d&^eD-P+G>BRJ-+qzI5&9eLQ>+g$HE< zL-R|2;e!zP3~pWQO^UMH$7#7%;>nQBZsE-fqdM%xfNtxbdo)0Daf@{iqm3X!Q=c1k z_MW$eB)twIMcb<7VPWiqQa7a8TXT09#x@2e9P)Q*<~=$rEz0z@+V+JV#Uk+7cdZW# zGsbM_kjjy{Z!#ZshJrc{(!l#Cxpy>w*NzyOV0(@_+YWB2FS-&o+s z+LLZn3Wu}=KA;Ef&3egCu&xp7f2I1m`#rPt(=m)>8rHp%EulP2iUdydYv?nh)3 zSh>Vh-wWfd>*1nfiPC_EgHEVX&xNg#s(af1)gsK&rDp9q8mib8(LdjwZ)VZ*Sfw`C zEj+E9uaZUZLQ+D1KFF-W@_4l!k_`4CaV|_vs~@OmV^f3o>tKY`JqU#~YppxM-u8?; zm^>!;$!U%N$k!yq0d9p_boa9cZ%EoGUIs>+Bbk;PS3HY4h2rIo7kU*dUz`Xpps5uecwzi*MNw z9DSFzB5#0n{HN@q&)?@`DtMA%UEV1}%fk_8xp2j0 z>2Z>dC8k{6c+#{Hl=67$C-<*;$-)67nIV_?+n2&VBV}NGYB!9hZm-pBQ?dn;qyxvnPI9y+DcpM%|D7^Gs^&!I5Y(H`%4Au|kCNYU z)Da%CCF;NDS`>#LAtF*YhFu2GuaX0uTPh2EIOlWC*CVlb4B4z`fUH7opn9@>B6|zay3SK>NlYygF!{N~;#L ze@zOxDaG&PQFvsDmPS8Z#*G>Cnh-=TS=1c!n(&2GKUyX+S@{1jtkiZj~%ZPyEf zVO^l@w6JfGjiL#&6MER@?xIimrm5=oN0J<%f%;dHnTgx$ZZV=j>1HI|C*atPhL_v`ZmV4oKw@xEk~`x-jC zStneIhuH85UfA*-r^?~iMj2vjPYH~0iLQXGh6uLC+Y>!qwJHp%(^;)|zqfA9FDu0L z(hmKrY3A7gDJ?#d7L#HCYL%qzSncD32z>xObHeksW?^2)rDE0RZz;GKuOIOY(CWqh z-{TQyUNZI`lf2G8iQzUaJB?2etl1=yW%yBtDONPQ;+V$sSZF>^KYxtHC_Q1hyAZek z&avLux|5$LTy`U+K~b>ZMYA8Qp{yUgY4gK1vE1860$fk2t;~&qk%iomx=8I?qJMt# z(bNTok?jlyW?AmOEJA${0N7-mM0=Fr8UOfsr%ps^D<{0kkTE9j7p7>}9{!r&TBsh; z%IV5b2CX>f+qgyxUesaj24T{}Wg=7ytvg|RDXTRsBB(hSFmM>(oBw6W=rUL7^VVA6 z9-~v4KE=CGBdt@hGWYcaPcpK1mZS5yL+dqEliy+JuZ*+YWMv;CZ_xlmx;(}^4hxuB z4O}p9Y~G;b(sweDl?F`tDw*zC&@>g6cK}A`kUzX}g>miPY1(rNIAm|Fia7R*1aA~8 zJVY$b@LojT%4PYS=fy}%wJ6EJRCy4xsUi|LMZpJaNkVccOv?jMhcd7$9_6%x1b%{7 zi!y>vE1tx!m}M?s33E7WHg6)^&4dVMGKK-dr{g@!OkT8b$mtzaT?+-llwAaI;bkLS zs2C9_v*%tqeYYrxjQj*u`rOT?bH7f7vhz+>X(Dfq8lm#8K3crRb~p^PHHG>zfC~@d zsXVQdT&gx3yfMV=2{V=}!0I$eHhdwOvGm{8(d9C87U2RL6)7*I=oAi1Sk7-go&OD~ zpzvQvz2i@Rf}_TAJ-PeX0Z(kXZ>g4OQ7A!9=N{)^WWqTdq9J69bc8V!{{#})_^b1j zj6nH&2Zxqahez_qZ03%N1buo;&QPW-;gEhaB@hd!P40FGF=0WhztplJ)?hoRO6)Et+9EIKwn{lmHIIw|} z&uuc4^6CW^eB>i72W~tu2EH$Y30k^QO@j{W$)R`9tDX^d&=fRGu6g3NiR5jk_G_*S zc4eV)y#J@JGy94nOTu_rAd8fOphAlvpyGle$PQ|jz|ak)vWH%KvhzeS($MozW5?CZrql=HC+1? z^r!m9Wf@8-gnk2U%b967`T_J8yN7c}m|brhner-O&vt=PIdM(Xs{hzsJaU^3n&*mc z#KU>o4L5NVsE3vSo$-3_tL8IO)SCh$qyR&@4}sRVeM&D%r@tTYOU?u((tQqyiV-5r zPC2sFJlSfrjb^H>8Ud0#4uMF5TG*$JyM=_*7yLKhQ486{?7w;kJ6uNHzy_G-aG1RF z3~a z+x(=P$urtY%{+})%Cc2UuDX03b>Jyb;2ka;*0G(mAiS%!(nT%bM%KMXdWP$zA1aA6 zwxbHP2t~7=a6}|IcqG+)L3I3L*?da1=4sd$>hqcp#u#8^eU^_5$8Hnuy%CP~N7u3c zQ%rsi;2U-c)E3rr4K#V?f_|aFq^hT7_-ry`R)g@KT$Hv-`PHbblWspOdBw$w|06-C z`a;*l@{!)Vh>rSiaMB4+St^fLdQxqhVevlYraP)Ik#49aa0e(mFSCX(pPCJjAsg=I zMgkBv2J}t4T4~{dUB1dEWI`f0{rqnp%;X^I{yEj`tEh@)Uzq4zn1|}g5s1rzS}e}6 zY|?gsiA>4T8Qjq74y)-OrF_wtENry;tC758oE=S2!HB(fHSHga+H&*akeWRuy15!z zmUhO)N*(yqK30)XnX4QIDz>Y!uqPkrA zIIY2Hioe{8lb5K^Dz$ZEXQbOzeQ>8zdkp`@KTif&51VDdOP48I5)FoGLy(+#`wE9 zyu{&=fZC<2$(s=YC$ZN9GI|G4A?VZj4~M+)rj+z9jupvL`~=VrxCj8lU4>~>={~6g zUEoZe4Uv5!XR!&?qK~vJtS!W_=;@*J%1<;VcM|PSL|@^cBv8lDxj{91u^fzS@GC`Z zIL;2r5C<{%KEDQSx8pVxwG-{o?+~4L@1;O_ySAA6N~?Q5fDFs(v)XvSm-8i2{IDWR zEi0yN5{|aZP^0SnfebOM7esAN45cjJaNQDxfA5BPY3KVw_4@~JpeoeB_?gujmv{mN zB-9JWmvh)~eXT7yaNbWR;Xo}$(bae__4+t6D?1n-2uwR*aq{%%k}EW=xN7XKmFb{L zpc!L-R~SjzQzTq9GFv_VsdgtCK|$#PmOB~fA%ua=Jo>?+K3^)@r1{#WwG>1j@=2gt z>NJTD_*)iwO5qNf1N}`S=eW`V>9e2-^5BrGC%Zq;WMjNRdfh=!LIOQ*19>pnW*_1k z`~{`lbE+qfoWy}7hHOv`GQ{&ZcVK=L5N~l`yXd~s8nqG!ND-uTFg{~Z@cvJ=?1S(8DVtWh@GBoQNQRAW+>WCk-a9gp;-u1AA&ih7e<17%;IKD zB1l)YTb{@8Pgt{I$jeOiC=AVDOzjD}Y%MX*o>Xg3E-uXa>$0`4mWayx-_lX(e@n~E z&&>Y6i$|sWS+#alSFIh(vme)0Yaupso89=mYR%WhWFm@qZ;!1&>3}IS4t(`Q6w+oR zmkv5=R=x2R;LZ)&8dOmP-*(1ZeahW+5lBXuMAQo`CN%%`srQ zz2)%0mdn?x8utZAllt`iHE2_B0%ahj*@C{zu(rIf)l%>o|nj(wffYMjMNt_En?*|UkYJ?r=ZloR*>He`Zj1&dV`)fh` zPMy7g<~L^G;2*(hXcj5|EWSfyXijS~f(8-dDB-m^@w{5PuHI5BAEyS+GsusbZ`H&y zFK7?9c=7Lxe9Ji8;%SIDQuUTCU4D(Hzu};f&!jO$qe}cHt}%SKOt@v%f)u*Qc?`-x z(`#;Iz>`LTl03H|vFDJOSHUtByA}Npi@dEKCTcmgx^0$~Xw`|FzVk;-M`Y>mwaV$5M)J&t#K`Nz&6^l@lobO>kz)f~{pT%EXQwrnN2>e)86L@ul9R)XTr zfQ-WK@ojBLXEKyIa7}7;ec)CZt&WhUORDobrgVjncXH|nBls@69^G_GdpTFukzI+! zJJn>^E;S!{LSMAqs;%-wo;|hJo>p8JnPYSJxb$1q-Tz!Sne=bYUoI5qZ94FIp}1RR z)itdAmXR9%%{p;kEgJX5M$88NMgJZtKcm-L)4w^kb^7V^nsi^DgpYpXtDkRIsQY&B zFOTf~)t>vSOy8b%j?K;O;2$3fb~*I<<7?-`)}PPyf4tMnA@mRXx$EbRH}qeVNiR5; RV_Wa0Uz>U8&;S0}{{jxg>w^FQ literal 0 HcmV?d00001 diff --git a/Scripts/Models (Under Development)/ffn.wts_01.pnl b/Scripts/Models (Under Development)/ffn.wts_01.pnl new file mode 100644 index 0000000000000000000000000000000000000000..20016bdf83137d428b9f459d6ec9f84065714191 GIT binary patch literal 28527 zcmb5WcT-m1_U(y)qM!(vBVa&LR7{xNs9*w7F(*)xBBFwbf(gkWIfEccqU4;PsyScC zeGUD8S9gCF=l*V0SJ$n+_j~FnKI~`jwPqM&%(>Ruw?Z5o?d+B>xBLJ6=Vs?&_aZs^ zReWmfp_msbFAly>cysxptKH-O*MAxI-;(V1KD^4;Z@hT_KK|{?TgmTU$3~~bzkBNy zm;5f#>(agO@Zhk!UUx6t33(D8eCJ;1-Dh4Y@4T*F3knJj^9sH2C^-DdwYTp-rrdg$ zoN~bbS<;f|w;vPVryY!bmmGUA@#CA6_!r5^FVdVNlVhXbz5S4q{4qMkIm!M%S2@3p z`Oj~Y9761Jk{qKWqu<5EMt^XQeQSPjPFnhK34@DDTJ~@mf4_N=694val2dNd^6;b; zc}dRUNiKUsmW0@cc;zOo{O`AnNlAOpH&;Eh=NoY+lUxh(^O9DFC%Nqnaag?D*(CR{ zu(0d@H~%gE5n`8;^^xE^D3UNU4K% zTiWaSCe&Rg90H?bc1zCf5UsSd^t9Nd#*zl@uDNK&xpzleWWox)EQLE*;iuIu?Hu%Yu-cY5!+fR{q|~`)bcf(*p9x{ybIc} zEKnNawO%H*TgJBpMmlnX1^uNz?E=3a*KSAYlP0lQIwBo9{#E{Hu{2tDr6XX;*^|=7 z1IG7g0oQkYcMVLCsR$jHyo=H$whJ4Rv}L`gPFlyMSpR;}P8sG#kxMVTX_*dcmZdXV z7Oj<9BRTrd^77Q_2SU4$QmU2LrOZovEiW08&bQZfUPr7|nzci# zPHL7kS=T=AS$FaRhPIu!eQwiNEH;F{Rvt`C&@qodX*qvj1>@@11uII5UaoO?v*@Ri zvdGNt&|*Y+qh9DHT$(rO1s#O zxyhXVTl&I1&^+w1{*%)0W^xt|Zw+QR-ApQJ%`0+iZFthUl%(}h4{!X>aUneTZuqs} z`@un8mu`mL4Ssm{e+?Wqyt03_nCTl|k-nS4lQx_5-DJ{t%PTjNg8yx|ot>|r_DRV} zsrg}PXM|?o4!V&pwrA3*c|Ua81g@n&rBA2*lD3d|Zxdv~Gd@u2EuH$LSugKimA{t$ zdGJz3r2Uxww8FLS^@aCe3257u2j0?pWt9+A6Teb*;|S9X)aqdE`<%%2`2kNg>yY;c{Nb%#0NJF6j9K!N*zDB4?-4R$-G0JX zGsC3&s`XvEf;aG_ER$ZdQYd-*SFV;e>#>aA4wsSj7{GQ%rnPsCHUbw6uUYdhxk&42 zOm34A9gxi3S|u|iT11leNR|J3lV*>#B2tD9`lWEutTx84^mLIP%UMd~w=Wa@EtQt{ z+plLd6LU;h1jj#uqv#AtlVeHwbFyBGw15b_;%l#iuG)@UZ8CL&fj3Hv7MToO&g1gW zOC`gvveMKRE8YH~GHN+#hKxM}UOO$u)pPHj^kQJKB|pKL^-`sU#Di3ClG3|cdy8;- z0Z2-T7A?Js{W>M@h}1pZ9?K(|m5oT%LcJ2Qxy|0a-9_ZX1$#^3L z^21%c5HkNB_EK==d0!K4#~ByKr+N^t{sLXII_<` z<)CW275H8kzBruc>T|xAFjB4KH_Y2Cqvxac-@tUQAd{M;m}5kn_U~D%zvbTz$q?J7 zolo~~c_nj}k7R3K^ed93!%86_dL5o({%Wy-Lmyx=+iMxOWFpkfSAN^eAF+9HkNz0> zv(sE8b6=!ry=3UH^jU`us5g*!-5`*^Ob49APHnj5$JfgaM@?p17o=CFS3os0`p4S; z>=AbFjun8ZSz9gy@yjB~vFvo7d-qGvNq*z(FN0F&FZohuff(%vB7w zQEW*v;1Uh2jl3dHwFNKC!=;4=Ce}nmX+z34<1!i3!VQj3HFuG)r0BNxII0aK+Ii?= z@@24N*a}#u!!M-blL^54Fk5~Qie=6`;)=7^UhGq6?#Y9j3D^-GLpUs0(tBSP*2=_w z0^de3jcDh0ZQ@s5KJ zq*I;iLGnuTw9yW#txK0|OIn}r)C#Qx%quM~ow(vFMc+_;wnKz%uscsCkRnc>lkSaD zlPtDfS}MO}-tj3`_lY4;7d$oNv!(eMp%xpK$b^}r3~gU2l{XLLefP&YkbKUWfv(l8 z&)Sr51UTIKc3)!jV{ESrTP4TG8QbSsmp8r*IT>L@fIGQS|Mm7VL}%(%tWl!P$tGcE zbUNl5%vB(NtaZ?OBVMUUH7SH9%+&?l*_HlUY}<8QvtymMGO!sJ{o{5+W+SX!lE34Q z=D0~Mg0z~z&vOqm57X?;Xv%-cgCaa*<38gu$s}iWRQiE8sa90MQht-q{E<%jd#5|5 z(BgHeo+~Bms~Oh}A=cm~|D?#0(hU)e%Vuv-HbG}@kr?@hnY6*Kl?cWKSEgs%a#MdE zPmn(!xKno;cmKCP+I++Nrj9MQa6z&4;n>+PGG&Kn_QoAaS*v47TwA$KN|FQxJ!EYn z3H-F;BLTt4ei!_wu4aTBQAeb{tr3Cd@B~@0^qDWrEOelV4|?Aj&Qit(u>DH5v0%0s>*7 z@@EN-$;CA`9aYeI%1@^V5?MHR;!Rj2sGTCcF?$Gga?>K0`*#BMm2gnGC6h_8NY1_W z-^F(O+DR#u%9G?GzpvjR)zO-Xc~yO4h%v%eF7Z~1Q>Royeea_@A7Ek zT{|{O3DJt*(IJyh9_j20NTC^^&^ayIu$g;ySY+g1?bG=a=tUSqr#!V86ZG8y=)F8Y zN#$o5|6*+eXSxooc}OTzePqVS-@88+wMdpsAKvNvm~YswJ84s}*7@KDJoH-!-H;!K z=%CJl$B860%3f^y&r9F#Pec1mBxC++bKZ;J)?e4(+P;=Ml3kk*!=fE7 z&#||^bZp+Ee<-Aetp_s2GYjuA?p}25Jr{Q;Bm;uKC4!Kc*7=<#K7Za^ro)dm;1!!1 zsoQr8kuirUw;xJeFT*|r{uQZvtvOr1qdlUQ=-*gJ_-nxmms%ZPvjf#rYaKrmv-9Kz zZ8z2&f3MZ?Iw(^Xx5<+6uiEWyAl%I@(U~gkmljxG2+9(S_Y2d2PXpAojHgG3UvdL* z4&8#jvl$sO?fX>!Tz+g~p+P%0Q}h+<)MqQ@){%5=P2>JRhE6AunH@$hqnWjK9YND! zhhme;m&=(}gM(&RT6ycayJn@C{6}&2?0N*2RYX~My89u3r5$P3Dj5s?&IJ{);1%Mo zM%%qVCRs0a=&h7LWnd!~xBd$R&1+MHDXi=g5|29{HW6|NIP0MztRt$3xA-km{tdv& zqbSu5$-VK?@1enob}UmAvpL!D(#E?|{CRciVq}*|5$5ldsu<&x%1h^^D)E+a?vf{| z*Hd=`?df}%;P{1=QqBbOH+rx6b|1^DjSsZ?{7HfbSqB}cYYc8SEVg~tEGGd--le4*} zj>kQGj?^lC!2LV!TwKphZNTvh$zzT~kD(Z9hTnq7S&;JUgiaiiJWGa;-!fbYuhnr~ z>8Zn5f_{PyA4UnbnA)iArVhp)<*PWYK906Q$vz|)VLL6EZ&RLKArrLzimzr!(_UB# z2hM%-kzd=mp1QyH(|N6OF<$s+x*L*z5WQ@YtwOtR+N}vRY~HQ)d!^#Gb}iNucti{B z$riCWNY7auRZbFegYBBVw0Os_km%LVWiFr7oHw730Ss*d1bq5CNX0>{7v(A zYRNS+Rwk}W&*lhtvlG<-DK~8JSWW1YmL*RwBpIDU!6TVWu^JJY!(|&8cE^cSZN#54 zcULO|{oXEO%fyLD%{Y~Ah~VEY{k?5HP6v-?JkE2Gvw=rxP4kdH50(;JP;-&_bmI0G z>+w@*SZc`oi}gO_B!SZWXffml(Wrm5SAIvMWOe#wkY=w4UyR*3tx3_T?Z}c=#y6m& zX-u#8pveyxolJj0K6IGyfv~7WW4SkcvC~XSDgs`Z`ORsM%U;GbMJ!yR@xtFBKnW(~ zqq=8tTczw9X4Cl#(Qmi#g{0SqwdsZNn6>_!lwC!fP2iBQT=Xx_-evRC-PseK&dBl5|I#qRHtaYou%4BY2D?4&1jdf}h6X#P%>1py`Tr!Unf^)dF9m2D%1lQw)0rgqAU2VkO4 zTu9cjwOsT_Yi>xF+KdWzJ;8+~w|q>X>gb4BqNRh$%#aNvN(E*dB}ni*7TLJ@Zm<-rHkxO2gU3PXOTY-4jo}|X zAZD!7ek-*7{-Kw6>F)|D@sR$nq~~@0cZ&|9_90AcWJRgYZ&*}9R=^Ulr9j$GBUP6L zo_s@)k0mhXL20@1MXG(V&WS`W0A{sQ=@mC>-V3{B-%QR|kq?d1zv~9nah9%K#HClv zkipi3ljc2Mi)hm#b-Q+JxlVG?R_2=>v36;^mAuVcf9_ueNT$U`OY2she@?&be(X6v zR|7b|?ot1`qPwhb-jHI?J7jYq)Gmz=<&XU#E?K{xaTlc!S6CRy)BE%Rct5}cY*Ks6 zU}yv?8}p6G&=u#GVd$*jl{zO49`}h*q>GMwYVJJ%Q1d+`{|>Y_VZ_!2vVK7Owp=E< zfx)cQILUfvq)49h+e_{nIzFXL@oR+Cd_u7|)2i#gA=A6byjsiahh%+oB-&>U?=(HK z0H|zw8&KrX68*zxeTyuJ*>JJ-D#|RnU1Jq@cYX`M7b+3#|8FFwm>ZwfOzXD{h zHp}lc9rZ+~9npUGIH^lCpZ`i(5gNS;Q%cv1pqrY18*saf%<2L|n$}QxxPGC(d;OYL zS=&}qsW0fjOLFv}WbEE|nV6inyy z&PQv$;Bnj4AWE-%0B;-zgy7YB9SOK-XgzP6{C1Yy^h<6+R>=G9yTjUZsyqiRpK0S+KU83+L(%{qIQ1dqGQ-}PsR`&ux|o{j+>KWz&zU)g%+;>I6{VyfF- z*KJVP9D!hNl}0zaCJmr&uoAXj+eYu8Tx%`ZXC&^!>5VMuY}c~qS`tpR2oQIES#<^< z`~l4lyeEVjbly9PiPnJlgC=&G5fI0v@}Z2ZpyH(_8+OwvvRoT^MGH!_@mhukH@+CqGd>hIUPfb60L&5JE~iFR$$zCA)gFGD+Y;KQe5X86Ni z7i8hARB1jq@3@$?lbXelksmNhciQV$fSMnkYG*zyWLf@u9zfEy)J7Cph3kWMq85fh z`>!&(jvuF<0RTzqNt7W?t`sk%1_9gUE|Ubqb~UJ-W$=vEqSm`a2y#ys5^0lv(XmSp zjXDMQvTJ4P>Tm_Z_%e+CCEN3;e&t zaMQ4K>^ddc*G)j=e$s#Ywy{RSl0+$i@k<)a`owmyqELQ*Cc%-j=%YGlwS5hP_(gxD zk_MVVIK;Mgyx#X>ZvtRqf<6Lf{Jr9brNj64pNQrk*lFV2YU8R?JVKi;F|np&e0$ou z9-#$?10Vgc)_OccaQ+TA%&z}TDKRExJ4}v7Z;)w^LkDrfCI*tPzaNLeKksD%)a<+= zH4xd7z0$Vku1?}Oo94VY3ly>iz)&I}IHldMV_>FM3iCEycq;9V#6#*OI_~{4zYIDS zdRz6{l0=YU$=)qU+N$RtaL{dHqV593T@_$fP|CrBmV^_8i3}jXPFw;w3wcJVPA(zZ z`$4HnE0oUFclou^%sLl(>s*P-!?6x@GxXGw_*t}LJxZ)9U}O7 ze~Xd+Yyof}W9NB9v9(SszzzNNhFPckzr?3mD*<60*-CBola9Y;8K(OQERJZ-`;QE@ zKF~#LcM;CLm_xIYPOa0~ZIZm*DdG+%?R2hopzLO{`iWYeI&~V?5W1(rkk9p zg>K7fG2h|;bU(E8G0!mW?Su?llg{d}$FhCz z)-ua&v$U>|;vh7PjfJZxewq=7kyjr6s99kj31OStk4U0>o{9@5UAtyEK}gGl###gM z^H;}{@0cvf4$xZnFS|`x9oJ63XWq`N{_K0i8u`#(Z(7VGW3i67&}hy4#Nx)yT^9lI zIH*}L? zwtz>o1Tg;W-nw~95SykfEk<%BD>m=?t6yMcJiwWyRXA4+@) zxKWX{K}8e+-NsmJv}B3Q?hiEan94M4VK??G?So1k{}grif+;*QeC=-E4l$XmB z4ELN4G!NhH?Y?UVv&4f$U8(deU4o`*72A8BPRLhb2Pk+l;;sI^Op}yG`@;6M(z_hW z8LYf*1AC0ElkWs%84~sn&hsDograU)vOn zEYp8mr1!&d(xZo#iVS>BzIRMP`oG%z$^K9vsz|E6k{-eV!$_L%GQ3vu_mU!{(xGEI z>aBEw1}V&E)AX-*s7@G#nIkjF+UKvWX1&Me6rJ-Y^U0`%PbXMj%jO$RKP)$#@`o1Q0@A}QeO;60E0L-8f#k=mbnZDQ zHQI%-bR-dkAG<^(te3z0;2EPTi>#N4MkQ(Q0kG}1v=D&Qd`J-jqk>?2#@83yw#+YE zt5?V`FO=7+$J+KI7=$DQS!<$%%naFL%<)FEJ(38bcC_HT!%IRB9Aclx4KB>nxpO9# zEbkOhrVmFhXtA~2Rc%4naBvAZ_Y9V;b037DD?O&8lETwt0S<@Y^KL6{wGf!g{A^VA&hHZj_@6GIitq%U@0 z;JF8NFpiiTwc_8&!qFvY4_F~Xdp$2owu|OjA)mBr_mWpsuof6X+kxTA_DU_~V6PR-%H_ae$*~KB^-FJStDhYPWqy60F`8#j{GWc1 zoT`Oajlyh(V7mEp7R}Mkj;|`Ryy1${rn8P6@ea^l8fR4H?S5`Dew~!sy5DI%B`_s5 zPYq+O@6CBSCMj%3aOG(YLeGHaq;y}3z6oziuC#35Ld>wLY5Cge=&9E+1opnoj;S7` zDjPy-J=aL7KQ`~=;U)Bnr?qVhBZkMe8)Q^@@Na_wzSVYz}MYE5OvI zLxJDvr8cuHVx5;#k8?8MW6;{yLvlVIL{0E*2HGD_-1b6MWc<2g@^iZsI2crsUsv{l zs;zhu!IZn{K!9nm&^YQKc|FvyqUjMbt%-d`;+Yz3$(`++aZnl~2`p-7A!z=CV0umle=c-Vme$OsAwofUW zNaHBXTDvF8pB%jK#s6U76A=ITW zxkWZ_JoHgU1L>L>s_S*s{wpMt`}S5hYJJ1grsc!Jo)rbljJ>#q zQRb{4`?n^7!9WmgrBb#7%3Qb-)jk0p)+UJEm-PzrjrFff|Z3ElEa9N;cKv z0!pqhz~p`4Q&#>`FcZW9f%&cjf?JGzE-lfY_ep_iq`>a4utd>u9tV9*nw?c_@bELN z+xbOv!WdBx8TWbjIqCFuHONV^w}3Q07ew3U^YP%_zOPd6zIl|8sJaGTHTBm_YVi5KG+ zM?v%pU*=k$gP%q_eCH%dwDSgR$^NbZn_Jgvm}I#h0B48_~+NC5We zSy~q=Rfq6XH$E_m#s?-;?QXb;0^uPIUT?8 zKyxi;c(2mg1GJ#~ljEYpA-ldB&t7?#eurWFcMg<4d$8=J=Dp@2^@sN0@)}?4=fgAV zr6(Q-)LX%uQXWX}4HGc_Y!5;C&PkI?ppy0tgTGtkBzhXiPTrGQjMQ3~$ z)#PkKG%1|58HU@+#q^i;=Mn9*qV_@N74JwcnmRNwHy+hLuFS~%EInq?carEJdfD8? z@cS-6;JdXh269bRI&>NPP$3QINlGH?hm6JRpLMRZ!_t?>UDX+wH=lga$~y_pGL4jb z6R*<;v^~m*b-$y5VAw(|2=sxhtk!(V;&}v9ZcNNN9ZUd@Hhd*7S+JNi9M!*JCz)uP z7-la*Hye>xX>SOCRQu@csyo;U%v;bgsr#aJEGk(Cz8zMA%MUMH`Fp{I14FVIwAqj5sWtZ4 z`5ZsWtkV{dNn6$|mj7ML3A^*@cz00x-je_OR!Q$UDihjeR?rqH+kb{#D%M@#qvs~c zOippn>OIFbXKA|pME`Aeb@n=qBeXs^@Lud!vSC}GW{2Y`cKBO9!K_6;`HBnLlGegl z(_j^lkryfRi5Kp(t16hfqKr0|=HLV^@OhwJkAgr;yOm<|2)+p&5au#)jAF3U6ilCF zc1yZ1Kcg8rDL{i|evg|Ol;dRr|F_IX7{HgpSU~*OGHt~E&riRQHg-3_3yIpenmqWa zT?UdWq%%0^xR#TUUSM275OH6g0I5`HroFPHQ!TbofN*TjRp60kfALc%o$%FxJv?C? zokxz$!DbCmRte=w`+)=IH`P3<^y(>K38*JvV`cQ+I<&7dU|SP^^(mlqUb1g**Zc&T zVe1jSa}eVXtof7vR3VQVKE4d|2Pm6ql?eC^$J~S+f8M=&&&=DCZC=_D?M>C2_*$FaNSugf}i26Vv zn1ZzBDfnZ?Tm*XBxV(h~?=X)+nSr9Y#BvM&$RH0`>amNH)=Nglw`$)ZJKqiF9?NfO zgEoXj5yrdlZofPA^8){P5AthC>JPFYN&j&cVE-EGC=|zp9X_t`H{y-1j?T)=F+d#h z40rVOARPwH&zkkta3p;A$Ed;l8)nPPWs2o6nk_uFKUC2hoR_s z_O|Qbo->P*I`ian?RJZG4_rkwWCL4f-+r_8I_%3u>g!ga#Azh4GhY&BC^evh{dboz z4c%?Z+&egLA&AXvwC`rIZ5#B8@mVu#k+E-IuLEBYFGz{l9VTD$jUPPr?2OuM8U$HC zboz_bAEZ{%7B}Yh*WtTeVB$L9_&yp3!P20^u1lrM^nA3LM|VUC2L<}9)2q!k@RH?= zelB&q0G-!uWqbs0pJO6b|1TKG*YJ=L3lBko_o+*CdMAsBw8M5s6W(TJvC3K_6G^7r z+=e4=5gOAzczDDp_j#H9B=g_M!9k0rOg^y3jvzeYWB*v2B@;llj($>h())vcKiNK6 z`9l8gw>ROq2^jBYGZQ&SKS|4;I0Z*Cv@z?)0ov+#padnTLn|YRy|I^+9ITD78iEe3 zk_pb2OiLFl56rmDD8}{`GM!Fe3tV`1a<|mqJY-xt&Ip0n7jVm*byafN4!e?3mRj$i zzAToHM;F%63}7OH)h5w{q3^1IeO)H+hma`eXDEixWY1)u8v0>m}o9-lVm=nRSDewA&OeKH*7wQj+#YJq-Py7cT#GZ0>)rZ1MfCvSptsTlR-1 z`TdWw<-mW+mjA=W&JHN(@s^1PD~VDY#lT!h$bOxthaw}-Ak4}bSP(&>WvfozyoF5I zD}~Q)$8JHgr)>{_SBn!26<6HXrd`X?P{kI7{(&d#euuo21;8QgeIQwP@%;GZ6?X#H z0*pRdbIp$4(E_$Pw`Qr-Z-^r(b@YY&jyo=`diBV8?hwdk$)jkIU&(Z&C(+?WQkw`SUVXK8{c~>N7`NKK`JAZlUn)ew=*8UpBcRo>O-)~+m?2YY5hzvfK*6(2T3$4Gr zoA~^T?VY5|J|zg|Zer=6_q@)pinH+>M9xj!HY7Ys>MDRO>{-Ui_nlQOs~Q;Rp>D)GFIl|55dr&0997rLb`6VPCTe zt7^mJQ%P*hwMl*g;nUIE^r1_50ZvB}FQ*$c{s7jE3MGLWUS2|x7u zZy3?S9}Sn5>(70XYe~ubH|Hq&t4M?x@_g{!)y;C3G$S|Y28kKjcH!v`A)98qf{=7qE38`Wn6&vMwg- zqa;GmaV&!g4gyaLhV}C=zk9@d^2jh`W$#w0 z2|8g|pQc%n^jr@$5vN@q%%hw9VX4ewF|Qy(fBC#N%5=x~go}qv(4UdgXPUw88423Hg90S( z7|;KE72`kAu@CNejLwLqongC9UBqmo?ChT)5zBDo;z75z6Hiyh!eL-$(+c*3%fBGv z3v}qz{FEmYszy_F>%?vyd5F4Z>bEuHk;BuMW|_0@%Kjk06`PrZ8MT@r8J1 zm>(&{65#09eQS?HkPJUdnqQgs4lLG@r`LGP!0oNz1f|XGrAKw{7HOGIc{8cMy{LYE zTkFCp?S8GIx?`E4jOnHl4c%ZkMmY%)%mlu-BB@J9KkL|8jJpb|@B9deH-dpn>6cI( zp7jHqq~My=?O?h^hL!6+g?}{5jOYONBiKv9`Va%Hy?auybqiqu1=pR?!JXQD9&wUz zRVyREl90@x-!Jt=IB?ox-QGfND2)eAXcn?d^fSEHLdns_@CzA22~gc^d4);ijfiU$ zV$8C{>!9REo0?-Nk&>BBQ(nmXvlkz(z6p^)K@)5(CkOjn!mK-lY@L)GAl}=pbBvn< z2I;^fdIMt3-$vOyEyz}BPs2~GFFC|)t>?hdt8F@M7btxPNF*v>XEYJbgCb;6<6($~ z1>G)fIrfH(EC(1EWtLXk?=d^)U2kHxKDQ${!Jf-Y0=^ier>ToqOXOELI~S~^gQn~_ z6rSXtl5{xgVaWe~+3~-1CjZEPRUJo7)p0C5>G=ORlOOP(s^kCSOul4XkoH5zr7sm* z6njFw4d86S2MVVXyS4M3i9ze6PO}<9FmaZPnjgMwlB%1MpH8;Uc%9&w1E7N#&v8TCSN?Khm~+Z4BYba=aDu7_c3Y z(Ma@#4nF^2c4`(Q)@7bcTl`-liC7o21JlRwUzwU?`bG-PY(*2bC$dTutdp0*;F}3n zzNLE=ncP8abR*%uroLyp<eGNg0&t;f6fe3XBOKnThX z4u5Ub`jv**5JpVt=4!Mc5@-y$FiG2qwgCLiL1hDMN_RR`>ys*dCL! zbp2BzHJgPh;&epw9)CK^m|051se^AK_hKG;W*a}D0_eZ-^*PJ+x>kM(LWa{6n7rnX z#oCskwqTu{BT`JYgw%KmYZk|cG3sF}0(*9`J-`!QvmIRa-mDK$C?ddDz}6h9A2N7{ z$upnH_>P09S{p0a=buVu9G2L)lgrBk+K4;L>#Rv3}`x1F(K1Nv#%t z@m>^Iqt2RtLT4iu-Mk<*Zu;x36kg!Gtkk>eua`0fJs!ZW)et{#gjodr1#b^+d$`It zBUAIYYw1q2TSiKkK#-NOBoaq@mW7>^IT?(%wz2WS$?dL@H4ZqfWTQ48GbJlEQnU8k zoqu4G;4gq;R&zKQ&WXetUDzDUmN4rxNn#p4^JHJWb^pXZzQ_jNR37mJDC0!}!DiqG zpC{|U4Q~FIjT65%NPZj~iWG7rFvoOZt=(Je#B-KhF2pYmf7}&@w6VY72Y(LB#1GK1 z@I9s2Ri517%ifhD6HdhLP_%V82=&5w{@-X(qh_+0sg9kB(DrqRaIVVTrPWV-FBmZ* ztsYXriy~xdpY_PvyhA4g7lWq!o({XbU;`jxFosQ!NALq@pD#So`m_4?BVX)22>s+U z%W+cTBPu8h!Yr^i!{D&)cHaoTVYU%-0tty(^cq1qLbct$R&8c4JYY2cB1OuU6Ux{* z>WGwXg*-}dKE+&?=3K(I{jmBL4$`2JyDvdD%jo3ho8z_JpG=W9w^T>cpP&_Y@4I^5 z)W3%JZt z`m2bGSyW84N$KRJs2ds#9rBy-ir8k12R{zk%k*X^5|H`jrpVt&SMXFDTxG)kE@u-F z0r^_CGvvFe`Yzg^Wgh1J02q5S{;4#Ck0hdT`YQahixc58V}1F`#Cf~Na+H!SK0vac zGP?np0uVKa|FNWXo=d|z@Y~>pF`D`S8L5C7}ng2 z*9`Di3*Q(F=U=SK^P(0444$@@MDz9fRU~>X4T}C%ZrZpX#&lkxf6kyITTFgbUDDnk zM?qf@J^R8&DSmx~t98I9DBOAF;sa(+>8Olq^AkhPwQD8U>C_tFZ<)7t9q0a4Ca9mw zbW*zX{E$KG#J*kU9Y|+-KKc)|njrtiug01kJB{(cK?YR!*Rpe$x^p*E8014V?JN*9 z^%hnn$jarP5B}(qBJ1s!_589ofKb}Wb4DK^25LWoemdvE>E`9cE^VE08BQSu$Bi@g zJ^)<*a_Ze^&MzlnfJz_Cie{~mwzK}+o{kc;IFdw^Rg%Jxz%nhp#CV7G=NC5hd9U7m zfzP{;IO6i|1%J0&HAC^VJ7CYwJI%5g@U9hCgtzL@ErAvna8$czm?0Tbw@h<=b=(ox+IE|_1pPVz za2dj_z2)-B4&bA$lE?cMe9mkCTWjxuRN!iuGA+Y;?!)7Tc$tH_(`BYHpSCTUjdCN=^555%`!Erg4MfUh9O^y$j7*Dgq;?PoUI4J$H zVC|O$={*#5MCiVNj z9UC?KJQA_fT6*N2Oxj~r?o{f0GVnQ*&7%wkeOtJ?5V=CHpoP%0NoBHm{R-P*%N|L_ zaW)XM*wcSKSTp&~FYEp$pE%mykp3kZzeLvmB6`5AU()Hq)L1V!EYTDt4QBxDF=SJ+ zlT(zo-!_?CpoJ@OM8D=4)T&gPu+=iNGud+dz;I_H`6J7c;i$yq0JFlF;39oG8}de` zt^KUG3~J4B#2!d-UaB2Dq|dzI<&E~xT8LP)7~=D{xv2Q^D!_Xq!fGHfAS{|0<#`Y& ze=!sFg|+GA(`Kw>3#0&2w7^b^W3-ux{I)jeM7Wml&MGjo_Z)4wCN1zc z1!vn48GcA;)(}j#4?1DiTqoZ!C7WixlcHm5ml8wgu}J25mstFKUz&F$(?jMRfflzZ zTo-i5^ki)xqg|x?%2nul`N>47^Eqi$2dc2v?|Q13(qR~yrfswVq@Z-=6s5b=dIJAl z);k?Kr}Zel7K4A!wS3EuPnV!Z3WZK-*>S)^#!FE^sLni2H$wbP?D~+i(WXn$EDc6t z6!R%;wfp@G&iI)9Fl=GfvRgXA`|WreLCYDLT(-Cuhunog0mIo#i>%xu+1|XmjMqn5 zK_`|WF0y|>A2x=_%g=UW%-@D8KkLtnykrD2>9lTs!JM-sQ12s4r?398Y*aawcpKF|slxt%!exbD>*s7c1QN`WKysIWF`)=-u%itm24idQ-OjyYWd0lkU006g`(@xGEtX_;2--Ft7mGm*xVOD#` z9ElU+p<&JgAB5~W$()VOUDcjd_6$SaJ=cB;jnUCTP@k{?vf+6@27Woeo>CC zS@@>*B{0d0pD1tHSKqmoy*`Fg`6lad^drBR-?Tas<67|)PulSM%!>8jm=^ZardwDg z+dCCQAmAo0uVLRvI{gEDU^dQ___9i{OBZ-X&d#+)=3Ve3=zc#(F91Dk_*$;xL56Q^ zL0`1*!R@2Qa<@R1$us9j9{5lHenZL?ahx96)VFyFhuj%NpXESl+YsTo{tF8-$6vxy z<;N*uW#HB=`a!<9G4*4pm-{Iq*hcx&;;+;j7{801D|@y6oaTKr8Abo@x76IwN}h|+ zoN2U6@y_KYhx>Hw_$MUI;0a!jm*o}X{J_M(bexpj15L6Fx@2lIw;g$*)vF2QJ;Ce4 zv>$QF&H?QYVG_f}_H4ApG?mo#|7l*<+}e`pddWB=ygww@qiqY~QGl_O7w5L!0=j38+bvkm2j~r31Wc=oG7HwV6&w1FMdaK|H!BgsKL{RNH(z%wyY>UyDd+GksbE{a7J9|+}&b(NP z33%UUoc{yZwua{OoYQmdcQH{lC7Ju!D(}KXY(!CU$jQ?j6Jo7tO2+JbAKkHb9p*I+ zr-A-F7G5;-!b^H**&nS(kbm!rujUqQlD&pBsRC&TS_a(+Nwo&J8}xXw`Y9TvoC=@) z+#5b`lMzFlHaA3d!Iony?oxhy)4!WP>%w)GSz%DPPqH5$vOM{zhlnla9HT-0iC}V` z0?^3IPWcxPw3o36!fJnl^c;9cm@|@EyL;E|5Ovko-soLRbnXmQl|P(4BE#>9^KQ#M zbcOu8DZ^gD_e=>NRKOh}Ky3Q{KXT>Tk%emS}6SfO`9A4ky zEsl%o7fbM((@Pll`pG?MWK%W*+fM$O%1)}!vm}EwoV|YL6ahm|c}~pMhu^3+v%5yJ zW8EI>&`uUU$WzW$v9742PZ>$YZBnl8`&t9zPv}GEoR)Sc+;*0Q6X>mmS;IVArTvnZ z!`a2`Y>1Kx7h=2LR5HBqH3U~Qu+^1qa2s_pK=Pg`3u-xDaoX__{|s0!nQY#s2+*q5 z3j3PE`B`YH%DRYea=hsq8^fMBpraeT_+;2R8hFQH6XFk?k3T?)=E(%(9Hx4$_xIeu z&L}9-`^-iUWDM!-;-=kO!szF|Ml! z{?glo%e1vxy8_Q$PD`+ z3o5UqP>wL5gJyRiZBRHOq*uLw{=eAywkua8Dzgf)P5 zmyDxBnC{N6yHVsmEV4ahVgGd4MlEDHC1=01L4%jL$uAIf^4RzLCi9eC1*S879*-D6 z9HE#fkK0Czr^;fzY*ZUR%QSFjJ9GM`*+|7PH~5oi z+hM*L)K-E{ZBlri@M_`+XdC1{poI=5S@O_EVAb*nbAy!cFWBk@AN5!iL$zto&Oo{( zB!?#*!FYNEAUwr&iMO43v!18a--{ws&5<8&)V`nW8ax|874hKWX=#qZQDEx~wIOc+ z^Pp>>8SHL8tbY~@eJOYqBh4O+rX4xit^-H>NP!m1Im`!-n6n>bPVt3x{;~FkPghBv zs^-|8^EkiJ5EtAyLAhY|^zr5iLxYtD{yp_4 zX{#+J(af7cDquO`@`RznWqOJ(BtcZ|}(D%~ILx{45NMUz`OmC-2z`>AE{d-}p z*%FZN`atTgpy^m(DBPe!N4e3s!S*tv7yE*k=ZxjRTPc`q=K(Xj%Ry-}xh|c6JmeJ# z9CBLKX!z1)Z0!58hi(zyFOlZ-=&M>{v2|7R*SJa9ce6eCJCRnv%;zCGDW-(WdV5m~ ztfL>34=)Aov+3LIK=~}}f5j5Y(`8^tyJjHV5l)}e_i6ue1Aqlf)@k|Y1JKwJ@R1H5 zmN5I-wRofadl0h;i{DD~mFZ(Uo?A4!+G(TBO4WJRb4b;`ikFl&eGIyt#w<+1Q>X-wlH>C3~@#h(#dw!1`Lp^eA9rZcG zW?RJ!^O|SR>t{J1U@c8MsMW8p8DUYFM9MAzV7!U>sl$rL;Xky3mEXLL&a&XdO_oaK z(d%j!5<9H$eOf_QAQ-x=Lyx6BMzhuclUr_mKmv5~f*!HG`?PnnmRw1uX!er<_cPM? znO!J=>5d?b2){~)7g;~$MdCMGJDn~@NzGv$v3U2=fixMv0eEiszRcxRWU{sY>pht6 zFJ?y5`#66NIASCY0w#W|2U22^RlsZm0_?^4Ey!>#m$!wytjmXR29oVaceq33DS24H zgB!~vBa(%tO_JlN!#LAg7xx9%HxbEytt*_rrinCy)5w?VGfr9>=?!Q{OU<^&^6P_% ze;MBL1Pr8~Xi{z2aV_;r^Il}w*@r7gwm(vAI=CHg;+=f7E7eN%NBr;#&Op+^E%?rX zD~swHAkW$NQP5PBWMxvrcJIyBO`;MW>UBe4P$TiIGK#FR^*Vc>xrC9HY-Cy<@tqz) z`#qS8W5~aGiAJ`ke+uka#`3;xo6Pfu4Iq%RWUu~=V-Qmdi_^gHU1b;lh4TW6d*~6R$IypPMFgi{>8PBeV8LS=cPp;?42WNjj`x*D3Cm@yf}o*l$*{c`dmWzMRbKa3ygVHgB>yvg84i=(M7>&z+*Lk$lP{ zILf8dgTPx{Vf`(|zOjM#-&>zg-;czu)^&(XBYXvxO+CVwRV5u8)u2_=kMfPY3x^4fcvVF6Rs z)CtB&qsD*VuCCq48&oLE5)&RSBfb6^qRDZW>16=Pm`M?^_oqyI@;_Zzm5)$PRB*-}Mx##k|Po9r}-Ei2b>EqhFEJswFim`tVwUE-g zm;qfwvivWS4eAkM z%;fZDD8A;g{@F>!td_9=`l(M%s8STqM45B<6{dkOlc@hsUFYJBRl3LV7fIt%j>uFp z;}B|6opOpw+a;Njyp%Gdl3OZNZ-`2{$sDx8qZlm=ZjZh6=G{r-aa6V88fKF?fQ z)ta#u_O|za?dN%Zm+$ZT;1@FSXCx$le4_)Tlg1e>@c581=k&k6#KJlv?U3#TV$uY2A|1n}CW5=mrHpf?d%Xz||KnS>?qttK z(la;-t=)#`eY%5YL&_vgv8vq>U22kIhFTcp_w_IyI<8zSuw{fc+P-k;Dm9XwjYECV1r-Utwi_ODa9K)T2|=(DAgI%{_?(u8?Fcs`fJ{lz-Ez2``Zj|lCn;XR zV0o!;Eekw6sP|n#YWsAyG(tuKVZr0-49?r3?Y{B!4$;RtQkD!A1Tw&J;M3fOOMpoN z@98q`&1Y0g^I`cZd%2Ne_d`JDJv;R&vhKLng~|*KY$3C9>R;B!0BCtT;^UQyGdze> z=Z4MF%agKr*0``H&?$GQ#5W0my(r)o8Bf-S)QXQ4c#Sil9IIHZQuErT3mgcgsd3f1fQWDxBHhKy`p`{H=Xy!nbLO^rAI=A z;i9#tp*}d=vl5zU(EeB`b8|9j-j&!R&sdK4%;Ol#X*5JzRengud2OaY!h?Kx{k>V$ zk+0GBQ}D;Qz#VmXgdU}*qu7m3d})#gs3t@X&f)ox-VG2(qc`U6>UFU)Uz2PoZ`wyj z%l9%o{O*0E-^lo0=_3(}4kud}#j|e|SN8{5%vmG#XHlp~mIdtzlx7>B6YZy+MVnaP zN96ubBV`bLP=39&d?PK5{HZ53KBxypiC5eVc&d>*RW}I{vm)TEx@9sD%=po%MQE}< zba+n@%TVYK(Ki)jx=gtHkq^~;&al=;#ry6y zGu11jt7SluD=f#qPOMp5ww1{}0##1lIL=$nSnGG5mB!o(Zq7V?euv;(0bR(LgMU3G zZF$f$=H(Feq1vtuxQ}9U!wyGoCXa`JYZf&4G*ZF89g#>CV;#LQ()kL+3x7=#e6SEj zqi`5CK4aQ~W(c|^)2rkIBINe1lW9p1>X4;xPI~B^J3edxoW8paAJ0H9@7P~}Na=fR zJ#HkMn&kFrTwX5`l`?lKpEL@Z-&B~~sWwQ$!=GlWzbu_r)}gVa}{XP*>pg5X*& zGwQS3u7v|{>Mh4^lY$nL>4>dx?8(nQ{P!tnpKT2F$%at}l9LV{JCy#HTSo2uGXrg( z$v}(CbnG`7Xy2x~Hao1}Gtm6q$Uy6qDckolu@SNE+#Rp&8xBYl*xf0O$e2ZrXZ~$2 zo!N>T&$OTC5-YP z*U4bip%YxzdE+ZxycDQ)ISFB9(td&?%GTkow(KfbmL89oe?Hkk9OsarRx4u>5KM?* z#d`P@8+PTN(=*kk0|%mc_O*2ETBozo93xq&=){$4BQm>UWwiEsxNyhPiU6QmFFav5 zc2XbJz@abJVvRXrke@5<`5xvaCS>^Q%hK%5u~!h-X&@*GspSMJ56aXzW)VP3Avm}# z@^n0?%qb1m)*&D}@gYnjX!ZFXX+r+vrd%g&l(Zp|0HPE%?yd4ljJjZ@+h`3$uRa5l zzF}&HeD^>L%)1{!<9=yz=Wlpa&!k@1U2|i3^4Z`v0kV&+aQV<&+VrG3O&{ZV*8~!Sj++`1sS9*%YzM^IN}Xo zCS*DiD;!Drm0Rtp9xLSX|HghTv)RnI{cksjTV>nvaf7&PMOVw&@$F#Z)-U#l`)sqg zzc|_HY5w7_gT+~9uQl^+PjxZBKJH2P>C5<<3!i;`xkKHTPk;K!t3G=&@0)*I>E3FH~Y=;jH}Ju|NFiF E2bn$tVE_OC literal 0 HcmV?d00001 diff --git a/Scripts/Models (Under Development)/ffn.wts_nep_1_lr_01.pnl b/Scripts/Models (Under Development)/ffn.wts_nep_1_lr_01.pnl new file mode 100644 index 0000000000000000000000000000000000000000..f5f9c4d160be061a5cd8f8f753177ef2066db972 GIT binary patch literal 28527 zcmb5WcT<*4*X~I$ASw!C!~jNA%sHN*qDWLk#4JG(6$J$a5hX{JAUS7{AUO!gRkhZ~ z+4I+Yftg?P%)6_mrl$70@28&oR=BP{yH_}lW3BE!Zv{Cnv9oh>vitx1v)0bdF8Y1! z%Y={5kHkf%L?2Fi{pR`=7rO`luYa-}GL!ARqeA$4NOV$C!rS;;@87+89-ETz?ycvG z_wN!t!~G+|Z~902U-P^g78>Cn74b0K|IXduh{v8O?>qx;Uc2TW>KW{J-#`4}&9_OZ zDYxFePdRk-ak72v+tkFQ^uw|5-akK_nEECqA^QFM=yd0&@1MuMd;1~feQIoqbF#yK zKIQy2?mxdxb_}x1PhJxHH1=KG^Vkp0&)=FqI43WSvS)@d$;+aa@!vPmDG6_nB`+^X zb_!2kQJCx;p1jgK$UbOukY_>is{d!mxRmrHzPUQefp5Gxo9t3tRG7RbJbA5mkmKTL z7n5B>Lqh}qFaBBlSCCywvRh2?%cW8OV~?GkHtvye&6BZP+Ga)RoZ1|pOTOe-S|Z&? zWcvJj&9^>$y{5k<$1%;4I+?yE!!r6rTP@ALDLvXPe|7ewrPIl0q@O=GO1HH|f2&PL z9F~8zbS&=pcA3{d(j_*@liBBwwfKDLyZaN!IcZZPy$r)EUjS;-&PUvkYjL3|srO?VU8a%A8hM zQYJZ)cTV!OPa7@C(kd;|N^R3#9kA>rcY{=aaFD;2{E?|g`(#uZ!jb}=e=vMdDV8)yiB{U3ahGaqrRJw=i&P~@jo36xCwSmr9o#BaVhhrV z2>bA}Px-~{`(0Wp^{3OUlnpW^S<-x3O0Dacm`a6I>m(L9s^9O*uy$#IR9S26H22im z82%(3Z?M~xj%cgYTDve$vsCi$PWdUeq=!qT=`dCuwgTk0=4wL(9$?z?``{*-d4fUu z#OAqHvh2JtVUzVrW-v@P<`|5y!gNrEzv`Tww2I9IcUNnRmdSwhSW!-xWFYCiDQl9@BD@5mr4Z|>DTrZQesK2 z7Hn~jb(C^OXx}fLm+tYDHY+S8DlI^NNV}!&@^@t#Q|rddNk?SC>FOTov!qu$6FzFk zCbNw7clGxf1}>CthN;@`oThDiQpEQBxW@sl(ZN))Es;vCOO(Hxq?(U3=>n^$)FBJY z#oxyOGdlQ!5p2>Z%^&bn?=EA-TOaPo!aJ7LZQW0MB^_GoCciB^9o3%j%{uvk&kySO z7HPS~^cwlwzqdCJ?+-Folk70RR4Kg2=e0ES)`J9TkwQxfU+640u6tv?+wnkZL*?f^ z!p0V!q;tn)>=SLH_;EhyTr}UQ;uDs2MYpvB6TwPzuf=R&s4Vi&brnRJ_?Y^(H z#YoR}X|=Fj!@=WP{pP79`PwJ#=cHJ^Gy9L)CJWk(lbW&a+K)1Q^{gLLi7t@kLrw$Q;{o z9oB^p`V-7*wlwQfz_)!F#w{+9GDifB;>dAkp@r9?PQ4&XCI9@>r6+)&HtV_$XRzGw zTFH`iP^+ZJYp<68KV#B%O9nRYj``2(r$LUN2W1R|yJEtjN6W*l6rBnMu1G+h=CdhY z{>i9=lv^??6IvwGIxU0v-}X}TnSO&70|MFBU7uIdwbPeb*&NBf-`aGDyim+v%k)A+bw75_dl;K$?`*N z=Q59lYTjw4yP3Z>YS|^Ji5A<}-I9ZmhIPPz-H~k&wthNK2&0v1o_F>l%4BBO#J}&K# z_=UnWycL5>4=m&CTeRWtAMTMbRt3j zz^&HNTiSbJ!v&%Y)Fz2!7N-0mbC&!)uZ2>lO?wk9X*?g$yrBak>``AQo6GtXvvaZsz9|mai0tK*4$1Yi#>%>7C^| zl7Xw9E!`6Y2if$yrMA!d_lyr;8@I@{##B}`f#t+@Kz=7nPo%2}wCEUj>Dre-W~~dp z74E%Sep>gKb{5WPOiUv60E96^Tt8*OlCD5$)y7vRWy12?C)qlt&A0TIb>PjlU@c1L zr&6}ZRY%g-XrZ-tjpl39Nf}7MYPs?U-|O_}kJdh!jM2(aaI1h{6kbb|u{Q~ZDP&=Z zE`&(Oky}>Ea~VaV=rDdskP4sUIt>`MSu*8w^tm+e1AV14O$MF)u||WXvw5s^EV%&53Xh zdA7#rp)rRn-vTCJ}84;4S`G5yhF46pW6BmISf=|oT4MNWL zP0M8@^s6i+Tj$QZlTi~!LD-NX4cEf7$CdGI2iIEqHwbIhThTE(7wVxu0O)M()cR$> zcfaA6oLouPCC+#8D-Qfa1F^HV4gHkC2%Kc}uJy@9UwPFgj7| zi@k^RpUz+9``~=;HEl?_2Q>mMnF%PD5#YVvdcR}4j$b6ct(}qbbEgah%RlRbRz1U= zSvZ7P$XZ9P*SV!4f4)lBNh$gI&Va#3<}?0kYmi~LDZBej+8}d}m8gGp{zXi(A$Q*= zI`Kk(-j*TDL)!Pnqh(32gZ_>(PR5p-RGasSfn6FhWxz@clVa!O7dViqwO_WbVgaM9 z1|CFet@b}6%Cy@@hHfCb2aFjVSy%sdVtPK)LgLO6cBAX~YnjDNerw}{z!i_JD``@j zzKfWxvt*L={H{eoM{tX6U(C(NGVSxtlH!af59DtK)4@{}(sm$J+c0#ROdi>R_N`vb zf*Bog4%x8*+414U!3R&nk}c=eGWzo2iw{=hCb3=8sbhxq9o#{Kh6nb>L^pM6yR^sZ zKSPGcjAk4-5cI(!Ab&c2U3S5Uii)-1Qo&IGyx-F5V4r<^`~@)TO^5*z%QJWy(A=i| zVDk89EOSTd?#3=(Wq#g3rsM${wO>F{A8A5j4_!bGlvyvpw5-=Ump}xc@byY`+`?*% zI(GhePX5X$>b~{76hG41ILp%&^ADquvcW^?ymnT`zXT8%K~4lufp&V}(lqVTzlk55R8&K#gTJa~JZApxnUwhH}nEBHwiA3(*GAQEb^7^2CN)<=FfH21Vw7BlOX zKk4$@lWd88sm15cc;Ubk=$sNA3Se~V*MG0Bo#|9N&CwLMLA|~l%h8mU&$<) z#pko7`8_PrYJGLl{41Xl<(G94s9%ujw^7Iz2>0L76?el-BG{!$=kv?GM&oX92Sm5Y{H=JKDOGbONgOo(Woc{E2p2H>Abw z?V7C=klVbGmA~G@^yEk(WU@ZwrPz>b7YMIblmXafy;u$c=4H1=E+Z12ZR zejLw0H4bQ@A`82v_4V@)Vtc(;+kNEs5fg9H z|N7tt9o+{GeFrz|&_+|Z7T32Q)CKKWP8bzhN8ydyw^DvhdyM8PlFqnYu}pKNmYtQc zO>5BQGJR27rPY27@!|L>ZYi}&uTbJQZ1joH>RklW2}=Ei>%J@)%)XuWE(o{6(w~q) z<2uPIx*`^BL;Ow1_e?}$Fgbh&b4`Ng&)l>;)qOoq#}8G)k! z_BLSS2Zz^YSy~UQwmtrW0@xhjA|XKo0*_Qd!!hZ)38Qoe8GAv7s1` zKxx(d4=3iMFvn%;jM)6?+YIZkZ2~Yd@M_Io<$Ii{M-aS|j@8>s>X;o=w0MW zNl?t9;`t|CZg`-Q-yxE|YmOAX*1iNoHX$M3F1?B}f=4SJfj}cKB`1=<)Zm9)ojVN- zgIwFSoiwl4oJh1oid5Zl#JGPSTF>;y>yIRDB_6TKoYXHr`tX`D3gv#s5*|0LBa6vs zCe?A>WrLx2Oxn)MdmMKdf&)xvWmejy-z&!YN|PugWR0P?vi(7^@e!yJ-~~$+f_9bY zW#&Kk>MLA0NQ9eMs|i_44FB*43}o_z%!Wc}*&u(a(~fmU5|O#maz9B29xXSI%lvo( zS=tqCBB@^6kHvlYUpv6x0oK=<@P3>kqnotW3q5~NbAq%f;*OYtDErmN z7YJ!Et^kCUHfQ9^Vu?(}+Zg7$l1@xatTw$*ff0ypnZ2PuUjb0)2UwHtWW(+~#2JqV zRocPoLQZO{s(a6)E#^0@*nyK_|LthD<` ziNXDc2|1hH&1YJ^a<@!cJ3_T%XXZz+)*F{ktF2 z{at21Qg4*MO-JHM(Pe*`-eJrUFa2NbLy1@VFvEAX)ZJ;~P?uh-gxjvd$RH$YMqavM(8yq93@v~^JXcAi^x$ru73E&$$rpVM#R zhXTIvOB*k)GHG~hlYUU<^OaPw0$m6=#1NE$7qjo#8~ehvjre|UL% z=c0MCwjzz%?%1PaH){D_>0g0?h=X>T6geo33Swu}vO7R{4TtDxlGwh{`tO(Tfhb$x zXD_)mEjcmZWv_MY^0!TRBa1p`FTIdizzVK0VgDsv^0(Q2SY)a~?b?%m&J}^~tTWn> z_W8J<6>^B={x08>%mnQsrCbLSj$p4$XgN3K>KBk_XK)Bj)HRD4JrW*IC)-9=fb&jE zsR72-c93-G1o0*w%s>&1-`FjoF92*W%ZD2po0)Tl126!T_1pWt~{RT`T=@ zF+i8SojK3k0%KaN^e4|K|A)1Ph(9l5C-&&b)q56M*A;vL3DITj-42laiI>?I7}mBM zyU)K&|6rg1L@v8WK6kjkx7NgN-F!2_6#YwACLRmBMef^lA?=HHe7TZpx^CIpc2(wY zAvJYAMau3m$s#nm46mW|=(zF{>F`7fXhN-x=Aq}M+~w^TttMzt$NBq~!E4mVr2Kmx z*u~;XP|=XkAObWCCk;Lm+wxEFSD_uV?y+QIy>{Q1g^dPT+odfc260`6DQ!Y(1Y(I_ z>q)vqZSn;7_v>%#>uu&-K`{I?dqPc8wiC!~yoxDdRQm5)7fiin1{jwkafA@vf7~Ev_BDDxmsowpx@RODRZoIW zC;90}ax#3S#o=1KOBy~}-`*SpMZbH$fvvChN!iq6#lGG``w+k}@~Km75HrFyg}J=f;F=0P?I{lw6VBdcGAbb(?f9 zgGixEJ@-U^o_ceM8NWU%StcMqFN=+zh8L@og9x|s=52Nha zT1iVJ9Ht3wDM-{`K>Gp??6-XGN?jmHo{NH8RD8v!0F6F{b!feZY1#ESWQL5$~C_`-Q^xg#LQlK~r$+_n@9X(OS7oJPe$+H*=81;CHFDRlD z3|`J`-r%?WlF6StI<@B6HLZ9LrUYo|H_gL(6le|FE7kFC08yc}|I|a1U4sIw25S z!rJY7qL5*zoQFio*cI(Qk3@i~%cL&=lKX4)@$~j|%|DKE%eLGSmH~KW+K1%WfHkwE zFA_NHu+Bg_qo7Lx@n^Jfos_z}tb|7XS@I`@ALaxcF-&%DmrNYi>Er0@C*kXk8Vmsj zFB$Pkv!7t3V?3G6%!{<)4gxBcL>a|l9nXo$Gg5foTWttM6RbzPq$YM1z2vA{@qR}z zYY%81!xwwBW1VR+_pf?HSPne}LfOk)A%6oYquBmA>-<0`x8VTtb>VIDUOrT89YIPp z>X- z8FnOACk%Mx1`?iaI`G9r90}3y`2M+?ZshBO2J^|_#aP6H)h@n|7_?t(8IBR2uU(YQ znr3|5==54!?0w_*u@yMKgVuWqQ`PcVnQ(!8yRGLkl0pk~0gDt`{`M!e6^l*U`+jBu zzptaY7RAv-!nGg(s>ER6r{(*1NUohp<;sm;j_mdL4EvhCT8oZ8rfK87F`jzASz4TZ zGqp1U7g8ZBVO?=zl@=b3Mi1@yNM?1MhYGu`FLa$NqDYZC=yHvYgs`ZV)r3NfH~SW^ z;*3)V99etu(lq6ny*RA-u@>%TNXo-pQ0<3Dx~s4DLUW^JIk`$^BYTSX?A4;>z6h^B zaTu|F6NV1m7k>NOA;3XOLXPgxfm4{pCRvW>H1~0w*={tEoTWcBi-Q@~62$i?c|J^v z7Eu$_P}CReJb#X-Tl5G;Dpidm>5u zeV{>i`MFfs(Z;Ej76k+;hj@bY@=Tl^&~aaBxQsJ_X&u^d23E+kKBViMRNNpUyRF?J zAnpvTJF9;z$Vvv2FH5I`ujFir^xlrY@fwhJ?x-I=`fIItt^F>F_H7~H=3ho$^#;UP z?cgdFF^^9X$fqohQgt^i)H#6(Zk>r|x~ECI#ns zk~ctDDHDbP{^{SHQW&|+NWp{h-RFy%J=Hwxv(H!R`LXw6O4=4FcnNki-X&Ee`>Uh$ zdBPXtR`R-+93^uPPhOvrydfqk^uKc!?oPzbkh{Tm@BH@;!)<)&@N)5F+oqQsSKAz( zyu}<>b27)(JYKFfN6!9_l^i*P>vFYIe?8O*>s%aVNo~*r6Na6k(zb<~ugs)l(^;o= zczIfA2<5)wLnAu8mM=s}uTO%u#bFy64b*NwLajH!D7uMv(r=E4j9NR;d8OL)pQ8pg zvOIS^S(US&WO$Rl0+8%@L`pgZ>Cgkoxk_lrEW>l8B>e)sMH4423XK$DqP~p*v(u#U z4Dx;X7Bo!x!&QcSrljN;M>8%kZ_Zf-UDscW+ZH*mK0{}1ndG=mBwf@-kLQlQpxBJ$ zo?uG3T4SC43R5pgQ>wDJF|i&3{t1QTrm%aTqT|nX{s@taG_6>=6>e++kJ918ffri$ z5ww$%ZGq=ApxJpohpEK4z_T*rkIpGOjt+EkDUQc#OtGt+nxd8bDXG+Nc$oIPgfb=?C*^ znLhJn?VC_DoZAg)TXtd-62S%@<~nQbMYyJdLo!`5^!jr+8y@Ur?o0BfH)Iymp%bs9 z&`#%#qb6C;uoJQw#A=tHhqdtA)nv;E3yQ>#I*Cs7Wb0$KC;R*s1(0FWmeoKhDU)|6 zSUQ)1*x8oHdDj)Iu4oTJa&@{E-nx&`Q!r=G&9@%`zy_zatF?6n7Eho?X_MT{+W_D_ zVCr|OvZn-ZUM061j~_isipr3K45EA%LJ2xnO=V2S;alKWQ-Y7kpJh6=f^V$fqK)y= zvWuMilqo-4VI*49)+$$4SSWqZjXZ$*%A|_U5%AMt9YXcb>OkBlFfc4=*OIH-fd|bA zUm5j?x#n;xWJpRgSq4GBMgD|KfnzGOwYlr0gY)JyeE5Z3irP{)64%c!-u|jtfxyL6 z)aMXzP~t}iPjVkRu#%rN@vS=>HcD;`sPhYCtxHE@OnBS*UOpG(e2PBAQcSF-L!py~Q8CokkQ3^6im;CyylWi~&gf`DCw*KY!`q8-c*-W3k;p!MODR2mQf%gAZAa{tO1&yQtP% zP>d-zGt4g3Ap1D_&nE{>n4SAWD-(0ZA(*Y2}^wIsgkYQ<7a*FrY4fut98#>+N zp&u6AUn=cB)RKj9`qR3&Lt3}$7`#7k?X(BYe)ZQn=7a6$UHlBUA=-bflYh%GHaS*LY6Q7Hq@0a+e+1Pm zoDzcc&q?EX+?Y?qNp{8_P_F^w7}fqAH8!qe$B^9|q4fTYLux3NXaN1uR(|&r7M%;V zmS@`Q@X0V@f9X=M_{d`hML3nZQSwhRr7UaPJ1W^Jt&b)8cdt!y`?k!O-gf83wQZTk z?%iR|7n8jW5M2`#Y4)8^7}yqJj(e7^p}wBrsBsHK+W?uDLe25S);}YRlEEF)e)J&t zv5<`QrPeh{tM%tyjIRs(bYhFP-XSgs6PdhDk`@pmM%@45PetqezmQBah^QU?M240a z?CFp5gI9k#2^Gz7kQ93#4m3H?=(_6=Z2y}=#3zx@);PyF_yz(9y}K|&2SPCEzGh`e zmJ5i#>r=cR0^m2aVq3fMZbm#Yjj8`h<-2S8%!D$kL zjOE|KzThez$2cVN53O}hdIxYUp^5x5oD!nk`p7B| zWAqkNqCbFt%aR^Gft#o!3-)XC<#c05Kb>5=cR!_1p=rG|NKTNn@242L{$vO1DT4$! zQTI3TTT0w#c1PWrPTuk>>@g$Y>STchcT@gijV@>Ty;lA%rPqO#elG!Lhp1)Q^Zy;8 z|3WtUAiNyD_-jMLayiy6owly0XX4;i4lp97 zo@sN6^^`5-XHs-%9mv>B4B52QWA|rsn3b3x(3Wuho@j^VS3Z1sMTX6R#{0pV{W$%; z$x!_R5>VH0P^;EjbzUp4ol1aNY#};uAX!IHg%I998N2z4=1vRPBW;JCtiO1MxJ7Q3 zzjyaH=LB@ZPV(K>9cS$KI~EU)V_ZJlzLd(a4I`Tl!QoV}sDStzerU<$1?j%2HJq#G z&meWh=TrEK>R#GBqO|SQDuPamj&DDc{?*hTEz;z#-7@kNKjN;umxdQdptICtTI!(r z#0p!BQn>!bW@*oiXI%x_6QM;1H~?T}n)8tW*D-^ZSYv z%xKPvY3_YMmZ%w{!r-{j&_n*0Gs9!f64xhwkg_Nvu;& z9!X!ER0c?mn}cthjvkcGqxQ7uK$CV48CZd@MlAQB{Wtw6-a4(@+L(YO8bLvE;xvn+ zMvm7mBQmd$ir?>PmmgU6`5ofKwiH-Eloo2?bM1B@z$UEKY3yp`U8B4nJGNI#jxc|y zwd5sjgT`o7h0VH0x3O{uW|}jdj2}P}Xm0-~rjx`8E7be}v(TN$sY4W7F|qEL;Om zj63=!Z#6Z?w(#WbDakuxq5}T^YmTdT!^8bUBRnJg?gah!stwQosy4h#wXriid6%g+ z989&b`#;sj|H+-0a2dO^lG)X{`7V2t60?+ZnO3yZ>eUJI2am8R@L-Kjx&YfY{egt} z!QRZD6>h#Mn|16R=9>oB?k)t!>(J4Xan{goCdZq=%i= zv76+fOnya5a5rJMldt<;cJ;YAp`EV2QS#@-M`?E5;|paq1Z>@%PDI7+l2U}F=Qdw} zKU;sU;fpfgd^YyTvQU5n)A2w@TGH5U^TntfbdFqGakF=*c`*XzMc{ zLv;DJ8Z$LZ^yjaX|*Hw%nCp)~@Zs0hCRTrSueR-QeY$yvO9+-tgpoDaqb3 zQNjP~ocrIa^7sE&-W@P`cQ8EJ=YLlDhyIgyD_7asN%3~+3?i`De6`&8_ktGP`E2&S z|DvCQrRya)6|i}$w(bOJnk-rHJgd3;s1?SnCnxon z_hn#r`G>DlyZjFpZf1>aHhj?AD`AZP<)d)A3aRNXku%zpa5y(Z<-wpd;_+oF{ zOXgi`zf6mcwi6zVq+ZYNq4z)>#lLqq}H61--&#FiUZwq@%*3!tYu}9%9`AM-H9=I?`jSk2)LZ-gcS_wA&7w-w z>|p3Nd(S6fzqf-NF$c{5rt*zjq~X7fiT^noHcYW;cJ!ydLA9%W!*)o!C*^`xuGpw0 z4w+A_m#5=h0FMC{@8fXq#*u@@0~H^^zVjKFA7asg;3%EUg@uwMA^?Ozw^X3%3P)Bb3;tyJpxw37w)lJRT>a>QbWNi?L>m8nk?+vi&+ zL%_PlBem_QkNWW70QA*)pn_wTm~F~b1Vo~T)~|z0(W8^`^vpRkAhpPp8*ZLRwMni^ znG*pesoZ+nYUdlK90u^9*)l?0<1>eF47rKsnEgh z?|q#PcqBhr@;~cgKpNYv+knS=lq6$oWx=q)hmAU9uOkVIbx@&>ILWWo(e#GD@yHRK zyt~14HK#A&*NQhsAS}tfuZ_>(xf(q2Arg(pE~81Va#YV~5~iJY@cn9b8W5FDN5Qcx z(hZ~XeIS!uXaXt!%6P(O3H2 zGP{#R2o`QDj*=Dy#>jk|8=n}qGX#^`u1CNAc1E+FQ&;Ki8XdZB%2AUS#LCcXt$1wb zyPBi6((9n}*w!^X{;+4iu z9g*mP%H^0EAR;V4r2L2S%Q)e?lx%sbjY}MTp8}(7S1HtwU1q{9f`ddN6iJH$_VI6Hqfi zM|Q=%C-6ZV={vXNT4vNeOPdefyr8wKIW4qCCUMG0@)8a>^Ea|=j+?RfKAc@8t%2s_QxJ2R-o#!s)H~D4J z2N>L?^{%EX_gNdi3fBjAaWdPHa2mPBs(Q4H5W|!WnP+_fqlakH%EVpVw?VaUhK?2j zL7YNs!v>x@d56>k{W)i`%>Q1LacQys<0|59H&Q@~IC{^xJ-!AtdZg{EzYK3Sov|#Q zkQe*pCW8W-CAz?g&u9ly z=%qB21BN+GbyB-o8cySdUGnS6Lm8A>m|*?;B~K9g16Y-08$`a3f6{RtoV4XWEsrt} zdUemV?lnTA{;;4xjG&i#(r>^-Y-x0q&U{`AOKe!V@!Z-)axT2QTn9FlHEvZr(0)Pwnh| zzcKp6_3h9Fg&1O8%g=ow%%-iaM*(;)YZqg&hg#u{VC}fU>^Dj+w*YgO6H~Sbpwr0Q zB*I~}RP3O!*FY73e_L>w*`UqmD4_L@(=JyRS7A<9x5-2}x+7Pc(JZkyrJLLIUmhYQK8T*h2P#VAcCXN0F;Q`=zRQsafs(eL_v}7c#a1EDoOGdxGg&zlj zPb$Y5ok*2-?;UW^&ZU~a@d%HiYI}D^ihzi!?!1F{ugCWRMdcB${ht${kng_tTUmSh~z zU$>3`g4&rVMJ{_x>|(Yb?_$WA4o|}@T$rnmKLU}kBbTa0@2{9z%IwH3Na>STI{-Io zV=g#@2(@q4nWp8)a{1w8cY=uYTp4vh$^wmw`j8p=VKb?SZ9XJv`OatZgLKT(!8_4w zSCM`f&RrtdZO~Prb>=!SN6E2W8?EqkYT4@Rsbsg!=P9@bYLlfSS1hV)GK}lKC-KXb1E;qgRa=2mY(-qTL;>L0p^2_YMqRb{&on3yJ zuh`PiLFJ(D?-b2PWY~Z7etQDj+GH4va{au^h^`rS7{DTFKWtjg9Ge9j=gviNPA&~v zsPvbfyR1NSW2tIdpw13t0~NA4ZCL00Dv{$4TKV;=m)6n*IBW`%BjL$MQ<9IxLS8yEvuX^N!sYgzRC!hMCFUdIlpL%4&UUnUuP{}!9Y(2At8H)7Jy2w{LbIDvO zr%rt*`Ke&+Dj$lu6Eb*`0{Z7iT2K>|@SLTrOaS%{IFdc&hM09HU*r5T>;6eNCri4xUCVji z{pqJUq5h4CeuGk~a@Jp0t<#Rkmn@y;JdS0>=s(;xM5vUK+Pzzr^4vI}aGP>y>*>Zb zjA);RPOR3xqX>o|B4pU?LzaPI2WSxM^yx^jFwq}u?&Pee&?(fuO$q4BL20{hJ%~o+ zjvZjN2qxnAmkhrC1{#x3+OUmioZCU`ij>+X-#-U(32|{AW{V|Dxl6PD8Ax#rsgArW z{A6u)c^RZN`&pq)J7s$JXSzv$NHHzjt8*R%(u9F`9a`s>tiP{haIQ{qc!18T@TFygmq#`yq!{@7{#br(dx~OR6G$FZH3)B-oylSR+Du=vN2g>u_7eea(-Qkg9lT5+6P0II9u6W`X= zNN}N$_d3+BlkSV#P@iXHGzy8;gt(iqeBOJ&^#!N~j;bz|vQyWO8AS2AMaov)K}Mr6 zDA6za?YnE1W-bv-YUOd>yTVb*UY6SZ1zAF2q!l|h+|a35(pqK?$d5q2X3Jpbf7*lH ztpZc^5Tav#G0bt;cs!nfsUck}z&9EweLi46p4<)K{(MW!(WbkA3=YsrSGu~qP^ZZ{ z_JTyo;vN_RgrbvBtzzTUSQ%Te$*_^Zx)vuhjnJbnVzzKFiB6&AINu(=f@A)?(zPnl zB>5jEt^?s$?rPbo5EH{^-Ms==p+aKx7dN^(wbd;2vy-;IWdF;MN}$PWu&{J5<}lZ> zTF_H1N-LS_2i%!l!m#Grlx>UKCvH-Ldh|bKRt7o4Nq{n=3<}Tj6_84ECVT9EwtWRn zXEBZxx;#gH{{6hT;=juhcZR|lplQ_m^I%7_R+9-t4aWDs!>I#q(CvhbxIDIxw0FC8 zB@;!>mKNr!zkMgr-hpU*g8UtIVv{P;N;|s?B(8*!aPZ(#CnQBHr`p(G36&mKKk}@B zGq1TZuSh@6KwCsniFBl8(zwmH4%(4>H32--a95ek@sg4|A;xji5kwFTo5L_oR!maB zewinGz~daDBk&iI`iwUSY1fVWh82x2?+ykd`=hwYi2PW zR?t;1v~?Zjz+XH)zUT=WdwCz@r1~UBJ*^dz&zlJB--2=b!Q?g>+yr7|TaNESwBa7- zG;r+CRq`i7dRBiilQDXKkO}`ob!bQCBR9#41brNn+{lOWj|QnE?;Ccm+mvXaDDW{_ ze)Nj|@MFQewugh1TXgz8?%Vw3lQ}P6Z>^WfZFCxo_%7i$idl*?9CjG;+W&?)pVl8H z!)!@O+Uj)!7%sCsk;ScN!OfO;;25+x?u!39p&Xg@@;rK+96;@5>CiWATX*z}x!scI zq~%VuPJr7v_^BK00~8L?8!e>W>ixb4&4OaF#2dt;}T~MSgdkthr`x(EbTFog%4)WCi-b1`|f) zb|cM3dj%SO5ifpnA@jo;KT zvSpWRf_q6WmksRaM#!H`?UmZ|d>qBLN~^E1vI=YERph`d2-uLsZ#uN^w%uBcN^55P zv}A3+!EIa1GvP8VsfzUs^z7bBRDZvdsZ{g8z51{2)B#;qy5n}4aXXLlL|aVC%pIzC z!o@~D|2s)hg#0BJj?4G8^7C4nR9f$~H5!g6-h(}DD=D|90=^Kb9oCTpgj1hZzgjeW zkN1V+=!O9!J7j3D%qHJY=5dFgk}YK~x#MUZc9p4vkC{ZLb==2Wrgm%AS97kZU;8i1 z=n5k90)+bOFxP{O%V%ja{<&{_#u_R=3|)qkI?Akt=@imBwAnJmBQPu1cLhfH&E4tC(pK?m6KK-s)Y z=1~o?cu>d89kg6&0mpi*7iZAOGxm{-u-N`SE>vrFVdd4@nTekGNmR~T4*Jh|lj~Yx zi>v+ZUfPfxzTqt{Ju9Thel?LAF8>}oNY!-%caEABy+XR8`LK2qyir=X-vl61u8+1n z;s2Kf@lq|)Y5~}4G{9cFQ$~*atfjI@q`#0sH1(PIx}k-4&v;<$co2>&O#iv^ZF}eB z4hhb)*j$ofEb*?GwybHnj-jg;wAhs;LfBH|tqnJ%(a{({dN+l@L*qCS@Zto4=ENk^ zF2|e@X&{*K$6$KvEqh<-d5@XpN9+|HTl&$o8OFU`8Y4l z3vxl4Zwx#wjfWrvy7MKGdmo10v*)(evdaq^lqsid0c$SbRx76 z*}wed{uJxzy`YT!%Iy&1vXDLtTf)_fZW{O2n!CrzX!d-w=_HnEZwxU(J$BGdCN9(C zTx1-a`6gW2ubNpky9AyJSv;RNw?YbnGa=7f4)?Zs#XZM5B~}Lhk~wp89S$hgW{fuP z<4?tph_sjAk>Tc?QLejw-|uEeXGm*FhYqcLFN4=%9&Nwy*4!z($6I(Jo?|?NJ5_!U zq&wYx(Fu^%@vnf#{30+^$-=5Yow#9eCz|)eK+^rxbhI#k%kZNA^9g#MZIr@yPb)nY zbGx1+I_kVk23LcKt>HI6Gm*>k^BLbN_k54Imp{iaLN7p;!#T@=nDeL6;1!&Sow#y; zwf6GyZ`${P+-SRk%`Z#uRjGZ3>rJ`QiMNrQTvdlbfslMHJh_zl7IJG~7^gLmt^=?y zYPcr-Idsu#jPvVSs6AO|?nt?W=&<_}sMoN25g5oOo zlkbJx=@^TR0=(qkHx;uZX7M0+@sIk9Wm3F4Tx zasRa?Bw{CW+_vxMF~+)>OAs*tmwdfsdK zvp)&q!loSLcqnb0m1P2)#BHjdX=}m1s5#|WKzX8X1Dh3_G+{>pi6HwXu46M9Rl2Br<1dq8II4B5>os(EfMPOw8|H1T0bZ6Jg2HLvx+jQu#ZL-QjHK+H!&W^?AHCgcv59wRJfHtdEw zJ=L~(9{@844xG5Kj`wMi-!zk}wdxFM+(-DgSO<@}y^bfU8m$92rSS0`KNE(}T)nW; zKk0wTik3?GA^q`caoL0PK<5CKo;1)NHO9`wN}W86n`t%luY(9mJxy8oQVMTq=?kSr zNi0#u^}G7NhP$LUL%R-sA|dETwH!R-5Tk`GNpsHKS+Nr(OO-xKQSXCa*ch77k;sc$ z`j%yGiIgl)X+CF&Y5fagX;O+EV<;z{f^CJYe>4zCfPn|1H~2y*#7chx$jHf2%EilY zSHlvFgDiTrf3xFi{$6j2%z7d!sHyy(jKDY(>|JZN=h1pD-NmaDeO? zdT7?#FU9sNGj^aLFKE#>?efCIrXx1<4eXp*iMpVzP_ZhV)U5#`u%^r%0tW`EbbiuE z|nO`$hwvQ$ctCi#hE$wg$q*fhe} zq2S%2-6ni$rSr?CRrlDV+)SE}KjJ-%jn;`5IvgPRzJ_+HmCGFmoNf|CF@6ZV+#N`0 z^t;ZSaNThk$VP!b_Pfq|49r_-wg1@a12J6Uig!D!C9lDgEX&niDnChkG^6o?01h;h z*!6&*)}PZ!e;B8Qr}KgzIIQ^@!;lVKc%WSjkvifTJlHbcs5m+4S*&voOt1B_wU0=XPI7O*B)SP1c$#TIjwQFr_O( zsqgZxLK!-88cHRl{Q^WpoE9 z|A8*Sdzi3?w&N4Vp_MK_*+k$Q%=^5bSq0#3qq~Ry%fjV%fTdmGQIs4-kJ)C9VHaKI z-hPRJh!7@ugnkV}7+q7#q1{D(W^2&4DuADvy?NWBR+(RRC`S9uxsRYU?{(Xsrdk27 zj)R&#P&C(S1APgoy2r$W^4h$!Isx`~o8OQ=*^;ZR_zS}#FG;>q2(pwPbFS4Ca%#J@ zor6lVyftwJ2i1)EAA*!-i~a92oMT)5^!iHaq8W}UUYyvq+I2DS8g>4)*N3%&PSm#G zmpXDLI2tUIiDg)&%KoE;9di8PyMcq|N##t8TV)Pm?@$H1Qq$d}j*kqV&OE1MoRr|! z;#vlTY!`0~lmH_+G7-8T`1tx&I8e~F#1LD){(GfhBV7rIP>O=I>m}PJ1Pm7=howA$ zq{(MO9q+h6GK>W&Y61P7I=X$cS;`?yJd}j9yR!!ML!B;Ike#9kX{QDBVgC068McDk1rjM-M z9C^(Nkv#K0iaBHYN^^;d1%w$3r;?tZLLJRL@4p9S{GRI)*!=J*dm?LOj|q0E+7#+5 zwdc&CZF6?25t3Z&RG#9=T!z(=PYlY^3zM`>%Xqmh3HSB97A1pK4c5B2%TQ!H=0}qC zos+72Qm~E$B`#>AYsm+AuG``qS_)?TPU#D(OrKG#e$m~K);}rV!6lF_l$5D^^pC&R z9X9sU5}$3G1Mf(I^}tJVoHg5>r`a|F)8<&sz#K~rh(QSpZUrn`y30@A178 zhH=0|B}=BYo7Zo4SP$PEmf6+m2H|se$H}zRv)gF8J`+;1B=zhk9{Ovy2LVY zNU%H{+M&6YA8PngDpcA}H<6o%ByF3FD$G>?3sr0GRsGHP347r!dhs*~X~8Jntt9~- zQ2ZWNTX;i;pHUq6#{)Q}C*DB}GH~OTyBld%WgX@i8k(#F$F;yZWjd9-4+~qn15c(n znKQkan7zQ=S8F*Pq7$c%?qoYYg&EK6U#{ct$YI=-Whd2_IjLe|{N!5Rln7l8kmiSV z{1fj`N&ZSru*aN?X*guk9XQP1FZpL!1EsM~ID$RTkN9opv>nZa%2>&RCY~#Y6K-mQ zi`fI7Kcr30E|Rb1)`!zi5gFA)aJhBBoMCRkZQPY1G%nJyYkYp&*#9ec<9DKq&81Ch z;-_2az;JPyUqA!PwKQ@OuqFYdwo@c=H~+VJ6L+04I{4MZ{%h@VIIYFNcMry1A9kGc z8u_SPX?FY~e}dxiH=AZ2JL9|&r!wow{ObT9sE=mCRzCdu;X4CHywD|W=Q`q!t!iHD z2;8=i$aHiKPUJ_M7f2XKUy=DQpmN13{_GAn46yp{ckH;H2JB{TJ&hmdWuy6yEi|+_jA1zs{h3uTtah;MOsdeN1a*#{HQ?+>oL$B}7-PkIvV2zP5`KNW^ zLC`vr6`K3t4N&+G)sArjwRs&_&nx*Ut+f0BNLL}#cg?TZ1Fu~dpskqTHIX#~RD=NzyjWJ#WgN2^uG@Jh8we*l8_`MS618nr}E@x0qXEUNd+1uooYw zaVK%AL)D$u z^SE`dwK+D4Vaj>WjGEoeEOMN~7rp0X;DT}(2_<9rP0N7!4kZ40S$KapGMb84n~v{s zdPNB@4Dn?pLM&g+h2Pk4~Dvtl%yIwRb%V0dVS- z*3Vw<5#WOBGac6YXM*aYKy!JRq(5gPfM>&yiYj3*D}@ ziM+`7y$h74E&q32=l)jXw#D%kN}{WMkU}T-Tpo7XlFPeX3Z-_+EeeI|?9hqW8>N*l zN=iz)kdShVo>b2+(t4gTemFm$KjHi*=QG-SJDcZhp7g%fyXKl}&N08^JH{MiOh4`1 zkywfdoJMfQh!3e?NwAzrYQmo-L)V$;JgLvMN)hI2=_?0A@u+P>n3(;>(=w5bIb-9I ztQ{%pq4px>!Up77i1z0zly*WptVhrFo_eg17}(VdFm%!YiB|3~JL-XjK0OA3c(*a) zij$GDG^hlPMA2Hxv*~>|oqyqM4b`qCJVKBHtuzamidX!y09^K z3M*2}&V$mZ~*%KEct{9eW`_G)^XLQeOaa+v7tGdfa+K45Mr&4fZ3L6 z2eb`TYmvcSOAr-^-tAa~m?6~~JO#6~Z_2c1Z4(}(?}H$y;AbBQ9TLu3dK!9GiRWS& z{!?G&ZX%ci55PXwvh5J`I0tj%gV^6fE0g8@QtDEz1|Ae5WSRCCX{A3JB`4O&{pln` zm5Oq!LlHX?yPW*EnV(ef6l$MNgicEvpjr@Nos0Kz14E+ccg#+9JKs6O3~Ek zfK<)dDBY!~b{W|$!=;7nXgl%RY~?cs<)iJckCE`Ktwm8f;iQH7$G? ze2YlttWsCER3dA2MR&J4#Pd^FOo;6Q_c~`f?SQzTKg}TjBd4)nNSSCwK5-$ z3@2U2rtQJ&O6pOpbnMw~aHbuBQt7jt;C?4J3w*BD>U@7UA9MREc5-uq%|!3C9p6PS zXXtldUt_x<<3-bht80*-d-K(O1hNdCA(J-f+c^L+TK%%7)FJUL8P z7vujA?J2#o4eiceP4BF&{AhQ8TRSUcl$u;VI_yMdCZJ4E@HD@q5O2s^Od*mUo%6wG z;A<~u!&JOmz|4U{vn)P1bEk((NAw2cZN-+s-L=ZkRgfcel48T8c;q?S|Vu?<G7#eN%|{lvkIhI>$}v&nOIwnu2|PFfj7OXeO=G@diSf9peh-un6GRTIM# z(mdN1%x-1&K4ooB3a3}Sj(hgO0>0TtgbWMxF1Mq7Zhk=$-w|)N*;1dW zYKs_kFhrpYX?#)HfmL2%Nw3(!Sbb@0U2dH^CoL7k zwY9Jf>s`k4)KGnhuB@ZdgyktcG%5eEFvPgKxsHc6>~gV*saR~CXWa>WxCP&VYpwDm zQ(u5LsMp6y`Yxy7%qj6MKMd>YmCn7$WK2Z#j0zoM8-~W-MQ}w>L%DN#Pqq1GkR%C(OABBYZv(Fu*9PDTc1B1{gM4GP_h)vDtQ)f34Wj>xNXcj71t#$yu*OC@sN#J1@JI#(Zw z#n7ySD`>zg<5Y|YT#NI-ETqQrG8RCayyk^F@cQw!D=&a}NG8r}D~H#$Q-8H!ul9wx zv&*14S{Hmq9|cZb-i1xRrGNT<9^GrW@twA0E}PFYwI{Wkl*=l`qqaR;`;X5=+<VuqZzV}gzDh)qwlYpBFi8H-Pow?XR3l;_D<=@0U`UNe6qGTo-P z=q?!Xh__mfLcjH%vcKsRl7Iwq)2EFqj|n@-+#H#M#DAbq6O5}D%&3M@jzL-;>nHJ& zs6);|?ty6a1l=sxhL|H<#RSId@SGW-JCigZ4UvgudYj72Ob9-aYd{(1-F&^ftbl2% zv74lo3{XQ(7?dVXs=oIHy#@`WIzx~Z484S@dF%_-VOnSmgEXE9bgi?!02J}r0E6i7AX1jXoRl+miYY(}`{g&V|`t}CqRF(}8g73xXJPeUrJq=oAe zw=LYA86~?)OLpg7S)ZOV(|_oB)xZ98!R4}_^uIhXF4z_Q<-oYB=Pq651h}MKi@!N9 z?yJM&xQPw05BT?!g2Q&qn*F4w_}H&6XVU%o!$a(aul{^{LfyCT{&UO0U%hkVMc238 zO;7RjTl@P%i>I&q^7vGLxclSxjo)8sp$YrL`)1njz+(H?<+8Kp(o=l>?0Oo1?fKuH F{V!9j78n2k literal 0 HcmV?d00001 diff --git a/autodiff_composition_matrix_wts.pnl b/autodiff_composition_matrix_wts.pnl new file mode 100644 index 0000000000000000000000000000000000000000..4053d03da1dd273cede4b73419a26ec5df177faf GIT binary patch literal 1071 zcmaJ=TTc^F5Z+!WWf6*qfFdXuBN#3!1`U`T!vh93sZ9x{=tJFZ3vK9y({71?#!Dh} z@qwg4s-gst5D)}{auGq2nP(q-)BoTfFfq}yZ6Oq+lkA>zGV{&ln;DCs({r5B$o(Zb zTrMXDCAX)?RpS(8vAVm<+tirN-Aa*%+WV9%x1}?<+vBwR#BQpQ{9#{rZ?zN%x~hF) zuj~{7t#l0v-8uU>eXX45d=4r;I& z*y$BzkN*NqAD~7n%^0M-m1dR;8bK@Y1N2bRrBm+hX2>j?mLWQ>&}=0#NKICnQ!eP@ z-mcQaX0y5FHjnc7`Y8&pp>42d7LH~Z5MIKr@>bQ*K_M3(g_MKkhhN(6qT+joFc|EHjLeaaePON_#U6O zbk)EL!DWoXc3zPg)(B33QU*(<@Bv0h|0x`?Gu<@4CB?A8s@Z~BoWWUg7B~7}{WQjK zp3oz1#$}$B9+hdnBi_b9SdO=`z^!%1ds66TL?5%#<0{d-pc$Y=?i`h@yDC|a@FPxp zFlNCef-9w$@DmFkWn~oNas_O$oFe#|T&#q3Yawp3@aMfIh&5b?&jfdHs}*J%`WRU- zn+Go-($S8O2}4AUH(;}#ag-VIUSlp-7#Dh6LZyL~q~J?JPpF9(TWN_*PddWY79ri3 z_4C*F20] = gain - # input[(input-bias)<=0] = gain * leak + # input[(input - bias) > 0] = gain + # input[(input - bias) <= 0] = gain * leak # MODIFIED 11/5/22 END return input diff --git a/psyneulink/core/components/functions/stateful/memoryfunctions.py b/psyneulink/core/components/functions/stateful/memoryfunctions.py index c6fb7d67731..5c13c251278 100644 --- a/psyneulink/core/components/functions/stateful/memoryfunctions.py +++ b/psyneulink/core/components/functions/stateful/memoryfunctions.py @@ -466,7 +466,7 @@ class ContentAddressableMemory(MemoryFunction): # ------------------------------ An entry is stored and retrieved as an array containing a set of `fields ` each of which is a 1d array. An array containing such entries can be used to initialize the contents of `memory ` by providing it in the **initializer** argument of the ContentAddressableMemory's - constructor, or in a call to its `reset ` method. The current contents of `memory + constructor, or in a call to its `reset ` method. The current contents of `memory ` can be inspected using the `memory ` attribute, which returns a list containing the current entries, each as a list containing all fields for that entry. The `memory_num_fields ` contains the number of fields expected for each diff --git a/psyneulink/core/components/mechanisms/modulatory/learning/learningmechanism.py b/psyneulink/core/components/mechanisms/modulatory/learning/learningmechanism.py index 2ae1da4c11b..e8cfca7b532 100644 --- a/psyneulink/core/components/mechanisms/modulatory/learning/learningmechanism.py +++ b/psyneulink/core/components/mechanisms/modulatory/learning/learningmechanism.py @@ -1313,7 +1313,7 @@ def _execute( # Get error_signals (from ERROR_SIGNAL InputPorts) and error_matrices relevant for the current execution: error_signal_indices = self.error_signal_indices error_signal_inputs = variable[error_signal_indices] - # FIX 7/22/19 [JDC]: MOVE THIS TO ITS OWN METHOD CALLED ON INITALIZATION AND UPDTATED AS NECESSARY + # FIX 7/22/19 [JDC]: MOVE THIS TO ITS OWN METHOD CALLED ON INITALIZATION AND UPDATED AS NECESSARY if self.error_matrices is None: # KAM 6/28/19 Hack to get the correct shape and contents for initial error matrix in backprop if self.function is BackPropagation or isinstance(self.function, BackPropagation): @@ -1354,7 +1354,6 @@ def _execute( ] ) learning_signal, error_signal = super()._execute(variable=function_variable, - # MODIFIED CROSS_PATHWAYS 7/22/19 END context=context, error_matrix=error_matrix, runtime_params=runtime_params, @@ -1368,7 +1367,7 @@ def _execute( and self.initialization_status != ContextFlags.INITIALIZING): print("\n{} weight change matrix: \n{}\n".format(self.name, summed_learning_signal)) - # Durning initialization return zeros so that the first "real" trial for Backprop does not start + # During initialization return zeros so that the first "real" trial for Backprop does not start # with the error computed during initialization if (self.in_composition and isinstance(self.function, BackPropagation) and diff --git a/psyneulink/core/compositions/composition.py b/psyneulink/core/compositions/composition.py index 724e3f2f403..9dcfc218ed7 100644 --- a/psyneulink/core/compositions/composition.py +++ b/psyneulink/core/compositions/composition.py @@ -3813,6 +3813,7 @@ def __init__( self._partially_added_nodes = [] self.disable_learning = disable_learning + self._runtime_learning_rate = None # graph and scheduler status attributes self.graph_consistent = True # Tracks if Composition is in runnable state (no dangling projections (what else?) @@ -10178,6 +10179,7 @@ def learn( targets: tc.optional(dict) = None, num_trials: tc.optional(int) = None, epochs: int = 1, + learning_rate = None, minibatch_size: int = 1, patience: tc.optional(int) = None, min_delta: int = 0, @@ -10226,6 +10228,12 @@ def learn( epochs : int (default=1) specifies the number of training epochs (that is, repetitions of the batched input set) to run with + learning_rate : float : default None + specifies the learning_rate used by all `learning pathways ` + when the Composition's learn method is called. This overrides the `learning_rate specified + for any individual Pathways at construction, but only applies for the current execution of + the learn method. + minibatch_size : int (default=1) specifies the size of the minibatches to use. The input trials will be batched and run, after which learning mechanisms with learning mode TRIAL will update weights @@ -10315,6 +10323,7 @@ def learn( targets=targets, num_trials=num_trials, epochs=epochs, + learning_rate=learning_rate, minibatch_size=minibatch_size, patience=patience, min_delta=min_delta, @@ -11210,7 +11219,7 @@ def execute( return self.get_output_values(context) def __call__(self, *args, **kwargs): - """Execute Composition of any args are provided; else simply return results of last execution. + """Execute Composition if any args are provided; else simply return results of last execution. This allows Composition, after it has been constructed, to be run simply by calling it directly. """ if not args and not kwargs: diff --git a/psyneulink/core/globals/utilities.py b/psyneulink/core/globals/utilities.py index 0adb9969835..61c987cb823 100644 --- a/psyneulink/core/globals/utilities.py +++ b/psyneulink/core/globals/utilities.py @@ -442,6 +442,16 @@ def iscompatible(candidate, reference=None, **kargs): warnings.simplefilter(action='ignore', category=FutureWarning) if reference is not None and (candidate == reference): return True + # if reference is not None: + # if (isinstance(reference, (bool, int, float)) + # and isinstance(candidate, (bool, int, float)) + # and candidate == reference): + # return True + # elif (isinstance(reference, (list, np.ndarray)) + # and isinstance(candidate, (list, np.ndarray)) and (candidate == reference).all()): + # return True + # elif is_iterable(reference) and is_iterable(candidate) and (candidate == reference): + # return True except ValueError: # raise UtilitiesError("Could not compare {0} and {1}".format(candidate, reference)) # IMPLEMENTATION NOTE: np.array generates the following error: diff --git a/psyneulink/library/compositions/autodiffcomposition.py b/psyneulink/library/compositions/autodiffcomposition.py index 7a001ae3b75..9c3518e2237 100644 --- a/psyneulink/library/compositions/autodiffcomposition.py +++ b/psyneulink/library/compositions/autodiffcomposition.py @@ -94,9 +94,10 @@ Logging ~~~~~~~ -Logging in AutodiffCompositions follows the same procedure as `logging in a Composition `. However, since an AutodiffComposition internally converts all of its mechanisms to an equivalent PyTorch model, -then its inner components are not actually executed. This means that there is limited support for logging parameters of components inside an AutodiffComposition; -Currently, the only supported parameters are: +Logging in AutodiffCompositions follows the same procedure as `logging in a Composition `. +However, since an AutodiffComposition internally converts all of its mechanisms to an equivalent PyTorch model, +then its inner components are not actually executed. This means that there is limited support for +logging parameters of components inside an AutodiffComposition; Currently, the only supported parameters are: 1) the `matrix` parameter of Projections @@ -132,8 +133,9 @@ """ import logging - +import os import numpy as np +from pathlib import Path, PosixPath try: import torch @@ -146,6 +148,9 @@ from psyneulink.library.compositions.pytorchmodelcreator import PytorchModelCreator from psyneulink.library.components.mechanisms.processing.objective.comparatormechanism import ComparatorMechanism +from psyneulink.core.components.mechanisms.processing.compositioninterfacemechanism import CompositionInterfaceMechanism +from psyneulink.core.components.mechanisms.modulatory.modulatorymechanism import ModulatoryMechanism_Base +from psyneulink.core.components.projections.modulatory.modulatoryprojection import ModulatoryProjection_Base from psyneulink.core.compositions.composition import Composition, NodeRole from psyneulink.core.compositions.composition import CompositionError from psyneulink.core.compositions.report \ @@ -159,6 +164,7 @@ from psyneulink.core import llvm as pnlvm + logger = logging.getLogger(__name__) @@ -185,7 +191,7 @@ class AutodiffComposition(Composition): --------- learning_rate : float : default 0.001 - the learning rate, which is passed to the optimizer. + the learning rate passed to the optimizer if none is specified in the learn method of the AutodiffComposition. disable_learning : bool: default False specifies whether the AutodiffComposition should disable learning when run in `learning mode @@ -259,6 +265,7 @@ def __init__(self, self.force_no_retain_graph = force_no_retain_graph self.loss = None self.disable_learning = disable_learning + self._runtime_learning_rate = None # keeps track of average loss per epoch self.losses = [] @@ -276,10 +283,10 @@ def __init__(self, # CLEANUP: move some of what's done in the methods below to a "validate_params" type of method @handle_external_context() - def _build_pytorch_representation(self, context=None): + def _build_pytorch_representation(self, context=None, refresh=False): if self.scheduler is None: self.scheduler = Scheduler(graph=self.graph_processing) - if self.parameters.pytorch_representation._get(context=context) is None: + if self.parameters.pytorch_representation._get(context=context) is None or refresh: model = PytorchModelCreator(composition=self, device=self.device, context=context) @@ -288,8 +295,9 @@ def _build_pytorch_representation(self, context=None): # Set up optimizer function old_opt = self.parameters.optimizer._get(context) - if old_opt is None: - opt = self._make_optimizer(self.optimizer_type, self.learning_rate, self.weight_decay, context) + learning_rate = self._runtime_learning_rate or self.learning_rate + if old_opt is None or refresh: + opt = self._make_optimizer(self.optimizer_type, learning_rate, self.weight_decay, context) self.parameters.optimizer._set(opt, context, skip_history=True, skip_log=True) # Set up loss function @@ -355,7 +363,10 @@ def autodiff_training(self, inputs, targets, context=None, scheduler=None): # compute total loss across output neurons for current trial tracked_loss = self.parameters.tracked_loss._get(context) if tracked_loss is None: - self.parameters.tracked_loss._set(torch.zeros(1, device=self.device).double(), context=context, skip_history=True, skip_log=True) + self.parameters.tracked_loss._set(torch.zeros(1, device=self.device).double(), + context=context, + skip_history=True, + skip_log=True) tracked_loss = self.parameters.tracked_loss._get(context) curr_tensor_inputs = {} @@ -368,10 +379,9 @@ def autodiff_training(self, inputs, targets, context=None, scheduler=None): curr_tensor_targets[component] = torch.tensor(target, device=self.device).double() # do forward computation on current inputs - curr_tensor_outputs = self.parameters.pytorch_representation._get(context).forward( - curr_tensor_inputs, - context, - ) + curr_tensor_outputs = self.parameters.pytorch_representation._get(context).forward(curr_tensor_inputs, + context, + ) for component in curr_tensor_outputs.keys(): # possibly add custom loss option, which is a loss function that takes many args @@ -385,7 +395,10 @@ def autodiff_training(self, inputs, targets, context=None, scheduler=None): component = input_port.all_afferents[0].sender.owner outputs.append(curr_tensor_outputs[component].detach().cpu().numpy().copy()) - self.parameters.tracked_loss_count._set(self.parameters.tracked_loss_count._get(context=context) + 1, context=context, skip_history=True, skip_log=True) + self.parameters.tracked_loss_count._set(self.parameters.tracked_loss_count._get(context=context) + 1, + context=context, + skip_history=True, + skip_log=True) return outputs def clear_losses(self, context=None): @@ -394,7 +407,7 @@ def clear_losses(self, context=None): def _update_learning_parameters(self, context): """ - Updates parameters based on trials ran since last update. + Updates parameters based on trials run since last update. """ optimizer = self.parameters.optimizer._get(context=context) optimizer.zero_grad() @@ -563,6 +576,120 @@ def execute(self, report_num=report_num ) + @handle_external_context(fallback_most_recent=True) + def save(self, path:PosixPath=None, directory:str=None, filename:str=None, context=None): + """Saves all weight matrices for all MappingProjections in the AutodiffComposition + + Arguments + --------- + path: Path, PosixPath or str : default None + path specification; must be a legal path specification in the filesystem. + directory: str : default ``current working directory`` + directory where `matrices ` for all MappingProjections + in the AutodiffComposition are saved. + filename: str : default ``_matrix_wts.pnl`` + filename in which `matrices ` for all MappingProjections + in the AutodiffComposition are saved. + .. note:: + Matrices are saved in + `PyTorch state_dict `_ format. + + Return + ------ + Path + + """ + if path: + try: + path = Path(path) + except: + raise AutodiffCompositionError(f"'{path}' (for saving weight matrices of ({self.name}) " + f"is not a legal path.") + else: + try: + if directory: + path = Path(directory) + else: + path = Path(os.getcwd()) + if filename: + # path = Path(path / filename) + path = Path(os.path.join(path / filename)) + else: + path = Path(os.path.join(path / f'{self.name}_matrix_wts.pnl')) + except IsADirectoryError: + raise AutodiffCompositionError(f"'{path}' (for saving weight matrices of ({self.name}) " + f"is not a legal path.") + proj_state = { + # p.name: p.parameters.matrix.get(context=context) + p.name: p.matrix.base + for p in self.projections + if not (isinstance(p, ModulatoryProjection_Base) + or isinstance(p.sender.owner, CompositionInterfaceMechanism) + or isinstance(p.receiver.owner, CompositionInterfaceMechanism) + or isinstance(p.sender.owner, ModulatoryMechanism_Base) + or isinstance(p.receiver.owner, ModulatoryMechanism_Base) + or p.sender.owner in self.get_nodes_by_role(NodeRole.LEARNING) + or p.receiver.owner in self.get_nodes_by_role(NodeRole.LEARNING) + )} + torch.save(proj_state, path) + return path + + @handle_external_context(fallback_most_recent=True) + def load(self, path:PosixPath=None, directory:str=None, filename:str=None, context=None): + """Loads all weights matrices for all MappingProjections in the AutodiffComposition from file + Arguments + --------- + path: Path : default None + Path for file in which `MappingProjection` `matrices ` are stored. + This must be a legal PosixPath object; if it is specified **directory** and **filename** are ignored. + directory: str : default ``current working directory`` + directory where `MappingProjection` `matrices ` are stored. + filename: str : default ``_matrix_wts.pnl`` + name of file in which `MappingProjection` `matrices ` are stored. + .. note:: + Matrices must be stored in + `PyTorch state_dict `_ format. + """ + if path: + if not isinstance(path,Path): + raise AutodiffCompositionError(f"'{path}' (for saving weight matrices of ({self.name}) " + f"is not a legal path.") + else: + try: + if directory: + path = Path(directory) + else: + path = Path(os.getcwd()) + if filename: + # path = Path(path / filename) + path = Path(os.path.join(path / filename)) + else: + # path = Path(path / f'{self.name}_matrix_wts.pnl') + path = Path(os.path.join(path , f'{self.name}_matrix_wts.pnl')) + except IsADirectoryError: + raise AutodiffCompositionError(f"'{path}' (for saving weight matrices of ({self.name}) " + f"is not a legal path.") + state = torch.load(path) + for projection in [p for p in self.projections + if not (isinstance(p, ModulatoryProjection_Base) + or isinstance(p.sender.owner, CompositionInterfaceMechanism) + or isinstance(p.receiver.owner, CompositionInterfaceMechanism) + or isinstance(p.sender.owner, ModulatoryMechanism_Base) + or isinstance(p.receiver.owner, ModulatoryMechanism_Base) + or p.sender.owner in self.get_nodes_by_role(NodeRole.LEARNING) + or p.receiver.owner in self.get_nodes_by_role(NodeRole.LEARNING) + )]: + matrix = state[projection.name] + if np.array(matrix).shape != projection.matrix.base.shape: + raise AutodiffCompositionError(f"Shape of matrix loaded for '{projection.name}' " + f"({np.array(matrix).shape}) " + f"does not match its shape ({projection.matrix.base.shape})") + projection.matrix.base = matrix + projection.parameters.matrix.set(matrix, context=context, override=True) + projection.parameter_ports['matrix'].parameters.value.set(matrix, context=context, override=True) + self._build_pytorch_representation(context=context, refresh=True) + # MODIFIED 11/8/22 END + def _get_state_ids(self): return super()._get_state_ids() + ["optimizer"] diff --git a/psyneulink/library/compositions/compositionrunner.py b/psyneulink/library/compositions/compositionrunner.py index d7039a1902e..8e7a757a353 100644 --- a/psyneulink/library/compositions/compositionrunner.py +++ b/psyneulink/library/compositions/compositionrunner.py @@ -129,6 +129,7 @@ def run_learning(self, targets: dict = None, num_trials: int = None, epochs: int = 1, + learning_rate = None, minibatch_size: int = 1, patience: int = None, min_delta: int = 0, @@ -139,7 +140,7 @@ def run_learning(self, execution_mode:pnlvm.ExecutionMode = pnlvm.ExecutionMode.Python, **kwargs): """ - Runs the composition repeatedly with the specified parameters + Runs the composition repeatedly with the specified parameters. Returns --------- @@ -150,6 +151,9 @@ def run_learning(self, else: self._is_llvm_mode = True + # This is used by local learning-related methods to override the default learning_rate set at construction. + self._composition._runtime_learning_rate = learning_rate + # Handle function and generator inputs if isgeneratorfunction(inputs): inputs = inputs() diff --git a/psyneulink/library/compositions/pytorchcomponents.py b/psyneulink/library/compositions/pytorchcomponents.py index 43122730437..e106272d91a 100644 --- a/psyneulink/library/compositions/pytorchcomponents.py +++ b/psyneulink/library/compositions/pytorchcomponents.py @@ -1,4 +1,4 @@ -from psyneulink.core.components.functions.nonstateful.transferfunctions import Linear, Logistic, ReLU +from psyneulink.core.components.functions.nonstateful.transferfunctions import Linear, Logistic, ReLU, SoftMax from psyneulink.library.compositions.pytorchllvmhelper import * from psyneulink.core.globals.log import LogCondition from psyneulink.core import llvm as pnlvm @@ -10,7 +10,8 @@ def pytorch_function_creator(function, device, context=None): """ Converts a PsyNeuLink function into an equivalent PyTorch lambda function. - NOTE: This is needed due to PyTorch limitations (see: https://github.com/PrincetonUniversity/PsyNeuLink/pull/1657#discussion_r437489990) + NOTE: This is needed due to PyTorch limitations + (see: https://github.com/PrincetonUniversity/PsyNeuLink/pull/1657#discussion_r437489990) """ def get_fct_param_value(param_name): val = function._get_current_parameter_value( @@ -38,6 +39,10 @@ def get_fct_param_value(param_name): return lambda x: (torch.max(input=(x - bias), other=torch.tensor([0], device=device).double()) * gain + torch.min(input=(x - bias), other=torch.tensor([0], device=device).double()) * leak) + elif isinstance(function, SoftMax): + gain = get_fct_param_value('gain') + return lambda x: (torch.softmax(x, len(x), other=torch.tensor([0], device=device).double())) + else: raise Exception(f"Function {function} is not currently supported in AutodiffCompositions!") diff --git a/psyneulink/library/compositions/pytorchmodelcreator.py b/psyneulink/library/compositions/pytorchmodelcreator.py index af809613bf4..916dfca438f 100644 --- a/psyneulink/library/compositions/pytorchmodelcreator.py +++ b/psyneulink/library/compositions/pytorchmodelcreator.py @@ -60,7 +60,8 @@ def __init__(self, composition, device, context=None): proj_recv.add_afferent(new_proj) self.projection_map[projection] = new_proj self.projections.append(new_proj) - self.params.append(new_proj.matrix) + + self._regenerate_paramlist() c = Context() try: @@ -81,6 +82,11 @@ def __init__(self, composition, device, context=None): __deepcopy__ = get_deepcopy_with_shared(shared_types=(Component, ComponentsMeta)) + def _regenerate_paramlist(self): + self.params = nn.ParameterList() + for proj in self.projections: + self.params.append(proj.matrix) + # generates llvm function for self.forward def _gen_llvm_function(self, *, ctx:pnlvm.LLVMBuilderContext, tags:frozenset): args = [ctx.get_state_struct_type(self._composition).as_pointer(), diff --git a/tests/composition/autodiff_composition_matrix_wts.pnl b/tests/composition/autodiff_composition_matrix_wts.pnl new file mode 100644 index 0000000000000000000000000000000000000000..4053d03da1dd273cede4b73419a26ec5df177faf GIT binary patch literal 1071 zcmaJ=TTc^F5Z+!WWf6*qfFdXuBN#3!1`U`T!vh93sZ9x{=tJFZ3vK9y({71?#!Dh} z@qwg4s-gst5D)}{auGq2nP(q-)BoTfFfq}yZ6Oq+lkA>zGV{&ln;DCs({r5B$o(Zb zTrMXDCAX)?RpS(8vAVm<+tirN-Aa*%+WV9%x1}?<+vBwR#BQpQ{9#{rZ?zN%x~hF) zuj~{7t#l0v-8uU>eXX45d=4r;I& z*y$BzkN*NqAD~7n%^0M-m1dR;8bK@Y1N2bRrBm+hX2>j?mLWQ>&}=0#NKICnQ!eP@ z-mcQaX0y5FHjnc7`Y8&pp>42d7LH~Z5MIKr@>bQ*K_M3(g_MKkhhN(6qT+joFc|EHjLeaaePON_#U6O zbk)EL!DWoXc3zPg)(B33QU*(<@Bv0h|0x`?Gu<@4CB?A8s@Z~BoWWUg7B~7}{WQjK zp3oz1#$}$B9+hdnBi_b9SdO=`z^!%1ds66TL?5%#<0{d-pc$Y=?i`h@yDC|a@FPxp zFlNCef-9w$@DmFkWn~oNas_O$oFe#|T&#q3Yawp3@aMfIh&5b?&jfdHs}*J%`WRU- zn+Go-($S8O2}4AUH(;}#ag-VIUSlp-7#Dh6LZyL~q~J?JPpF9(TWN_*PddWY79ri3 z_4C*F2 Date: Tue, 1 Nov 2022 11:48:42 -0400 Subject: [PATCH 049/127] tests/functions/transfer: Use numpy instead of python math module Signed-off-by: Jan Vesely --- tests/functions/test_transfer.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/functions/test_transfer.py b/tests/functions/test_transfer.py index e0c0066295e..5d4cbe27854 100644 --- a/tests/functions/test_transfer.py +++ b/tests/functions/test_transfer.py @@ -4,8 +4,6 @@ import psyneulink.core.globals.keywords as kw import pytest -from math import e, pi, sqrt - SIZE=10 np.random.seed(0) test_var = np.random.rand(SIZE) @@ -25,7 +23,7 @@ tanh_helper = (RAND1 * (test_var + RAND2 - RAND3) + RAND4) tanh_helper = np.tanh(tanh_helper) -gaussian_helper = e**(-(test_var - RAND2)**2 / (2 * RAND1**2)) / sqrt(2 * pi * RAND1) +gaussian_helper = np.e**(-(test_var - RAND2)**2 / (2 * RAND1**2)) / np.sqrt(2 * np.pi * RAND1) gaussian_helper = RAND3 * gaussian_helper + RAND4 def gaussian_distort_helper(seed): From 135a7866c837234f2413439bc6a69252562a8857 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Tue, 8 Nov 2022 22:12:04 -0500 Subject: [PATCH 050/127] tests/transfer: Use lambda function to get the name of tested function Signed-off-by: Jan Vesely --- tests/functions/test_transfer.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/tests/functions/test_transfer.py b/tests/functions/test_transfer.py index 5d4cbe27854..fb64981d64d 100644 --- a/tests/functions/test_transfer.py +++ b/tests/functions/test_transfer.py @@ -86,19 +86,11 @@ def test_execute(func, variable, params, expected, benchmark, func_mode): (Functions.Tanh, test_var, {'gain':RAND1, 'bias':RAND2, 'offset':RAND3, 'scale':RAND4}, tanh_derivative_helper), ] -derivative_names = [ - "LINEAR_DERIVATIVE", - "EXPONENTIAL_DERIVATIVE", - "LOGISTIC_DERIVATIVE", - "RELU_DERIVATIVE", - "TANH_DERIVATIVE", -] - @pytest.mark.function @pytest.mark.transfer_function @pytest.mark.benchmark -@pytest.mark.parametrize("func, variable, params, expected", derivative_test_data, ids=derivative_names) -def test_execute_derivative(func, variable, params, expected, benchmark, func_mode): +@pytest.mark.parametrize("func, variable, params, expected", derivative_test_data, ids=lambda x: getattr(x, 'name', None)) +def test_transfer_derivative(func, variable, params, expected, benchmark, func_mode): f = func(default_variable=variable, **params) benchmark.group = "TransferFunction " + func.componentName + " Derivative" if func_mode == 'Python': From a36b9f921b86866b08aafdd2aa7175d8b7fed79c Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Wed, 9 Nov 2022 00:16:30 -0500 Subject: [PATCH 051/127] llvm: Add sin, cos builtins PTX can't select llvm intrinsics, use builtin wrapper functions. Signed-off-by: Jan Vesely --- psyneulink/core/llvm/builder_context.py | 1 + psyneulink/core/llvm/builtins.py | 6 ++++-- psyneulink/core/llvm/jit_engine.py | 2 ++ tests/llvm/test_builtins_intrinsics.py | 4 +++- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/psyneulink/core/llvm/builder_context.py b/psyneulink/core/llvm/builder_context.py index 8695d1e5347..3398a2d974a 100644 --- a/psyneulink/core/llvm/builder_context.py +++ b/psyneulink/core/llvm/builder_context.py @@ -56,6 +56,7 @@ def module_count(): _BUILTIN_PREFIX = "__pnl_builtin_" _builtin_intrinsics = frozenset(('pow', 'log', 'exp', 'tanh', 'coth', 'csch', + 'sin', 'cos', 'is_close_float', 'is_close_double', 'mt_rand_init', 'philox_rand_init')) diff --git a/psyneulink/core/llvm/builtins.py b/psyneulink/core/llvm/builtins.py index 30973992713..977e941bdbe 100644 --- a/psyneulink/core/llvm/builtins.py +++ b/psyneulink/core/llvm/builtins.py @@ -448,7 +448,7 @@ def setup_coth(ctx): exp_f = ctx.get_builtin("exp", [x.type]) # (e**2x + 1)/(e**2x - 1) is faster but doesn't handle large inputs (exp -> Inf) well (Inf/Inf = NaN) # (1 + (2/(exp(2*x) - 1))) is a bit slower but handles large inputs better - # (e**2x + 1)/(e**2x - 1) + _2x = builder.fmul(x.type(2), x) e2x = builder.call(exp_f, [_2x]) den = builder.fsub(e2x, e2x.type(1)) @@ -463,6 +463,8 @@ def setup_pnl_intrinsics(ctx): double_intr_ty = ir.FunctionType(ctx.float_ty, (ctx.float_ty, ctx.float_ty)) # Create function declarations + ir.Function(ctx.module, single_intr_ty, name=_BUILTIN_PREFIX + "cos") + ir.Function(ctx.module, single_intr_ty, name=_BUILTIN_PREFIX + "sin") ir.Function(ctx.module, single_intr_ty, name=_BUILTIN_PREFIX + "exp") ir.Function(ctx.module, single_intr_ty, name=_BUILTIN_PREFIX + "log") ir.Function(ctx.module, double_intr_ty, name=_BUILTIN_PREFIX + "pow") @@ -483,7 +485,7 @@ def _generate_intrinsic_wrapper(module, name, ret, args): def _generate_cpu_builtins_module(_float_ty): """Generate function wrappers for log, exp, and pow intrinsics.""" module = ir.Module(name="cpu_builtins") - for intrinsic in ('exp', 'log'): + for intrinsic in ('sin', 'cos', 'exp', 'log'): _generate_intrinsic_wrapper(module, intrinsic, _float_ty, [_float_ty]) _generate_intrinsic_wrapper(module, "pow", _float_ty, [_float_ty, _float_ty]) diff --git a/psyneulink/core/llvm/jit_engine.py b/psyneulink/core/llvm/jit_engine.py index 23815b9aa45..a5db35e2864 100644 --- a/psyneulink/core/llvm/jit_engine.py +++ b/psyneulink/core/llvm/jit_engine.py @@ -279,6 +279,8 @@ def _init(self): _ptx_builtin_source = """ +__device__ {type} __pnl_builtin_sin({type} a) {{ return sin(a); }} +__device__ {type} __pnl_builtin_cos({type} a) {{ return cos(a); }} __device__ {type} __pnl_builtin_log({type} a) {{ return log(a); }} __device__ {type} __pnl_builtin_exp({type} a) {{ return exp(a); }} __device__ {type} __pnl_builtin_pow({type} a, {type} b) {{ return pow(a, b); }} diff --git a/tests/llvm/test_builtins_intrinsics.py b/tests/llvm/test_builtins_intrinsics.py index 307ccdabc5d..0b7b4dc7bdd 100644 --- a/tests/llvm/test_builtins_intrinsics.py +++ b/tests/llvm/test_builtins_intrinsics.py @@ -25,8 +25,10 @@ (lambda x: 1.0 / np.sinh(x), (450,), "__pnl_builtin_csch", 1 / np.sinh(450)), #~900 is the limit after which exp(x) used in csch formula returns inf (lambda x: 1.0 / np.sinh(x), (900,), "__pnl_builtin_csch", 1 / np.sinh(900)), + (np.sin, (x,), "__pnl_builtin_sin", np.sin(x)), + (np.cos, (x,), "__pnl_builtin_cos", np.cos(x)), ], ids=["EXP", "Large EXP", "LOG", "POW", "TANH", "Large TANH", "COTH", "Large COTH", - "CSCH", "Large CSCH", "xLarge CSCH"]) + "CSCH", "Large CSCH", "xLarge CSCH", "SIN", "COS"]) def test_builtin_op(benchmark, op, args, builtin, result, func_mode): if func_mode == 'Python': f = op From 1cb0c5cce60bf3a003beda7cc91c2b7106470e93 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Tue, 8 Nov 2022 23:09:29 -0500 Subject: [PATCH 052/127] llvm, functions/Transfer: Implement compiled Angle function Signed-off-by: Jan Vesely --- .../nonstateful/transferfunctions.py | 73 ++++++++++++------- tests/functions/test_transfer.py | 2 - 2 files changed, 48 insertions(+), 27 deletions(-) diff --git a/psyneulink/core/components/functions/nonstateful/transferfunctions.py b/psyneulink/core/components/functions/nonstateful/transferfunctions.py index 774da8a96ba..afd8d3e83d2 100644 --- a/psyneulink/core/components/functions/nonstateful/transferfunctions.py +++ b/psyneulink/core/components/functions/nonstateful/transferfunctions.py @@ -1741,27 +1741,6 @@ def __init__(self, prefs=prefs, ) - # def _gen_llvm_transfer(self, builder, index, ctx, vi, vo, params, state, *, tags:frozenset): - # ptri = builder.gep(vi, [ctx.int32_ty(0), index]) - # ptro = builder.gep(vo, [ctx.int32_ty(0), index]) - # slope_ptr = pnlvm.helpers.get_param_ptr(builder, self, params, SLOPE) - # intercept_ptr = pnlvm.helpers.get_param_ptr(builder, self, params, INTERCEPT) - # - # slope = pnlvm.helpers.load_extract_scalar_array_one(builder, slope_ptr) - # intercept = pnlvm.helpers.load_extract_scalar_array_one(builder, intercept_ptr) - # - # - # if "derivative" in tags: - # # f'(x) = m - # val = slope - # else: - # # f(x) = mx + b - # val = builder.load(ptri) - # val = builder.fmul(val, slope) - # val = builder.fadd(val, intercept) - # - # builder.store(val, ptro) - def _function(self, variable=None, context=None, @@ -1818,13 +1797,57 @@ def _angle(self, value): angle[0] = np.cos(value[0]) prod = np.product([np.sin(value[k]) for k in range(1, dim - 1)]) n_prod = prod - for j in range(dim - 2): - n_prod /= np.sin(value[j + 1]) - amt = n_prod * np.cos(value[j + 1]) - angle[j + 1] = amt + for j in range(1, dim - 1): + n_prod /= np.sin(value[j]) + amt = n_prod * np.cos(value[j]) + angle[j] = amt angle[dim - 1] = prod return angle + def _gen_llvm_function_body(self, ctx, builder, params, state, arg_in, arg_out, *, tags:frozenset): + assert isinstance(arg_in.type.pointee, pnlvm.ir.ArrayType) + assert isinstance(arg_out.type.pointee, pnlvm.ir.ArrayType) + assert len(arg_in.type.pointee) + 1 == len(arg_out.type.pointee) + + # The first cos + res0_ptr = builder.gep(arg_out, [ctx.int32_ty(0), ctx.int32_ty(0)]) + val0_ptr = builder.gep(arg_in, [ctx.int32_ty(0), ctx.int32_ty(0)]) + val0 = builder.load(val0_ptr) + cos_f = ctx.get_builtin("cos", [val0.type]) + cos_val0 = builder.call(cos_f, [val0]) + builder.store(cos_val0, res0_ptr) + + # calculate suffix product + sin_f = ctx.get_builtin("sin", [val0.type]) + prod_ptr = builder.alloca(val0.type) + builder.store(prod_ptr.type.pointee(1.0), prod_ptr) + + dim_m1 = ctx.int32_ty(len(arg_out.type.pointee) - 1) + with pnlvm.helpers.for_loop(builder, dim_m1.type(1), dim_m1, dim_m1.type(1), id="suff_prod") as (b, idx): + #revert the index to go from the end + idx = b.sub(dim_m1, idx) + + prod = b.load(prod_ptr) + val_ptr = b.gep(arg_in, [ctx.int32_ty(0), idx]) + val = b.load(val_ptr) + + # calculate suffix product of sin(input) + val_sin = b.call(sin_f, [val]) + new_prod = b.fmul(prod, val_sin) + b.store(new_prod, prod_ptr) + + # output value is suffix product * cos(val) + val_cos = b.call(cos_f, [val]) + res = b.fmul(prod, val_cos) + res_ptr = b.gep(arg_out, [ctx.int32_ty(0), idx]) + b.store(res, res_ptr) + + # The last element is just the suffix product * 1 + last_ptr = builder.gep(arg_out, [ctx.int32_ty(0), dim_m1]) + builder.store(builder.load(prod_ptr), last_ptr) + + return builder + # @handle_external_context() # def derivative(self, input=None, output=None, context=None): # """ diff --git a/tests/functions/test_transfer.py b/tests/functions/test_transfer.py index fb64981d64d..4604314f8c9 100644 --- a/tests/functions/test_transfer.py +++ b/tests/functions/test_transfer.py @@ -62,8 +62,6 @@ def gaussian_distort_helper(seed): @pytest.mark.benchmark @pytest.mark.parametrize("func, variable, params, expected", test_data) def test_execute(func, variable, params, expected, benchmark, func_mode): - if 'Angle' in func.componentName and func_mode != 'Python': - pytest.skip('Angle not yet supported by LLVM or PTX') benchmark.group = "TransferFunction " + func.componentName f = func(default_variable=variable, **params) ex = pytest.helpers.get_func_execution(f, func_mode) From 71b4316b69b50fcf003926668e30f49a71f3667a Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Tue, 8 Nov 2022 23:42:17 -0500 Subject: [PATCH 053/127] functions/Transfer: Consider 'bias' parameter in 0 check in ReLU derivative function Signed-off-by: Jan Vesely --- .../nonstateful/transferfunctions.py | 25 ++++++++----------- tests/functions/test_transfer.py | 2 +- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/psyneulink/core/components/functions/nonstateful/transferfunctions.py b/psyneulink/core/components/functions/nonstateful/transferfunctions.py index afd8d3e83d2..a93918d7788 100644 --- a/psyneulink/core/components/functions/nonstateful/transferfunctions.py +++ b/psyneulink/core/components/functions/nonstateful/transferfunctions.py @@ -1579,12 +1579,12 @@ def _gen_llvm_transfer(self, builder, index, ctx, vi, vo, params, state, *, tags # Maxnum for some reason needs full function prototype max_f = ctx.get_builtin("maxnum", [ctx.float_ty]) var = builder.load(ptri) + val = builder.fsub(var, bias) if "derivative" in tags: - predicate = builder.fcmp_ordered('>', var, var.type(0)) + predicate = builder.fcmp_ordered('>', val, val.type(0)) val = builder.select(predicate, gain, builder.fmul(gain, leak)) else: - val = builder.fsub(var, bias) val1 = builder.fmul(val, gain) val2 = builder.fmul(val1, leak) @@ -1593,7 +1593,7 @@ def _gen_llvm_transfer(self, builder, index, ctx, vi, vo, params, state, *, tags builder.store(val, ptro) @handle_external_context() - def derivative(self, input, output=None, context=None): + def derivative(self, variable, output=None, context=None): """ derivative(input) @@ -1613,18 +1613,13 @@ def derivative(self, input, output=None, context=None): """ gain = self._get_current_parameter_value(GAIN, context) leak = self._get_current_parameter_value(LEAK, context) - # MODIFIED 11/5/22 OLD: - input = np.asarray(input).copy() - input[input>0] = gain - input[input<=0] = gain * leak - # # MODIFIED 11/5/22 NEW: - # bias = self._get_current_parameter_value(BIAS, context) - # input = np.asarray(input).copy() - # input[(input - bias) > 0] = gain - # input[(input - bias) <= 0] = gain * leak - # MODIFIED 11/5/22 END - - return input + bias = self._get_current_parameter_value(BIAS, context) + + value = np.empty_like(variable) + value[(variable - bias) > 0] = gain + value[(variable - bias) <= 0] = gain * leak + + return value # ********************************************************************************************************************** # Angle diff --git a/tests/functions/test_transfer.py b/tests/functions/test_transfer.py index 4604314f8c9..cd8b6061f09 100644 --- a/tests/functions/test_transfer.py +++ b/tests/functions/test_transfer.py @@ -80,7 +80,7 @@ def test_execute(func, variable, params, expected, benchmark, func_mode): (Functions.Linear, test_var, {'slope':RAND1, 'intercept':RAND2}, RAND1), (Functions.Exponential, test_var, {'scale':RAND1, 'rate':RAND2}, RAND1 * RAND2 * np.exp(RAND2 * test_var)), (Functions.Logistic, test_var, {'gain':RAND1, 'x_0':RAND2, 'offset':RAND3, 'scale':RAND4}, RAND1 * RAND4 * logistic_helper * (1 - logistic_helper)), - (Functions.ReLU, test_var, {'gain':RAND1, 'bias':RAND2, 'leak':RAND3}, np.where(test_var > 0, RAND1, RAND1 * RAND3)), + (Functions.ReLU, test_var, {'gain':RAND1, 'bias':RAND2, 'leak':RAND3}, np.where((test_var - RAND2) > 0, RAND1, RAND1 * RAND3)), (Functions.Tanh, test_var, {'gain':RAND1, 'bias':RAND2, 'offset':RAND3, 'scale':RAND4}, tanh_derivative_helper), ] From dc6c0cd85d0681a6367051ec317dcdf913b88217 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Tue, 8 Nov 2022 23:51:58 -0500 Subject: [PATCH 054/127] llvm, functions/SoftMax: Pass output_type as argument This will allow generating softmax with different output type than the one set by the SoftMax Function instance. Signed-off-by: Jan Vesely --- .../functions/nonstateful/transferfunctions.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/psyneulink/core/components/functions/nonstateful/transferfunctions.py b/psyneulink/core/components/functions/nonstateful/transferfunctions.py index a93918d7788..0c863ff8c49 100644 --- a/psyneulink/core/components/functions/nonstateful/transferfunctions.py +++ b/psyneulink/core/components/functions/nonstateful/transferfunctions.py @@ -2644,7 +2644,7 @@ def __gen_llvm_exp_div(self, builder, index, ctx, vi, vo, gain, exp_sum): builder.store(val, ptro) - def __gen_llvm_apply(self, ctx, builder, params, state, arg_in, arg_out, tags:frozenset): + def __gen_llvm_apply(self, ctx, builder, params, state, arg_in, arg_out, output_type, tags:frozenset): exp_sum_ptr = builder.alloca(ctx.float_ty) builder.store(exp_sum_ptr.type.pointee(0), exp_sum_ptr) @@ -2657,7 +2657,7 @@ def __gen_llvm_apply(self, ctx, builder, params, state, arg_in, arg_out, tags:fr exp_sum = builder.load(exp_sum_ptr) - if self.output == ALL: + if output_type == ALL: with pnlvm.helpers.array_ptr_loop(builder, arg_in, "exp_div") as args: self.__gen_llvm_exp_div(ctx=ctx, vi=arg_in, vo=arg_out, gain=gain, exp_sum=exp_sum, *args) @@ -2671,14 +2671,14 @@ def __gen_llvm_apply(self, ctx, builder, params, state, arg_in, arg_out, tags:fr one_hot_out = arg_out one_hot_in = builder.alloca(one_hot_f.args[2].type.pointee) - if self.output in {MAX_VAL, MAX_INDICATOR}: + if output_type in {MAX_VAL, MAX_INDICATOR}: with pnlvm.helpers.array_ptr_loop(builder, arg_in, "exp_div") as (b, i): self.__gen_llvm_exp_div(ctx=ctx, vi=arg_in, vo=one_hot_in, gain=gain, exp_sum=exp_sum, builder=b, index=i) builder.call(one_hot_f, [one_hot_p, one_hot_s, one_hot_in, one_hot_out]) - elif self.output == PROB: + elif output_type in PROB: one_hot_in_data = builder.gep(one_hot_in, [ctx.int32_ty(0), ctx.int32_ty(0)]) one_hot_in_dist = builder.gep(one_hot_in, [ctx.int32_ty(0), ctx.int32_ty(1)]) @@ -2693,21 +2693,23 @@ def __gen_llvm_apply(self, ctx, builder, params, state, arg_in, arg_out, tags:fr builder.call(one_hot_f, [one_hot_p, one_hot_s, one_hot_in, one_hot_out]) else: - assert False, "Unsupported output in {}: {}".format(self, self.output) + assert False, "Unsupported output in {}: {}".format(self, out_type) return builder - def _gen_llvm_function_body(self, ctx, builder, params, state, arg_in, arg_out, *, tags:frozenset): + def _gen_llvm_function_body(self, ctx, builder, params, state, arg_in, arg_out, output_type=None, *, tags:frozenset): + output_type = self.output if output_type is None else output_type + if self.parameters.per_item.get(): assert isinstance(arg_in.type.pointee.element, pnlvm.ir.ArrayType) assert isinstance(arg_out.type.pointee.element, pnlvm.ir.ArrayType) for i in range(arg_in.type.pointee.count): inner_in = builder.gep(arg_in, [ctx.int32_ty(0), ctx.int32_ty(i)]) inner_out = builder.gep(arg_out, [ctx.int32_ty(0), ctx.int32_ty(i)]) - builder = self.__gen_llvm_apply(ctx, builder, params, state, inner_in, inner_out, tags=tags) + builder = self.__gen_llvm_apply(ctx, builder, params, state, inner_in, inner_out, output_type, tags=tags) return builder else: - return self.__gen_llvm_apply(ctx, builder, params, state, arg_in, arg_out, tags=tags) + return self.__gen_llvm_apply(ctx, builder, params, state, arg_in, arg_out, output_type, tags=tags) def apply_softmax(self, input_value, gain, output_type): # Modulate input_value by gain From cae1465b319539f17bb651dd7961ede88a7c4022 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Wed, 9 Nov 2022 02:05:23 -0500 Subject: [PATCH 055/127] llvm, functions/SoftMax: Implement compiled 'derivative' variant Signed-off-by: Jan Vesely --- .../nonstateful/transferfunctions.py | 45 +++++++++++++++++-- tests/functions/test_transfer.py | 6 +++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/psyneulink/core/components/functions/nonstateful/transferfunctions.py b/psyneulink/core/components/functions/nonstateful/transferfunctions.py index 0c863ff8c49..da3bfd300b0 100644 --- a/psyneulink/core/components/functions/nonstateful/transferfunctions.py +++ b/psyneulink/core/components/functions/nonstateful/transferfunctions.py @@ -2697,8 +2697,47 @@ def __gen_llvm_apply(self, ctx, builder, params, state, arg_in, arg_out, output_ return builder + def _gen_llvm_function_derivative_body(self, ctx, builder, params, state, arg_in, arg_out, *, tags:frozenset): + assert "derivative" in tags + forward_tags = tags.difference({"derivative"}) + all_out = builder.alloca(arg_out.type.pointee) + builder = self._gen_llvm_function_body(ctx, builder, params, state, arg_in, all_out, output_type=ALL, tags=forward_tags) + + max_pos_ptr = builder.alloca(ctx.int32_ty) + builder.store(max_pos_ptr.type.pointee(-1), max_pos_ptr) + max_val_ptr = builder.alloca(arg_out.type.pointee.element) + builder.store(max_val_ptr.type.pointee(float("NaN")), max_val_ptr) + + with pnlvm.helpers.array_ptr_loop(builder, all_out, id="max") as (b, idx): + val_ptr = b.gep(all_out, [ctx.int32_ty(0), idx]) + val = b.load(val_ptr) + max_val = b.load(max_val_ptr) + new_max = b.fcmp_unordered(">", val, max_val) + with b.if_then(new_max): + b.store(val, max_val_ptr) + b.store(idx, max_pos_ptr) + + max_val = builder.load(max_val_ptr) + max_pos = builder.load(max_pos_ptr) + + with pnlvm.helpers.array_ptr_loop(builder, all_out, id="derivative") as (b, idx): + val_ptr = b.gep(all_out, [ctx.int32_ty(0), idx]) + val = b.load(val_ptr) + is_max_pos = b.icmp_unsigned("==", idx, max_pos) + + d = b.select(is_max_pos, val.type(1), val.type(0)) + dv = b.fsub(d, max_val) + val = b.fmul(val, dv) + + out_ptr = b.gep(arg_out, [ctx.int32_ty(0), idx]) + b.store(val, out_ptr) + + return builder + def _gen_llvm_function_body(self, ctx, builder, params, state, arg_in, arg_out, output_type=None, *, tags:frozenset): output_type = self.output if output_type is None else output_type + if "derivative" in tags: + return self._gen_llvm_function_derivative_body(ctx, builder, params, state, arg_in, arg_out, tags=tags) if self.parameters.per_item.get(): assert isinstance(arg_in.type.pointee.element, pnlvm.ir.ArrayType) @@ -2781,8 +2820,8 @@ def derivative(self, input=None, output=None, context=None): """ output_type = self._get_current_parameter_value(OUTPUT_TYPE, context) - size = len(output) - sm = self.function(output, params={OUTPUT_TYPE: ALL}, context=context) + size = len(input) + sm = self.function(input, params={OUTPUT_TYPE: ALL}, context=context) sm = np.squeeze(sm) if output_type == ALL: @@ -2800,7 +2839,7 @@ def derivative(self, input=None, output=None, context=None): # Return 1d array of derivatives for max element (i.e., the one chosen by SoftMax) derivative = np.empty(size) # Get the element of output returned as non-zero when output_type is not ALL - index_of_max = int(np.where(output == np.max(output))[0][0]) + index_of_max = int(np.where(sm == np.max(sm))[0]) max_val = sm[index_of_max] for i in range(size): if i == index_of_max: diff --git a/tests/functions/test_transfer.py b/tests/functions/test_transfer.py index cd8b6061f09..66a3d0838ab 100644 --- a/tests/functions/test_transfer.py +++ b/tests/functions/test_transfer.py @@ -82,6 +82,12 @@ def test_execute(func, variable, params, expected, benchmark, func_mode): (Functions.Logistic, test_var, {'gain':RAND1, 'x_0':RAND2, 'offset':RAND3, 'scale':RAND4}, RAND1 * RAND4 * logistic_helper * (1 - logistic_helper)), (Functions.ReLU, test_var, {'gain':RAND1, 'bias':RAND2, 'leak':RAND3}, np.where((test_var - RAND2) > 0, RAND1, RAND1 * RAND3)), (Functions.Tanh, test_var, {'gain':RAND1, 'bias':RAND2, 'offset':RAND3, 'scale':RAND4}, tanh_derivative_helper), + (Functions.SoftMax, test_var, {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_VAL}, 'per_item': False}, + [-0.010680386821751537, -0.011118109698906909, -0.01082040340318878, -0.010670257514724047, -0.010362498859374309, + -0.010933660158663306, -0.010397412260182806, -0.011602329078808718, 0.09684744183944892, -0.010262384043848513]), + (Functions.SoftMax, test_var, {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_INDICATOR}, 'per_item': False}, + [-0.010680386821751537, -0.011118109698906909, -0.01082040340318878, -0.010670257514724047, -0.010362498859374309, + -0.010933660158663306, -0.010397412260182806, -0.011602329078808718, 0.09684744183944892, -0.010262384043848513]), ] @pytest.mark.function From 56d64f757d5e8e6b26982d0a0f78e15e16342a91 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Wed, 9 Nov 2022 12:06:05 -0500 Subject: [PATCH 056/127] tests/Buffer: Consolidate Buffer Function test Parametrize testing of 'noise' and 'rate'. Drop use of deque. Signed-off-by: Jan Vesely --- tests/functions/test_buffer.py | 110 ++++++++++----------------------- 1 file changed, 33 insertions(+), 77 deletions(-) diff --git a/tests/functions/test_buffer.py b/tests/functions/test_buffer.py index b088cf8a6a6..2c39dc4acc9 100644 --- a/tests/functions/test_buffer.py +++ b/tests/functions/test_buffer.py @@ -1,6 +1,5 @@ import numpy as np import pytest -from collections import deque from psyneulink.core.compositions.composition import Composition from psyneulink.core.components.functions.nonstateful.distributionfunctions import NormalDist @@ -13,92 +12,49 @@ class TestBuffer(): def test_buffer_standalone(self): B = Buffer() val = B.execute(1.0) - assert np.allclose(deque(np.atleast_1d(1.0)), val) + assert np.allclose(np.atleast_1d(1.0), val) @pytest.mark.benchmark(group="BufferFunction") - def test_buffer_standalone_rate_float(self, benchmark): - B = Buffer(history=3, rate = 0.1) - B.execute([1,2,3]) - B.execute([4,5,6]) - B.execute([7,8,9]) - val = B.execute([10,11,12]) - assert np.allclose(deque(np.atleast_1d([ 0.04, 0.05, 0.06], [ 0.7, 0.8, 0.9], [10, 11, 12])), val) - if benchmark.enabled: - benchmark(B.execute, [1, 2, 3]) - - @pytest.mark.benchmark(group="BufferFunction") - def test_buffer_standalone_rate_list(self, benchmark): - B = Buffer(history=3, rate = [0.1, 0.5, 0.9]) - B.execute([1,2,3]) - B.execute([4,5,6]) - B.execute([7,8,9]) - val = B.execute([10,11,12]) - assert np.allclose(deque(np.atleast_1d([ 0.04, 1.25, 4.86], [ 0.7, 4. , 8.1], [10, 11, 12])), val) - if benchmark.enabled: - benchmark(B.execute, [1, 2, 3]) - - @pytest.mark.benchmark(group="BufferFunction") - def test_buffer_standalone_rate_ndarray(self, benchmark): - B = Buffer(history=3, rate = np.array([0.1, 0.5, 0.9])) - B.execute([1,2,3]) - B.execute([4,5,6]) - B.execute([7,8,9]) - val = B.execute([10,11,12]) - assert np.allclose(deque(np.atleast_1d([ 0.04, 1.25, 4.86], [ 0.7, 4. , 8.1], [10, 11, 12])), val) - if benchmark.enabled: - benchmark(B.execute, [1, 2, 3]) - - @pytest.mark.benchmark(group="BufferFunction") - def test_buffer_standalone_noise_float(self, benchmark): - B = Buffer(history=3, rate = 1.0, noise=10.0) - B.execute([1,2,3]) - B.execute([4,5,6]) - B.execute([7,8,9]) - val = B.execute([10,11,12]) - assert np.allclose(deque(np.atleast_1d([ 24., 25., 26.], [ 17., 18., 19.], [10, 11, 12])), val) - if benchmark.enabled: - benchmark(B.execute, [1, 2, 3]) - - @pytest.mark.benchmark(group="BufferFunction") - def test_buffer_standalone_noise_list(self, benchmark): - B = Buffer(history=3, rate = 1.0, noise=[10.0, 20.0, 30.0]) - B.execute([1,2,3]) - B.execute([4,5,6]) - B.execute([7,8,9]) - val = B.execute([10,11,12]) - assert np.allclose(deque(np.atleast_1d([ 24., 45., 66.], [ 17., 28., 39.], [10, 11, 12])), val) - if benchmark.enabled: - benchmark(B.execute, [1, 2, 3]) - - @pytest.mark.benchmark(group="BufferFunction") - def test_buffer_standalone_noise_ndarray(self, benchmark): - B = Buffer(history=3, rate = 1.0, noise=[10.0, 20.0, 30.0]) - B.execute([1,2,3]) - B.execute([4,5,6]) - B.execute([7,8,9]) - val = B.execute([10,11,12]) - assert np.allclose(deque(np.atleast_1d([ 24., 45., 66.], [ 17., 28., 39.], [10, 11, 12])), val) + @pytest.mark.parametrize("rate, expected", + [ + (0.1, [[0.04, 0.05, 0.06], [0.7, 0.8, 0.9], [10, 11, 12]]), + ([0.1, 0.5, 0.9], [[0.04, 1.25, 4.86], [ 0.7, 4., 8.1], [10, 11, 12]]), + (np.array([0.1, 0.5, 0.9]), [[0.04, 1.25, 4.86], [ 0.7, 4., 8.1], [10, 11, 12]]), + ], ids=["float", "list", "ndarray"]) + def test_buffer_standalone_rate(self, benchmark, rate, expected): + B = Buffer(history=3, rate=rate) + B.execute([1, 2, 3]) + B.execute([4, 5, 6]) + B.execute([7, 8, 9]) + val = B.execute([10, 11, 12]) + assert np.allclose(expected, val) if benchmark.enabled: benchmark(B.execute, [1, 2, 3]) + @pytest.mark.parametrize("noise, expected", + [ + (10.0, [[ 24., 25., 26.], [17., 18., 19.], [10, 11, 12]]), + ([10.0, 20.0, 30.0], [[ 24., 45., 66.], [17., 28., 39.], [10, 11, 12]]), + (np.array([10.0, 20.0, 30.0]), [[ 24., 45., 66.], [17., 28., 39.], [10, 11, 12]]), + (NormalDist(seed=0, standard_deviation=0.1), [[4.02430687, 4.91927251, 5.95087965], + [7.09586966, 7.91823773, 8.86077491], + [10, 11, 12]]), + ], ids=["float", "list", "ndarray", "function"]) @pytest.mark.benchmark(group="BufferFunction") - def test_buffer_standalone_noise_function(self, benchmark): - B = Buffer(history=3, rate = 1.0, noise=NormalDist(standard_deviation=0.1)) + def test_buffer_standalone_noise_float(self, benchmark, noise, expected): + B = Buffer(history=3, rate=1.0, noise=noise) B.execute([1, 2, 3]) B.execute([4, 5, 6]) B.execute([7, 8, 9]) - val = B.execute([10,11,12]) - assert np.allclose(deque(np.atleast_1d([[4.02430687, 4.91927251, 5.95087965], - [7.09586966, 7.91823773, 8.86077491], - [10, 11, 12]])), val) + val = B.execute([10, 11, 12]) + assert np.allclose(expected, val) if benchmark.enabled: benchmark(B.execute, [1, 2, 3]) @pytest.mark.benchmark(group="BufferFunction") def test_buffer_standalone_noise_function_in_array(self, benchmark): B = Buffer(history=3) - # Set noise parameter ouside of a constructor to avoid problems - # with extra copying + # Set noise parameter outside of the constructor to avoid problems with extra copying B.parameters.noise.set([10, NormalDist(standard_deviation=0.1), 20]) B.execute([1, 2, 3]) B.execute([4, 5, 6]) @@ -120,8 +76,8 @@ def __call__(self): return self.count counter_f = CallCount() - # Set noise parameter ouside of a constructor to avoid problems - # with extra copying. This test fails if noise is passed to constructor + # Set noise parameter outside of the constructor to avoid problems with extra copying + # This test fails if noise is passed to constructor B = Buffer(history=3) B.parameters.noise.set([10, counter_f, 20]) B.execute([1, 2, 3]) @@ -140,9 +96,9 @@ def test_buffer_initializer_len_3(self, benchmark): B = Buffer(default_variable=[[0.0], [1.0], [2.0]], initializer=[[0.0], [1.0], [2.0]], history=3) - assert np.allclose(B.execute(3.0), deque([[1.0], [2.0], np.array([3.])])) - assert np.allclose(B.execute(4.0), deque([[2.0], np.array([3.]), np.array([4.])])) - assert np.allclose(B.execute(5.0), deque([np.array([3.]), np.array([4.]), np.array([5.])])) + assert np.allclose(B.execute(3.0), [[1.0], [2.0], np.array([3.])]) + assert np.allclose(B.execute(4.0), [[2.0], np.array([3.]), np.array([4.])]) + assert np.allclose(B.execute(5.0), [np.array([3.]), np.array([4.]), np.array([5.])]) if benchmark.enabled: benchmark(B.execute, 5.0) From fdc5d681dd69e2739f30bab53c1a676ce6b9b17b Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Wed, 9 Nov 2022 15:53:50 -0500 Subject: [PATCH 057/127] tests/Integrator: Use more than one iteration in test using stateful functions Signed-off-by: Jan Vesely --- tests/mechanisms/test_integrator_mechanism.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/mechanisms/test_integrator_mechanism.py b/tests/mechanisms/test_integrator_mechanism.py index 5369cb14f92..36a1197446c 100644 --- a/tests/mechanisms/test_integrator_mechanism.py +++ b/tests/mechanisms/test_integrator_mechanism.py @@ -442,8 +442,9 @@ def test_FitzHughNagumo_simple_scalar(self, benchmark, mech_mode): function=FitzHughNagumoIntegrator()) ex = pytest.helpers.get_mech_execution(I, mech_mode) + ex(var) val = ex(var) - assert np.allclose(val[0], [0.05127053]) + assert np.allclose(val, [[0.10501801629915011], [0.10501801629915011], [0.10501801629915011]]) if benchmark.enabled: benchmark(ex, var) @@ -457,8 +458,11 @@ def test_FitzHughNagumo_simple_vector(self, benchmark, mech_mode): function=FitzHughNagumoIntegrator) ex = pytest.helpers.get_mech_execution(I, mech_mode) + ex(var) val = ex(var) - assert np.allclose(val[0], [0.05127053, 0.15379818]) + assert np.allclose(val, [[[0.10501801629915011, 0.3151109244983909]], + [[0.10501801629915011, 0.3151109244983909]], + [[0.10501801629915011, 0.3151109244983909]]]) if benchmark.enabled: benchmark(ex, var) @@ -602,8 +606,9 @@ def test_integrator_no_function(self, benchmark, mech_mode): I = IntegratorMechanism() ex = pytest.helpers.get_mech_execution(I, mech_mode) + ex([10]) val = ex([10]) - assert np.allclose(val, [[5.0]]) + assert np.allclose(val, [[7.5]]) if benchmark.enabled: benchmark(ex, [10]) From 3577740c9f0539b013e593e671f7c44323d336d3 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Wed, 9 Nov 2022 17:05:38 -0500 Subject: [PATCH 058/127] tests/DDM: Run more than one iteration in tests that use stateful Functions Signed-off-by: Jan Vesely --- tests/mechanisms/test_ddm_mechanism.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/mechanisms/test_ddm_mechanism.py b/tests/mechanisms/test_ddm_mechanism.py index 44a7013eefd..223e0185a90 100644 --- a/tests/mechanisms/test_ddm_mechanism.py +++ b/tests/mechanisms/test_ddm_mechanism.py @@ -258,6 +258,7 @@ def test_DDM_Integrator_Bogacz(benchmark, mech_mode, prng): T.parameters.random_state.set(_SeededPhilox([0])) ex = pytest.helpers.get_mech_execution(T, mech_mode) + ex(stim) val = ex(stim)[0] assert np.allclose(val, [1.0]) if benchmark.enabled: @@ -291,9 +292,9 @@ def test_DDM_Integrator_Bogacz(benchmark, mech_mode, prng): @pytest.mark.mechanism @pytest.mark.benchmark(group="DDM") @pytest.mark.parametrize("noise, expected", [ - (0., 10), - (np.sqrt(0.5), 8.194383551861414), - (np.sqrt(2.0), 6.388767103722829), + (0., 20), + (np.sqrt(0.5), 18.40852795454561), + (np.sqrt(2.0), 16.817055909091223), ], ids=["0", "0.5", "2.0"]) def test_DDM_noise(mech_mode, benchmark, noise, expected): T = DDM( @@ -307,6 +308,7 @@ def test_DDM_noise(mech_mode, benchmark, noise, expected): ) ex = pytest.helpers.get_mech_execution(T, mech_mode) + ex([10]) val = ex([10]) assert np.allclose(val[0][0], expected) if benchmark.enabled: @@ -421,7 +423,7 @@ def test_DDM_input_fn(): @pytest.mark.mechanism @pytest.mark.benchmark(group="DDM") @pytest.mark.parametrize("rate, expected", [ - (5, 50), (5., 50), ([5], 50), (-5.0, -50), + (5, 100), (5., 100), ([5], 100), (-5.0, -100), ], ids=["int", "float", "list", "negative"]) # ****** # Should negative pass? @@ -439,6 +441,7 @@ def test_DDM_rate(benchmark, rate, expected, mech_mode): ) ex = pytest.helpers.get_mech_execution(T, mech_mode) + ex(stim) val = float(ex(stim)[0][0]) assert val == expected if benchmark.enabled: From eac14d2ed2d5523ce48c3d34eea878b1832806d0 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Wed, 9 Nov 2022 17:29:02 -0500 Subject: [PATCH 059/127] tests/RecurrentTransferMechanism: Consolidate test of different input formats Signed-off-by: Jan Vesely --- .../test_recurrent_transfer_mechanism.py | 54 +++++-------------- 1 file changed, 14 insertions(+), 40 deletions(-) diff --git a/tests/mechanisms/test_recurrent_transfer_mechanism.py b/tests/mechanisms/test_recurrent_transfer_mechanism.py index c8b9a96c329..78d114971b3 100644 --- a/tests/mechanisms/test_recurrent_transfer_mechanism.py +++ b/tests/mechanisms/test_recurrent_transfer_mechanism.py @@ -100,35 +100,24 @@ def test_recurrent_mech_check_proj_attrs(self): @pytest.mark.mechanism @pytest.mark.recurrent_transfer_mechanism @pytest.mark.benchmark(group="RecurrentTransferMechanism") - def test_recurrent_mech_inputs_list_of_ints(self, benchmark, mech_mode): - R = RecurrentTransferMechanism( - name='R', - default_variable=[0, 0, 0, 0] - ) + @pytest.mark.parametrize("variable, params", + [ + pytest.param(([10, 12, 0, -1], [1, 2, 3, 0]), {'size': 4}, id="list_of_ints"), + pytest.param(([1.0, 1.2, 0., -1.3], [1., 5., 3., 0.]), {'size': 4}, id="list_of_floats"), + pytest.param(([10], [10]), {}, id="no_init_params"), + ]) + def test_recurrent_mech_inputs(self, benchmark, params, variable, mech_mode): + R = RecurrentTransferMechanism(name='R', **params) EX = pytest.helpers.get_mech_execution(R, mech_mode) - val1 = EX([10, 12, 0, -1]) - val2 = EX([1, 2, 3, 0]) + val1 = EX(variable[0]) + val2 = benchmark(EX, variable[1]) # The outputs match inputs because recurrent projection is - # not used when executing: mech is reset each time - np.testing.assert_allclose(val1, [[10.0, 12.0, 0, -1]]) - np.testing.assert_allclose(val2, [[1, 2, 3, 0]]) - if benchmark.enabled: - benchmark(EX, [1, 2, 3, 0]) - - @pytest.mark.mechanism - @pytest.mark.recurrent_transfer_mechanism - @pytest.mark.benchmark(group="RecurrentTransferMechanism") - def test_recurrent_mech_inputs_list_of_floats(self, benchmark, mech_mode): - R = RecurrentTransferMechanism( - name='R', - size=4 - ) - EX = pytest.helpers.get_mech_execution(R, mech_mode) - - val = benchmark(EX, [10.0, 10.0, 10.0, 10.0]) - np.testing.assert_allclose(val, [[10.0, 10.0, 10.0, 10.0]]) + # not used when executing standalone mechanism: + # the mechanism is reset each time + np.testing.assert_allclose(val1, [variable[0]]) + np.testing.assert_allclose(val2, [variable[1]]) @pytest.mark.mechanism @pytest.mark.recurrent_transfer_mechanism @@ -191,21 +180,6 @@ def test_recurrent_mech_lci(self, benchmark, mech_mode): # for i in range(len(val[0])): # np.testing.assert_allclose(val[0][i], expected[0][i]) - @pytest.mark.mechanism - @pytest.mark.recurrent_transfer_mechanism - @pytest.mark.benchmark(group="RecurrentTransferMechanism") - def test_recurrent_mech_no_inputs(self, benchmark, mech_mode): - R = RecurrentTransferMechanism( - name='R' - ) - np.testing.assert_allclose(R.defaults.variable, [[0]]) - EX = pytest.helpers.get_mech_execution(R, mech_mode) - - val = EX([10]) - np.testing.assert_allclose(val, [[10.]]) - if benchmark.enabled: - benchmark(EX, [1]) - def test_recurrent_mech_inputs_list_of_strings(self): with pytest.raises(MechanismError) as error_text: R = RecurrentTransferMechanism( From 10aef59cd6107c74c7942ee4bb1802a957917247 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Wed, 9 Nov 2022 18:37:01 -0500 Subject: [PATCH 060/127] tests/TransferMechanism: Consolidate testing of different mechanism functions Logistic, ReLU, Exponential, and SoftMax all follow the same pattern. Signed-off-by: Jan Vesely --- tests/mechanisms/test_transfer_mechanism.py | 94 +++++---------------- 1 file changed, 21 insertions(+), 73 deletions(-) diff --git a/tests/mechanisms/test_transfer_mechanism.py b/tests/mechanisms/test_transfer_mechanism.py index f9b98489291..f089d6c20b9 100644 --- a/tests/mechanisms/test_transfer_mechanism.py +++ b/tests/mechanisms/test_transfer_mechanism.py @@ -229,8 +229,8 @@ def test_transfer_mech_mismatched_shape_noise(self): integrator_mode=True ) T.execute() - assert 'Noise parameter' in str(error_text.value) and "does not match default variable" in str( - error_text.value) + assert 'Noise parameter' in str(error_text.value) + assert "does not match default variable" in str(error_text.value) @pytest.mark.mechanism @pytest.mark.transfer_mechanism @@ -412,87 +412,35 @@ def sum_all_elements(variable): @pytest.mark.mechanism @pytest.mark.transfer_mechanism @pytest.mark.benchmark(group="TransferMechanism Logistic") - def test_transfer_mech_logistic_fun(self, benchmark, mech_mode): + @pytest.mark.parametrize("func,variables,expected", + [ + # Operations on vector elements are independent so we only provide one value + pytest.param(Logistic, [0], [0.5], id="Logistic"), + pytest.param(ReLU, [0, 1, -1], [0., 1, 0.], id="ReLU"), + pytest.param(Exponential, [0, 1, -1], [1., 2.71828183, 0.36787944], id="Exponential"), + pytest.param(SoftMax, [0, 1, -1], [1. / VECTOR_SIZE, 1. / VECTOR_SIZE, 1. / VECTOR_SIZE], id="SoftMax"), + ]) + def test_transfer_mech_func(self, benchmark, func, variables, expected, mech_mode): T = TransferMechanism( name='T', - default_variable=[0 for i in range(VECTOR_SIZE)], - function=Logistic(), + default_variable=np.zeros(VECTOR_SIZE), + function=func, integration_rate=1.0, integrator_mode=True ) EX = pytest.helpers.get_mech_execution(T, mech_mode) - var = [0 for i in range(VECTOR_SIZE)] - val = EX(var) - assert np.allclose(val, [[0.5 for i in range(VECTOR_SIZE)]]) - if benchmark.enabled: - benchmark(EX, var) - - @pytest.mark.mechanism - @pytest.mark.transfer_mechanism - @pytest.mark.benchmark(group="TransferMechanism ReLU") - def test_transfer_mech_relu_fun(self, benchmark, mech_mode): - - T = TransferMechanism( - name='T', - default_variable=[0 for i in range(VECTOR_SIZE)], - function=ReLU(), - integration_rate=1.0, - integrator_mode=True - ) - EX = pytest.helpers.get_mech_execution(T, mech_mode) - - val1 = EX([0 for i in range(VECTOR_SIZE)]) - val2 = EX([1 for i in range(VECTOR_SIZE)]) - val3 = EX([-1 for i in range(VECTOR_SIZE)]) - - assert np.allclose(val1, [[0.0 for i in range(VECTOR_SIZE)]]) - assert np.allclose(val2, [[1.0 for i in range(VECTOR_SIZE)]]) - assert np.allclose(val3, [[0.0 for i in range(VECTOR_SIZE)]]) + vals = [] + for var in variables[:-1]: + vals.append(EX([var] * VECTOR_SIZE)) + vals.append(EX([variables[-1]] * VECTOR_SIZE)) + assert len(vals) == len(expected) + for val, exp in zip(vals, expected): + assert np.allclose(val, [[exp]] * VECTOR_SIZE) if benchmark.enabled: - benchmark(EX, [0 for i in range(VECTOR_SIZE)]) - - @pytest.mark.mechanism - @pytest.mark.transfer_mechanism - @pytest.mark.benchmark(group="TransferMechanism Exponential") - def test_transfer_mech_exponential_fun(self, benchmark, mech_mode): - - T = TransferMechanism( - name='T', - default_variable=[0 for i in range(VECTOR_SIZE)], - function=Exponential(), - integration_rate=1.0, - integrator_mode=True - ) - EX = pytest.helpers.get_mech_execution(T, mech_mode) - - var = [0 for i in range(VECTOR_SIZE)] - val = EX(var) - assert np.allclose(val, [[1.0 for i in range(VECTOR_SIZE)]]) - if benchmark.enabled: - benchmark(EX, var) - - @pytest.mark.mechanism - @pytest.mark.transfer_mechanism - @pytest.mark.benchmark(group="TransferMechanism SoftMax") - def test_transfer_mech_softmax_fun(self, benchmark, mech_mode): - - T = TransferMechanism( - name='T', - default_variable=[0 for i in range(VECTOR_SIZE)], - function=SoftMax(), - integration_rate=1.0, - integrator_mode=True - ) - EX = pytest.helpers.get_mech_execution(T, mech_mode) - - var = [0 for i in range(VECTOR_SIZE)] - val = EX(var) - assert np.allclose(val, [[1.0 / VECTOR_SIZE for i in range(VECTOR_SIZE)]]) - if benchmark.enabled: - benchmark(EX, var) + benchmark(EX, variables[0]) @pytest.mark.mechanism @pytest.mark.transfer_mechanism From 58f02e06dc890b73655b38a5935224248c5b51cb Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Wed, 9 Nov 2022 19:44:25 -0500 Subject: [PATCH 061/127] tests/TransferMechanism: Run more than one iteration in tests that use integrator mode Use rate other than 1 so observe the effect of integrator_mode = True. Signed-off-by: Jan Vesely --- tests/mechanisms/test_transfer_mechanism.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/mechanisms/test_transfer_mechanism.py b/tests/mechanisms/test_transfer_mechanism.py index f089d6c20b9..fad6f1e21ea 100644 --- a/tests/mechanisms/test_transfer_mechanism.py +++ b/tests/mechanisms/test_transfer_mechanism.py @@ -54,15 +54,16 @@ def test_transfer_mech_inputs_list_of_floats(self, benchmark, mech_mode): T = TransferMechanism( name='T', default_variable=[0 for i in range(VECTOR_SIZE)], - integration_rate=1.0, + integration_rate=0.5, integrator_mode=True ) T.reset_stateful_function_when = Never() var = [10.0 for i in range(VECTOR_SIZE)] EX = pytest.helpers.get_mech_execution(T, mech_mode) + EX(var) val = EX(var) - assert np.allclose(val, [[10.0 for i in range(VECTOR_SIZE)]]) + assert np.allclose(val, [[7.5 for i in range(VECTOR_SIZE)]]) if benchmark.enabled: benchmark(EX, var) @@ -149,15 +150,16 @@ def test_transfer_mech_array_var_float_noise(self, benchmark, mech_mode): default_variable=[0 for i in range(VECTOR_SIZE)], function=Linear(), noise=5.0, - integration_rate=1.0, + integration_rate=0.5, integrator_mode=True ) T.reset_stateful_function_when = Never() EX = pytest.helpers.get_mech_execution(T, mech_mode) - var = [0 for i in range(VECTOR_SIZE)] + var = [1 for i in range(VECTOR_SIZE)] + EX(var) val = EX(var) - assert np.allclose(val, [[5.0 for i in range(VECTOR_SIZE)]]) + assert np.allclose(val, [[8.25 for i in range(VECTOR_SIZE)]]) if benchmark.enabled: benchmark(EX, var) @@ -203,16 +205,17 @@ def test_transfer_mech_array_var_normal_array_noise2(self, benchmark, mech_mode) name='T', default_variable=[0 for i in range(VECTOR_SIZE)], function=Linear(), - noise=[5.0 for i in range(VECTOR_SIZE)], - integration_rate=1.0, + noise=[5.0 + i for i in range(VECTOR_SIZE)], + integration_rate=0.3, integrator_mode=True ) T.reset_stateful_function_when = Never() EX = pytest.helpers.get_mech_execution(T, mech_mode) var = [0 for i in range(VECTOR_SIZE)] + EX(var) val = EX(var) - assert np.allclose(val, [[5.0 for i in range(VECTOR_SIZE)]]) + assert np.allclose(val, [[8.5 + (i * 1.7) for i in range(VECTOR_SIZE)]]) if benchmark.enabled: benchmark(EX, var) From 940319e84221272356778e304e999e8ae4f655e1 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Wed, 9 Nov 2022 22:34:08 -0500 Subject: [PATCH 062/127] tests/composition: Use 'num_trials' instead of manually calling 'execute' Use comp_mode instead of manulyl listing supported modes (not LLVMRun, PTXRun for 'execute') Signed-off-by: Jan Vesely --- tests/composition/test_composition.py | 93 +++++++++------------------ 1 file changed, 30 insertions(+), 63 deletions(-) diff --git a/tests/composition/test_composition.py b/tests/composition/test_composition.py index 1120e94d108..b614616c13e 100644 --- a/tests/composition/test_composition.py +++ b/tests/composition/test_composition.py @@ -3928,12 +3928,7 @@ def test_run_recurrent_transfer_mechanism(self, benchmark, comp_mode): @pytest.mark.composition @pytest.mark.benchmark(group="Recurrent") - @pytest.mark.parametrize("mode", [pnl.ExecutionMode.Python, - pytest.param(pnl.ExecutionMode.LLVM, marks=pytest.mark.llvm), - pytest.param(pnl.ExecutionMode.LLVMExec, marks=pytest.mark.llvm), - pytest.param(pnl.ExecutionMode.PTXExec, marks=[pytest.mark.llvm, pytest.mark.cuda]), - ]) - def test_run_recurrent_transfer_mechanism_hetero(self, benchmark, mode): + def test_run_recurrent_transfer_mechanism_hetero(self, benchmark, comp_mode): comp = Composition() R = RecurrentTransferMechanism(size=1, function=Logistic(), @@ -3942,28 +3937,21 @@ def test_run_recurrent_transfer_mechanism_hetero(self, benchmark, mode): comp.add_node(R) comp._analyze_graph() sched = Scheduler(composition=comp) - val = comp.execute(inputs={R: [[3.0]]}, execution_mode=mode) + val = comp.run(inputs={R: [[3.0]]}, num_trials=1, execution_mode=comp_mode) assert np.allclose(val, [[0.95257413]]) - val = comp.execute(inputs={R: [[4.0]]}, execution_mode=mode) + val = comp.run(inputs={R: [[4.0]]}, num_trials=1, execution_mode=comp_mode) assert np.allclose(val, [[0.98201379]]) - # execute 10 times - for i in range(10): - val = comp.execute(inputs={R: [[5.0]]}, execution_mode=mode) - + # execute 10 trials + val = comp.run(inputs={R: [[5.0]]}, num_trials=10, execution_mode=comp_mode) assert np.allclose(val, [[0.99330715]]) if benchmark.enabled: - benchmark(comp.execute, inputs={R: [[1.0]]}, execution_mode=mode) + benchmark(comp.run, inputs={R: [[1.0]]}, execution_mode=comp_mode) @pytest.mark.composition @pytest.mark.benchmark(group="Recurrent") - @pytest.mark.parametrize("mode", [pnl.ExecutionMode.Python, - pytest.param(pnl.ExecutionMode.LLVM, marks=pytest.mark.llvm), - pytest.param(pnl.ExecutionMode.LLVMExec, marks=pytest.mark.llvm), - pytest.param(pnl.ExecutionMode.PTXExec, marks=[pytest.mark.llvm, pytest.mark.cuda]), - ]) - def test_run_recurrent_transfer_mechanism_integrator(self, benchmark, mode): + def test_run_recurrent_transfer_mechanism_integrator(self, benchmark, comp_mode): comp = Composition() R = RecurrentTransferMechanism(size=1, function=Logistic(), @@ -3974,55 +3962,42 @@ def test_run_recurrent_transfer_mechanism_integrator(self, benchmark, mode): comp.add_node(R) comp._analyze_graph() sched = Scheduler(composition=comp) - val = comp.execute(inputs={R: [[3.0]]}, execution_mode=mode) + val = comp.run(inputs={R: [[3.0]]}, num_trials=1, execution_mode=comp_mode) assert np.allclose(val, [[0.50749944]]) - val = comp.execute(inputs={R: [[4.0]]}, execution_mode=mode) + val = comp.run(inputs={R: [[4.0]]}, num_trials=1, execution_mode=comp_mode) assert np.allclose(val, [[0.51741795]]) - # execute 10 times - for i in range(10): - val = comp.execute(inputs={R: [[5.0]]}, execution_mode=mode) - + # execute 10 trials + val = comp.run(inputs={R: [[5.0]]}, num_trials=10, execution_mode=comp_mode) assert np.allclose(val, [[0.6320741]]) if benchmark.enabled: - benchmark(comp.execute, inputs={R: [[1.0]]}, execution_mode=mode) + benchmark(comp.run, inputs={R: [[1.0]]}, execution_mode=comp_mode) @pytest.mark.composition @pytest.mark.benchmark(group="Recurrent") - @pytest.mark.parametrize("mode", [pnl.ExecutionMode.Python, - pytest.param(pnl.ExecutionMode.LLVM, marks=pytest.mark.llvm), - pytest.param(pnl.ExecutionMode.LLVMExec, marks=pytest.mark.llvm), - pytest.param(pnl.ExecutionMode.PTXExec, marks=[pytest.mark.llvm, pytest.mark.cuda]), - ]) - def test_run_recurrent_transfer_mechanism_vector_2(self, benchmark, mode): + def test_run_recurrent_transfer_mechanism_vector_2(self, benchmark, comp_mode): comp = Composition() R = RecurrentTransferMechanism(size=2, function=Logistic()) comp.add_node(R) comp._analyze_graph() sched = Scheduler(composition=comp) - val = comp.execute(inputs={R: [[1.0, 2.0]]}, execution_mode=mode) + val = comp.run(inputs={R: [[1.0, 2.0]]}, num_trials=1, execution_mode=comp_mode) assert np.allclose(val, [[0.81757448, 0.92414182]]) - val = comp.execute(inputs={R: [[1.0, 2.0]]}, execution_mode=mode) + val = comp.run(inputs={R: [[1.0, 2.0]]}, num_trials=1, execution_mode=comp_mode) assert np.allclose(val, [[0.87259959, 0.94361816]]) - # execute 10 times - for i in range(10): - val = comp.execute(inputs={R: [[1.0, 2.0]]}, execution_mode=mode) + # execute 10 trials + val = comp.run(inputs={R: [[1.0, 2.0]]}, num_trials=10, execution_mode=comp_mode) assert np.allclose(val, [[0.87507549, 0.94660049]]) if benchmark.enabled: - benchmark(comp.execute, inputs={R: [[1.0, 2.0]]}, execution_mode=mode) + benchmark(comp.run, inputs={R: [[1.0, 2.0]]}, execution_mode=comp_mode) @pytest.mark.composition @pytest.mark.benchmark(group="Recurrent") - @pytest.mark.parametrize("mode", [pnl.ExecutionMode.Python, - pytest.param(pnl.ExecutionMode.LLVM, marks=pytest.mark.llvm), - pytest.param(pnl.ExecutionMode.LLVMExec, marks=pytest.mark.llvm), - pytest.param(pnl.ExecutionMode.PTXExec, marks=[pytest.mark.llvm, pytest.mark.cuda]), - ]) - def test_run_recurrent_transfer_mechanism_hetero_2(self, benchmark, mode): + def test_run_recurrent_transfer_mechanism_hetero_2(self, benchmark, comp_mode): comp = Composition() R = RecurrentTransferMechanism(size=2, function=Logistic(), @@ -4031,28 +4006,21 @@ def test_run_recurrent_transfer_mechanism_hetero_2(self, benchmark, mode): comp.add_node(R) comp._analyze_graph() sched = Scheduler(composition=comp) - val = comp.execute(inputs={R: [[1.0, 2.0]]}, execution_mode=mode) + val = comp.run(inputs={R: [[1.0, 2.0]]}, num_trials=1, execution_mode=comp_mode) assert np.allclose(val, [[0.5, 0.73105858]]) - val = comp.execute(inputs={R: [[1.0, 2.0]]}, execution_mode=mode) + val = comp.run(inputs={R: [[1.0, 2.0]]}, num_trials=1, execution_mode=comp_mode) assert np.allclose(val, [[0.3864837, 0.73105858]]) - # execute 10 times - for i in range(10): - val = comp.execute(inputs={R: [[1.0, 2.0]]}, execution_mode=mode) - + # execute 10 trials + val = comp.run(inputs={R: [[1.0, 2.0]]}, num_trials=10, execution_mode=comp_mode) assert np.allclose(val, [[0.36286875, 0.78146724]]) if benchmark.enabled: - benchmark(comp.execute, inputs={R: [[1.0, 2.0]]}, execution_mode=mode) + benchmark(comp.run, inputs={R: [[1.0, 2.0]]}, execution_mode=comp_mode) @pytest.mark.composition @pytest.mark.benchmark(group="Recurrent") - @pytest.mark.parametrize("mode", [pnl.ExecutionMode.Python, - pytest.param(pnl.ExecutionMode.LLVM, marks=pytest.mark.llvm), - pytest.param(pnl.ExecutionMode.LLVMExec, marks=pytest.mark.llvm), - pytest.param(pnl.ExecutionMode.PTXExec, marks=[pytest.mark.llvm, pytest.mark.cuda]), - ]) - def test_run_recurrent_transfer_mechanism_integrator_2(self, benchmark, mode): + def test_run_recurrent_transfer_mechanism_integrator_2(self, benchmark, comp_mode): comp = Composition() R = RecurrentTransferMechanism(size=2, function=Logistic(), @@ -4063,19 +4031,18 @@ def test_run_recurrent_transfer_mechanism_integrator_2(self, benchmark, mode): comp.add_node(R) comp._analyze_graph() sched = Scheduler(composition=comp) - val = comp.execute(inputs={R: [[1.0, 2.0]]}, execution_mode=mode) + val = comp.run(inputs={R: [[1.0, 2.0]]}, num_trials=1, execution_mode=comp_mode) assert np.allclose(val, [[0.5, 0.50249998]]) - val = comp.execute(inputs={R: [[1.0, 2.0]]}, execution_mode=mode) + val = comp.run(inputs={R: [[1.0, 2.0]]}, num_trials=1, execution_mode=comp_mode) assert np.allclose(val, [[0.4999875, 0.50497484]]) - # execute 10 times - for i in range(10): - val = comp.execute(inputs={R: [[1.0, 2.0]]}, execution_mode=mode) + # execute 10 trials + val = comp.run(inputs={R: [[1.0, 2.0]]}, num_trials=10, execution_mode=comp_mode) assert np.allclose(val, [[0.49922843, 0.52838607]]) if benchmark.enabled: - benchmark(comp.execute, inputs={R: [[1.0, 2.0]]}, execution_mode=mode) + benchmark(comp.run, inputs={R: [[1.0, 2.0]]}, execution_mode=comp_mode) def test_run_termination_condition_custom_context(self): D = pnl.DDM(function=pnl.DriftDiffusionIntegrator, execute_until_finished=False) From 0732440b12b7f7e3ef74fe5f291707cf6a5a8ab7 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Wed, 9 Nov 2022 14:53:26 -0500 Subject: [PATCH 063/127] tests: Overload 'benchmark' fixture with custom invocation Returns the result of the first iteration instead of the last when benchmarking. Signed-off-by: Jan Vesely --- conftest.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/conftest.py b/conftest.py index 94a4de81cc4..3d9c8ec1d91 100644 --- a/conftest.py +++ b/conftest.py @@ -112,6 +112,29 @@ def comp_mode_no_llvm(): # dummy fixture to allow 'comp_mode' filtering pass +class FirstBench(): + def __init__(self, benchmark): + super().__setattr__("benchmark", benchmark) + + def __call__(self, f, *args, **kwargs): + res = [] + # Compute the first result if benchmark is enabled + if self.benchmark.enabled: + res.append(f(*args, **kwargs)) + + res.append(self.benchmark(f, *args, **kwargs)) + return res[0] + + def __getattr__(self, attr): + return getattr(self.benchmark, attr) + + def __setattr__(self, attr, val): + return setattr(self.benchmark, attr, val) + +@pytest.fixture +def benchmark(benchmark): + return FirstBench(benchmark) + @pytest.helpers.register def llvm_current_fp_precision(): float_ty = pnlvm.LLVMBuilderContext.get_current().float_ty From 4bf42150a6c0dc34c817f60ceb23fda2d6d5e9f5 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Wed, 9 Nov 2022 14:55:07 -0500 Subject: [PATCH 064/127] tests/functions: Use the benchmark fixture unconditionally Signed-off-by: Jan Vesely --- tests/functions/test_buffer.py | 22 +++++++--------------- tests/functions/test_distribution.py | 5 +---- tests/functions/test_fhn_integrator.py | 5 +---- tests/functions/test_integrator.py | 5 +---- tests/functions/test_memory.py | 4 +--- tests/functions/test_optimization.py | 5 +---- tests/functions/test_selection.py | 4 +--- tests/functions/test_transfer.py | 4 +--- 8 files changed, 14 insertions(+), 40 deletions(-) diff --git a/tests/functions/test_buffer.py b/tests/functions/test_buffer.py index 2c39dc4acc9..e3e8c4bb2c1 100644 --- a/tests/functions/test_buffer.py +++ b/tests/functions/test_buffer.py @@ -26,10 +26,8 @@ def test_buffer_standalone_rate(self, benchmark, rate, expected): B.execute([1, 2, 3]) B.execute([4, 5, 6]) B.execute([7, 8, 9]) - val = B.execute([10, 11, 12]) + val = benchmark(B.execute, [10, 11, 12]) assert np.allclose(expected, val) - if benchmark.enabled: - benchmark(B.execute, [1, 2, 3]) @pytest.mark.parametrize("noise, expected", [ @@ -46,10 +44,8 @@ def test_buffer_standalone_noise_float(self, benchmark, noise, expected): B.execute([1, 2, 3]) B.execute([4, 5, 6]) B.execute([7, 8, 9]) - val = B.execute([10, 11, 12]) + val = benchmark(B.execute, [10, 11, 12]) assert np.allclose(expected, val) - if benchmark.enabled: - benchmark(B.execute, [1, 2, 3]) @pytest.mark.benchmark(group="BufferFunction") def test_buffer_standalone_noise_function_in_array(self, benchmark): @@ -59,13 +55,11 @@ def test_buffer_standalone_noise_function_in_array(self, benchmark): B.execute([1, 2, 3]) B.execute([4, 5, 6]) B.execute([7, 8, 9]) - val = B.execute([10, 11, 12]) + val = benchmark(B.execute, [10, 11, 12]) expected_val = [[24, 4.693117564500052, 46], [17, 7.744647273059847, 29], [10, 11, 12]] for v_v, v_e in zip(val, expected_val): for v, e in zip(v_v, v_e): assert np.allclose(v, e) - if benchmark.enabled: - benchmark(B.execute, [1, 2, 3]) def test_buffer_standalone_noise_function_invocation(self): class CallCount: @@ -98,9 +92,8 @@ def test_buffer_initializer_len_3(self, benchmark): history=3) assert np.allclose(B.execute(3.0), [[1.0], [2.0], np.array([3.])]) assert np.allclose(B.execute(4.0), [[2.0], np.array([3.]), np.array([4.])]) - assert np.allclose(B.execute(5.0), [np.array([3.]), np.array([4.]), np.array([5.])]) - if benchmark.enabled: - benchmark(B.execute, 5.0) + val = benchmark(B.execute, 5.0) + assert np.allclose(val, [np.array([3.]), np.array([4.]), np.array([5.])]) @pytest.mark.benchmark(group="BufferFunction") def test_buffer_as_function_of_processing_mech(self, benchmark): @@ -108,12 +101,11 @@ def test_buffer_as_function_of_processing_mech(self, benchmark): P = ProcessingMechanism(function=Buffer(default_variable=[[0.0]], initializer=[0.0], history=3)) - val = P.execute(1.0) + val = benchmark(P.execute, 1.0) # NOTE: actual output is [0, [[1]]] assert np.allclose(np.asfarray(val), [[0., 1.]]) - if benchmark.enabled: - benchmark(P.execute, 5.0) + # fails due to value and variable problems when Buffer is the function of a mechanism # P = ProcessingMechanism(function=Buffer(default_variable=[[0.0], [1.0], [2.0]], # initializer=[[0.0], [1.0], [2.0]], diff --git a/tests/functions/test_distribution.py b/tests/functions/test_distribution.py index 2b0d111d2c3..1075490b6b4 100644 --- a/tests/functions/test_distribution.py +++ b/tests/functions/test_distribution.py @@ -122,12 +122,9 @@ def test_execute(func, variable, params, prng, llvm_skip, expected, benchmark, f f.parameters.random_state.set(prng([0])) ex = pytest.helpers.get_func_execution(f, func_mode) - res = ex(variable) + res = benchmark(ex, variable) if pytest.helpers.llvm_current_fp_precision() == 'fp32': assert np.allclose(res, expected) else: np.testing.assert_allclose(res, expected) - - if benchmark.enabled: - benchmark(ex, variable) diff --git a/tests/functions/test_fhn_integrator.py b/tests/functions/test_fhn_integrator.py index 445ec8d9697..d117ed40d16 100644 --- a/tests/functions/test_fhn_integrator.py +++ b/tests/functions/test_fhn_integrator.py @@ -56,11 +56,8 @@ def test_basic(func, variable, integration_method, params, expected, benchmark, res = EX(variable) res = EX(variable) - res = EX(variable) + res = benchmark(EX, variable) assert np.allclose(res[0], expected[0]) assert np.allclose(res[1], expected[1]) assert np.allclose(res[2], expected[2]) - - if benchmark.enabled: - benchmark(EX, variable) diff --git a/tests/functions/test_integrator.py b/tests/functions/test_integrator.py index 30771433a4f..ff007c6afe3 100644 --- a/tests/functions/test_integrator.py +++ b/tests/functions/test_integrator.py @@ -191,14 +191,11 @@ def test_execute(func, func_mode, variable, noise, params, benchmark): ex(variable) ex(variable) - res = ex(variable) + res = benchmark(ex, variable) expected = func[1](f.initializer, variable, 3, noise, **params) for r, e in zip(res, expected): assert np.allclose(r, e) - if benchmark.enabled: - benchmark(ex, variable) - def test_integrator_function_no_default_variable_and_params_len_more_than_1(): I = Functions.AdaptiveIntegrator(rate=[.1, .2, .3]) diff --git a/tests/functions/test_memory.py b/tests/functions/test_memory.py index 92d736fda8a..3eefdcc0b14 100644 --- a/tests/functions/test_memory.py +++ b/tests/functions/test_memory.py @@ -149,11 +149,9 @@ def test_basic(func, variable, params, expected, benchmark, func_mode): EX = pytest.helpers.get_func_execution(f, func_mode) EX(variable) - res = EX(variable) + res = benchmark(EX, variable) assert np.allclose(res[0], expected[0]) assert np.allclose(res[1], expected[1]) - if benchmark.enabled: - benchmark(EX, variable) #endregion diff --git a/tests/functions/test_optimization.py b/tests/functions/test_optimization.py index 2069c2b2356..6f4669f9766 100644 --- a/tests/functions/test_optimization.py +++ b/tests/functions/test_optimization.py @@ -82,13 +82,10 @@ def test_grid_search(obj_func, metric, normalize, direction, selection, benchmar seed=0, save_values=False) EX = pytest.helpers.get_func_execution(f, func_mode) - res = EX(variable) + res = benchmark(EX, variable) assert np.allclose(res[0], result[0]) assert np.allclose(res[1], result[1]) if func_mode == 'Python': assert np.allclose(res[2], result[2]) assert np.allclose(res[3], result[3]) - - if benchmark.enabled: - benchmark(EX, variable) diff --git a/tests/functions/test_selection.py b/tests/functions/test_selection.py index 8fe21b1c5b2..8fc4f3c6408 100644 --- a/tests/functions/test_selection.py +++ b/tests/functions/test_selection.py @@ -76,8 +76,6 @@ def test_basic(func, variable, params, expected, benchmark, func_mode): EX = pytest.helpers.get_func_execution(f, func_mode) EX(variable) - res = EX(variable) + res = benchmark(EX, variable) assert np.allclose(res, expected) - if benchmark.enabled: - benchmark(EX, variable) diff --git a/tests/functions/test_transfer.py b/tests/functions/test_transfer.py index e0c0066295e..d0230987ea9 100644 --- a/tests/functions/test_transfer.py +++ b/tests/functions/test_transfer.py @@ -70,10 +70,8 @@ def test_execute(func, variable, params, expected, benchmark, func_mode): f = func(default_variable=variable, **params) ex = pytest.helpers.get_func_execution(f, func_mode) - res = ex(variable) + res = benchmark(ex, variable) assert np.allclose(res, expected) - if benchmark.enabled: - benchmark(ex, variable) logistic_helper = RAND4 / (1 + np.exp(-(RAND1 * (test_var - RAND2)) + RAND3)) From 3d573291ea78bf213b99a95a66aa28d6018207b9 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Wed, 9 Nov 2022 19:05:11 -0500 Subject: [PATCH 065/127] tests/mechanisms: Use the benchmark fixture unconditionally Signed-off-by: Jan Vesely --- tests/mechanisms/test_control_mechanism.py | 5 +- tests/mechanisms/test_ddm_mechanism.py | 32 ++++------ tests/mechanisms/test_episodic_memory.py | 4 +- tests/mechanisms/test_integrator_mechanism.py | 24 ++------ tests/mechanisms/test_lca.py | 18 +++--- .../test_recurrent_transfer_mechanism.py | 17 +++--- tests/mechanisms/test_transfer_mechanism.py | 61 ++++++------------- 7 files changed, 53 insertions(+), 108 deletions(-) diff --git a/tests/mechanisms/test_control_mechanism.py b/tests/mechanisms/test_control_mechanism.py index 365825fa6a4..1c2ff3405ec 100644 --- a/tests/mechanisms/test_control_mechanism.py +++ b/tests/mechanisms/test_control_mechanism.py @@ -84,16 +84,13 @@ def test_lc_control_mech_basic(self, benchmark, mech_mode): ) EX = pytest.helpers.get_mech_execution(LC, mech_mode) - val = EX([10.0]) + val = benchmark(EX, [10.0]) # All values are the same because LCControlMechanism assigns all of its ControlSignals to the same value # (the 1st item of its function's value). # FIX: 6/6/19 - Python returns 3d array but LLVM returns 2d array # (np.allclose bizarrely passes for LLVM because all the values are the same) assert np.allclose(val, [[[3.00139776]], [[3.00139776]], [[3.00139776]], [[3.00139776]]]) - if benchmark.enabled: - benchmark(EX, [10.0]) - @pytest.mark.composition def test_lc_control_modulated_mechanisms_all(self): diff --git a/tests/mechanisms/test_ddm_mechanism.py b/tests/mechanisms/test_ddm_mechanism.py index 223e0185a90..6f39c04d19f 100644 --- a/tests/mechanisms/test_ddm_mechanism.py +++ b/tests/mechanisms/test_ddm_mechanism.py @@ -126,18 +126,15 @@ def test_threshold_stops_accumulation(self, mech_mode, variable, expected, bench decision_variables = [] time_points = [] - for i in range(5): - output = ex([variable]) - decision_variables.append(output[0][0]) - time_points.append(output[1][0]) + results = [] + for i in range(4): + results.append(ex([variable])) - # decision variable accumulation stops - assert np.allclose(decision_variables, expected) + results.append(benchmark(ex,[variable])) + # decision variable accumulation stops # time accumulation does not stop - assert np.allclose(time_points, [1.0, 2.0, 3.0, 4.0, 5.0]) - if benchmark.enabled: - benchmark(ex, [variable]) + assert np.allclose(results, [[[b], [a + 1.0]] for a,b in enumerate(expected)]) # def test_threshold_stops_accumulation_multiple_variables(self): # D = IntegratorMechanism(name='DDM', @@ -259,10 +256,8 @@ def test_DDM_Integrator_Bogacz(benchmark, mech_mode, prng): ex = pytest.helpers.get_mech_execution(T, mech_mode) ex(stim) - val = ex(stim)[0] + val = benchmark(ex, stim)[0] assert np.allclose(val, [1.0]) - if benchmark.enabled: - benchmark(ex, stim) # ------------------------------------------------------------------------------------------------ # # TEST 3 @@ -309,10 +304,8 @@ def test_DDM_noise(mech_mode, benchmark, noise, expected): ex = pytest.helpers.get_mech_execution(T, mech_mode) ex([10]) - val = ex([10]) + val = benchmark(ex, [10]) assert np.allclose(val[0][0], expected) - if benchmark.enabled: - benchmark(ex, [10]) # ------------------------------------------------------------------------------------------------ @@ -442,10 +435,8 @@ def test_DDM_rate(benchmark, rate, expected, mech_mode): ex = pytest.helpers.get_mech_execution(T, mech_mode) ex(stim) - val = float(ex(stim)[0][0]) + val = float(benchmark(ex, stim)[0][0]) assert val == expected - if benchmark.enabled: - benchmark(ex, stim) # ------------------------------------------------------------------------------------------------ # INVALID RATES: @@ -656,13 +647,12 @@ def test_DDM_in_composition(benchmark, comp_mode): C = pnl.Composition() C.add_linear_processing_pathway([M]) inputs = {M: [10]} - val = C.run(inputs, num_trials=2, execution_mode=comp_mode) + val = benchmark(C.run, inputs, num_trials=2, execution_mode=comp_mode) + # FIXME: Python version returns dtype=object val = np.asfarray(val) assert np.allclose(val[0], [2.0]) assert np.allclose(val[1], [0.2]) - if benchmark.enabled: - benchmark(C.run, inputs, num_trials=2, execution_mode=comp_mode) @pytest.mark.composition diff --git a/tests/mechanisms/test_episodic_memory.py b/tests/mechanisms/test_episodic_memory.py index c50b89cc364..41f6d0bae21 100644 --- a/tests/mechanisms/test_episodic_memory.py +++ b/tests/mechanisms/test_episodic_memory.py @@ -52,11 +52,9 @@ def test_with_dictionary_memory(variable, func, params, expected, benchmark, mec EX = pytest.helpers.get_mech_execution(m, mech_mode) EX(variable) - res = EX(variable) + res = benchmark(EX, variable) assert np.allclose(res[0], expected[0]) assert np.allclose(res[1], expected[1]) - if benchmark.enabled: - benchmark(EX, variable) # TEST WITH ContentAddressableMemory *********************************************************************************** diff --git a/tests/mechanisms/test_integrator_mechanism.py b/tests/mechanisms/test_integrator_mechanism.py index 36a1197446c..41e247425cc 100644 --- a/tests/mechanisms/test_integrator_mechanism.py +++ b/tests/mechanisms/test_integrator_mechanism.py @@ -392,10 +392,8 @@ def test_integrator_multiple_input(self, benchmark, mech_mode): ) ex = pytest.helpers.get_mech_execution(I, mech_mode) - val = ex([[1], [2]]) + val = benchmark(ex, [[1], [2]]) assert np.allclose(val, [[3]]) - if benchmark.enabled: - benchmark(ex, [[1], [2]]) @pytest.mark.mimo @pytest.mark.mechanism @@ -408,10 +406,8 @@ def test_integrator_multiple_output(self, benchmark, mech_mode): ) ex = pytest.helpers.get_mech_execution(I, mech_mode) - val = ex([5]) + val = benchmark(ex, [5]) assert np.allclose(val, [[2.5], [2.5]]) - if benchmark.enabled: - benchmark(ex, [5]) @pytest.mark.mimo @pytest.mark.mechanism @@ -427,10 +423,8 @@ def test_integrator_multiple_input_output(self, benchmark, mech_mode): ) ex = pytest.helpers.get_mech_execution(I, mech_mode) - val = ex([[1], [2]]) + val = benchmark(ex, [[1], [2]]) assert np.allclose(val, [[5], [3]]) - if benchmark.enabled: - benchmark(ex, [[1], [2]]) @pytest.mark.mechanism @pytest.mark.integrator_mechanism @@ -443,10 +437,8 @@ def test_FitzHughNagumo_simple_scalar(self, benchmark, mech_mode): ex = pytest.helpers.get_mech_execution(I, mech_mode) ex(var) - val = ex(var) + val = benchmark(ex, var) assert np.allclose(val, [[0.10501801629915011], [0.10501801629915011], [0.10501801629915011]]) - if benchmark.enabled: - benchmark(ex, var) @pytest.mark.mechanism @pytest.mark.integrator_mechanism @@ -459,12 +451,10 @@ def test_FitzHughNagumo_simple_vector(self, benchmark, mech_mode): ex = pytest.helpers.get_mech_execution(I, mech_mode) ex(var) - val = ex(var) + val = benchmark(ex, var) assert np.allclose(val, [[[0.10501801629915011, 0.3151109244983909]], [[0.10501801629915011, 0.3151109244983909]], [[0.10501801629915011, 0.3151109244983909]]]) - if benchmark.enabled: - benchmark(ex, var) @pytest.mark.mechanism @pytest.mark.integrator_mechanism @@ -607,10 +597,8 @@ def test_integrator_no_function(self, benchmark, mech_mode): ex = pytest.helpers.get_mech_execution(I, mech_mode) ex([10]) - val = ex([10]) + val = benchmark(ex, [10]) assert np.allclose(val, [[7.5]]) - if benchmark.enabled: - benchmark(ex, [10]) class TestIntegratorInputs: # Part 1: VALID INPUT: diff --git a/tests/mechanisms/test_lca.py b/tests/mechanisms/test_lca.py index 875c53a3f76..1dc08750638 100644 --- a/tests/mechanisms/test_lca.py +++ b/tests/mechanisms/test_lca.py @@ -39,7 +39,7 @@ def test_LCAMechanism_length_1(self, benchmark, comp_mode): # - - - - - - - - - - - - - - - - - - - - - - - - - - - C.run(inputs={T: [1.0]}, num_trials=3, execution_mode=comp_mode) + benchmark(C.run, inputs={T: [1.0]}, num_trials=3, execution_mode=comp_mode) # - - - - - - - TRIAL 1 - - - - - - - @@ -56,9 +56,7 @@ def test_LCAMechanism_length_1(self, benchmark, comp_mode): # new_transfer_input = 0.265 + ( 0.5 * 0.265 + 3.0 * 0.53 + 0.0 + 1.0)*0.1 + 0.0 = 0.53725 # f(new_transfer_input) = 0.53725 * 2.0 = 1.0745 - assert np.allclose(C.results, [[[0.2]], [[0.51]], [[0.9905]]]) - if benchmark.enabled: - benchmark(C.run, inputs={T: [1.0]}, num_trials=3, execution_mode=comp_mode) + assert np.allclose(C.results[:3], [[[0.2]], [[0.51]], [[0.9905]]]) @pytest.mark.composition @pytest.mark.lca_mechanism @@ -91,7 +89,7 @@ def test_LCAMechanism_length_2(self, benchmark, comp_mode): # - - - - - - - - - - - - - - - - - - - - - - - - - - - C.run(inputs={T: [1.0, 2.0]}, num_trials=3, execution_mode=comp_mode) + benchmark(C.run, inputs={T: [1.0, 2.0]}, num_trials=3, execution_mode=comp_mode) # - - - - - - - TRIAL 1 - - - - - - - @@ -117,9 +115,7 @@ def test_LCAMechanism_length_2(self, benchmark, comp_mode): # new_transfer_input_2 = 0.51 + ( 0.5 * 0.51 + 3.0 * 1.02 - 1.0*0.45 + 2.0)*0.1 + 0.0 = 0.9965 # f(new_transfer_input_2) = 0.9965 * 2.0 = 1.463 - assert np.allclose(C.results, [[[0.2, 0.4]], [[0.43, 0.98]], [[0.6705, 1.833]]]) - if benchmark.enabled: - benchmark(C.run, inputs={T: [1.0, 2.0]}, num_trials=3, execution_mode=comp_mode) + assert np.allclose(C.results[:3], [[[0.2, 0.4]], [[0.43, 0.98]], [[0.6705, 1.833]]]) @pytest.mark.composition def test_equivalance_of_threshold_and_when_finished_condition(self): @@ -161,10 +157,9 @@ def test_LCAMechanism_threshold(self, benchmark, comp_mode): lca = LCAMechanism(size=2, leak=0.5, threshold=0.7) comp = Composition() comp.add_node(lca) - result = comp.run(inputs={lca:[1,0]}, execution_mode=comp_mode) + + result = benchmark(comp.run, inputs={lca:[1,0]}, execution_mode=comp_mode) assert np.allclose(result, [0.70005431, 0.29994569]) - if benchmark.enabled: - benchmark(comp.run, inputs={lca:[1,0]}, execution_mode=comp_mode) @pytest.mark.composition def test_LCAMechanism_threshold_with_max_vs_next(self): @@ -189,6 +184,7 @@ def test_LCAMechanism_threshold_with_convergence(self, benchmark, comp_mode): lca = LCAMechanism(size=3, leak=0.5, threshold=0.01, threshold_criterion=CONVERGENCE) comp = Composition() comp.add_node(lca) + result = comp.run(inputs={lca:[0,1,2]}, execution_mode=comp_mode) assert np.allclose(result, [[0.19153799, 0.5, 0.80846201]]) if comp_mode is pnl.ExecutionMode.Python: diff --git a/tests/mechanisms/test_recurrent_transfer_mechanism.py b/tests/mechanisms/test_recurrent_transfer_mechanism.py index 78d114971b3..bb6e431f05e 100644 --- a/tests/mechanisms/test_recurrent_transfer_mechanism.py +++ b/tests/mechanisms/test_recurrent_transfer_mechanism.py @@ -133,15 +133,16 @@ def test_recurrent_mech_integrator(self, benchmark, mech_mode): val1 = EX([[1.0, 2.0]]) val2 = EX([[1.0, 2.0]]) + # execute 10 times - for i in range(10): - val10 = EX([[1.0, 2.0]]) + for i in range(9): + EX([[1.0, 2.0]]) + + val10 = benchmark(EX, [[1.0, 2.0]]) assert np.allclose(val1, [[0.50249998, 0.50499983]]) assert np.allclose(val2, [[0.50497484, 0.50994869]]) assert np.allclose(val10, [[0.52837327, 0.55656439]]) - if benchmark.enabled: - benchmark(EX, [[1.0, 2.0]]) @pytest.mark.mechanism @pytest.mark.recurrent_transfer_mechanism @@ -158,14 +159,14 @@ def test_recurrent_mech_lci(self, benchmark, mech_mode): val1 = EX([[1.0, 2.0]]) val2 = EX([[1.0, 2.0]]) # execute 10 times - for i in range(10): - val10 = EX([[1.0, 2.0]]) + for i in range(9): + EX([[1.0, 2.0]]) + + val10 = benchmark(EX, [[1.0, 2.0]]) assert np.allclose(val1, [[0.1, 0.2]]) assert np.allclose(val2, [[0.196, 0.392]]) assert np.allclose(val10, [[0.96822561, 1.93645121]]) - if benchmark.enabled: - benchmark(EX, [[1.0, 2.0]]) # def test_recurrent_mech_inputs_list_of_fns(self): # R = RecurrentTransferMechanism( diff --git a/tests/mechanisms/test_transfer_mechanism.py b/tests/mechanisms/test_transfer_mechanism.py index fad6f1e21ea..e7f1f465551 100644 --- a/tests/mechanisms/test_transfer_mechanism.py +++ b/tests/mechanisms/test_transfer_mechanism.py @@ -62,10 +62,8 @@ def test_transfer_mech_inputs_list_of_floats(self, benchmark, mech_mode): EX = pytest.helpers.get_mech_execution(T, mech_mode) EX(var) - val = EX(var) + val = benchmark(EX, var) assert np.allclose(val, [[7.5 for i in range(VECTOR_SIZE)]]) - if benchmark.enabled: - benchmark(EX, var) #@pytest.mark.mechanism #@pytest.mark.transfer_mechanism @@ -158,10 +156,8 @@ def test_transfer_mech_array_var_float_noise(self, benchmark, mech_mode): var = [1 for i in range(VECTOR_SIZE)] EX(var) - val = EX(var) + val = benchmark(EX, var) assert np.allclose(val, [[8.25 for i in range(VECTOR_SIZE)]]) - if benchmark.enabled: - benchmark(EX, var) @pytest.mark.mechanism @pytest.mark.transfer_mechanism @@ -214,10 +210,8 @@ def test_transfer_mech_array_var_normal_array_noise2(self, benchmark, mech_mode) var = [0 for i in range(VECTOR_SIZE)] EX(var) - val = EX(var) + val = benchmark(EX, var) assert np.allclose(val, [[8.5 + (i * 1.7) for i in range(VECTOR_SIZE)]]) - if benchmark.enabled: - benchmark(EX, var) @pytest.mark.mechanism @pytest.mark.transfer_mechanism @@ -437,13 +431,11 @@ def test_transfer_mech_func(self, benchmark, func, variables, expected, mech_mod vals = [] for var in variables[:-1]: vals.append(EX([var] * VECTOR_SIZE)) - vals.append(EX([variables[-1]] * VECTOR_SIZE)) + vals.append(benchmark(EX, [variables[-1]] * VECTOR_SIZE)) assert len(vals) == len(expected) for val, exp in zip(vals, expected): assert np.allclose(val, [[exp]] * VECTOR_SIZE) - if benchmark.enabled: - benchmark(EX, variables[0]) @pytest.mark.mechanism @pytest.mark.transfer_mechanism @@ -528,10 +520,8 @@ def test_transfer_mech_array_assignments_mech_rate(self, benchmark, mech_mode): var = [1 for i in range(VECTOR_SIZE)] EX(var) - val = EX(var) + val = benchmark(EX, var) assert np.allclose(val, [[ 0., 0.19, 0.36, 0.51]]) - if benchmark.enabled: - benchmark(EX, var) @pytest.mark.mechanism @pytest.mark.transfer_mechanism @@ -548,10 +538,8 @@ def test_transfer_mech_array_assignments_fct_rate(self, benchmark, mech_mode): var = [1 for i in range(VECTOR_SIZE)] EX(var) - val = EX(var) + val = benchmark(EX, var) assert np.allclose(val, [[ 0., 0.19, 0.36, 0.51]]) - if benchmark.enabled: - benchmark(EX, var) @pytest.mark.mechanism @pytest.mark.transfer_mechanism @@ -569,10 +557,8 @@ def test_transfer_mech_array_assignments_fct_over_mech_rate(self, benchmark, mec var = [1 for i in range(VECTOR_SIZE)] EX(var) - val = EX(var) + val = benchmark(EX, var) assert np.allclose(val, [[ 0., 0.0975, 0.19, 0.2775]]) - if benchmark.enabled: - benchmark(EX, var) def test_transfer_mech_array_assignments_wrong_size_mech_rate(self): @@ -618,10 +604,8 @@ def test_transfer_mech_array_assignments_mech_init_val(self, benchmark, mech_mod var = [1 for i in range(VECTOR_SIZE)] EX(var) - val = EX(var) + val = benchmark(EX, var) assert np.allclose(val, [[ 0.75, 0.775, 0.8, 0.825]]) - if benchmark.enabled: - benchmark(EX, var) @pytest.mark.mechanism @@ -641,10 +625,8 @@ def test_transfer_mech_array_assignments_fct_initzr(self, benchmark, mech_mode): var = [1 for i in range(VECTOR_SIZE)] EX(var) - val = EX(var) + val = benchmark(EX, var) assert np.allclose(val, [[ 0.75, 0.775, 0.8, 0.825]]) - if benchmark.enabled: - benchmark(EX, var) @pytest.mark.mechanism @@ -665,10 +647,9 @@ def test_transfer_mech_array_assignments_fct_initlzr_over_mech_init_val(self, be var = [1 for i in range(VECTOR_SIZE)] EX(var) - val = EX(var) + val = benchmark(EX, var) assert np.allclose(val, [[ 0.75, 0.775, 0.8, 0.825]]) - if benchmark.enabled: - benchmark(EX, var) + def test_transfer_mech_array_assignments_wrong_size_mech_init_val(self): @@ -757,10 +738,9 @@ def test_transfer_mech_array_assignments_mech_noise(self, benchmark, mech_mode): var = [1 for i in range(VECTOR_SIZE)] EX(var) - val = EX(var) + val = benchmark(EX, var) assert np.allclose(val, [[ 0.75, 0.9, 1.05, 1.2 ]]) - if benchmark.enabled: - benchmark(EX, var) + @pytest.mark.mechanism @pytest.mark.transfer_mechanism @@ -778,10 +758,9 @@ def test_transfer_mech_array_assignments_fct_noise(self, benchmark, mech_mode): var = [1 for i in range(VECTOR_SIZE)] EX(var) - val = EX(var) + val = benchmark(EX, var) assert np.allclose(val, [[ 0.75, 0.9, 1.05, 1.2 ]]) - if benchmark.enabled: - benchmark(EX, var) + @pytest.mark.mechanism @pytest.mark.transfer_mechanism @@ -800,10 +779,9 @@ def test_transfer_mech_array_assignments_fct_over_mech_noise(self, benchmark, me var = [1 for i in range(VECTOR_SIZE)] EX(var) - val = EX(var) + val = benchmark(EX, var) assert np.allclose(val, [[ 0.75, 0.825, 0.9, 0.975]]) - if benchmark.enabled: - benchmark(EX, var) + # def test_transfer_mech_array_assignments_wrong_size_mech_noise(self, benchmark, mode): def test_transfer_mech_array_assignments_wrong_size_mech_noise(self): @@ -855,14 +833,11 @@ def test_transfer_mech_integration_rate_0_8(self, benchmark, mech_mode): EX = pytest.helpers.get_mech_execution(T, mech_mode) val1 = EX([1 for i in range(VECTOR_SIZE)]) - val2 = EX([1 for i in range(VECTOR_SIZE)]) + val2 = benchmark(EX, [1 for i in range(VECTOR_SIZE)]) assert np.allclose(val1, [[0.8 for i in range(VECTOR_SIZE)]]) assert np.allclose(val2, [[0.96 for i in range(VECTOR_SIZE)]]) - if benchmark.enabled: - benchmark(EX, [0 for i in range(VECTOR_SIZE)]) - @pytest.mark.mechanism @pytest.mark.transfer_mechanism @pytest.mark.benchmark(group="TransferMechanism Linear TimeConstant=1") From 490752dcaff212d45e6c3e2b7a730a614a7d3096 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Wed, 9 Nov 2022 22:45:18 -0500 Subject: [PATCH 066/127] tests/composition: Use the benchmark fixture unconditionally Signed-off-by: Jan Vesely --- tests/composition/test_autodiffcomposition.py | 33 ++++-------- tests/composition/test_composition.py | 53 ++++--------------- tests/composition/test_control.py | 26 +++------ tests/composition/test_gating.py | 6 +-- 4 files changed, 30 insertions(+), 88 deletions(-) diff --git a/tests/composition/test_autodiffcomposition.py b/tests/composition/test_autodiffcomposition.py index d04ffe6f210..6eba3a2b288 100644 --- a/tests/composition/test_autodiffcomposition.py +++ b/tests/composition/test_autodiffcomposition.py @@ -384,9 +384,9 @@ def test_optimizer_specs(self, learning_rate, weight_decay, optimizer_type, expe # results_before_proc = xor.run(inputs={xor_in:xor_inputs}, # targets={xor_out:xor_targets}, # epochs=10) - results_before_proc = xor.learn(inputs={"inputs": {xor_in:xor_inputs}, - "targets": {xor_out:xor_targets}, - "epochs": 10}, execution_mode=autodiff_mode) + results_before_proc = benchmark(xor.learn, inputs={"inputs": {xor_in:xor_inputs}, + "targets": {xor_out:xor_targets}, + "epochs": 10}, execution_mode=autodiff_mode) # fp32 results are different due to rounding if pytest.helpers.llvm_current_fp_precision() == 'fp32' and \ @@ -398,11 +398,6 @@ def test_optimizer_specs(self, learning_rate, weight_decay, optimizer_type, expe if learning_rate != 1.5 or autodiff_mode == pnl.ExecutionMode.Python: assert np.allclose(results_before_proc, expected) - if benchmark.enabled: - benchmark(xor.learn, inputs={"inputs": {xor_in:xor_inputs}, - "targets": {xor_out:xor_targets}, - "epochs": 10}, execution_mode=autodiff_mode) - # test whether pytorch parameters and projections are kept separate (at diff. places in memory) def test_params_stay_separate(self, autodiff_mode): @@ -511,7 +506,7 @@ def test_xor_training_correctness(self, eps, calls, opt, autodiff_mode, benchmar [[0], [1], [1], [0]]) if calls == 'single': - results = xor.learn(inputs={"inputs": {xor_in:xor_inputs}, + results = benchmark(xor.learn, inputs={"inputs": {xor_in:xor_inputs}, "targets": {xor_out:xor_targets}, "epochs": eps}, execution_mode=autodiff_mode) @@ -519,18 +514,14 @@ def test_xor_training_correctness(self, eps, calls, opt, autodiff_mode, benchmar input_dict = {"inputs": {xor_in: xor_inputs}, "targets": {xor_out: xor_targets}, "epochs": 1} - for i in range(eps): - results = xor.learn(inputs=input_dict, execution_mode=autodiff_mode) + for i in range(eps - 1): + xor.learn(inputs=input_dict, execution_mode=autodiff_mode) + results = benchmark(xor.learn, inputs=input_dict, execution_mode=autodiff_mode) assert len(results) == len(expected) for r, t in zip(results, expected): assert np.allclose(r[0], t) - if benchmark.enabled: - benchmark(xor.learn, inputs={"inputs": {xor_in: xor_inputs}, - "targets": {xor_out: xor_targets}, - "epochs": eps}, execution_mode=autodiff_mode) - # tests whether semantic network created as autodiff composition learns properly @pytest.mark.benchmark(group="Semantic net") @@ -700,9 +691,9 @@ def test_semantic_net_training_correctness(self, eps, opt, autodiff_mode, benchm targets_dict[out_sig_can].append(truth_can[i]) # TRAIN THE MODEL - results = sem_net.learn(inputs={'inputs': inputs_dict, - 'targets': targets_dict, - 'epochs': eps}, execution_mode=autodiff_mode) + results = benchmark(sem_net.learn, inputs={'inputs': inputs_dict, + 'targets': targets_dict, + 'epochs': eps}, execution_mode=autodiff_mode) # CHECK CORRECTNESS expected = [[[0.13455769, 0.12924714, 0.13288172, 0.1404659 , 0.14305814, @@ -830,10 +821,6 @@ def test_semantic_net_training_correctness(self, eps, opt, autodiff_mode, benchm for res, exp in zip(results, expected): for r, e in zip(res, exp): assert np.allclose(r, e) - if benchmark.enabled: - benchmark(sem_net.learn, inputs={'inputs': inputs_dict, - 'targets': targets_dict, - 'epochs': eps}, execution_mode=autodiff_mode) def test_pytorch_equivalence_with_autodiff_composition(self, autodiff_mode): iSs = np.array( diff --git a/tests/composition/test_composition.py b/tests/composition/test_composition.py index b614616c13e..49e60e00f21 100644 --- a/tests/composition/test_composition.py +++ b/tests/composition/test_composition.py @@ -2605,12 +2605,9 @@ def test_3_mechanisms_frozen_values(self, benchmark, comp_mode): inputs_dict = {A: [4.0]} sched = Scheduler(composition=comp) - output = comp.run(inputs=inputs_dict, scheduler=sched, execution_mode=comp_mode) + output = benchmark(comp.run, inputs=inputs_dict, scheduler=sched, execution_mode=comp_mode) assert np.allclose(output, 320) - if benchmark.enabled: - benchmark(comp.run, inputs=inputs_dict, scheduler=sched, execution_mode=comp_mode) - @pytest.mark.control @pytest.mark.composition @pytest.mark.benchmark(group="Control composition scalar") @@ -2645,12 +2642,9 @@ def test_3_mechanisms_2_origins_1_multi_control_1_terminal(self, benchmark, comp inputs_dict = {B: [4.0]} - output = comp.run(inputs=inputs_dict, execution_mode=comp_mode) + output = benchmark(comp.run, inputs=inputs_dict, execution_mode=comp_mode) assert np.allclose(output, 354.19328716) - if benchmark.enabled: - benchmark(comp.run, inputs=inputs_dict, execution_mode=comp_mode) - @pytest.mark.control @pytest.mark.composition @pytest.mark.benchmark(group="Control composition scalar") @@ -2685,12 +2679,9 @@ def test_3_mechanisms_2_origins_1_additive_control_1_terminal(self, benchmark, c inputs_dict = {B: [4.0]} sched = Scheduler(composition=comp) - output = comp.run(inputs=inputs_dict, scheduler=sched, execution_mode=comp_mode) + output = benchmark(comp.run, inputs=inputs_dict, scheduler=sched, execution_mode=comp_mode) assert np.allclose(output, 650.83865743) - if benchmark.enabled: - benchmark(comp.run, inputs=inputs_dict, scheduler=sched, execution_mode=comp_mode) - @pytest.mark.control @pytest.mark.composition @pytest.mark.benchmark(group="Control composition scalar") @@ -2725,10 +2716,8 @@ def test_3_mechanisms_2_origins_1_override_control_1_terminal(self, benchmark, c inputs_dict = {B: [4.0]} - output = comp.run(inputs=inputs_dict, execution_mode=comp_mode) + output = benchmark(comp.run, inputs=inputs_dict, execution_mode=comp_mode) assert np.allclose(output, 150.83865743) - if benchmark.enabled: - benchmark(comp.run, inputs=inputs_dict, execution_mode=comp_mode) @pytest.mark.control @pytest.mark.composition @@ -2765,12 +2754,9 @@ def test_3_mechanisms_2_origins_1_disable_control_1_terminal(self, benchmark, co inputs_dict = {B: [4.0]} sched = Scheduler(composition=comp) - output = comp.run(inputs=inputs_dict, scheduler=sched, execution_mode=comp_mode) + output = benchmark(comp.run, inputs=inputs_dict, scheduler=sched, execution_mode=comp_mode) assert np.allclose(output, 600) - if benchmark.enabled: - benchmark(comp.run, inputs=inputs_dict, scheduler=sched, execution_mode=comp_mode) - @pytest.mark.composition @pytest.mark.benchmark(group="Transfer") def xtest_transfer_mechanism(self, benchmark, comp_mode): @@ -3918,13 +3904,11 @@ def test_run_recurrent_transfer_mechanism(self, benchmark, comp_mode): sched = Scheduler(composition=comp) output1 = comp.run(inputs={A: [[1.0, 2.0, 3.0]]}, scheduler=sched, execution_mode=comp_mode) assert np.allclose([5.0, 10.0, 15.0], output1) - output2 = comp.run(inputs={A: [[1.0, 2.0, 3.0]]}, scheduler=sched, execution_mode=comp_mode) + output2 = benchmark(comp.run, inputs={A: [[1.0, 2.0, 3.0]]}, scheduler=sched, execution_mode=comp_mode) # Using the hollow matrix: (10 + 15 + 1) * 5 = 130, # ( 5 + 15 + 2) * 5 = 110, # ( 5 + 10 + 3) * 5 = 90 assert np.allclose([130.0, 110.0, 90.0], output2) - if benchmark.enabled: - benchmark(comp.run, inputs={A: [[1.0, 2.0, 3.0]]}, scheduler=sched, execution_mode=comp_mode) @pytest.mark.composition @pytest.mark.benchmark(group="Recurrent") @@ -3943,12 +3927,9 @@ def test_run_recurrent_transfer_mechanism_hetero(self, benchmark, comp_mode): assert np.allclose(val, [[0.98201379]]) # execute 10 trials - val = comp.run(inputs={R: [[5.0]]}, num_trials=10, execution_mode=comp_mode) + val = benchmark(comp.run, inputs={R: [[5.0]]}, num_trials=10, execution_mode=comp_mode) assert np.allclose(val, [[0.99330715]]) - if benchmark.enabled: - benchmark(comp.run, inputs={R: [[1.0]]}, execution_mode=comp_mode) - @pytest.mark.composition @pytest.mark.benchmark(group="Recurrent") def test_run_recurrent_transfer_mechanism_integrator(self, benchmark, comp_mode): @@ -3968,12 +3949,9 @@ def test_run_recurrent_transfer_mechanism_integrator(self, benchmark, comp_mode) assert np.allclose(val, [[0.51741795]]) # execute 10 trials - val = comp.run(inputs={R: [[5.0]]}, num_trials=10, execution_mode=comp_mode) + val = benchmark(comp.run, inputs={R: [[5.0]]}, num_trials=10, execution_mode=comp_mode) assert np.allclose(val, [[0.6320741]]) - if benchmark.enabled: - benchmark(comp.run, inputs={R: [[1.0]]}, execution_mode=comp_mode) - @pytest.mark.composition @pytest.mark.benchmark(group="Recurrent") def test_run_recurrent_transfer_mechanism_vector_2(self, benchmark, comp_mode): @@ -3988,13 +3966,10 @@ def test_run_recurrent_transfer_mechanism_vector_2(self, benchmark, comp_mode): assert np.allclose(val, [[0.87259959, 0.94361816]]) # execute 10 trials - val = comp.run(inputs={R: [[1.0, 2.0]]}, num_trials=10, execution_mode=comp_mode) + val = benchmark(comp.run, inputs={R: [[1.0, 2.0]]}, num_trials=10, execution_mode=comp_mode) assert np.allclose(val, [[0.87507549, 0.94660049]]) - if benchmark.enabled: - benchmark(comp.run, inputs={R: [[1.0, 2.0]]}, execution_mode=comp_mode) - @pytest.mark.composition @pytest.mark.benchmark(group="Recurrent") def test_run_recurrent_transfer_mechanism_hetero_2(self, benchmark, comp_mode): @@ -4012,12 +3987,9 @@ def test_run_recurrent_transfer_mechanism_hetero_2(self, benchmark, comp_mode): assert np.allclose(val, [[0.3864837, 0.73105858]]) # execute 10 trials - val = comp.run(inputs={R: [[1.0, 2.0]]}, num_trials=10, execution_mode=comp_mode) + val = benchmark(comp.run, inputs={R: [[1.0, 2.0]]}, num_trials=10, execution_mode=comp_mode) assert np.allclose(val, [[0.36286875, 0.78146724]]) - if benchmark.enabled: - benchmark(comp.run, inputs={R: [[1.0, 2.0]]}, execution_mode=comp_mode) - @pytest.mark.composition @pytest.mark.benchmark(group="Recurrent") def test_run_recurrent_transfer_mechanism_integrator_2(self, benchmark, comp_mode): @@ -4037,13 +4009,10 @@ def test_run_recurrent_transfer_mechanism_integrator_2(self, benchmark, comp_mod assert np.allclose(val, [[0.4999875, 0.50497484]]) # execute 10 trials - val = comp.run(inputs={R: [[1.0, 2.0]]}, num_trials=10, execution_mode=comp_mode) + val = benchmark(comp.run, inputs={R: [[1.0, 2.0]]}, num_trials=10, execution_mode=comp_mode) assert np.allclose(val, [[0.49922843, 0.52838607]]) - if benchmark.enabled: - benchmark(comp.run, inputs={R: [[1.0, 2.0]]}, execution_mode=comp_mode) - def test_run_termination_condition_custom_context(self): D = pnl.DDM(function=pnl.DriftDiffusionIntegrator, execute_until_finished=False) comp = pnl.Composition() diff --git a/tests/composition/test_control.py b/tests/composition/test_control.py index c6eedd34694..bf8ca75f1a5 100644 --- a/tests/composition/test_control.py +++ b/tests/composition/test_control.py @@ -1914,11 +1914,8 @@ def test_multilevel_ocm_gridsearch_conflicting_directions(self, mode, benchmark) intensity_cost_function=pnl.Linear(slope=0.0), allocation_samples=pnl.SampleSpec(start=1.0, stop=5.0, num=5))]) ) - results = ocomp.run([5], execution_mode=mode) - assert np.allclose(results, [[50]]) - - if benchmark.enabled: - benchmark(ocomp.run, [5], execution_mode=mode) + result = benchmark(ocomp.run, [5], execution_mode=mode) + assert np.allclose(result, [[50]]) @pytest.mark.control @pytest.mark.composition @@ -1981,11 +1978,8 @@ def test_multilevel_ocm_gridsearch_maximize(self, mode, benchmark): stop=5.0, num=5))]) ) - results = ocomp.run([5], execution_mode=mode) - assert np.allclose(results, [[70]]) - - if benchmark.enabled: - benchmark(ocomp.run, [5], execution_mode=mode) + result = benchmark(ocomp.run, [5], execution_mode=mode) + assert np.allclose(result, [[70]]) @pytest.mark.control @pytest.mark.composition @@ -2048,11 +2042,8 @@ def test_multilevel_ocm_gridsearch_minimize(self, mode, benchmark): stop=5.0, num=5))]) ) - results = ocomp.run([5], execution_mode=mode) - assert np.allclose(results, [[5]]) - - if benchmark.enabled: - benchmark(ocomp.run, [5], execution_mode=mode) + result = benchmark(ocomp.run, [5], execution_mode=mode) + assert np.allclose(result, [[5]]) def test_two_tier_ocm(self): integrationConstant = 0.8 # Time Constant @@ -2275,12 +2266,9 @@ def test_multilevel_control(self, comp_mode, benchmark): iComp.add_controller(iController) assert iComp.controller == iController assert oComp.controller == oController - res = oComp.run(inputs=[5], execution_mode=comp_mode) + res = benchmark(oComp.run, inputs=[5], execution_mode=comp_mode) assert np.allclose(res, [40]) - if benchmark.enabled: - benchmark(oComp.run, [5], execution_mode=comp_mode) - @pytest.mark.control @pytest.mark.composition def test_recurrent_control(self, comp_mode): diff --git a/tests/composition/test_gating.py b/tests/composition/test_gating.py index 486f1c04fbb..e27cda9f4b7 100644 --- a/tests/composition/test_gating.py +++ b/tests/composition/test_gating.py @@ -40,7 +40,7 @@ def test_gating(benchmark, comp_mode): comp.add_linear_processing_pathway(p_pathway) comp.add_node(Gating_Mechanism) - comp.run(num_trials=4, inputs=stim_list, execution_mode=comp_mode) + benchmark(comp.run, num_trials=4, inputs=stim_list, execution_mode=comp_mode) expected_results = [ [np.array([0., 0., 0.])], @@ -49,9 +49,7 @@ def test_gating(benchmark, comp_mode): [np.array([2.53788284, 2.53788284, 2.53788284])] ] - np.testing.assert_allclose(comp.results, expected_results) - if benchmark.enabled: - benchmark(comp.run, num_trials=4, inputs=stim_list, execution_mode=comp_mode) + np.testing.assert_allclose(comp.results[:4], expected_results) # DEPRECATED FUNCTIONALITY 9/26/19 # @pytest.mark.composition From a59008fe6f867feb2d8de88639e7cd0da6b33d76 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Wed, 9 Nov 2022 23:17:00 -0500 Subject: [PATCH 067/127] tests/models: Use the benchmark fixture unconditionally Signed-off-by: Jan Vesely --- tests/models/test_bi_percepts.py | 23 +++++++++-------------- tests/models/test_botvinick.py | 6 +++--- tests/models/test_greedy_agent.py | 31 +++++-------------------------- 3 files changed, 17 insertions(+), 43 deletions(-) diff --git a/tests/models/test_bi_percepts.py b/tests/models/test_bi_percepts.py index 5d819d890bf..5a2fd94e181 100644 --- a/tests/models/test_bi_percepts.py +++ b/tests/models/test_bi_percepts.py @@ -32,6 +32,8 @@ pytest.param(8, 10, expected_8_10, id="8-10"), ]) def test_necker_cube(benchmark, comp_mode, n_nodes, n_time_steps, expected): + benchmark.group = "Necker Cube {}-{}".format(n_nodes, n_time_steps) + # this code only works for N_PERCEPTS == 2 ALL_PERCEPTS = ['a', 'b'] @@ -124,13 +126,6 @@ def get_node(percept, node_id): for node_ in bp_comp.nodes } - # run the model - res = bp_comp.run(input_dict, num_trials=n_time_steps, execution_mode=comp_mode) - if pytest.helpers.llvm_current_fp_precision() == 'fp32': - assert np.allclose(res, expected) - else: - np.testing.assert_allclose(res, expected) - # Test that order of CIM ports follows order of Nodes in self.nodes for i in range(n_nodes): a_name = "a-{}".format(i) @@ -140,9 +135,12 @@ def get_node(percept, node_id): assert b_name in bp_comp.input_CIM.input_ports.names[i + n_nodes] assert b_name in bp_comp.output_CIM.output_ports.names[i + n_nodes] - if benchmark.enabled: - benchmark.group = "Necker Cube {}-{}".format(n_nodes, n_time_steps) - benchmark(bp_comp.run, input_dict, num_trials=n_time_steps, execution_mode=comp_mode) + # run the model + res = benchmark(bp_comp.run, input_dict, num_trials=n_time_steps, execution_mode=comp_mode) + if pytest.helpers.llvm_current_fp_precision() == 'fp32': + assert np.allclose(res, expected) + else: + np.testing.assert_allclose(res, expected) @pytest.mark.model @@ -222,7 +220,7 @@ def test_vectorized_necker_cube(benchmark, comp_mode): node4: np.random.random((1,16)) } - result = comp2.run(input_dict, num_trials=10, execution_mode=comp_mode) + result = benchmark(comp2.run, input_dict, num_trials=10, execution_mode=comp_mode) assert np.allclose(result, [[ 2636.29181172, -662.53579899, 2637.35386946, -620.15550833, -595.55319772, 2616.74310649, -442.74286574, 2588.4778162 , @@ -232,6 +230,3 @@ def test_vectorized_necker_cube(benchmark, comp_mode): 2590.69244696, -555.19824432, 2591.63200098, -509.58072358, -2618.88711219, 682.65814776, -2620.18294962, 640.09719335, 615.39758884, -2599.45663784, 462.67291695, -2570.99427346]]) - - if benchmark.enabled: - benchmark(comp2.run, input_dict, num_trials=10, execution_mode=comp_mode) diff --git a/tests/models/test_botvinick.py b/tests/models/test_botvinick.py index 02cfe6e3d9e..aa5064fcfef 100644 --- a/tests/models/test_botvinick.py +++ b/tests/models/test_botvinick.py @@ -189,10 +189,12 @@ def run(mode): # Comp results include concatenation of both the above runs results.append(comp.results) + # cleanup the results of the most recently used context id + comp.results = [] return results - res = run(comp_mode) + res = benchmark(run, comp_mode) # the corresponding output port indices in composition results # these were 0 and 1 in the prior version of the test response_results_index = 3 @@ -283,5 +285,3 @@ def run(mode): assert np.allclose(res[1][-1][response_decision_energy_index], [1.87232903]) assert np.allclose(res[2][ntrials0 - 1][response_decision_energy_index], [0.94440397]) assert np.allclose(res[2][-1][response_decision_energy_index], [0.90033387]) - if benchmark.enabled: - benchmark(run, comp_mode) diff --git a/tests/models/test_greedy_agent.py b/tests/models/test_greedy_agent.py index 1ee9c192628..d74eb33df83 100644 --- a/tests/models/test_greedy_agent.py +++ b/tests/models/test_greedy_agent.py @@ -52,15 +52,8 @@ def test_simplified_greedy_agent(benchmark, comp_mode): for projection in greedy_action_mech.projections: agent_comp.add_projection(projection) - run_results = agent_comp.run(inputs={player:[[619,177]], - prey:[[419,69]]}, - execution_mode=comp_mode) + run_results = benchmark(agent_comp.run, inputs={player:[[619,177]],prey:[[419,69]]}, execution_mode=comp_mode) assert np.allclose(run_results, [[-200, -108]]) - if benchmark.enabled: - benchmark(agent_comp.run, **{'inputs':{ - player:[[619,177]], - prey:[[419,69]], - }, 'execution_mode':comp_mode}) @pytest.mark.model @pytest.mark.benchmark(group="Greedy Agant Random") @@ -94,19 +87,8 @@ def test_simplified_greedy_agent_random(benchmark, comp_mode): for projection in greedy_action_mech.projections: agent_comp.add_projection(projection) - run_results = agent_comp.run(inputs={player:[[619,177]], - prey:[[419,69]]}, - execution_mode=comp_mode) - # KDM 12/4/19: modified results due to global seed offset of - # GaussianDistort assignment. - # to produce old numbers, run get_global_seed once before creating - # each Mechanism with GaussianDistort above + run_results = benchmark(agent_comp.run, inputs={player:[[619, 177]], prey:[[419, 69]]}, execution_mode=comp_mode) assert np.allclose(run_results, [[-199.5484223217141, -107.79361870517444]]) - if benchmark.enabled: - benchmark(agent_comp.run, **{'inputs':{ - player:[[619,177]], - prey:[[419,69]], - }, 'execution_mode':comp_mode}) @pytest.mark.model @pytest.mark.benchmark(group="Predator Prey") @@ -232,7 +214,7 @@ def action_fn(variable): predator_pos:[[-0.03479106, -0.47666293]], prey_pos:[[-0.60836214, 0.1760381 ]], } - run_results = agent_comp.run(inputs=input_dict, num_trials=2, execution_mode=mode) + run_results = benchmark(agent_comp.run, inputs=input_dict, num_trials=2, execution_mode=mode) if len(samples) == 2: if prng == 'Default': @@ -247,12 +229,9 @@ def action_fn(variable): else: assert False, "Unknown PRNG!" - if mode == pnl.ExecutionMode.Python: - # FIXEM: The results are 'close' for both Philox and MT, + if mode == pnl.ExecutionMode.Python and not benchmark.enabled: + # FIXME: The results are 'close' for both Philox and MT, # because they're dominated by costs assert np.allclose(np.asfarray(ocm.function.saved_values).flatten(), [-2.66258741, -22027.9970321, -22028.17515945, -44053.59867802, -22028.06045185, -44053.4048842, -44053.40736234, -66078.90687915]) - - if benchmark.enabled: - benchmark(agent_comp.run, inputs=input_dict, execution_mode=mode) From d4f5478df6b4dee4a64e4ce897cc5f72d4324864 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Thu, 10 Nov 2022 02:52:10 -0500 Subject: [PATCH 068/127] models/Botvinnick: Use top level import Signed-off-by: Jan Vesely --- tests/models/test_botvinick.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/tests/models/test_botvinick.py b/tests/models/test_botvinick.py index aa5064fcfef..d7b1634a904 100644 --- a/tests/models/test_botvinick.py +++ b/tests/models/test_botvinick.py @@ -14,8 +14,6 @@ # Note that this script implements a slightly different Figure than in the original Figure in the paper. # However, this implementation is identical with a plot we created with an old MATLAB code which was used for the # conflict monitoring simulations. -import psyneulink.core.components.functions.nonstateful.objectivefunctions -import psyneulink.core.components.functions.nonstateful.transferfunctions @pytest.mark.model @@ -30,20 +28,20 @@ def test_botvinick_model(benchmark, comp_mode, reps): # Linear input layer # colors: ('red', 'green'), words: ('RED','GREEN') colors_input_layer = pnl.TransferMechanism(size=3, - function=psyneulink.core.components.Linear, + function=pnl.Linear, name='COLORS_INPUT') words_input_layer = pnl.TransferMechanism(size=3, - function=psyneulink.core.components.Linear, + function=pnl.Linear, name='WORDS_INPUT') task_input_layer = pnl.TransferMechanism(size=2, - function=psyneulink.core.components.Linear, + function=pnl.Linear, name='TASK_INPUT') # Task layer, tasks: ('name the color', 'read the word') task_layer = pnl.RecurrentTransferMechanism(size=2, - function=psyneulink.core.components.Logistic, + function=pnl.Logistic, hetero=-2, integrator_mode=True, integration_rate=0.01, @@ -52,14 +50,14 @@ def test_botvinick_model(benchmark, comp_mode, reps): # Hidden layer # colors: ('red','green', 'neutral') words: ('RED','GREEN', 'NEUTRAL') colors_hidden_layer = pnl.RecurrentTransferMechanism(size=3, - function=psyneulink.core.components.Logistic(x_0=4.0), # bias 4.0 is -4.0 in the paper see Docs for description + function=pnl.Logistic(x_0=4.0), # bias 4.0 is -4.0 in the paper see Docs for description integrator_mode=True, hetero=-2, integration_rate=0.01, # cohen-huston text says 0.01 name='COLORS_HIDDEN') words_hidden_layer = pnl.RecurrentTransferMechanism(size=3, - function=psyneulink.core.components.Logistic(x_0=4.0), + function=pnl.Logistic(x_0=4.0), integrator_mode=True, hetero=-2, integration_rate=0.01, @@ -67,14 +65,14 @@ def test_botvinick_model(benchmark, comp_mode, reps): # Response layer, responses: ('red', 'green') response_layer = pnl.RecurrentTransferMechanism(size=2, - function=psyneulink.core.components.Logistic, + function=pnl.Logistic, hetero=-2.0, integrator_mode=True, integration_rate=0.01, output_ports = [pnl.RESULT, {pnl.NAME: 'DECISION_ENERGY', pnl.VARIABLE: (pnl.OWNER_VALUE,0), - pnl.FUNCTION: psyneulink.core.components.Stability( + pnl.FUNCTION: pnl.Stability( default_variable = np.array([0.0, 0.0]), metric = pnl.ENERGY, matrix = np.array([[0.0, -4.0], From e462f10978df7e1490c67d57ef5947dd21367523 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Thu, 10 Nov 2022 21:23:54 -0500 Subject: [PATCH 069/127] docs: Remove invalid escape sequences (#2531) Some escape sequences were ignored so we can just remove the escaping '\', others are needed for rest/sphinx and should be double escaped. User documentation should ideally use raw strings, but that is a bigger change. Check that the required warning is present at any position in "test_model_based_num_estimates". Use raw string when listing code in versioneer.py. Change warning for invalid escape sequences to error. Signed-off-by: Jan Vesely --- psyneulink/core/components/functions/function.py | 2 +- .../components/functions/stateful/integratorfunctions.py | 4 ++-- .../core/components/functions/stateful/memoryfunctions.py | 6 +++--- psyneulink/core/compositions/composition.py | 2 +- psyneulink/core/compositions/pathway.py | 2 +- setup.cfg | 1 + tests/composition/test_control.py | 4 ++-- versioneer.py | 2 +- 8 files changed, 12 insertions(+), 11 deletions(-) diff --git a/psyneulink/core/components/functions/function.py b/psyneulink/core/components/functions/function.py index b5d551ea9e0..3b592428886 100644 --- a/psyneulink/core/components/functions/function.py +++ b/psyneulink/core/components/functions/function.py @@ -1216,7 +1216,7 @@ class RandomMatrix(): .. technical_note:: A call to the class calls `random_matrix `, passing **sender_size** and **receiver_size** to `random_matrix ` as its **num_rows** and **num_cols** - arguments, respectively, and passing the `center `\-0.5 and `range ` + arguments, respectively, and passing the `center `-0.5 and `range ` attributes specified at construction to `random_matrix ` as its **offset** and **scale** arguments, respectively. diff --git a/psyneulink/core/components/functions/stateful/integratorfunctions.py b/psyneulink/core/components/functions/stateful/integratorfunctions.py index a20d289ad95..83613c87f40 100644 --- a/psyneulink/core/components/functions/stateful/integratorfunctions.py +++ b/psyneulink/core/components/functions/stateful/integratorfunctions.py @@ -429,7 +429,7 @@ class AccumulatorIntegrator(IntegratorFunction): # ---------------------------- so that, with each call to `function `, the accumulated value increases by: .. math:: - increment \\cdot rate^{time\\ step}. + increment \\cdot rate^{time\\_step}. Thus, accumulation increases lineary in steps of `increment ` if `rate `\\=1.0, and exponentially otherwise. @@ -2216,7 +2216,7 @@ class DriftDiffusionIntegrator(IntegratorFunction): # ------------------------- offset : float, list or 1d array : default 0.0 specifies constant value added to integral in each call to `function ` - if it's absolute value is below `threshold `\; + if it's absolute value is below `threshold `; if it is a list or array, it must be the same length as `variable ` (see `offset ` for details). diff --git a/psyneulink/core/components/functions/stateful/memoryfunctions.py b/psyneulink/core/components/functions/stateful/memoryfunctions.py index 5c13c251278..37253ab5d7e 100644 --- a/psyneulink/core/components/functions/stateful/memoryfunctions.py +++ b/psyneulink/core/components/functions/stateful/memoryfunctions.py @@ -501,7 +501,7 @@ class ContentAddressableMemory(MemoryFunction): # ------------------------------ the entry closest to `variable ` is retrieved from is retrieved from `memory `. The entry is chosen by calling, in order: - * `distance_function `\: generates a list of and compares + * `distance_function `: generates a list of and compares `distances ` between `variable ` and each entry in `memory `, possibly weighted by `distance_field_weights `, as follows: @@ -528,7 +528,7 @@ class ContentAddressableMemory(MemoryFunction): # ------------------------------ between `variable ` and entries for those fields are not included in the averaging of distances by field. - * `selection_function `\: called with the list of distances + * `selection_function `: called with the list of distances to determine which entries to select for consideration. If more than on entry from `memory ` is identified, `equidistant_entries_select ` is used to determine which to retrieve. If no @@ -765,7 +765,7 @@ class ContentAddressableMemory(MemoryFunction): # ------------------------------ noise : float, list, 2d array, or Function : default 0.0 specifies random value(s) added to `variable ` before storing in - `memory `\; if a list or 2d array, it must be the same shape as `variable + `memory `; if a list or 2d array, it must be the same shape as `variable ContentAddressableMemory.variable>` (see `noise ` for details). initializer : 3d array or list : default None diff --git a/psyneulink/core/compositions/composition.py b/psyneulink/core/compositions/composition.py index 9dcfc218ed7..6c345800027 100644 --- a/psyneulink/core/compositions/composition.py +++ b/psyneulink/core/compositions/composition.py @@ -9828,7 +9828,7 @@ def run( 2d list of values of OUTPUT Nodes at end of last trial : list[list] each item in the list is the `output_values ` for an `OUTPUT` `Node ` of the Composition, listed in the order listed in `get_nodes_by_role - `\(`NodeRole.OUTPUT `). + `\\ (`NodeRole.OUTPUT `). .. note:: The `results ` attribute of the Composition contains a list of the outputs for all diff --git a/psyneulink/core/compositions/pathway.py b/psyneulink/core/compositions/pathway.py index 98b423201d7..978b2fc00bb 100644 --- a/psyneulink/core/compositions/pathway.py +++ b/psyneulink/core/compositions/pathway.py @@ -252,7 +252,7 @@ ` method), they can be specified in a list, in which each item of the list can be any of the forms above, or one of the following: - * **Pathway** object or constructor: Pathway(pathway=\ `Pathway specification `,...). + * **Pathway** object or constructor: Pathway(pathway=\\ `Pathway specification `,...). .. .. _Pathway_Specification_Dictionary: * **dict**: {name : Pathway) -- in which **name** is a str and **Pathway** is a Pathway object or constuctor, diff --git a/setup.cfg b/setup.cfg index fbd3d06c3bb..12d3191ded7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -67,6 +67,7 @@ xfail_strict = True filterwarnings = error:Creating an ndarray from ragged nested sequences \(which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes\) is deprecated.*:numpy.VisibleDeprecationWarning + error:Invalid escape sequence ignore:Multiple ParameterPorts:UserWarning [pycodestyle] diff --git a/tests/composition/test_control.py b/tests/composition/test_control.py index bf8ca75f1a5..b6b89cb28ce 100644 --- a/tests/composition/test_control.py +++ b/tests/composition/test_control.py @@ -3566,7 +3566,7 @@ def test_model_based_num_estimates(self, num_estimates, rand_var): warning_msg = f'"\'OptimizationControlMechanism-0\' has \'num_estimates = {num_estimates}\' specified, ' \ f'but its \'agent_rep\' (\'comp\') has no random variables: ' \ f'\'RANDOMIZATION_CONTROL_SIGNAL\' will not be created, and num_estimates set to None."' - with pytest.warns(warning_type) as warning: + with pytest.warns(warning_type) as warnings: ocm = pnl.OptimizationControlMechanism(agent_rep=comp, state_features=[A.input_port], objective_mechanism=objective_mech, @@ -3574,7 +3574,7 @@ def test_model_based_num_estimates(self, num_estimates, rand_var): num_estimates=num_estimates, control_signals=[control_signal]) if warning_type: - assert repr(warning[5].message.args[0]) == warning_msg + assert any(warning_msg == repr(w.message.args[0]) for w in warnings) comp.add_controller(ocm) inputs = {A: [[[1.0]]]} diff --git a/versioneer.py b/versioneer.py index 64fea1c8927..13901fcd1b9 100644 --- a/versioneer.py +++ b/versioneer.py @@ -418,7 +418,7 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, return stdout, p.returncode -LONG_VERSION_PY['git'] = ''' +LONG_VERSION_PY['git'] = r''' # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (built by setup.py sdist) and build From 20fdbf3351e42bb045560673d853d0389ec2cb9e Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Thu, 10 Nov 2022 23:11:56 -0500 Subject: [PATCH 070/127] functios/ReLU: Rename 'variable' arguemnt to 'input' This is the standard form for derivatives. Signed-off-by: Jan Vesely --- .../components/functions/nonstateful/transferfunctions.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/psyneulink/core/components/functions/nonstateful/transferfunctions.py b/psyneulink/core/components/functions/nonstateful/transferfunctions.py index da3bfd300b0..b00bd15422b 100644 --- a/psyneulink/core/components/functions/nonstateful/transferfunctions.py +++ b/psyneulink/core/components/functions/nonstateful/transferfunctions.py @@ -1593,7 +1593,7 @@ def _gen_llvm_transfer(self, builder, index, ctx, vi, vo, params, state, *, tags builder.store(val, ptro) @handle_external_context() - def derivative(self, variable, output=None, context=None): + def derivative(self, input=None, output=None, context=None): """ derivative(input) @@ -1615,9 +1615,9 @@ def derivative(self, variable, output=None, context=None): leak = self._get_current_parameter_value(LEAK, context) bias = self._get_current_parameter_value(BIAS, context) - value = np.empty_like(variable) - value[(variable - bias) > 0] = gain - value[(variable - bias) <= 0] = gain * leak + value = np.empty_like(input) + value[(input - bias) > 0] = gain + value[(input - bias) <= 0] = gain * leak return value From a663d2cf57a78363c009bf9159e2a3bfe6da1b5d Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Thu, 10 Nov 2022 23:16:05 -0500 Subject: [PATCH 071/127] functions/SoftMax: Restore correct computation of derivation Commit cae1465b319539f17bb651dd7961ede88a7c4022 ("llvm, functions/SoftMax: Implement compiled 'derivative' variant") incorrectly assumed that the use of 'output' was an oversight. It wasn't SoftMax derivative can take advantage of results if available. This change restores the original functionality and adds a path to compute the results if output is None. This is used for testing where the results would need to be calculated anyway. The compiled variant is adapted in the same way, and the test are updated to reflect the new results. Signed-off-by: Jan Vesely --- .../nonstateful/transferfunctions.py | 21 +++++++++++++++---- tests/functions/test_transfer.py | 8 +++---- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/psyneulink/core/components/functions/nonstateful/transferfunctions.py b/psyneulink/core/components/functions/nonstateful/transferfunctions.py index b00bd15422b..381d80d9c6b 100644 --- a/psyneulink/core/components/functions/nonstateful/transferfunctions.py +++ b/psyneulink/core/components/functions/nonstateful/transferfunctions.py @@ -2700,8 +2700,18 @@ def __gen_llvm_apply(self, ctx, builder, params, state, arg_in, arg_out, output_ def _gen_llvm_function_derivative_body(self, ctx, builder, params, state, arg_in, arg_out, *, tags:frozenset): assert "derivative" in tags forward_tags = tags.difference({"derivative"}) + + # SoftMax derivative is calculated from the results. Recalculate them here + base_out = builder.alloca(arg_out.type.pointee) + builder = self._gen_llvm_function_body(ctx, builder, params, state, arg_in, base_out, output_type=self.output, tags=forward_tags) + + all_out = builder.alloca(arg_out.type.pointee) - builder = self._gen_llvm_function_body(ctx, builder, params, state, arg_in, all_out, output_type=ALL, tags=forward_tags) + builder = self._gen_llvm_function_body(ctx, builder, params, state, base_out, all_out, output_type=ALL, tags=forward_tags) + + # The rest of the algorithm is for MAX_VAL and MAX_INDICATOR only + assert self.output in {MAX_VAL, MAX_INDICATOR}, \ + "Derivative of SoftMax is only implemented for MAX_VAL and MAX_INDICATOR! ({})".format(self.output) max_pos_ptr = builder.alloca(ctx.int32_ty) builder.store(max_pos_ptr.type.pointee(-1), max_pos_ptr) @@ -2819,9 +2829,12 @@ def derivative(self, input=None, output=None, context=None): derivative of values returned by SoftMax : 1d or 2d array (depending on *OUTPUT_TYPE* of SoftMax) """ + if output is None: + output = self.function(input, context=context) + output_type = self._get_current_parameter_value(OUTPUT_TYPE, context) - size = len(input) - sm = self.function(input, params={OUTPUT_TYPE: ALL}, context=context) + size = len(output) + sm = self.function(output, params={OUTPUT_TYPE: ALL}, context=context) sm = np.squeeze(sm) if output_type == ALL: @@ -2839,7 +2852,7 @@ def derivative(self, input=None, output=None, context=None): # Return 1d array of derivatives for max element (i.e., the one chosen by SoftMax) derivative = np.empty(size) # Get the element of output returned as non-zero when output_type is not ALL - index_of_max = int(np.where(sm == np.max(sm))[0]) + index_of_max = int(np.where(output == np.max(output))[0][0]) max_val = sm[index_of_max] for i in range(size): if i == index_of_max: diff --git a/tests/functions/test_transfer.py b/tests/functions/test_transfer.py index 47dc5c0a94e..3c4cef5beef 100644 --- a/tests/functions/test_transfer.py +++ b/tests/functions/test_transfer.py @@ -81,11 +81,11 @@ def test_execute(func, variable, params, expected, benchmark, func_mode): (Functions.ReLU, test_var, {'gain':RAND1, 'bias':RAND2, 'leak':RAND3}, np.where((test_var - RAND2) > 0, RAND1, RAND1 * RAND3)), (Functions.Tanh, test_var, {'gain':RAND1, 'bias':RAND2, 'offset':RAND3, 'scale':RAND4}, tanh_derivative_helper), (Functions.SoftMax, test_var, {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_VAL}, 'per_item': False}, - [-0.010680386821751537, -0.011118109698906909, -0.01082040340318878, -0.010670257514724047, -0.010362498859374309, - -0.010933660158663306, -0.010397412260182806, -0.011602329078808718, 0.09684744183944892, -0.010262384043848513]), + [-0.010211427111966652, -0.010211427111966652, -0.010211427111966652, -0.010211427111966652, -0.010211427111966652, + -0.010211427111966652, -0.010211427111966652, -0.010211427111966652, 0.09190284400769985, -0.010211427111966652]), (Functions.SoftMax, test_var, {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_INDICATOR}, 'per_item': False}, - [-0.010680386821751537, -0.011118109698906909, -0.01082040340318878, -0.010670257514724047, -0.010362498859374309, - -0.010933660158663306, -0.010397412260182806, -0.011602329078808718, 0.09684744183944892, -0.010262384043848513]), + [-0.012062786611097685, -0.012062786611097685, -0.012062786611097685, -0.012062786611097685, -0.012062786611097685, + -0.012062786611097685, -0.012062786611097685, -0.012062786611097685, 0.10856507949987917, -0.012062786611097685]), ] @pytest.mark.function From e54eb253502db7825a3b66f2289fb567d66e8752 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Sat, 12 Nov 2022 21:59:12 -0500 Subject: [PATCH 072/127] functions/SoftMax: Add support for 2d inputs to SoftMax derivative (#2534) * llvm/jit: Generate module IR separately from parsing it to binary module This produces more readable reports in case of errors. Signed-off-by: Jan Vesely * tests/TransferFunction: Do not convert used variable/parameters to list Signed-off-by: Jan Vesely * tests/SoftMax: Add per-item=True tests Signed-off-by: Jan Vesely * tests/SoftMax: Add per-item tests with single element Signed-off-by: Jan Vesely * functions/SoftMax/derivative: Use correct max index for 2D output Remove outer dimension in compiled code. Add tests. Signed-off-by: Jan Vesely Signed-off-by: Jan Vesely --- .../nonstateful/transferfunctions.py | 11 ++++- psyneulink/core/llvm/jit_engine.py | 6 ++- tests/functions/test_transfer.py | 47 +++++++++++++++---- 3 files changed, 51 insertions(+), 13 deletions(-) diff --git a/psyneulink/core/components/functions/nonstateful/transferfunctions.py b/psyneulink/core/components/functions/nonstateful/transferfunctions.py index 381d80d9c6b..9812ddc5f0e 100644 --- a/psyneulink/core/components/functions/nonstateful/transferfunctions.py +++ b/psyneulink/core/components/functions/nonstateful/transferfunctions.py @@ -2699,6 +2699,7 @@ def __gen_llvm_apply(self, ctx, builder, params, state, arg_in, arg_out, output_ def _gen_llvm_function_derivative_body(self, ctx, builder, params, state, arg_in, arg_out, *, tags:frozenset): assert "derivative" in tags + assert arg_in.type == arg_out.type forward_tags = tags.difference({"derivative"}) # SoftMax derivative is calculated from the results. Recalculate them here @@ -2713,6 +2714,11 @@ def _gen_llvm_function_derivative_body(self, ctx, builder, params, state, arg_in assert self.output in {MAX_VAL, MAX_INDICATOR}, \ "Derivative of SoftMax is only implemented for MAX_VAL and MAX_INDICATOR! ({})".format(self.output) + if not pnlvm.helpers.is_scalar(arg_out.type.pointee.element): + assert len(arg_out.type.pointee) == 1 + arg_out = builder.gep(arg_out, [ctx.int32_ty(0), ctx.int32_ty(0)]) + all_out = builder.gep(all_out, [ctx.int32_ty(0), ctx.int32_ty(0)]) + max_pos_ptr = builder.alloca(ctx.int32_ty) builder.store(max_pos_ptr.type.pointee(-1), max_pos_ptr) max_val_ptr = builder.alloca(arg_out.type.pointee.element) @@ -2833,9 +2839,10 @@ def derivative(self, input=None, output=None, context=None): output = self.function(input, context=context) output_type = self._get_current_parameter_value(OUTPUT_TYPE, context) - size = len(output) sm = self.function(output, params={OUTPUT_TYPE: ALL}, context=context) sm = np.squeeze(sm) + size = len(sm) + assert (len(output) == 1 and len(output[0]) == size) or len(output) == size if output_type == ALL: # Return full Jacobian matrix of derivatives @@ -2852,7 +2859,7 @@ def derivative(self, input=None, output=None, context=None): # Return 1d array of derivatives for max element (i.e., the one chosen by SoftMax) derivative = np.empty(size) # Get the element of output returned as non-zero when output_type is not ALL - index_of_max = int(np.where(output == np.max(output))[0][0]) + index_of_max = int(np.where(output == np.max(output))[-1][0]) max_val = sm[index_of_max] for i in range(size): if i == index_of_max: diff --git a/psyneulink/core/llvm/jit_engine.py b/psyneulink/core/llvm/jit_engine.py index a5db35e2864..73fbf36683b 100644 --- a/psyneulink/core/llvm/jit_engine.py +++ b/psyneulink/core/llvm/jit_engine.py @@ -128,16 +128,18 @@ def _ptx_jit_constructor(): def _try_parse_module(module): + module_text_ir = str(module) + if "dump-llvm-gen" in debug_env: with open(module.name + '.generated.ll', 'w') as dump_file: - dump_file.write(str(module)) + dump_file.write(module_text_ir) # IR module is not the same as binding module. # "assembly" in this case is LLVM IR assembly. # This is intentional design decision to ease # compatibility between LLVM versions. try: - mod = binding.parse_assembly(str(module)) + mod = binding.parse_assembly(module_text_ir) mod.verify() except Exception as e: print("ERROR: llvm parsing failed: {}".format(e)) diff --git a/tests/functions/test_transfer.py b/tests/functions/test_transfer.py index 3c4cef5beef..f62691234fe 100644 --- a/tests/functions/test_transfer.py +++ b/tests/functions/test_transfer.py @@ -16,9 +16,10 @@ RAND3 = np.random.rand() RAND4 = np.random.rand() -softmax_helper = RAND1 * test_var -softmax_helper = softmax_helper - np.max(softmax_helper) -softmax_helper = np.exp(softmax_helper) / np.sum(np.exp(softmax_helper)) +softmax_helper = RAND1 * test_var +softmax_helper = softmax_helper - np.max(softmax_helper) +softmax_helper = np.exp(softmax_helper) / np.sum(np.exp(softmax_helper)) +softmax_helper2 = np.array((softmax_helper, softmax_helper)).reshape(2, -1) tanh_helper = (RAND1 * (test_var + RAND2 - RAND3) + RAND4) tanh_helper = np.tanh(tanh_helper) @@ -44,17 +45,35 @@ def gaussian_distort_helper(seed): [0.85314409, 0.00556188, 0.01070476, 0.0214405, 0.05559454, 0.08091079, 0.21657281, 0.19296643, 0.21343805, 0.92738261, 0.00483101], id="ANGLE"), + pytest.param(Functions.Gaussian, test_var, {'standard_deviation':RAND1, 'bias':RAND2, 'scale':RAND3, 'offset':RAND4}, gaussian_helper, id="GAUSSIAN"), - pytest.param(Functions.GaussianDistort, test_var.tolist(), {'bias': RAND1, 'variance':RAND2, 'offset':RAND3, 'scale':RAND4 }, gaussian_distort_helper(0), id="GAUSSIAN DISTORT GLOBAL SEED"), - pytest.param(Functions.GaussianDistort, test_var.tolist(), {'bias': RAND1, 'variance':RAND2, 'offset':RAND3, 'scale':RAND4, 'seed':0 }, gaussian_distort_helper(0), id="GAUSSIAN DISTORT"), + pytest.param(Functions.GaussianDistort, test_var, {'bias': RAND1, 'variance':RAND2, 'offset':RAND3, 'scale':RAND4 }, gaussian_distort_helper(0), id="GAUSSIAN DISTORT GLOBAL SEED"), + pytest.param(Functions.GaussianDistort, test_var, {'bias': RAND1, 'variance':RAND2, 'offset':RAND3, 'scale':RAND4, 'seed':0 }, gaussian_distort_helper(0), id="GAUSSIAN DISTORT"), + + # SoftMax 1D input pytest.param(Functions.SoftMax, test_var, {'gain':RAND1, 'per_item': False}, softmax_helper, id="SOFT_MAX ALL"), - pytest.param(Functions.SoftMax, test_var, {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_VAL}, 'per_item': False}, np.where(softmax_helper == np.max(softmax_helper), np.max(softmax_helper), 0), id="SOFT_MAX MAX_VAL"), + pytest.param(Functions.SoftMax, test_var, {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_VAL}, 'per_item': False}, np.where(softmax_helper == np.max(softmax_helper), softmax_helper, 0), id="SOFT_MAX MAX_VAL"), pytest.param(Functions.SoftMax, test_var, {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_INDICATOR}, 'per_item': False}, np.where(softmax_helper == np.max(softmax_helper), 1, 0), id="SOFT_MAX MAX_INDICATOR"), pytest.param(Functions.SoftMax, test_var, {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.PROB}, 'per_item': False}, [0.0, 0.0, 0.0, 0.0, test_var[4], 0.0, 0.0, 0.0, 0.0, 0.0], id="SOFT_MAX PROB"), - pytest.param(Functions.LinearMatrix, test_var.tolist(), {'matrix':test_matrix.tolist()}, np.dot(test_var, test_matrix), id="LINEAR_MATRIX SQUARE"), - pytest.param(Functions.LinearMatrix, test_var.tolist(), {'matrix':test_matrix_l.tolist()}, np.dot(test_var, test_matrix_l), id="LINEAR_MATRIX WIDE"), - pytest.param(Functions.LinearMatrix, test_var.tolist(), {'matrix':test_matrix_s.tolist()}, np.dot(test_var, test_matrix_s), id="LINEAR_MATRIX TALL"), + + # SoftMax 2D testing per-item + pytest.param(Functions.SoftMax, [test_var], {'gain':RAND1, 'per_item': True}, [softmax_helper], id="SOFT_MAX ALL 2D"), + pytest.param(Functions.SoftMax, [test_var], {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_VAL}, 'per_item': True}, [np.where(softmax_helper == np.max(softmax_helper), softmax_helper, 0)], id="SOFT_MAX MAX_VAL 2D"), + pytest.param(Functions.SoftMax, [test_var], {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_INDICATOR}, 'per_item': True}, [np.where(softmax_helper == np.max(softmax_helper), 1, 0)], id="SOFT_MAX MAX_INDICATOR 2D"), + pytest.param(Functions.SoftMax, [test_var], {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.PROB}, 'per_item': True}, + [[0.0, 0.0, 0.0, 0.0, test_var[4], 0.0, 0.0, 0.0, 0.0, 0.0]], id="SOFT_MAX PROB 2D"), + + # SoftMax per-item with 2 elements in input + pytest.param(Functions.SoftMax, [test_var, test_var], {'gain':RAND1, 'per_item': True}, softmax_helper2, id="SOFT_MAX ALL PER_ITEM"), + pytest.param(Functions.SoftMax, [test_var, test_var], {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_VAL}, 'per_item': True}, + np.where(softmax_helper2 == np.max(softmax_helper2), softmax_helper2, 0), id="SOFT_MAX MAX_VAL PER_ITEM"), + pytest.param(Functions.SoftMax, [test_var, test_var], {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_INDICATOR}, 'per_item': True}, + np.where(softmax_helper2 == np.max(softmax_helper2), 1, 0), id="SOFT_MAX MAX_INDICATOR PER_ITEM"), + + pytest.param(Functions.LinearMatrix, test_var, {'matrix':test_matrix}, np.dot(test_var, test_matrix), id="LINEAR_MATRIX SQUARE"), + pytest.param(Functions.LinearMatrix, test_var, {'matrix':test_matrix_l}, np.dot(test_var, test_matrix_l), id="LINEAR_MATRIX WIDE"), + pytest.param(Functions.LinearMatrix, test_var, {'matrix':test_matrix_s}, np.dot(test_var, test_matrix_s), id="LINEAR_MATRIX TALL"), ] @pytest.mark.function @@ -80,12 +99,22 @@ def test_execute(func, variable, params, expected, benchmark, func_mode): (Functions.Logistic, test_var, {'gain':RAND1, 'x_0':RAND2, 'offset':RAND3, 'scale':RAND4}, RAND1 * RAND4 * logistic_helper * (1 - logistic_helper)), (Functions.ReLU, test_var, {'gain':RAND1, 'bias':RAND2, 'leak':RAND3}, np.where((test_var - RAND2) > 0, RAND1, RAND1 * RAND3)), (Functions.Tanh, test_var, {'gain':RAND1, 'bias':RAND2, 'offset':RAND3, 'scale':RAND4}, tanh_derivative_helper), + + # SoftMax per-item=False (Functions.SoftMax, test_var, {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_VAL}, 'per_item': False}, [-0.010211427111966652, -0.010211427111966652, -0.010211427111966652, -0.010211427111966652, -0.010211427111966652, -0.010211427111966652, -0.010211427111966652, -0.010211427111966652, 0.09190284400769985, -0.010211427111966652]), (Functions.SoftMax, test_var, {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_INDICATOR}, 'per_item': False}, [-0.012062786611097685, -0.012062786611097685, -0.012062786611097685, -0.012062786611097685, -0.012062786611097685, -0.012062786611097685, -0.012062786611097685, -0.012062786611097685, 0.10856507949987917, -0.012062786611097685]), + + # SoftMax per-tem=True 2D single element + (Functions.SoftMax, [test_var], {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_VAL}, 'per_item': True}, + [-0.010211427111966652, -0.010211427111966652, -0.010211427111966652, -0.010211427111966652, -0.010211427111966652, + -0.010211427111966652, -0.010211427111966652, -0.010211427111966652, 0.09190284400769985, -0.010211427111966652]), + (Functions.SoftMax, test_var, {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_INDICATOR}, 'per_item': False}, + [-0.012062786611097685, -0.012062786611097685, -0.012062786611097685, -0.012062786611097685, -0.012062786611097685, + -0.012062786611097685, -0.012062786611097685, -0.012062786611097685, 0.10856507949987917, -0.012062786611097685]), ] @pytest.mark.function From f30f747071c65f40e6050a3ee5a255dcd1673248 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Thu, 10 Nov 2022 03:08:08 -0500 Subject: [PATCH 073/127] utils, projections/autoassociative: Don't use np.float It's an obsolete alias to 'float', use that instead. Signed-off-by: Jan Vesely --- psyneulink/core/globals/utilities.py | 2 +- .../projections/pathway/autoassociativeprojection.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/psyneulink/core/globals/utilities.py b/psyneulink/core/globals/utilities.py index 61c987cb823..34308ee3ab2 100644 --- a/psyneulink/core/globals/utilities.py +++ b/psyneulink/core/globals/utilities.py @@ -1031,7 +1031,7 @@ def type_match(value, value_type): return value if value_type in {int, np.integer, np.int64, np.int32}: return int(value) - if value_type in {float, np.float, np.float64, np.float32}: + if value_type in {float, np.float64, np.float32}: return float(value) if value_type is np.ndarray: return np.array(value) diff --git a/psyneulink/library/components/projections/pathway/autoassociativeprojection.py b/psyneulink/library/components/projections/pathway/autoassociativeprojection.py index 98c9948ca5d..1938d5e4bb8 100644 --- a/psyneulink/library/components/projections/pathway/autoassociativeprojection.py +++ b/psyneulink/library/components/projections/pathway/autoassociativeprojection.py @@ -382,11 +382,11 @@ def get_hetero_matrix(raw_hetero, size): # similar to get_hetero_matrix() above def get_auto_matrix(raw_auto, size): if isinstance(raw_auto, numbers.Number): - return np.diag(np.full(size, raw_auto, dtype=np.float)) + return np.diag(np.full(size, raw_auto, dtype=float)) elif ((isinstance(raw_auto, np.ndarray) and raw_auto.ndim == 1) or (isinstance(raw_auto, list) and np.array(raw_auto).ndim == 1)): if len(raw_auto) == 1: - return np.diag(np.full(size, raw_auto[0], dtype=np.float)) + return np.diag(np.full(size, raw_auto[0], dtype=float)) else: if len(raw_auto) != size: return None From 04d04386b5d0a7049fdb3ba4ff4859f5faf65bb8 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Sat, 12 Nov 2022 21:51:27 -0500 Subject: [PATCH 074/127] functions/TransferFunction/Angle: Optimize python version Original time: 247 usecs. Optimizations: * Use numpy array operations for sin, these are faster than Python iterations. Time: 214 usecs * Use numpy array operations for sin and cos, these are faster than Python iterations. Time: 191 usecs * Use numpy cumsum instead of complete product with partial divisions, this is both faster and more accurate. it also avoids potential issue with division by zero. Time: 180 usecs Signed-off-by: Jan Vesely --- .../functions/nonstateful/transferfunctions.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/psyneulink/core/components/functions/nonstateful/transferfunctions.py b/psyneulink/core/components/functions/nonstateful/transferfunctions.py index 9812ddc5f0e..a11a6e7bf9e 100644 --- a/psyneulink/core/components/functions/nonstateful/transferfunctions.py +++ b/psyneulink/core/components/functions/nonstateful/transferfunctions.py @@ -1789,14 +1789,16 @@ def _angle(self, value): value = np.squeeze(value) dim = len(value) + 1 angle = np.zeros(dim) - angle[0] = np.cos(value[0]) - prod = np.product([np.sin(value[k]) for k in range(1, dim - 1)]) - n_prod = prod + sin_value = np.sin(value) + cos_value = np.cos(value) + angle[0] = cos_value[0] + prod_a = np.cumprod(np.flip(sin_value))[:-1] + angle[dim - 1] = prod_a[-1] + prod_a[-1] = 1. + + # going down from the top of cumprod we skip: 2 edge values +1 extra for output size for j in range(1, dim - 1): - n_prod /= np.sin(value[j]) - amt = n_prod * np.cos(value[j]) - angle[j] = amt - angle[dim - 1] = prod + angle[j] = prod_a[dim -3 -j] * cos_value[j] return angle def _gen_llvm_function_body(self, ctx, builder, params, state, arg_in, arg_out, *, tags:frozenset): From a58dc57aceef9639d8b2fcb9689341a5bbbdeb13 Mon Sep 17 00:00:00 2001 From: "lgtm-com[bot]" <43144390+lgtm-com[bot]@users.noreply.github.com> Date: Sat, 12 Nov 2022 22:35:02 -0500 Subject: [PATCH 075/127] Add CodeQL workflow for GitHub code scanning (#2533) Co-authored-by: LGTM Migrator --- .github/workflows/codeql.yml | 41 ++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000000..146255ce292 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,41 @@ +name: "CodeQL" + +on: + push: + branches: [ "master", "devel" ] + pull_request: + branches: [ "master" ] + schedule: + - cron: "14 21 * * 5" + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ python ] + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + queries: +security-and-quality + + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{ matrix.language }}" From 2b23c43e74e2a239f55246f52c50dbcd6153c587 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Sat, 12 Nov 2022 22:41:16 -0500 Subject: [PATCH 076/127] github-actions/coedql: Run codeql analysis on PRs to devel Signed-off-by: Jan Vesely --- .github/workflows/codeql.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 146255ce292..79b66e82f71 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -4,7 +4,7 @@ on: push: branches: [ "master", "devel" ] pull_request: - branches: [ "master" ] + branches: [ "master", "devel" ] schedule: - cron: "14 21 * * 5" From 6f111d7d8ca2fde6097a16e14eb62a4fe186e8df Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Sat, 12 Nov 2022 22:51:09 -0500 Subject: [PATCH 077/127] tests/models/PredatorPrey: Don't use the same position in default variable Avoids zero values in distance computation during initialization. Signed-off-by: Jan Vesely --- tests/models/test_greedy_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/models/test_greedy_agent.py b/tests/models/test_greedy_agent.py index d74eb33df83..a12a4f99dd4 100644 --- a/tests/models/test_greedy_agent.py +++ b/tests/models/test_greedy_agent.py @@ -161,7 +161,7 @@ def action_fn(variable): # note: unitization is done in main loop greedy_action_mech = pnl.ProcessingMechanism(function=action_fn, input_ports=["predator", "player", "prey"], - default_variable=[[0,0],[0,0],[0,0]], name="ACTION") + default_variable=[[0, 1], [0, -1], [1, 0]], name="ACTION") direct_move = ComparatorMechanism(name='DIRECT MOVE',sample=player_pos, target=prey_pos) From 2018cc1117fd0fc566f408b699e56789a53a95f1 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Sun, 13 Nov 2022 01:32:13 -0500 Subject: [PATCH 078/127] utilities: Check that the compared value is not an empty list Empty list creates empty elementwise comparison which is false and deprecated. Add an explicit check to avoid that. Signed-off-by: Jan Vesely --- psyneulink/core/globals/utilities.py | 3 ++- tests/ports/test_input_ports.py | 20 ++++++++++--------- .../test_projection_specifications.py | 3 +-- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/psyneulink/core/globals/utilities.py b/psyneulink/core/globals/utilities.py index 34308ee3ab2..9d45134e953 100644 --- a/psyneulink/core/globals/utilities.py +++ b/psyneulink/core/globals/utilities.py @@ -440,7 +440,8 @@ def iscompatible(candidate, reference=None, **kargs): try: with warnings.catch_warnings(): warnings.simplefilter(action='ignore', category=FutureWarning) - if reference is not None and (candidate == reference): + # np.array(...).size > 0 checks for empty list. Everything else create single element (dtype=obejct) array + if reference is not None and np.array(candidate, dtype=object).size > 0 and (candidate == reference): return True # if reference is not None: # if (isinstance(reference, (bool, int, float)) diff --git a/tests/ports/test_input_ports.py b/tests/ports/test_input_ports.py index a2c1d807a71..bd2568b018a 100644 --- a/tests/ports/test_input_ports.py +++ b/tests/ports/test_input_ports.py @@ -112,19 +112,21 @@ def test_default_input(self, default_input): comp = pnl.Composition(nodes=(m, pnl.NodeRole.INTERNAL)) assert pnl.NodeRole.INTERNAL in comp.get_roles_by_node(m) assert pnl.NodeRole.INPUT not in comp.get_roles_by_node(m) - assert not m.path_afferents + + assert not m.path_afferents # No path_afferents since internal_only is set by default_input + + if default_input is None: - with pytest.warns(UserWarning) as warning: # Warn, since default_input is NOT set + with pytest.warns(UserWarning) as warnings: # Warn, since default_input is NOT set comp.run() - assert repr(warning[1].message.args[0]) == '"InputPort (\'INTERNAL_NODE\') of \'TransferMechanism-0\' ' \ - 'doesn\'t have any afferent Projections."' - assert m.input_port.value == variable # For Mechanisms other than controller, default_variable seems - assert m.value == variable # to still be used even though default_input is NOT set + assert any(repr(w.message.args[0]) == '"InputPort (\'INTERNAL_NODE\') of \'TransferMechanism-0\' ' + 'doesn\'t have any afferent Projections."' + for w in warnings) else: - assert not m.path_afferents # No path_afferents since internal_only is set by default_input comp.run() # No warning since default_input is set - assert m.input_port.value == variable - assert m.value == variable + + assert m.input_port.value == variable # For Mechanisms other than controller, default_variable seems + assert m.value == variable # to still be used even though default_input is NOT set def test_no_efferents(self): A = pnl.InputPort() diff --git a/tests/projections/test_projection_specifications.py b/tests/projections/test_projection_specifications.py index 02edd207534..a7338f2efe7 100644 --- a/tests/projections/test_projection_specifications.py +++ b/tests/projections/test_projection_specifications.py @@ -480,8 +480,7 @@ def test_no_warning_when_matrix_specified(self): ) c.add_linear_processing_pathway([m0, p0, m1]) for warn in w: - if r'elementwise comparison failed; returning scalar instead' in warn.message.args[0]: - raise + assert 'elementwise comparison failed; returning scalar instead' not in warn.message.args[0] # KDM: this is a good candidate for pytest.parametrize def test_masked_mapping_projection(self): From c13a0cb48dcf83539b27a78c5827b9e2b274692c Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Sun, 13 Nov 2022 14:58:25 -0500 Subject: [PATCH 079/127] github-actions: Run only the latest instance of the main CI workflows (#2537) One per branch/PR for testing, docs, and codeql analysis. Cancel old jobs, even those in progress. Signed-off-by: Jan Vesely --- .github/workflows/codeql.yml | 7 +++++++ .github/workflows/pnl-ci-docs.yml | 7 +++++++ .github/workflows/pnl-ci.yml | 7 +++++++ 3 files changed, 21 insertions(+) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 79b66e82f71..4520507bb21 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -8,6 +8,13 @@ on: schedule: - cron: "14 21 * * 5" +# run only the latest instance of this workflow job for the current branch/PR +# cancel older runs +# fall back to run id if not available (run id is unique -> no cancellations) +concurrency: + group: ci-${{ github.ref || github.run_id }}-${{ github.workflow }} + cancel-in-progress: true + jobs: analyze: name: Analyze diff --git a/.github/workflows/pnl-ci-docs.yml b/.github/workflows/pnl-ci-docs.yml index 507befbc8d0..e85043e7f24 100644 --- a/.github/workflows/pnl-ci-docs.yml +++ b/.github/workflows/pnl-ci-docs.yml @@ -8,6 +8,13 @@ on: - 'v*' pull_request: +# run only the latest instance of this workflow job for the current branch/PR +# cancel older runs +# fall back to run id if not available (run id is unique -> no cancellations) +concurrency: + group: ci-${{ github.ref || github.run_id }}-${{ github.workflow }} + cancel-in-progress: true + jobs: docs-build: runs-on: ${{ matrix.os }} diff --git a/.github/workflows/pnl-ci.yml b/.github/workflows/pnl-ci.yml index 95f4d611a88..be57f480190 100644 --- a/.github/workflows/pnl-ci.yml +++ b/.github/workflows/pnl-ci.yml @@ -11,6 +11,13 @@ on: - 'v**' pull_request: +# run only the latest instance of this workflow job for the current branch/PR +# cancel older runs +# fall back to run id if not available (run id is unique -> no cancellations) +concurrency: + group: ci-${{ github.ref || github.run_id }}-${{ github.workflow }} + cancel-in-progress: true + jobs: build: runs-on: ${{ matrix.os }} From f4dda146b0d5638acc1285ec9f2f5eada2396926 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Sun, 13 Nov 2022 16:12:17 -0500 Subject: [PATCH 080/127] functions/SoftMax: Use "ALL" results to calculate derivative directly The results can be provided from outside, or calculated locally. Adjust the compiled variant, and the test results to match. Signed-off-by: Jan Vesely --- .../functions/nonstateful/transferfunctions.py | 18 ++++++++---------- tests/functions/test_transfer.py | 18 +++++++++--------- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/psyneulink/core/components/functions/nonstateful/transferfunctions.py b/psyneulink/core/components/functions/nonstateful/transferfunctions.py index 9812ddc5f0e..1a19589fdfe 100644 --- a/psyneulink/core/components/functions/nonstateful/transferfunctions.py +++ b/psyneulink/core/components/functions/nonstateful/transferfunctions.py @@ -2702,13 +2702,10 @@ def _gen_llvm_function_derivative_body(self, ctx, builder, params, state, arg_in assert arg_in.type == arg_out.type forward_tags = tags.difference({"derivative"}) - # SoftMax derivative is calculated from the results. Recalculate them here - base_out = builder.alloca(arg_out.type.pointee) - builder = self._gen_llvm_function_body(ctx, builder, params, state, arg_in, base_out, output_type=self.output, tags=forward_tags) - - + # SoftMax derivative is calculated from the "ALL" results. + # Those can provided from outside, but we don't support receiving data in arg_out all_out = builder.alloca(arg_out.type.pointee) - builder = self._gen_llvm_function_body(ctx, builder, params, state, base_out, all_out, output_type=ALL, tags=forward_tags) + builder = self._gen_llvm_function_body(ctx, builder, params, state, arg_in, all_out, output_type=ALL, tags=forward_tags) # The rest of the algorithm is for MAX_VAL and MAX_INDICATOR only assert self.output in {MAX_VAL, MAX_INDICATOR}, \ @@ -2836,14 +2833,15 @@ def derivative(self, input=None, output=None, context=None): """ if output is None: - output = self.function(input, context=context) + output = self.function(input, params={OUTPUT_TYPE: ALL}, context=context) + else: + assert not any(o == 0 for o in output) - output_type = self._get_current_parameter_value(OUTPUT_TYPE, context) - sm = self.function(output, params={OUTPUT_TYPE: ALL}, context=context) - sm = np.squeeze(sm) + sm = np.squeeze(output) size = len(sm) assert (len(output) == 1 and len(output[0]) == size) or len(output) == size + output_type = self._get_current_parameter_value(OUTPUT_TYPE, context) if output_type == ALL: # Return full Jacobian matrix of derivatives derivative = np.empty([size, size]) diff --git a/tests/functions/test_transfer.py b/tests/functions/test_transfer.py index f62691234fe..2289258595c 100644 --- a/tests/functions/test_transfer.py +++ b/tests/functions/test_transfer.py @@ -102,19 +102,19 @@ def test_execute(func, variable, params, expected, benchmark, func_mode): # SoftMax per-item=False (Functions.SoftMax, test_var, {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_VAL}, 'per_item': False}, - [-0.010211427111966652, -0.010211427111966652, -0.010211427111966652, -0.010211427111966652, -0.010211427111966652, - -0.010211427111966652, -0.010211427111966652, -0.010211427111966652, 0.09190284400769985, -0.010211427111966652]), + [-0.010680386821751537, -0.011118109698906909, -0.01082040340318878, -0.010670257514724047, -0.010362498859374309, + -0.010933660158663306, -0.010397412260182806, -0.011602329078808718, 0.09684744183944892, -0.010262384043848513]), (Functions.SoftMax, test_var, {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_INDICATOR}, 'per_item': False}, - [-0.012062786611097685, -0.012062786611097685, -0.012062786611097685, -0.012062786611097685, -0.012062786611097685, - -0.012062786611097685, -0.012062786611097685, -0.012062786611097685, 0.10856507949987917, -0.012062786611097685]), + [-0.010680386821751537, -0.011118109698906909, -0.01082040340318878, -0.010670257514724047, -0.010362498859374309, + -0.010933660158663306, -0.010397412260182806, -0.011602329078808718, 0.09684744183944892, -0.010262384043848513]), # SoftMax per-tem=True 2D single element (Functions.SoftMax, [test_var], {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_VAL}, 'per_item': True}, - [-0.010211427111966652, -0.010211427111966652, -0.010211427111966652, -0.010211427111966652, -0.010211427111966652, - -0.010211427111966652, -0.010211427111966652, -0.010211427111966652, 0.09190284400769985, -0.010211427111966652]), - (Functions.SoftMax, test_var, {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_INDICATOR}, 'per_item': False}, - [-0.012062786611097685, -0.012062786611097685, -0.012062786611097685, -0.012062786611097685, -0.012062786611097685, - -0.012062786611097685, -0.012062786611097685, -0.012062786611097685, 0.10856507949987917, -0.012062786611097685]), + [[-0.010680386821751537, -0.011118109698906909, -0.01082040340318878, -0.010670257514724047, -0.010362498859374309, + -0.010933660158663306, -0.010397412260182806, -0.011602329078808718, 0.09684744183944892, -0.010262384043848513]]), + (Functions.SoftMax, [test_var], {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_INDICATOR}, 'per_item': True}, + [[-0.010680386821751537, -0.011118109698906909, -0.01082040340318878, -0.010670257514724047, -0.010362498859374309, + -0.010933660158663306, -0.010397412260182806, -0.011602329078808718, 0.09684744183944892, -0.010262384043848513]]), ] @pytest.mark.function From c02f34d2745e8eee775145d5fbfbbadc277f20e3 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Sun, 13 Nov 2022 19:27:02 -0500 Subject: [PATCH 081/127] tests/functions/Transfer: Add derivative test when using OUTPUT_MDOE=ALL Python only. Signed-off-by: Jan Vesely --- tests/functions/test_transfer.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/functions/test_transfer.py b/tests/functions/test_transfer.py index 2289258595c..b3b3ec80c2e 100644 --- a/tests/functions/test_transfer.py +++ b/tests/functions/test_transfer.py @@ -107,6 +107,17 @@ def test_execute(func, variable, params, expected, benchmark, func_mode): (Functions.SoftMax, test_var, {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_INDICATOR}, 'per_item': False}, [-0.010680386821751537, -0.011118109698906909, -0.01082040340318878, -0.010670257514724047, -0.010362498859374309, -0.010933660158663306, -0.010397412260182806, -0.011602329078808718, 0.09684744183944892, -0.010262384043848513]), + (Functions.SoftMax, test_var, {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.ALL}, 'per_item': False}, + [[ 0.08863569, -0.01005855, -0.00978921, -0.00965338, -0.00937495, -0.00989168, -0.00940653, -0.01049662, -0.01068039, -0.00928437], + [-0.01005855, 0.09185608, -0.01019041, -0.01004901, -0.00975917, -0.01029708, -0.00979205, -0.01092681, -0.01111811, -0.00966488], + [-0.00978921, -0.01019041, 0.08966934, -0.00977993, -0.00949785, -0.01002135, -0.00952985, -0.01063423, -0.0108204, -0.00940609], + [-0.00965338, -0.01004901, -0.00977993, 0.08856078, -0.00936606, -0.0098823, -0.00939761, -0.01048667, -0.01067026, -0.00927557], + [-0.00937495, -0.00975917, -0.00949785, -0.00936606, 0.08627659, -0.00959726, -0.00912656, -0.0101842, -0.0103625, -0.00900804], + [-0.00989168, -0.01029708, -0.01002135, -0.0098823, -0.00959726, 0.09050301, -0.0096296, -0.01074554, -0.01093366, -0.00950454], + [-0.00940653, -0.00979205, -0.00952985, -0.00939761, -0.00912656, -0.0096296, 0.08653653, -0.01021852, -0.01039741, -0.00903839], + [-0.01049662, -0.01092681, -0.01063423, -0.01048667, -0.0101842, -0.01074554, -0.01021852, 0.09538073, -0.01160233, -0.01008581], + [-0.01068039, -0.01111811, -0.0108204, -0.01067026, -0.0103625, -0.01093366, -0.01039741, -0.01160233, 0.09684744, -0.01026238], + [-0.00928437, -0.00966488, -0.00940609, -0.00927557, -0.00900804, -0.00950454, -0.00903839, -0.01008581, -0.01026238, 0.08553008]]), # SoftMax per-tem=True 2D single element (Functions.SoftMax, [test_var], {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_VAL}, 'per_item': True}, @@ -115,6 +126,17 @@ def test_execute(func, variable, params, expected, benchmark, func_mode): (Functions.SoftMax, [test_var], {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_INDICATOR}, 'per_item': True}, [[-0.010680386821751537, -0.011118109698906909, -0.01082040340318878, -0.010670257514724047, -0.010362498859374309, -0.010933660158663306, -0.010397412260182806, -0.011602329078808718, 0.09684744183944892, -0.010262384043848513]]), + (Functions.SoftMax, [test_var], {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.ALL}, 'per_item': True}, + [[ 0.08863569, -0.01005855, -0.00978921, -0.00965338, -0.00937495, -0.00989168, -0.00940653, -0.01049662, -0.01068039, -0.00928437], + [-0.01005855, 0.09185608, -0.01019041, -0.01004901, -0.00975917, -0.01029708, -0.00979205, -0.01092681, -0.01111811, -0.00966488], + [-0.00978921, -0.01019041, 0.08966934, -0.00977993, -0.00949785, -0.01002135, -0.00952985, -0.01063423, -0.0108204, -0.00940609], + [-0.00965338, -0.01004901, -0.00977993, 0.08856078, -0.00936606, -0.0098823, -0.00939761, -0.01048667, -0.01067026, -0.00927557], + [-0.00937495, -0.00975917, -0.00949785, -0.00936606, 0.08627659, -0.00959726, -0.00912656, -0.0101842, -0.0103625, -0.00900804], + [-0.00989168, -0.01029708, -0.01002135, -0.0098823, -0.00959726, 0.09050301, -0.0096296, -0.01074554, -0.01093366, -0.00950454], + [-0.00940653, -0.00979205, -0.00952985, -0.00939761, -0.00912656, -0.0096296, 0.08653653, -0.01021852, -0.01039741, -0.00903839], + [-0.01049662, -0.01092681, -0.01063423, -0.01048667, -0.0101842, -0.01074554, -0.01021852, 0.09538073, -0.01160233, -0.01008581], + [-0.01068039, -0.01111811, -0.0108204, -0.01067026, -0.0103625, -0.01093366, -0.01039741, -0.01160233, 0.09684744, -0.01026238], + [-0.00928437, -0.00966488, -0.00940609, -0.00927557, -0.00900804, -0.00950454, -0.00903839, -0.01008581, -0.01026238, 0.08553008]]), ] @pytest.mark.function @@ -122,6 +144,9 @@ def test_execute(func, variable, params, expected, benchmark, func_mode): @pytest.mark.benchmark @pytest.mark.parametrize("func, variable, params, expected", derivative_test_data, ids=lambda x: getattr(x, 'name', None)) def test_transfer_derivative(func, variable, params, expected, benchmark, func_mode): + if func == Functions.SoftMax and params['params'][kw.OUTPUT_TYPE] == kw.ALL and func_mode != "Python": + pytest.skip("Compiled derivative using 'ALL' is not implemented") + f = func(default_variable=variable, **params) benchmark.group = "TransferFunction " + func.componentName + " Derivative" if func_mode == 'Python': From b39bd921d74fcda244e30aab4c81390808bca2d3 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Sun, 13 Nov 2022 19:43:17 -0500 Subject: [PATCH 082/127] tests/TransferFunction: Use keywords instead of hardcoded strings as parameters Don't use nested 'params' Signed-off-by: Jan Vesely --- tests/functions/test_transfer.py | 71 +++++++++++++++++--------------- 1 file changed, 37 insertions(+), 34 deletions(-) diff --git a/tests/functions/test_transfer.py b/tests/functions/test_transfer.py index b3b3ec80c2e..65d161e1ce1 100644 --- a/tests/functions/test_transfer.py +++ b/tests/functions/test_transfer.py @@ -35,45 +35,48 @@ def gaussian_distort_helper(seed): test_data = [ - pytest.param(Functions.Linear, test_var, {'slope':RAND1, 'intercept':RAND2}, test_var * RAND1 + RAND2, id="LINEAR"), - pytest.param(Functions.Exponential, test_var, {'scale':RAND1, 'rate':RAND2}, RAND1 * np.exp(RAND2 * test_var), id="EXPONENTIAL"), - pytest.param(Functions.Logistic, test_var, {'gain':RAND1, 'x_0':RAND2, 'offset':RAND3, 'scale':RAND4}, RAND4 / (1 + np.exp(-(RAND1 * (test_var - RAND2)) + RAND3)), id="LOGISTIC"), - pytest.param(Functions.Tanh, test_var, {'gain':RAND1, 'bias':RAND2, 'x_0':RAND3, 'offset':RAND4}, tanh_helper, id="TANH"), - pytest.param(Functions.ReLU, test_var, {'gain':RAND1, 'bias':RAND2, 'leak':RAND3}, np.maximum(RAND1 * (test_var - RAND2), RAND3 * RAND1 *(test_var - RAND2)), id="RELU"), + pytest.param(Functions.Linear, test_var, {kw.SLOPE:RAND1, kw.INTERCEPT:RAND2}, test_var * RAND1 + RAND2, id="LINEAR"), + pytest.param(Functions.Exponential, test_var, {kw.SCALE:RAND1, kw.RATE:RAND2}, RAND1 * np.exp(RAND2 * test_var), id="EXPONENTIAL"), + pytest.param(Functions.Logistic, test_var, {kw.GAIN:RAND1, kw.X_0:RAND2, kw.OFFSET:RAND3, kw.SCALE:RAND4}, RAND4 / (1 + np.exp(-(RAND1 * (test_var - RAND2)) + RAND3)), id="LOGISTIC"), + pytest.param(Functions.Tanh, test_var, {kw.GAIN:RAND1, kw.BIAS:RAND2, kw.X_0:RAND3, kw.OFFSET:RAND4}, tanh_helper, id="TANH"), + pytest.param(Functions.ReLU, test_var, {kw.GAIN:RAND1, kw.BIAS:RAND2, kw.LEAK:RAND3}, np.maximum(RAND1 * (test_var - RAND2), RAND3 * RAND1 *(test_var - RAND2)), id="RELU"), + # Angle doesn't have a helper using 'test_var', hardcode the input as well pytest.param(Functions.Angle, [0.5488135, 0.71518937, 0.60276338, 0.54488318, 0.4236548, 0.64589411, 0.43758721, 0.891773, 0.96366276, 0.38344152], {}, [0.85314409, 0.00556188, 0.01070476, 0.0214405, 0.05559454, 0.08091079, 0.21657281, 0.19296643, 0.21343805, 0.92738261, 0.00483101], id="ANGLE"), - pytest.param(Functions.Gaussian, test_var, {'standard_deviation':RAND1, 'bias':RAND2, 'scale':RAND3, 'offset':RAND4}, gaussian_helper, id="GAUSSIAN"), - pytest.param(Functions.GaussianDistort, test_var, {'bias': RAND1, 'variance':RAND2, 'offset':RAND3, 'scale':RAND4 }, gaussian_distort_helper(0), id="GAUSSIAN DISTORT GLOBAL SEED"), - pytest.param(Functions.GaussianDistort, test_var, {'bias': RAND1, 'variance':RAND2, 'offset':RAND3, 'scale':RAND4, 'seed':0 }, gaussian_distort_helper(0), id="GAUSSIAN DISTORT"), + pytest.param(Functions.Gaussian, test_var, {kw.STANDARD_DEVIATION:RAND1, kw.BIAS:RAND2, kw.SCALE:RAND3, kw.OFFSET:RAND4}, gaussian_helper, id="GAUSSIAN"), + pytest.param(Functions.GaussianDistort, test_var, {kw.BIAS: RAND1, kw.VARIANCE:RAND2, kw.OFFSET:RAND3, kw.SCALE:RAND4 }, gaussian_distort_helper(0), id="GAUSSIAN DISTORT GLOBAL SEED"), + pytest.param(Functions.GaussianDistort, test_var, {kw.BIAS: RAND1, kw.VARIANCE:RAND2, kw.OFFSET:RAND3, kw.SCALE:RAND4, 'seed':0 }, gaussian_distort_helper(0), id="GAUSSIAN DISTORT"), # SoftMax 1D input - pytest.param(Functions.SoftMax, test_var, {'gain':RAND1, 'per_item': False}, softmax_helper, id="SOFT_MAX ALL"), - pytest.param(Functions.SoftMax, test_var, {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_VAL}, 'per_item': False}, np.where(softmax_helper == np.max(softmax_helper), softmax_helper, 0), id="SOFT_MAX MAX_VAL"), - pytest.param(Functions.SoftMax, test_var, {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_INDICATOR}, 'per_item': False}, np.where(softmax_helper == np.max(softmax_helper), 1, 0), id="SOFT_MAX MAX_INDICATOR"), - pytest.param(Functions.SoftMax, test_var, {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.PROB}, 'per_item': False}, + pytest.param(Functions.SoftMax, test_var, {kw.GAIN:RAND1, kw.PER_ITEM:False}, softmax_helper, id="SOFT_MAX ALL"), + pytest.param(Functions.SoftMax, test_var, {kw.GAIN:RAND1, kw.OUTPUT_TYPE:kw.MAX_VAL, kw.PER_ITEM:False}, np.where(softmax_helper == np.max(softmax_helper), softmax_helper, 0), id="SOFT_MAX MAX_VAL"), + pytest.param(Functions.SoftMax, test_var, {kw.GAIN:RAND1, kw.OUTPUT_TYPE:kw.MAX_INDICATOR, kw.PER_ITEM:False}, np.where(softmax_helper == np.max(softmax_helper), 1, 0), id="SOFT_MAX MAX_INDICATOR"), + pytest.param(Functions.SoftMax, test_var, {kw.GAIN:RAND1, kw.OUTPUT_TYPE:kw.PROB, kw.PER_ITEM:False}, [0.0, 0.0, 0.0, 0.0, test_var[4], 0.0, 0.0, 0.0, 0.0, 0.0], id="SOFT_MAX PROB"), # SoftMax 2D testing per-item - pytest.param(Functions.SoftMax, [test_var], {'gain':RAND1, 'per_item': True}, [softmax_helper], id="SOFT_MAX ALL 2D"), - pytest.param(Functions.SoftMax, [test_var], {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_VAL}, 'per_item': True}, [np.where(softmax_helper == np.max(softmax_helper), softmax_helper, 0)], id="SOFT_MAX MAX_VAL 2D"), - pytest.param(Functions.SoftMax, [test_var], {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_INDICATOR}, 'per_item': True}, [np.where(softmax_helper == np.max(softmax_helper), 1, 0)], id="SOFT_MAX MAX_INDICATOR 2D"), - pytest.param(Functions.SoftMax, [test_var], {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.PROB}, 'per_item': True}, + pytest.param(Functions.SoftMax, [test_var], {kw.GAIN:RAND1, kw.PER_ITEM:True}, [softmax_helper], id="SOFT_MAX ALL 2D"), + pytest.param(Functions.SoftMax, [test_var], {kw.GAIN:RAND1, kw.OUTPUT_TYPE:kw.MAX_VAL, kw.PER_ITEM:True}, + [np.where(softmax_helper == np.max(softmax_helper), softmax_helper, 0)], id="SOFT_MAX MAX_VAL 2D"), + pytest.param(Functions.SoftMax, [test_var], {kw.GAIN:RAND1, kw.OUTPUT_TYPE:kw.MAX_INDICATOR, kw.PER_ITEM:True}, + [np.where(softmax_helper == np.max(softmax_helper), 1, 0)], id="SOFT_MAX MAX_INDICATOR 2D"), + pytest.param(Functions.SoftMax, [test_var], {kw.GAIN:RAND1, kw.OUTPUT_TYPE:kw.PROB, kw.PER_ITEM:True}, [[0.0, 0.0, 0.0, 0.0, test_var[4], 0.0, 0.0, 0.0, 0.0, 0.0]], id="SOFT_MAX PROB 2D"), # SoftMax per-item with 2 elements in input - pytest.param(Functions.SoftMax, [test_var, test_var], {'gain':RAND1, 'per_item': True}, softmax_helper2, id="SOFT_MAX ALL PER_ITEM"), - pytest.param(Functions.SoftMax, [test_var, test_var], {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_VAL}, 'per_item': True}, + pytest.param(Functions.SoftMax, [test_var, test_var], {kw.GAIN:RAND1, kw.PER_ITEM: True}, softmax_helper2, id="SOFT_MAX ALL PER_ITEM"), + pytest.param(Functions.SoftMax, [test_var, test_var], {kw.GAIN:RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_VAL}, kw.PER_ITEM: True}, np.where(softmax_helper2 == np.max(softmax_helper2), softmax_helper2, 0), id="SOFT_MAX MAX_VAL PER_ITEM"), - pytest.param(Functions.SoftMax, [test_var, test_var], {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_INDICATOR}, 'per_item': True}, + pytest.param(Functions.SoftMax, [test_var, test_var], {kw.GAIN:RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_INDICATOR}, kw.PER_ITEM: True}, np.where(softmax_helper2 == np.max(softmax_helper2), 1, 0), id="SOFT_MAX MAX_INDICATOR PER_ITEM"), - pytest.param(Functions.LinearMatrix, test_var, {'matrix':test_matrix}, np.dot(test_var, test_matrix), id="LINEAR_MATRIX SQUARE"), - pytest.param(Functions.LinearMatrix, test_var, {'matrix':test_matrix_l}, np.dot(test_var, test_matrix_l), id="LINEAR_MATRIX WIDE"), - pytest.param(Functions.LinearMatrix, test_var, {'matrix':test_matrix_s}, np.dot(test_var, test_matrix_s), id="LINEAR_MATRIX TALL"), + pytest.param(Functions.LinearMatrix, test_var, {kw.MATRIX:test_matrix}, np.dot(test_var, test_matrix), id="LINEAR_MATRIX SQUARE"), + pytest.param(Functions.LinearMatrix, test_var, {kw.MATRIX:test_matrix_l}, np.dot(test_var, test_matrix_l), id="LINEAR_MATRIX WIDE"), + pytest.param(Functions.LinearMatrix, test_var, {kw.MATRIX:test_matrix_s}, np.dot(test_var, test_matrix_s), id="LINEAR_MATRIX TALL"), ] @pytest.mark.function @@ -94,20 +97,20 @@ def test_execute(func, variable, params, expected, benchmark, func_mode): tanh_derivative_helper = (1 - np.tanh(tanh_derivative_helper)**2) * RAND4 * RAND1 derivative_test_data = [ - (Functions.Linear, test_var, {'slope':RAND1, 'intercept':RAND2}, RAND1), - (Functions.Exponential, test_var, {'scale':RAND1, 'rate':RAND2}, RAND1 * RAND2 * np.exp(RAND2 * test_var)), - (Functions.Logistic, test_var, {'gain':RAND1, 'x_0':RAND2, 'offset':RAND3, 'scale':RAND4}, RAND1 * RAND4 * logistic_helper * (1 - logistic_helper)), - (Functions.ReLU, test_var, {'gain':RAND1, 'bias':RAND2, 'leak':RAND3}, np.where((test_var - RAND2) > 0, RAND1, RAND1 * RAND3)), - (Functions.Tanh, test_var, {'gain':RAND1, 'bias':RAND2, 'offset':RAND3, 'scale':RAND4}, tanh_derivative_helper), + (Functions.Linear, test_var, {kw.SLOPE:RAND1, kw.INTERCEPT:RAND2}, RAND1), + (Functions.Exponential, test_var, {kw.SCALE:RAND1, kw.RATE:RAND2}, RAND1 * RAND2 * np.exp(RAND2 * test_var)), + (Functions.Logistic, test_var, {kw.GAIN:RAND1, kw.X_0:RAND2, kw.OFFSET:RAND3, kw.SCALE:RAND4}, RAND1 * RAND4 * logistic_helper * (1 - logistic_helper)), + (Functions.ReLU, test_var, {kw.GAIN:RAND1, kw.BIAS:RAND2, kw.LEAK:RAND3}, np.where((test_var - RAND2) > 0, RAND1, RAND1 * RAND3)), + (Functions.Tanh, test_var, {kw.GAIN:RAND1, kw.BIAS:RAND2, kw.OFFSET:RAND3, kw.SCALE:RAND4}, tanh_derivative_helper), # SoftMax per-item=False - (Functions.SoftMax, test_var, {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_VAL}, 'per_item': False}, + (Functions.SoftMax, test_var, {kw.GAIN:RAND1, kw.OUTPUT_TYPE:kw.MAX_VAL, kw.PER_ITEM:False}, [-0.010680386821751537, -0.011118109698906909, -0.01082040340318878, -0.010670257514724047, -0.010362498859374309, -0.010933660158663306, -0.010397412260182806, -0.011602329078808718, 0.09684744183944892, -0.010262384043848513]), - (Functions.SoftMax, test_var, {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_INDICATOR}, 'per_item': False}, + (Functions.SoftMax, test_var, {kw.GAIN:RAND1, kw.OUTPUT_TYPE:kw.MAX_INDICATOR, kw.PER_ITEM:False}, [-0.010680386821751537, -0.011118109698906909, -0.01082040340318878, -0.010670257514724047, -0.010362498859374309, -0.010933660158663306, -0.010397412260182806, -0.011602329078808718, 0.09684744183944892, -0.010262384043848513]), - (Functions.SoftMax, test_var, {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.ALL}, 'per_item': False}, + (Functions.SoftMax, test_var, {kw.GAIN:RAND1, kw.OUTPUT_TYPE:kw.ALL, kw.PER_ITEM:False}, [[ 0.08863569, -0.01005855, -0.00978921, -0.00965338, -0.00937495, -0.00989168, -0.00940653, -0.01049662, -0.01068039, -0.00928437], [-0.01005855, 0.09185608, -0.01019041, -0.01004901, -0.00975917, -0.01029708, -0.00979205, -0.01092681, -0.01111811, -0.00966488], [-0.00978921, -0.01019041, 0.08966934, -0.00977993, -0.00949785, -0.01002135, -0.00952985, -0.01063423, -0.0108204, -0.00940609], @@ -120,13 +123,13 @@ def test_execute(func, variable, params, expected, benchmark, func_mode): [-0.00928437, -0.00966488, -0.00940609, -0.00927557, -0.00900804, -0.00950454, -0.00903839, -0.01008581, -0.01026238, 0.08553008]]), # SoftMax per-tem=True 2D single element - (Functions.SoftMax, [test_var], {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_VAL}, 'per_item': True}, + (Functions.SoftMax, [test_var], {kw.GAIN:RAND1, kw.OUTPUT_TYPE:kw.MAX_VAL, kw.PER_ITEM:True}, [[-0.010680386821751537, -0.011118109698906909, -0.01082040340318878, -0.010670257514724047, -0.010362498859374309, -0.010933660158663306, -0.010397412260182806, -0.011602329078808718, 0.09684744183944892, -0.010262384043848513]]), - (Functions.SoftMax, [test_var], {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_INDICATOR}, 'per_item': True}, + (Functions.SoftMax, [test_var], {kw.GAIN:RAND1, kw.OUTPUT_TYPE:kw.MAX_INDICATOR, kw.PER_ITEM:True}, [[-0.010680386821751537, -0.011118109698906909, -0.01082040340318878, -0.010670257514724047, -0.010362498859374309, -0.010933660158663306, -0.010397412260182806, -0.011602329078808718, 0.09684744183944892, -0.010262384043848513]]), - (Functions.SoftMax, [test_var], {'gain':RAND1, 'params':{kw.OUTPUT_TYPE:kw.ALL}, 'per_item': True}, + (Functions.SoftMax, [test_var], {kw.GAIN:RAND1, kw.OUTPUT_TYPE:kw.ALL, kw.PER_ITEM:True}, [[ 0.08863569, -0.01005855, -0.00978921, -0.00965338, -0.00937495, -0.00989168, -0.00940653, -0.01049662, -0.01068039, -0.00928437], [-0.01005855, 0.09185608, -0.01019041, -0.01004901, -0.00975917, -0.01029708, -0.00979205, -0.01092681, -0.01111811, -0.00966488], [-0.00978921, -0.01019041, 0.08966934, -0.00977993, -0.00949785, -0.01002135, -0.00952985, -0.01063423, -0.0108204, -0.00940609], @@ -144,7 +147,7 @@ def test_execute(func, variable, params, expected, benchmark, func_mode): @pytest.mark.benchmark @pytest.mark.parametrize("func, variable, params, expected", derivative_test_data, ids=lambda x: getattr(x, 'name', None)) def test_transfer_derivative(func, variable, params, expected, benchmark, func_mode): - if func == Functions.SoftMax and params['params'][kw.OUTPUT_TYPE] == kw.ALL and func_mode != "Python": + if func == Functions.SoftMax and params[kw.OUTPUT_TYPE] == kw.ALL and func_mode != "Python": pytest.skip("Compiled derivative using 'ALL' is not implemented") f = func(default_variable=variable, **params) From ef52d0a47e678bdd957914638b178683129f344a Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Sun, 13 Nov 2022 19:54:31 -0500 Subject: [PATCH 083/127] tests/TransferFunction: Add type of output to names of SoftMax tests Signed-off-by: Jan Vesely --- tests/functions/test_transfer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functions/test_transfer.py b/tests/functions/test_transfer.py index 65d161e1ce1..854c09bfa88 100644 --- a/tests/functions/test_transfer.py +++ b/tests/functions/test_transfer.py @@ -145,7 +145,7 @@ def test_execute(func, variable, params, expected, benchmark, func_mode): @pytest.mark.function @pytest.mark.transfer_function @pytest.mark.benchmark -@pytest.mark.parametrize("func, variable, params, expected", derivative_test_data, ids=lambda x: getattr(x, 'name', None)) +@pytest.mark.parametrize("func, variable, params, expected", derivative_test_data, ids=lambda x: getattr(x, 'name', None) or getattr(x, 'get', lambda p, q: None)(kw.OUTPUT_TYPE, None)) def test_transfer_derivative(func, variable, params, expected, benchmark, func_mode): if func == Functions.SoftMax and params[kw.OUTPUT_TYPE] == kw.ALL and func_mode != "Python": pytest.skip("Compiled derivative using 'ALL' is not implemented") From 957bb6157c7b454c9c46ceba2756e48310932563 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Sun, 13 Nov 2022 20:01:55 -0500 Subject: [PATCH 084/127] llvm, TransferFunction/SoftMax: Add compiled implementation of derivative using base output Fix assertion in Python SoftmaxDerivative. Add tests. Signed-off-by: Jan Vesely --- .../nonstateful/transferfunctions.py | 15 ++++++---- tests/functions/test_transfer.py | 29 +++++++++++++++++++ 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/psyneulink/core/components/functions/nonstateful/transferfunctions.py b/psyneulink/core/components/functions/nonstateful/transferfunctions.py index 1a19589fdfe..6e556b78e19 100644 --- a/psyneulink/core/components/functions/nonstateful/transferfunctions.py +++ b/psyneulink/core/components/functions/nonstateful/transferfunctions.py @@ -2698,14 +2698,17 @@ def __gen_llvm_apply(self, ctx, builder, params, state, arg_in, arg_out, output_ return builder def _gen_llvm_function_derivative_body(self, ctx, builder, params, state, arg_in, arg_out, *, tags:frozenset): - assert "derivative" in tags + assert "derivative" in tags or "derivative_out" in tags assert arg_in.type == arg_out.type - forward_tags = tags.difference({"derivative"}) + forward_tags = tags.difference({"derivative", "derivative_out"}) # SoftMax derivative is calculated from the "ALL" results. # Those can provided from outside, but we don't support receiving data in arg_out - all_out = builder.alloca(arg_out.type.pointee) - builder = self._gen_llvm_function_body(ctx, builder, params, state, arg_in, all_out, output_type=ALL, tags=forward_tags) + if "derivative_out" in tags: + all_out = arg_in + else: + all_out = builder.alloca(arg_out.type.pointee) + builder = self._gen_llvm_function_body(ctx, builder, params, state, arg_in, all_out, output_type=ALL, tags=forward_tags) # The rest of the algorithm is for MAX_VAL and MAX_INDICATOR only assert self.output in {MAX_VAL, MAX_INDICATOR}, \ @@ -2749,7 +2752,7 @@ def _gen_llvm_function_derivative_body(self, ctx, builder, params, state, arg_in def _gen_llvm_function_body(self, ctx, builder, params, state, arg_in, arg_out, output_type=None, *, tags:frozenset): output_type = self.output if output_type is None else output_type - if "derivative" in tags: + if "derivative" in tags or "derivative_out" in tags: return self._gen_llvm_function_derivative_body(ctx, builder, params, state, arg_in, arg_out, tags=tags) if self.parameters.per_item.get(): @@ -2835,7 +2838,7 @@ def derivative(self, input=None, output=None, context=None): if output is None: output = self.function(input, params={OUTPUT_TYPE: ALL}, context=context) else: - assert not any(o == 0 for o in output) + assert not np.any(np.equal(0, output)) sm = np.squeeze(output) size = len(sm) diff --git a/tests/functions/test_transfer.py b/tests/functions/test_transfer.py index 854c09bfa88..1c26fe7b9d3 100644 --- a/tests/functions/test_transfer.py +++ b/tests/functions/test_transfer.py @@ -163,6 +163,35 @@ def test_transfer_derivative(func, variable, params, expected, benchmark, func_m assert np.allclose(res, expected) +derivative_out_test_data = [ + (Functions.SoftMax, softmax_helper, {kw.GAIN:RAND1, kw.OUTPUT_TYPE:kw.MAX_VAL, kw.PER_ITEM:False}, + [-0.010680386821751537, -0.011118109698906909, -0.01082040340318878, -0.010670257514724047, -0.010362498859374309, + -0.010933660158663306, -0.010397412260182806, -0.011602329078808718, 0.09684744183944892, -0.010262384043848513]), + (Functions.SoftMax, [softmax_helper], {kw.GAIN:RAND1, kw.OUTPUT_TYPE:kw.MAX_VAL, kw.PER_ITEM:True}, + [[-0.010680386821751537, -0.011118109698906909, -0.01082040340318878, -0.010670257514724047, -0.010362498859374309, + -0.010933660158663306, -0.010397412260182806, -0.011602329078808718, 0.09684744183944892, -0.010262384043848513]]), +] +@pytest.mark.function +@pytest.mark.transfer_function +@pytest.mark.benchmark +@pytest.mark.parametrize("func, variable, params, expected", derivative_out_test_data, ids=lambda x: getattr(x, 'name', None) or getattr(x, 'get', lambda p, q: None)(kw.OUTPUT_TYPE, None)) +def test_transfer_derivative_out(func, variable, params, expected, benchmark, func_mode): + if func == Functions.SoftMax and params[kw.OUTPUT_TYPE] == kw.ALL and func_mode != "Python": + pytest.skip("Compiled derivative using 'ALL' is not implemented") + + f = func(default_variable=variable, **params) + benchmark.group = "TransferFunction " + func.componentName + " Derivative" + if func_mode == 'Python': + def ex(x): + return f.derivative(input=None, output=x) + elif func_mode == 'LLVM': + ex = pnlvm.execution.FuncExecution(f, tags=frozenset({"derivative_out"})).execute + elif func_mode == 'PTX': + ex = pnlvm.execution.FuncExecution(f, tags=frozenset({"derivative_out"})).cuda_execute + + res = benchmark(ex, variable) + assert np.allclose(res, expected) + def test_transfer_with_costs_function(): f = Functions.TransferWithCosts() result = f(1) From f933b5d8f844facf90646b4398692ecfb26b2493 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Sun, 13 Nov 2022 20:21:59 -0500 Subject: [PATCH 085/127] llvm, TransferFunction/ReLU: Add implementation of derivative using base output Both Python and compiled. Add tests. Signed-off-by: Jan Vesely --- .../functions/nonstateful/transferfunctions.py | 17 ++++++++++++----- tests/functions/test_transfer.py | 5 ++++- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/psyneulink/core/components/functions/nonstateful/transferfunctions.py b/psyneulink/core/components/functions/nonstateful/transferfunctions.py index 6e556b78e19..c622909cf3b 100644 --- a/psyneulink/core/components/functions/nonstateful/transferfunctions.py +++ b/psyneulink/core/components/functions/nonstateful/transferfunctions.py @@ -1579,9 +1579,12 @@ def _gen_llvm_transfer(self, builder, index, ctx, vi, vo, params, state, *, tags # Maxnum for some reason needs full function prototype max_f = ctx.get_builtin("maxnum", [ctx.float_ty]) var = builder.load(ptri) - val = builder.fsub(var, bias) + if "derivative_out" in tags: + val = builder.fdiv(var, gain) + else: + val = builder.fsub(var, bias) - if "derivative" in tags: + if "derivative" in tags or "derivative_out" in tags: predicate = builder.fcmp_ordered('>', val, val.type(0)) val = builder.select(predicate, gain, builder.fmul(gain, leak)) else: @@ -1615,10 +1618,14 @@ def derivative(self, input=None, output=None, context=None): leak = self._get_current_parameter_value(LEAK, context) bias = self._get_current_parameter_value(BIAS, context) - value = np.empty_like(input) - value[(input - bias) > 0] = gain - value[(input - bias) <= 0] = gain * leak + if input is not None: + # Use input if provided + variable = np.array(input) - bias + else: + # Infer input from output + variable = np.array(output) / gain + value = np.where(variable > 0, gain, gain * leak) return value # ********************************************************************************************************************** diff --git a/tests/functions/test_transfer.py b/tests/functions/test_transfer.py index 1c26fe7b9d3..d07ea9fc301 100644 --- a/tests/functions/test_transfer.py +++ b/tests/functions/test_transfer.py @@ -27,6 +27,8 @@ gaussian_helper = np.e**(-(test_var - RAND2)**2 / (2 * RAND1**2)) / np.sqrt(2 * np.pi * RAND1) gaussian_helper = RAND3 * gaussian_helper + RAND4 +relu_helper = np.maximum(RAND1 * (test_var - RAND2), RAND3 * RAND1 *(test_var - RAND2)) + def gaussian_distort_helper(seed): state = np.random.RandomState([seed]) # compensate for construction @@ -39,7 +41,7 @@ def gaussian_distort_helper(seed): pytest.param(Functions.Exponential, test_var, {kw.SCALE:RAND1, kw.RATE:RAND2}, RAND1 * np.exp(RAND2 * test_var), id="EXPONENTIAL"), pytest.param(Functions.Logistic, test_var, {kw.GAIN:RAND1, kw.X_0:RAND2, kw.OFFSET:RAND3, kw.SCALE:RAND4}, RAND4 / (1 + np.exp(-(RAND1 * (test_var - RAND2)) + RAND3)), id="LOGISTIC"), pytest.param(Functions.Tanh, test_var, {kw.GAIN:RAND1, kw.BIAS:RAND2, kw.X_0:RAND3, kw.OFFSET:RAND4}, tanh_helper, id="TANH"), - pytest.param(Functions.ReLU, test_var, {kw.GAIN:RAND1, kw.BIAS:RAND2, kw.LEAK:RAND3}, np.maximum(RAND1 * (test_var - RAND2), RAND3 * RAND1 *(test_var - RAND2)), id="RELU"), + pytest.param(Functions.ReLU, test_var, {kw.GAIN:RAND1, kw.BIAS:RAND2, kw.LEAK:RAND3}, relu_helper, id="RELU"), # Angle doesn't have a helper using 'test_var', hardcode the input as well pytest.param(Functions.Angle, [0.5488135, 0.71518937, 0.60276338, 0.54488318, 0.4236548, 0.64589411, 0.43758721, 0.891773, 0.96366276, 0.38344152], {}, @@ -164,6 +166,7 @@ def test_transfer_derivative(func, variable, params, expected, benchmark, func_m derivative_out_test_data = [ + (Functions.ReLU, relu_helper, {kw.GAIN:RAND1, kw.BIAS:RAND2, kw.LEAK:RAND3}, np.where((test_var - RAND2) > 0, RAND1, RAND1 * RAND3)), (Functions.SoftMax, softmax_helper, {kw.GAIN:RAND1, kw.OUTPUT_TYPE:kw.MAX_VAL, kw.PER_ITEM:False}, [-0.010680386821751537, -0.011118109698906909, -0.01082040340318878, -0.010670257514724047, -0.010362498859374309, -0.010933660158663306, -0.010397412260182806, -0.011602329078808718, 0.09684744183944892, -0.010262384043848513]), From 9aad3f1811767742e30e553c6d90743241d58cdb Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Sun, 13 Nov 2022 20:32:49 -0500 Subject: [PATCH 086/127] llvm, TransferFunction/Logistic: Add compiled implementation of derivative using base output Add tests. Signed-off-by: Jan Vesely --- .../nonstateful/transferfunctions.py | 19 ++++++++++--------- tests/functions/test_transfer.py | 5 +++-- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/psyneulink/core/components/functions/nonstateful/transferfunctions.py b/psyneulink/core/components/functions/nonstateful/transferfunctions.py index c622909cf3b..26af897252b 100644 --- a/psyneulink/core/components/functions/nonstateful/transferfunctions.py +++ b/psyneulink/core/components/functions/nonstateful/transferfunctions.py @@ -961,16 +961,17 @@ def _gen_llvm_transfer(self, builder, index, ctx, vi, vo, params, state, *, tags exp_f = ctx.get_builtin("exp", [ctx.float_ty]) val = builder.load(ptri) - val = builder.fadd(val, bias) - val = builder.fsub(val, x_0) - val = builder.fmul(val, gain) - val = builder.fsub(offset, val) - val = builder.call(exp_f, [val]) - val = builder.fadd(ctx.float_ty(1), val) - val = builder.fdiv(ctx.float_ty(1), val) - val = builder.fmul(val, scale) + if "derivative_out" not in tags: + val = builder.fadd(val, bias) + val = builder.fsub(val, x_0) + val = builder.fmul(val, gain) + val = builder.fsub(offset, val) + val = builder.call(exp_f, [val]) + val = builder.fadd(ctx.float_ty(1), val) + val = builder.fdiv(ctx.float_ty(1), val) + val = builder.fmul(val, scale) - if "derivative" in tags: + if "derivative" in tags or "derivative_out" in tags: # f(x) = g * s * o * (1-o) function_val = val val = builder.fsub(ctx.float_ty(1), function_val) diff --git a/tests/functions/test_transfer.py b/tests/functions/test_transfer.py index d07ea9fc301..a61564097fa 100644 --- a/tests/functions/test_transfer.py +++ b/tests/functions/test_transfer.py @@ -28,6 +28,7 @@ gaussian_helper = RAND3 * gaussian_helper + RAND4 relu_helper = np.maximum(RAND1 * (test_var - RAND2), RAND3 * RAND1 *(test_var - RAND2)) +logistic_helper = RAND4 / (1 + np.exp(-(RAND1 * (test_var - RAND2)) + RAND3)) def gaussian_distort_helper(seed): state = np.random.RandomState([seed]) @@ -39,7 +40,7 @@ def gaussian_distort_helper(seed): test_data = [ pytest.param(Functions.Linear, test_var, {kw.SLOPE:RAND1, kw.INTERCEPT:RAND2}, test_var * RAND1 + RAND2, id="LINEAR"), pytest.param(Functions.Exponential, test_var, {kw.SCALE:RAND1, kw.RATE:RAND2}, RAND1 * np.exp(RAND2 * test_var), id="EXPONENTIAL"), - pytest.param(Functions.Logistic, test_var, {kw.GAIN:RAND1, kw.X_0:RAND2, kw.OFFSET:RAND3, kw.SCALE:RAND4}, RAND4 / (1 + np.exp(-(RAND1 * (test_var - RAND2)) + RAND3)), id="LOGISTIC"), + pytest.param(Functions.Logistic, test_var, {kw.GAIN:RAND1, kw.X_0:RAND2, kw.OFFSET:RAND3, kw.SCALE:RAND4}, logistic_helper, id="LOGISTIC"), pytest.param(Functions.Tanh, test_var, {kw.GAIN:RAND1, kw.BIAS:RAND2, kw.X_0:RAND3, kw.OFFSET:RAND4}, tanh_helper, id="TANH"), pytest.param(Functions.ReLU, test_var, {kw.GAIN:RAND1, kw.BIAS:RAND2, kw.LEAK:RAND3}, relu_helper, id="RELU"), # Angle doesn't have a helper using 'test_var', hardcode the input as well @@ -94,7 +95,6 @@ def test_execute(func, variable, params, expected, benchmark, func_mode): assert np.allclose(res, expected) -logistic_helper = RAND4 / (1 + np.exp(-(RAND1 * (test_var - RAND2)) + RAND3)) tanh_derivative_helper = (RAND1 * (test_var + RAND2) + RAND3) tanh_derivative_helper = (1 - np.tanh(tanh_derivative_helper)**2) * RAND4 * RAND1 @@ -166,6 +166,7 @@ def test_transfer_derivative(func, variable, params, expected, benchmark, func_m derivative_out_test_data = [ + (Functions.Logistic, logistic_helper, {kw.GAIN:RAND1, kw.X_0:RAND2, kw.OFFSET:RAND3, kw.SCALE:RAND4}, RAND1 * RAND4 * logistic_helper * (1 - logistic_helper)), (Functions.ReLU, relu_helper, {kw.GAIN:RAND1, kw.BIAS:RAND2, kw.LEAK:RAND3}, np.where((test_var - RAND2) > 0, RAND1, RAND1 * RAND3)), (Functions.SoftMax, softmax_helper, {kw.GAIN:RAND1, kw.OUTPUT_TYPE:kw.MAX_VAL, kw.PER_ITEM:False}, [-0.010680386821751537, -0.011118109698906909, -0.01082040340318878, -0.010670257514724047, -0.010362498859374309, From 12a6f6a3ef978937fa9c1be6c083f7aad83e73fc Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Sun, 13 Nov 2022 22:48:23 -0500 Subject: [PATCH 087/127] tests/TransferFunction: Assert that func_mode is only one of the three valid options Signed-off-by: Jan Vesely --- tests/functions/test_transfer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/functions/test_transfer.py b/tests/functions/test_transfer.py index a61564097fa..b816f06d198 100644 --- a/tests/functions/test_transfer.py +++ b/tests/functions/test_transfer.py @@ -160,6 +160,8 @@ def test_transfer_derivative(func, variable, params, expected, benchmark, func_m ex = pnlvm.execution.FuncExecution(f, tags=frozenset({"derivative"})).execute elif func_mode == 'PTX': ex = pnlvm.execution.FuncExecution(f, tags=frozenset({"derivative"})).cuda_execute + else: + assert False, "unknown function mode: {}".format(func_mode) res = benchmark(ex, variable) assert np.allclose(res, expected) @@ -192,6 +194,8 @@ def ex(x): ex = pnlvm.execution.FuncExecution(f, tags=frozenset({"derivative_out"})).execute elif func_mode == 'PTX': ex = pnlvm.execution.FuncExecution(f, tags=frozenset({"derivative_out"})).cuda_execute + else: + assert False, "unknown function mode: {}".format(func_mode) res = benchmark(ex, variable) assert np.allclose(res, expected) From a8a5ba828dc6d7dd8592a5e64dc90b559b8436ab Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Mon, 14 Nov 2022 13:59:48 -0500 Subject: [PATCH 088/127] tests, functions/SoftMax: Don't use nested 'params' dir to set parameters Two instances were missed in b39bd921d74fcda244e30aab4c81390808bca2d3 Signed-off-by: Jan Vesely --- tests/functions/test_transfer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/functions/test_transfer.py b/tests/functions/test_transfer.py index b816f06d198..15db649b2fb 100644 --- a/tests/functions/test_transfer.py +++ b/tests/functions/test_transfer.py @@ -72,9 +72,9 @@ def gaussian_distort_helper(seed): # SoftMax per-item with 2 elements in input pytest.param(Functions.SoftMax, [test_var, test_var], {kw.GAIN:RAND1, kw.PER_ITEM: True}, softmax_helper2, id="SOFT_MAX ALL PER_ITEM"), - pytest.param(Functions.SoftMax, [test_var, test_var], {kw.GAIN:RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_VAL}, kw.PER_ITEM: True}, + pytest.param(Functions.SoftMax, [test_var, test_var], {kw.GAIN:RAND1, kw.OUTPUT_TYPE:kw.MAX_VAL, kw.PER_ITEM: True}, np.where(softmax_helper2 == np.max(softmax_helper2), softmax_helper2, 0), id="SOFT_MAX MAX_VAL PER_ITEM"), - pytest.param(Functions.SoftMax, [test_var, test_var], {kw.GAIN:RAND1, 'params':{kw.OUTPUT_TYPE:kw.MAX_INDICATOR}, kw.PER_ITEM: True}, + pytest.param(Functions.SoftMax, [test_var, test_var], {kw.GAIN:RAND1, kw.OUTPUT_TYPE:kw.MAX_INDICATOR, kw.PER_ITEM: True}, np.where(softmax_helper2 == np.max(softmax_helper2), 1, 0), id="SOFT_MAX MAX_INDICATOR PER_ITEM"), pytest.param(Functions.LinearMatrix, test_var, {kw.MATRIX:test_matrix}, np.dot(test_var, test_matrix), id="LINEAR_MATRIX SQUARE"), From dd88b4ab35307729cb6cb0df217255e7248f6ba7 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Mon, 14 Nov 2022 14:13:11 -0500 Subject: [PATCH 089/127] functions/SoftMax: Use 'squeezed' version of output when calculating Jacobian Otherwise the length is 1, producing results only in the first column. Fixes computation on 2d input. Use numpy ndindex instead of nested for loop. Signed-off-by: Jan Vesely --- .../functions/nonstateful/transferfunctions.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/psyneulink/core/components/functions/nonstateful/transferfunctions.py b/psyneulink/core/components/functions/nonstateful/transferfunctions.py index 26af897252b..4c09a109d6f 100644 --- a/psyneulink/core/components/functions/nonstateful/transferfunctions.py +++ b/psyneulink/core/components/functions/nonstateful/transferfunctions.py @@ -2856,13 +2856,12 @@ def derivative(self, input=None, output=None, context=None): if output_type == ALL: # Return full Jacobian matrix of derivatives derivative = np.empty([size, size]) - for j in range(size): - for i, val in zip(range(size), output): - if i == j: - d = 1 - else: - d = 0 - derivative[j, i] = sm[i] * (d - sm[j]) + for i, j in np.ndindex(size, size): + if i == j: + d = 1 + else: + d = 0 + derivative[j, i] = sm[i] * (d - sm[j]) elif output_type in {MAX_VAL, MAX_INDICATOR}: # Return 1d array of derivatives for max element (i.e., the one chosen by SoftMax) From ab193876b9f6c635fda5bd771cee60df835ab20d Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Wed, 16 Nov 2022 15:50:57 -0500 Subject: [PATCH 090/127] llvm/codegen: Use visit_Constant instead of visit_{Num,NamedConstant} if available The latter are deprecated in Python 3.8+. Signed-off-by: Jan Vesely --- psyneulink/core/llvm/codegen.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/psyneulink/core/llvm/codegen.py b/psyneulink/core/llvm/codegen.py index 76f29f8bbfb..9d192f4e744 100644 --- a/psyneulink/core/llvm/codegen.py +++ b/psyneulink/core/llvm/codegen.py @@ -68,11 +68,6 @@ def np_cmp(builder, x, y): if v is np: self.register[k] = numpy_handlers - name_constants = { - True: ctx.bool_ty(1), - False: ctx.bool_ty(0), - } - self.name_constants = name_constants super().__init__() def _update_debug_metadata(self, builder: ir.IRBuilder, node:ast.AST): @@ -239,9 +234,6 @@ def _convert(builder, x): return val[node.attr] - def visit_Num(self, node): - return self.ctx.float_ty(node.n) - def visit_Assign(self, node): value = self.visit(node.value) @@ -259,10 +251,24 @@ def visit_Assign(self, node): assert self.is_lval(target) self.builder.store(value, target) + # visit_Constant is supported in Python3.8+ + def visit_Constant(self, node): + # Only True/False are currently supported as named constants + # Call deprecated visit_* methods to maintain coverage + if node.value is True or node.value is False: + return self.visit_NameConstant(node) + + return self.visit_Num(node) + + # deprecated in Python3.8+ def visit_NameConstant(self, node): - val = self.name_constants[node.value] - assert val, f"Failed to convert NameConstant {node.value}" - return val + # Only True and False are supported atm + assert node.value is True or node.value is False + return self.ctx.bool_ty(node.value) + + # deprecated in Python3.8+ + def visit_Num(self, node): + return self.ctx.float_ty(node.n) def visit_Tuple(self, node:ast.AST): elements = (self.visit(element) for element in node.elts) From aa37609d4756bf3a95a8dd41ca14e0e75fce7e41 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Thu, 17 Nov 2022 21:59:19 -0500 Subject: [PATCH 091/127] tests/EpisodicMemoryMechanism: Use 'size' instead of 'content_size' in construction The latter is deprecated. Signed-off-by: Jan Vesely --- tests/mechanisms/test_episodic_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/mechanisms/test_episodic_memory.py b/tests/mechanisms/test_episodic_memory.py index 41f6d0bae21..c8a19d8cfb2 100644 --- a/tests/mechanisms/test_episodic_memory.py +++ b/tests/mechanisms/test_episodic_memory.py @@ -48,7 +48,7 @@ @pytest.mark.parametrize('variable, func, params, expected', test_data, ids=names) def test_with_dictionary_memory(variable, func, params, expected, benchmark, mech_mode): f = func(seed=0, **params) - m = EpisodicMemoryMechanism(content_size=len(variable[0]), assoc_size=len(variable[1]), function=f) + m = EpisodicMemoryMechanism(size=len(variable[0]), assoc_size=len(variable[1]), function=f) EX = pytest.helpers.get_mech_execution(m, mech_mode) EX(variable) From 8a804f61803eacd4d59da343d0df4b8f687a00bb Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Thu, 17 Nov 2022 22:09:38 -0500 Subject: [PATCH 092/127] tests/EpisodicMemoryMechanism: Check for compiled variant sooner No reason to test the construction again with no execution. Simplify generation of test names. Signed-off-by: Jan Vesely --- tests/mechanisms/test_episodic_memory.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/mechanisms/test_episodic_memory.py b/tests/mechanisms/test_episodic_memory.py index c8a19d8cfb2..479becb96ee 100644 --- a/tests/mechanisms/test_episodic_memory.py +++ b/tests/mechanisms/test_episodic_memory.py @@ -195,21 +195,21 @@ def test_with_dictionary_memory(variable, func, params, expected, benchmark, mec ] # Allows names to be with each test_data set -names = [test_data[i][0] for i in range(len(test_data))] +names = [td[0] for td in test_data] @pytest.mark.parametrize('name, func, func_params, mech_params, test_var,' 'input_port_names, output_port_names, expected_output', test_data, ids=names) def test_with_contentaddressablememory(name, func, func_params, mech_params, test_var, input_port_names, output_port_names, expected_output, mech_mode): + if mech_mode != 'Python': + pytest.skip("Compiled execution not yet implemented for ContentAddressableMemory") + f = func(seed=0, **func_params) # EpisodicMemoryMechanism(function=f, **mech_params) em = EpisodicMemoryMechanism(function=f, **mech_params) assert em.input_ports.names == input_port_names assert em.output_ports.names == output_port_names - if mech_mode != 'Python': - pytest.skip("PTX not yet implemented for ContentAddressableMemory") - EX = pytest.helpers.get_mech_execution(em, mech_mode) From 5c0342f2d28a24f18eafed2a9b3635ffc1f60692 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Thu, 17 Nov 2022 23:15:57 -0500 Subject: [PATCH 093/127] tests/TransferMechanism: Check for warning messages when there's a conflict between mech and function parameter Signed-off-by: Jan Vesely --- tests/mechanisms/test_transfer_mechanism.py | 63 ++++++++++++--------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/tests/mechanisms/test_transfer_mechanism.py b/tests/mechanisms/test_transfer_mechanism.py index e7f1f465551..5c0203a96b7 100644 --- a/tests/mechanisms/test_transfer_mechanism.py +++ b/tests/mechanisms/test_transfer_mechanism.py @@ -546,13 +546,16 @@ def test_transfer_mech_array_assignments_fct_rate(self, benchmark, mech_mode): @pytest.mark.benchmark(group="TransferMechanism Parameter Array Assignments") def test_transfer_mech_array_assignments_fct_over_mech_rate(self, benchmark, mech_mode): - T = TransferMechanism( - name='T', - default_variable=[0 for i in range(VECTOR_SIZE)], - integrator_mode=True, - integrator_function=AdaptiveIntegrator(rate=[i / 20 for i in range(VECTOR_SIZE)]), - integration_rate=[i / 10 for i in range(VECTOR_SIZE)] - ) + with pytest.warns(UserWarning) as warnings: + T = TransferMechanism( + name='T', + default_variable=[0 for i in range(VECTOR_SIZE)], + integrator_mode=True, + integrator_function=AdaptiveIntegrator(rate=[i / 20 for i in range(VECTOR_SIZE)]), + integration_rate=[i / 10 for i in range(VECTOR_SIZE)] + ) + assert any(str(w.message).startswith('Specification of the "integration_rate" parameter') + for w in warnings), "Warnings: {}".format([str(w.message) for w in warnings]) EX = pytest.helpers.get_mech_execution(T, mech_mode) var = [1 for i in range(VECTOR_SIZE)] @@ -633,19 +636,23 @@ def test_transfer_mech_array_assignments_fct_initzr(self, benchmark, mech_mode): @pytest.mark.transfer_mechanism @pytest.mark.benchmark(group="TransferMechanism Parameter Array Assignments") def test_transfer_mech_array_assignments_fct_initlzr_over_mech_init_val(self, benchmark, mech_mode): - T = TransferMechanism( - name='T', - default_variable=[0 for i in range(VECTOR_SIZE)], - integrator_mode=True, - integrator_function=AdaptiveIntegrator( - default_variable=[0 for i in range(VECTOR_SIZE)], - initializer=[i / 10 for i in range(VECTOR_SIZE)] - ), - initial_value=[i / 10 for i in range(VECTOR_SIZE)] - ) - EX = pytest.helpers.get_mech_execution(T, mech_mode) + with pytest.warns(UserWarning) as warnings: + T = TransferMechanism( + name='T', + default_variable=[0 for i in range(VECTOR_SIZE)], + integrator_mode=True, + integrator_function=AdaptiveIntegrator( + default_variable=[0 for i in range(VECTOR_SIZE)], + initializer=[i / 10 for i in range(VECTOR_SIZE)] + ), + initial_value=[i / 10 for i in range(VECTOR_SIZE)] + ) + assert any(str(w.message).startswith('Specification of the "initial_value" parameter') + for w in warnings), "Warnings: {}".format([str(w.message) for w in warnings]) + EX = pytest.helpers.get_mech_execution(T, mech_mode) var = [1 for i in range(VECTOR_SIZE)] + EX(var) val = benchmark(EX, var) assert np.allclose(val, [[ 0.75, 0.775, 0.8, 0.825]]) @@ -768,16 +775,20 @@ def test_transfer_mech_array_assignments_fct_noise(self, benchmark, mech_mode): # FIXME: Incorrect T.integrator_function.defaults.variable reported def test_transfer_mech_array_assignments_fct_over_mech_noise(self, benchmark, mech_mode): - T = TransferMechanism( - name='T', - default_variable=[0 for i in range(VECTOR_SIZE)], - integrator_mode=True, - integrator_function=AdaptiveIntegrator(noise=[i / 20 for i in range(VECTOR_SIZE)]), - noise=[i / 10 for i in range(VECTOR_SIZE)] - ) - EX = pytest.helpers.get_mech_execution(T, mech_mode) + with pytest.warns(UserWarning) as warnings: + T = TransferMechanism( + name='T', + default_variable=[0 for i in range(VECTOR_SIZE)], + integrator_mode=True, + integrator_function=AdaptiveIntegrator(noise=[i / 20 for i in range(VECTOR_SIZE)]), + noise=[i / 10 for i in range(VECTOR_SIZE)] + ) + assert any(str(w.message).startswith('Specification of the "noise" parameter') + for w in warnings), "Warnings: {}".format([str(w.message) for w in warnings]) + EX = pytest.helpers.get_mech_execution(T, mech_mode) var = [1 for i in range(VECTOR_SIZE)] + EX(var) val = benchmark(EX, var) assert np.allclose(val, [[ 0.75, 0.825, 0.9, 0.975]]) From dbef51c141fa506f9516eb5f7fdc43f8f0d71ae7 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Thu, 17 Nov 2022 23:34:52 -0500 Subject: [PATCH 094/127] tests/scheduler/conditions: Drop 'DECISION_TIME' output port It's not used in the test and PsyNeuLink warns about it. Signed-off-by: Jan Vesely --- tests/scheduling/test_scheduler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/scheduling/test_scheduler.py b/tests/scheduling/test_scheduler.py index 0c09eb33590..69aba949a04 100644 --- a/tests/scheduling/test_scheduler.py +++ b/tests/scheduling/test_scheduler.py @@ -1592,7 +1592,8 @@ def test_scheduler_conditions(self, comp_mode, condition, scale, expected_result time_step_size=1.0), reset_stateful_function_when=pnl.AtTrialStart(), execute_until_finished=False, - output_ports=[pnl.DECISION_VARIABLE, pnl.RESPONSE_TIME], + # Use only the decision variable in this test + output_ports=[pnl.DECISION_VARIABLE], name='DDM') response = pnl.ProcessingMechanism(size=2, name="GATE") From 5c1a6f78973c35cf22f6168dc6e3afc582eda4a6 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Tue, 15 Nov 2022 23:25:51 -0500 Subject: [PATCH 095/127] utilities: Add fastpath when checking if object is compatible with itself This is a common situation in tests that assign the same object to e.g. default variable, and then variable for execution. Signed-off-by: Jan Vesely --- psyneulink/core/globals/utilities.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/psyneulink/core/globals/utilities.py b/psyneulink/core/globals/utilities.py index 9d45134e953..56de8404219 100644 --- a/psyneulink/core/globals/utilities.py +++ b/psyneulink/core/globals/utilities.py @@ -459,6 +459,12 @@ def iscompatible(candidate, reference=None, **kargs): # ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all() pass + # If the two are the same thing, can settle it right here + # This is a common pattern for tests that use the same structure + # as default variable and variable + if reference is not None and candidate is reference: + return True + # If args not provided, assign to default values # if not specified in args, use these: # args[kwCompatibilityType] = list From fc13a633c7e402d1ebf65fb38e4913dedad0270a Mon Sep 17 00:00:00 2001 From: jdcpni Date: Fri, 18 Nov 2022 21:48:36 -0500 Subject: [PATCH 096/127] Nback (#2540) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add autodiff save/load functionality * - * Update autodiffcomposition.py * Update test_autodiffcomposition.py * Merge branch 'devel' of https://github.com/PrincetonUniversity/PsyNeuLink into devel * • Autodiff: - add save and load methods (from Samyak) - test_autodiffcomposition.py: add test_autodiff_saveload, but commented out for now, as it may be causing hanging on PR * • Autodiff: - add save and load methods (from Samyak) - test_autodiffcomposition.py: add test_autodiff_saveload, but commented out for now, as it may be causing hanging on PR * - * - * • pytorchcomponents.py: - pytorch_function_creator: add SoftMax • transferfunctions.py: - disable changes to ReLU.derivative for now * • utilities.py: - iscompatible: attempt to replace try and except, commented out for now * - * - * • autodiffcomposition.py: - save and load: augment file and directory handling - exclude processing of any ModulatoryProjections * - * - * - * • autodiffcomposition.py save(): add projection.matrix.base = matrix (fixes test_autodiff_saveload) * - * • autodiffcomposition.py: - save: return path • test_autodiffcomposition.py: - test_autodiff_saveload: modify to use current working directory rather than tmp * • autodiffcomposition.py: - save() and load(): ignore CIM, learning, and other modulation-related projections * • autodiffcomposition.py: - load(): change test for path (failing on Windows) from PosixPath to Path * • autodiffcomposition.py: - add _runtime_learning_rate attribute - _build_pytorch_representation(): use _runtime_learning_rate attribute for optimizer if provided in call to learn else use learning_rate specified at construction • compositionrunner.py: - assign learning_rate to _runtime_learning_rate attribute if specified in call to learn * - * [skip ci] * [skip ci] * [skip ci] • autodiffcomposition.py: load(): add testing for match of matrix shape * [skip ci] • N-back: - reset em after each run - save and load weights - torch epochs = batch size (number training stimuli) * num_epochs * [skip ci] * [skip ci] * Feat/add pathway default matrix (#2518) * • compositioninterfacemechanism.py: - _get_source_node_for_input_CIM: restore (modeled on _get_source_of_modulation_for_parameter_CIM) but NEEDS TESTS - _get_source_of_modulation_for_parameter_CIM: clean up comments, NEEDS TESTS * - * - * - * - * - * - * • Nback - EM uses ContentAddressableMemory (instead of DictionaryMemory) - Implements FFN for comparison of current and retrieved stimulus and context • Project: replace all instances of "RETREIVE" with "RETRIEVE" * • objectivefunctions.py - add cosine_similarity (needs compiled version) * • Project: make COSINE_SIMILARITY a synonym of COSINE • nback_CAM_FFN: - refactor to implement FFN and task input - assign termination condition for execution that is dependent on control - ContentAddressableMemory: selection_function=SoftMax(output=MAX_INDICATOR, gain=SOFT_MAX_TEMP) • DriftOnASphereIntegrator: - add dimension as dependency for initializer parameter * - * - * - * - * - * - * - * - * - * - * - * - * - * - * • test_integrator.py: Added identicalness test for DriftOnASphereIntegrator agains nback-paper implementation. * - * - * Parameters: allow _validate_ methods to reference other parameters (#2512) * • Scripts: - Updated N-back to use objective_mechanism, with commented out code for version that doesn't use it once bug is fixed - Deleted N-back_WITH_OBJECTIVE_MECH.py * • Scripts: - Updated N-back to use objective_mechanism, with commented out code for version that doesn't use it once bug is fixed - Deleted N-back_WITH_OBJECTIVE_MECH.py * • N-back.py: - added stimulus generation per nback-paper protocol * - N-back.py tstep(s) -> trial(s) * - * - * • N-back.py - comp -> nback_model - implement stim_set() method * - * • N-back.py: - added training set generation * - * - * • N-back.py - modularized script * - * - * - * - * • showgraph.py: - _assign_processing_components(): fix bug in which nested graphs not highlighted in animation. * • showgraph.py * composition.py - add further description of animation, including note that animation of nested Compostions is limited. * • showgraph.py * composition.py - add animation to N-back doc * • autodiffcomposition.py - __init__(): move pathways arg to beginning, to capture positional assignment (i.e. w/o kw) * - * • N-back.py - ffn: implement as autodiff; still needs small random initial weight assignment * • pathway.py - implement default_projection attribute * • pathway.py - implement default_projection attribute * • utilities.py: random_matrxi: refactored to allow negative values and use keyword ZERO_CENTER * • projection.py RandomMatrix: added class that can be used to pass a function as matrix spec * • utilities.py - RandomMatrix moved here from projection.py • function.py - get_matrix(): added support for RandomMatrix spec * • port.py - _parse_port_spec(): added support for RandomMatrix * • port.py - _parse_port_spec(): added support for RandomMatrix * • utilities.py - is_matrix(): modified to support random_matrix and RandomMatrix * • composition.py - add_linear_processing_pathway: add support for default_matrix argument (replaces default for MappingProjection for any otherwise unspecified projections) though still not used. * - * - RandomMatrix: moved from Utilities to Function * - * [skip ci] * [skip ci] * [skip ci] • N-back.py - clean up script * [skip ci] • N-back.py - further script clean-up * [skip ci] * [skip ci] * [skip ci] * [skip ci] • BeukersNBackModel.rst: - Overview written - Needs other sections completed * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] • N-back.py: - replace functions of TransferMechanisms with ReLU - replace function of Decision Mechanisms with SoftMax - more doc cleanup * [skip ci] • N-back.py: - replace functions of TransferMechanisms with ReLU - replace function of Decision Mechanisms with SoftMax - more doc cleanup * [skip ci] * - * - * [skip ci] * [skip ci] • composition.py: implement default_projection_matrix in add_XXX_pathway() methods * [skip ci] • composition.py: implement default_projection_matrix in add_XXX_pathway() methods * [skip ci] • test_composition.py: - add test_pathway_tuple_specs() * - * - * [skip ci] * [skip ci] * [skip ci] * - Co-authored-by: jdcpni Co-authored-by: Katherine Mantel * Feat/add pathway default matrix (#2519) * • compositioninterfacemechanism.py: - _get_source_node_for_input_CIM: restore (modeled on _get_source_of_modulation_for_parameter_CIM) but NEEDS TESTS - _get_source_of_modulation_for_parameter_CIM: clean up comments, NEEDS TESTS * - * - * - * - * - * - * • Nback - EM uses ContentAddressableMemory (instead of DictionaryMemory) - Implements FFN for comparison of current and retrieved stimulus and context • Project: replace all instances of "RETREIVE" with "RETRIEVE" * • objectivefunctions.py - add cosine_similarity (needs compiled version) * • Project: make COSINE_SIMILARITY a synonym of COSINE • nback_CAM_FFN: - refactor to implement FFN and task input - assign termination condition for execution that is dependent on control - ContentAddressableMemory: selection_function=SoftMax(output=MAX_INDICATOR, gain=SOFT_MAX_TEMP) • DriftOnASphereIntegrator: - add dimension as dependency for initializer parameter * - * - * - * - * - * - * - * - * - * - * - * - * - * - * • test_integrator.py: Added identicalness test for DriftOnASphereIntegrator agains nback-paper implementation. * - * - * Parameters: allow _validate_ methods to reference other parameters (#2512) * • Scripts: - Updated N-back to use objective_mechanism, with commented out code for version that doesn't use it once bug is fixed - Deleted N-back_WITH_OBJECTIVE_MECH.py * • Scripts: - Updated N-back to use objective_mechanism, with commented out code for version that doesn't use it once bug is fixed - Deleted N-back_WITH_OBJECTIVE_MECH.py * • N-back.py: - added stimulus generation per nback-paper protocol * - N-back.py tstep(s) -> trial(s) * - * - * • N-back.py - comp -> nback_model - implement stim_set() method * - * • N-back.py: - added training set generation * - * - * • N-back.py - modularized script * - * - * - * - * • showgraph.py: - _assign_processing_components(): fix bug in which nested graphs not highlighted in animation. * • showgraph.py * composition.py - add further description of animation, including note that animation of nested Compostions is limited. * • showgraph.py * composition.py - add animation to N-back doc * • autodiffcomposition.py - __init__(): move pathways arg to beginning, to capture positional assignment (i.e. w/o kw) * - * • N-back.py - ffn: implement as autodiff; still needs small random initial weight assignment * • pathway.py - implement default_projection attribute * • pathway.py - implement default_projection attribute * • utilities.py: random_matrxi: refactored to allow negative values and use keyword ZERO_CENTER * • projection.py RandomMatrix: added class that can be used to pass a function as matrix spec * • utilities.py - RandomMatrix moved here from projection.py • function.py - get_matrix(): added support for RandomMatrix spec * • port.py - _parse_port_spec(): added support for RandomMatrix * • port.py - _parse_port_spec(): added support for RandomMatrix * • utilities.py - is_matrix(): modified to support random_matrix and RandomMatrix * • composition.py - add_linear_processing_pathway: add support for default_matrix argument (replaces default for MappingProjection for any otherwise unspecified projections) though still not used. * - * - RandomMatrix: moved from Utilities to Function * - * [skip ci] * [skip ci] * [skip ci] • N-back.py - clean up script * [skip ci] • N-back.py - further script clean-up * [skip ci] * [skip ci] * [skip ci] * [skip ci] • BeukersNBackModel.rst: - Overview written - Needs other sections completed * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] • N-back.py: - replace functions of TransferMechanisms with ReLU - replace function of Decision Mechanisms with SoftMax - more doc cleanup * [skip ci] • N-back.py: - replace functions of TransferMechanisms with ReLU - replace function of Decision Mechanisms with SoftMax - more doc cleanup * [skip ci] * - * - * [skip ci] * [skip ci] • composition.py: implement default_projection_matrix in add_XXX_pathway() methods * [skip ci] • composition.py: implement default_projection_matrix in add_XXX_pathway() methods * [skip ci] • test_composition.py: - add test_pathway_tuple_specs() * - * - * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] • composition.py: - add_linear_processing_pathway: fixed bug when Reinforcement or TDLearning are specified • test_composition.py: - test_pathway_tuple_specs: add tests for Reinforcement and TDLearning * • composition.py: - add_linear_processing_pathway: fixed bug when Reinforcement or TDLearning are specified • test_composition.py: - test_pathway_tuple_specs: add tests for Reinforcement and TDLearning Co-authored-by: jdcpni Co-authored-by: Katherine Mantel * autodiff: Use most recent context while save/load * tests/autodiff: Use portable path join * autodiff: Add assertions for save/load * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] * • autodiffcomposition, test_autodiff_saveload: - merged from feat/autodiff_save * - * - * - * • autodiffcomposition.py - fix path assignment bug * - * • N-back mods * • N-back: reimplementing get_run_inputs * - * - * - * - * • N-back.py - refactoring of generate_stim_seq: continuous presentation of stimuli balancing of trial_types (with fill-in) return trial_type_seq * [skip ci] * Merge branch 'devel' of https://github.com/PrincetonUniversity/PsyNeuLink into devel  Conflicts:  psyneulink/library/compositions/autodiffcomposition.py * Merge branch 'devel' of https://github.com/PrincetonUniversity/PsyNeuLink into devel  Conflicts:  psyneulink/library/compositions/autodiffcomposition.py * [skip ci] * [skip ci] • N-back.py - docstring mods * [skip ci] * [skip ci] • N-back.py: add Kane stimuli (2back) * [skip ci] * [skip ci] * [skip ci] * • N-back.py - add analyze_results() * [skip ci] • N-back.py - add analyze_results() * [skip ci] • N-back.py: - analyze_results: fully implemented * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] * [skip ci] • composition.py, pytorchmodelcreator.py - modify loss_spec to use keywords for loss • Nback.py: - Autodiff loss_spec = CROSS_ENTROPY * [skip ci] • pytorchmodelcreator.py: - _gen_llvm_training_function_body(): - add support for loss_type = CROSS_ENTROPY • compiledloss.py: - _gen_loss_function: add support for CROSS_ENTROPYLoss - needs to be debugged - need differential implemented * [skip ci] * [skip ci] * [skip ci] • composition.py: - _create_terminal_backprop_learning_components: - support loss_function = CROSS_ENTROPY • combinationfunctions.py: - LinearCombination: add CROSS_ENTROPY as operation * [skip ci] * [skip ci] * [skip ci] * [skip ci] • transferfunctions.py: - ReLU: modified derivative to use infer input from output if provided (needed for BackPropagation) * [skip ci] * [skip ci] * [skip ci] * [skip ci] * - * • transferfunctions.py: - SoftMax.derivative: fixes NOTE: LLVM needs to be modified accordingly • test_transfer.py: - test_transfer_derivative: modify tests to match changes to SoftMax NOTE: LLVM tests don't pass * [skip ci] * [skip ci] * [skip ci] • composition.py - docstring mods re: Autodiff * [skip ci] * Merge branch 'nback' of https://github.com/PrincetonUniversity/PsyNeuLink into nback  Conflicts:  Scripts/Models (Under Development)/Nback/nback.py • composition.py: - run(): try addining Report context for LLVM execution * [skip ci] • composition.py - add Report for compiled mode • compiledloss.py: - CROSS_ENTROPYLoss: _gen_loss_function(): fixed bug, now runs _gen_inject_loss_differential(): dummy copied from MSELoss -- NEEDS TO BE FIXED • transferfunctions.py: - ReLU: added compiled support for derivative using output • test_transfer.py: - test_transfer_derivative_out: test derivatives with output instead of input as arg * Merge branch 'nback' of https://github.com/PrincetonUniversity/PsyNeuLink into nback  Conflicts:  Scripts/Models (Under Development)/Nback/nback.py • composition.py: - run(): try addining Report context for LLVM execution * [skip ci] * [skip ci] * [skip ci] * [skip ci] • Merge branch 'nback' of https://github.com/PrincetonUniversity/PsyNeuLink into nback • composition.py: - docstring mods regarding Autodiff learning * [skip ci] * • composition.py - more docstrings re: Autodiff * [skip ci] • composition.py - table for learning execution modes * [skip ci] • llvm/__init__.py - ExecuteMode: add PyTorch as synonym for Python • autodiffcomposition.py - docstrring refactoring * [skip ci] * [skip ci] • composition.py, autodiffcomposition.py - docstring mods * [skip ci] • composition.py, autodiffcomposition.py - docstring mods * [skip ci] • composition.py, autodiffcomposition.py - docstring mods * [skip ci] * [skip ci] • test_transfer.py: - get rid of duplicative test * Merge branch 'devel' of https://github.com/PrincetonUniversity/PsyNeuLink into devel Conflicts: .github/actions/install-pnl/action.yml .github/actions/on-branch/action.yml .github/workflows/pnl-ci-docs.yml .github/workflows/pnl-ci.yml .github/workflows/test-release.yml Scripts/Models (Under Development)/N-back.py * Merge branch 'devel' of https://github.com/PrincetonUniversity/PsyNeuLink into devel Conflicts: .github/actions/install-pnl/action.yml .github/actions/on-branch/action.yml .github/workflows/pnl-ci-docs.yml .github/workflows/pnl-ci.yml .github/workflows/test-release.yml Scripts/Models (Under Development)/N-back.py * • test_learning.py: - test_xor_training_identicalness_standard_composition_vs_PyTorch_vs_LLVM: replaces test_xor_training_identicalness_standard_composition_vs_Autodiff * • learningfunctions.py - BackPropagation: - fix bug in which derivative for default loss (MSE) was computed using L0 - add explicit specification for L0 loss • composition.py: - _create_terminal_backprop_learning_components: - add explicit assignment of output_port[SUM] for L0 loss • test_learning.py: - test_multilayer: - fix bug in which SSE was assigned as loss, but oputput_port[MSE] was used for objective_mechanism - replace with explicit L0 loss and ouput_port[SUM] for objective_mechanism * [skip ci] • learningfunctions.py - _create_non_terminal_backprop_learning_components: - fixed bug in which loss function for hidden layers was set to MSE rather than simple L0 * [skip ci] • All tests pass * [skip ci] • test_learning.py: - test_multilayer_truth(): parameterize test with expected results * [skip ci] • test_learning.py: - test_multilayer_truth(): test for L0, SSE and MSE * [skip ci] • All tests pass * [skip ci] • All tests pass * [skip ci] * [skip ci] • keywords.py - add Loss enum • llvm.rst - add ExecutionMode • Project - replace MSE, SSE, L0, L1, CROSS_ENTROPY, KL_DIV, NLL and POISSON_NLL with Loss enum members * [skip ci] • composition.py: - run(): add warning for use of PyTorch with Composition * [skip ci] * [skip ci] • composition.py: - run(): commented out warning, as can't distinguish ExecutionMode.PyTorch from ExecutionMode.Python * - * • test_learning.py - clean up test_xor_training_identicalness_standard_composition_vs_PyTorch_and_LLVM * • test_learning.py - clean up test_xor_training_identicalness_standard_composition_vs_PyTorch_and_LLVM Co-authored-by: SamKG Co-authored-by: Katherine Mantel Co-authored-by: jdcpni --- .../N-back/N-back MODULARIZED.py | 231 ----- .../N-back/N-back.py | 537 ------------ .../N-back/Nback Notebook.ipynb | 188 ---- .../N-back/Nback.py | 507 ----------- .../N-back/ffn.wts_nep_6250_lr_01.pnl | Bin 28463 -> 0 bytes .../{N-back => Nback}/SphericalDrift Tests.py | 0 .../{N-back => Nback}/__init__.py | 0 .../Nback/nback.ipynb | 365 ++++++++ .../Models (Under Development)/Nback/nback.py | 822 ++++++++++++++++++ ... MEMORY (fnn)_matrix_wts_20stim_2500ep.pnl | Bin .../results}/ffn.wts_nep_1_lr_01.pnl | Bin .../Nback/results/ffn.wts_nep_6250_lr_001.pnl | Bin 0 -> 29039 bytes .../Nback/results/ffn.wts_nep_6250_lr_01.pnl | Bin 0 -> 29103 bytes .../nback.results_nep_1_lr_01.pnl.npy | Bin .../nback.results_nep_6250_lr_001.pnl.npy | Bin 0 -> 7232 bytes .../nback.results_nep_6250_lr_01.pnl.npy | Bin 0 -> 7232 bytes .../Nback/stim/Archive.zip | Bin 0 -> 192951 bytes .../Nback/stim/Kane stimuli.py | 50 ++ .../stim/__init__.py} | 0 .../Nback/stim/ckm_2_back_a.doc | Bin 0 -> 22131 bytes .../Nback/stim/ckm_2_back_b.doc | Bin 0 -> 22016 bytes .../Nback/stim/ckm_2_back_c.doc | Bin 0 -> 21504 bytes .../Nback/stim/ckm_2_back_d.doc | Bin 0 -> 20992 bytes .../Nback/stim/ckm_2_back_e.doc | Bin 0 -> 32768 bytes .../Nback/stim/ckm_2_back_f.doc | Bin 0 -> 31232 bytes .../Nback/stim/ckm_2_back_g.doc | Bin 0 -> 34816 bytes .../Nback/stim/ckm_2_back_h.doc | Bin 0 -> 24064 bytes .../Nback/stim/ckm_3_back_a.doc | Bin 0 -> 22016 bytes .../Nback/stim/ckm_3_back_b.doc | Bin 0 -> 21504 bytes .../Nback/stim/ckm_3_back_c.doc | Bin 0 -> 23040 bytes .../Nback/stim/ckm_3_back_d.doc | Bin 0 -> 22131 bytes .../Nback/stim/ckm_3_back_e.doc | Bin 0 -> 34816 bytes .../Nback/stim/ckm_3_back_f.doc | Bin 0 -> 33280 bytes .../Nback/stim/ckm_3_back_g.doc | Bin 0 -> 35328 bytes .../Nback/stim/ckm_3_back_h.doc | Bin 0 -> 21504 bytes .../nback.results_nep_6250_lr_01.pnl.npy | Bin 1664 -> 1664 bytes docs/source/Keywords.rst | 2 +- docs/source/LLVM.rst | 7 + .../nonstateful/combinationfunctions.py | 30 +- .../nonstateful/learningfunctions.py | 79 +- .../nonstateful/transferfunctions.py | 66 +- psyneulink/core/compositions/composition.py | 524 ++++++----- psyneulink/core/compositions/report.py | 66 +- psyneulink/core/globals/keywords.py | 65 +- psyneulink/core/globals/utilities.py | 9 +- psyneulink/core/llvm/__init__.py | 44 + .../objective/comparatormechanism.py | 31 +- .../compositions/autodiffcomposition.py | 290 ++++-- .../library/compositions/compiledloss.py | 70 +- .../compositions/pytorchmodelcreator.py | 8 +- tests/composition/test_autodiffcomposition.py | 234 ++--- tests/composition/test_learning.py | 324 +++++-- tests/composition/test_show_graph.py | 2 +- tests/functions/test_transfer.py | 6 +- tests/log/test_log.py | 2 +- tests/log/test_rpc.py | 3 +- 56 files changed, 2487 insertions(+), 2075 deletions(-) delete mode 100644 Scripts/Models (Under Development)/N-back/N-back MODULARIZED.py delete mode 100644 Scripts/Models (Under Development)/N-back/N-back.py delete mode 100644 Scripts/Models (Under Development)/N-back/Nback Notebook.ipynb delete mode 100644 Scripts/Models (Under Development)/N-back/Nback.py delete mode 100644 Scripts/Models (Under Development)/N-back/ffn.wts_nep_6250_lr_01.pnl rename Scripts/Models (Under Development)/{N-back => Nback}/SphericalDrift Tests.py (100%) rename Scripts/Models (Under Development)/{N-back => Nback}/__init__.py (100%) create mode 100644 Scripts/Models (Under Development)/Nback/nback.ipynb create mode 100644 Scripts/Models (Under Development)/Nback/nback.py rename Scripts/Models (Under Development)/{N-back => Nback/results}/WORKING MEMORY (fnn)_matrix_wts_20stim_2500ep.pnl (100%) rename Scripts/Models (Under Development)/{N-back => Nback/results}/ffn.wts_nep_1_lr_01.pnl (100%) create mode 100644 Scripts/Models (Under Development)/Nback/results/ffn.wts_nep_6250_lr_001.pnl create mode 100644 Scripts/Models (Under Development)/Nback/results/ffn.wts_nep_6250_lr_01.pnl rename Scripts/Models (Under Development)/{N-back => Nback/results}/nback.results_nep_1_lr_01.pnl.npy (100%) create mode 100644 Scripts/Models (Under Development)/Nback/results/nback.results_nep_6250_lr_001.pnl.npy create mode 100644 Scripts/Models (Under Development)/Nback/results/nback.results_nep_6250_lr_01.pnl.npy create mode 100644 Scripts/Models (Under Development)/Nback/stim/Archive.zip create mode 100644 Scripts/Models (Under Development)/Nback/stim/Kane stimuli.py rename Scripts/Models (Under Development)/{N-back/N-back_WITH_OBJECTIVE_MECH.py => Nback/stim/__init__.py} (100%) create mode 100644 Scripts/Models (Under Development)/Nback/stim/ckm_2_back_a.doc create mode 100644 Scripts/Models (Under Development)/Nback/stim/ckm_2_back_b.doc create mode 100644 Scripts/Models (Under Development)/Nback/stim/ckm_2_back_c.doc create mode 100644 Scripts/Models (Under Development)/Nback/stim/ckm_2_back_d.doc create mode 100644 Scripts/Models (Under Development)/Nback/stim/ckm_2_back_e.doc create mode 100644 Scripts/Models (Under Development)/Nback/stim/ckm_2_back_f.doc create mode 100644 Scripts/Models (Under Development)/Nback/stim/ckm_2_back_g.doc create mode 100644 Scripts/Models (Under Development)/Nback/stim/ckm_2_back_h.doc create mode 100644 Scripts/Models (Under Development)/Nback/stim/ckm_3_back_a.doc create mode 100644 Scripts/Models (Under Development)/Nback/stim/ckm_3_back_b.doc create mode 100644 Scripts/Models (Under Development)/Nback/stim/ckm_3_back_c.doc create mode 100644 Scripts/Models (Under Development)/Nback/stim/ckm_3_back_d.doc create mode 100644 Scripts/Models (Under Development)/Nback/stim/ckm_3_back_e.doc create mode 100644 Scripts/Models (Under Development)/Nback/stim/ckm_3_back_f.doc create mode 100644 Scripts/Models (Under Development)/Nback/stim/ckm_3_back_g.doc create mode 100644 Scripts/Models (Under Development)/Nback/stim/ckm_3_back_h.doc rename Scripts/Models (Under Development)/{N-back => }/nback.results_nep_6250_lr_01.pnl.npy (69%) create mode 100644 docs/source/LLVM.rst diff --git a/Scripts/Models (Under Development)/N-back/N-back MODULARIZED.py b/Scripts/Models (Under Development)/N-back/N-back MODULARIZED.py deleted file mode 100644 index 74dcf24e51d..00000000000 --- a/Scripts/Models (Under Development)/N-back/N-back MODULARIZED.py +++ /dev/null @@ -1,231 +0,0 @@ -import numpy as np -from psyneulink import * -# from psyneulink.core.scheduling.condition import When -from graph_scheduler import * - -# TODO: -# - from nback-paper: -# - get ffn weights -# - import stimulus generation code -# - retrain on full set of 1,2,3,4,5 back -# - validate against nback-paper results -# - DriftOnASphereIntegrator: fix for noise=0 -# - write test that compares DriftOnASphereIntegrator with spherical_drift code in nback-paper - -# FROM nback-paper: -# 'smtemp':8, -# 'stim_weight':0.05, -# 'hrate':0.04 -# SDIM = 20 -# indim = 2 * (CDIM + SDIM) -# hiddim = SDIM * 4 - -# TEST: -# Structural parameters: -NUM_TASKS=3 -# Test: -STIM_SIZE=1 -# Replicate model: -# STIM_SIZE=20 -# ---------- -CONTEXT_SIZE=25 -HIDDEN_SIZE=STIM_SIZE*4 - -# Execution parameters -# Test: -CONTEXT_DRIFT_RATE=.1 -CONTEXT_DRIFT_NOISE=.00000000001 -# Replicate model: -# CONTEXT_DRIFT_RATE=.25 -# CONTEXT_DRIFT_NOISE=.075 -# ---- -NUM_TRIALS=20 -NBACK=2 -TOLERANCE=.5 -STIM_WEIGHT=.05 -HAZARD_RATE=0.04 -SOFT_MAX_TEMP=1/8 - -# # MODEL: -# STIM_SIZE=25 -# CONTEXT_SIZE=20 -# CONTEXT_DRIFT_RATE=.25 -# CONTEXT_DRIFT_NOISE=.075 -# NUM_TRIALS = 25 - -def control_function(outcome): - """Evaluate response and set ControlSignal for EM[store_prob] accordingly. - - outcome[0] = ffn output - If ffn_output signifies a MATCH: - set EM[store_prob]=1 (as prep encoding stimulus in EM on next trial) - terminate trial - If ffn_output signifies a NON-MATCH: - set EM[store_prob]=0 (as prep for another retrieval from EM without storage) - continue trial - - Notes: - - outcome is passed as 2d array with a single 1d length 2 entry, such that output[0] = ffn output - - ffn output: [1,0]=MATCH, [0,1]=NON-MATCH - - return value is used by: - - control Mechanism to set ControlSignal for EM[store_prob] (per above) - - terminate_trial(), which is used by Condition specified as termination_processing for comp.run(), - to determine whether to end or continue trial - - """ - ffn_output = outcome[0] - if ffn_output[1] > ffn_output[0]: - return 1 - else: # NON-MATCH: - return 0 - return None - - -def terminate_trial(ctl_mech): - """Determine whether to continue or terminate trial. - Determination is made in control_function (assigned as function of control Mechanism): - - terminate if match or hazard rate is realized - - continue if non-match or hazard rate is not realized - """ - if ctl_mech.value==1 or np.random.random() > HAZARD_RATE: - return 1 # terminate - else: - return 0 # continue - - -def construct_model(num_tasks, stim_size, context_size, hidden_size, display=False): - - # Mechanisms: - stim = TransferMechanism(name='STIM', size=STIM_SIZE) - context = ProcessingMechanism(name='CONTEXT', - function=DriftOnASphereIntegrator( - initializer=np.random.random(CONTEXT_SIZE-1), - noise=CONTEXT_DRIFT_NOISE, - dimension=CONTEXT_SIZE)) - task = ProcessingMechanism(name="TASK", size=NUM_TASKS) - em = EpisodicMemoryMechanism(name='EPISODIC MEMORY (dict)', - # default_variable=[[0]*STIM_SIZE, [0]*CONTEXT_SIZE], - input_ports=[{NAME:"STIMULUS_FIELD", - SIZE:STIM_SIZE}, - {NAME:"CONTEXT_FIELD", - SIZE:CONTEXT_SIZE}], - function=ContentAddressableMemory( - initializer=[[[0]*STIM_SIZE, [0]*CONTEXT_SIZE]], - distance_field_weights=[STIM_WEIGHT, 1-STIM_WEIGHT], - equidistant_entries_select=NEWEST, - selection_function=SoftMax(output=MAX_INDICATOR, - gain=SOFT_MAX_TEMP)), - ) - stim_comparator = ComparatorMechanism(name='STIM COMPARATOR', - # sample=STIM_SIZE, target=STIM_SIZE - input_ports=[{NAME:"CURRENT_STIMULUS", SIZE:STIM_SIZE}, - {NAME:"RETRIEVED_STIMULUS", SIZE:STIM_SIZE}], - ) - context_comparator = ComparatorMechanism(name='CONTEXT COMPARATOR', - # sample=np.zeros(STIM_SIZE), - # target=np.zeros(CONTEXT_SIZE) - input_ports=[{NAME:"CURRENT_CONTEXT", SIZE:CONTEXT_SIZE}, - {NAME:"RETRIEVED_CONTEXT", SIZE:CONTEXT_SIZE}], - function=Distance(metric=COSINE)) - - # QUESTION: GET INFO ABOUT INPUT FUNCTIONS FROM ANDRE: - input_current_stim = TransferMechanism(size=STIM_SIZE, function=Linear, name="CURRENT STIMULUS") # function=Logistic) - input_current_context = TransferMechanism(size=STIM_SIZE, function=Linear, name="CURRENT CONTEXT") # function=Logistic) - input_retrieved_stim = TransferMechanism(size=STIM_SIZE, function=Linear, name="RETRIEVED STIMULUS") # function=Logistic) - input_retrieved_context = TransferMechanism(size=STIM_SIZE, function=Linear, name="RETRIEVED CONTEXT") # function=Logistic) - input_task = TransferMechanism(size=NUM_TASKS, function=Linear, name="CURRENT TASK") # function=Logistic) - hidden = TransferMechanism(size=HIDDEN_SIZE, function=Logistic, name="HIDDEN LAYER") - decision = ProcessingMechanism(size=2, name="DECISION LAYER") - - control = ControlMechanism(name="READ/WRITE CONTROLLER", - monitor_for_control=decision, - function=control_function, - control=(STORAGE_PROB, em),) - - # Compositions: - ffn = Composition([{input_current_stim, - input_current_context, - input_retrieved_stim, - input_retrieved_context, - input_task}, - hidden, decision], - name="WORKING MEMORY (fnn)") - comp = Composition(nodes=[stim, context, task, em, ffn, control], - name="N-back Model") - comp.add_projection(MappingProjection(), stim, input_current_stim) - comp.add_projection(MappingProjection(), context, input_current_context) - comp.add_projection(MappingProjection(), task, input_task) - comp.add_projection(MappingProjection(), em.output_ports["RETRIEVED_STIMULUS_FIELD"], input_retrieved_stim) - comp.add_projection(MappingProjection(), em.output_ports["RETRIEVED_CONTEXT_FIELD"], input_retrieved_context) - comp.add_projection(MappingProjection(), stim, em.input_ports["STIMULUS_FIELD"]) - comp.add_projection(MappingProjection(), context, em.input_ports["CONTEXT_FIELD"]) - comp.add_projection(MappingProjection(), decision, control) - - if display: - comp.show_graph() - # comp.show_graph(show_cim=True, - # show_node_structure=ALL, - # show_dimensions=True) - - # Execution: - - # Define a function that detects when the a Mechanism's value has converged, such that the change in all of the - # elements of its value attribute from the last execution (given by its delta attribute) falls below ``epsilon`` - # - # def converge(mech, thresh): - # return all(abs(v) <= thresh for v in mech.delta) - # - # # Add Conditions to the ``color_hidden`` and ``word_hidden`` Mechanisms that depend on the converge function: - # epsilon = 0.01 - # Stroop_model.scheduler.add_condition(color_hidden, When(converge, task, epsilon))) - # Stroop_model.scheduler.add_condition(word_hidden, When(converge, task, epsilon))) - return comp - - -def execute_model(model): - input_dict = {model.nodes['STIM']: np.array(list(range(NUM_TRIALS))).reshape(NUM_TRIALS,1)+1, - model.nodes['CONTEXT']:[[CONTEXT_DRIFT_RATE]]*NUM_TRIALS, - model.nodes['TASK']: np.array([[0,0,1]]*NUM_TRIALS)} - model.run(inputs=input_dict, - termination_processing={TimeScale.TRIAL: - Condition(terminate_trial, # termination function - model.nodes["READ/WRITE CONTROLLER"])}, # function arg - report_output=ReportOutput.ON - ) - -nback_model = construct_model(display=True) -execute_model(nback_model) - - -# TEST OF SPHERICAL DRIFT: -# stims = np.array([x[0] for x in em.memory]) -# contexts = np.array([x[1] for x in em.memory]) -# cos = Distance(metric=COSINE) -# dist = Distance(metric=EUCLIDEAN) -# diffs = [np.sum([contexts[i+1] - contexts[1]]) for i in range(NUM_TRIALS)] -# diffs_1 = [np.sum([contexts[i+1] - contexts[i]]) for i in range(NUM_TRIALS)] -# diffs_2 = [np.sum([contexts[i+2] - contexts[i]]) for i in range(NUM_TRIALS-1)] -# dots = [[contexts[i+1] @ contexts[1]] for i in range(NUM_TRIALS)] -# dot_diffs_1 = [[contexts[i+1] @ contexts[i]] for i in range(NUM_TRIALS)] -# dot_diffs_2 = [[contexts[i+2] @ contexts[i]] for i in range(NUM_TRIALS-1)] -# angle = [cos([contexts[i+1], contexts[1]]) for i in range(NUM_TRIALS)] -# angle_1 = [cos([contexts[i+1], contexts[i]]) for i in range(NUM_TRIALS)] -# angle_2 = [cos([contexts[i+2], contexts[i]]) for i in range(NUM_TRIALS-1)] -# euclidean = [dist([contexts[i+1], contexts[1]]) for i in range(NUM_TRIALS)] -# euclidean_1 = [dist([contexts[i+1], contexts[i]]) for i in range(NUM_TRIALS)] -# euclidean_2 = [dist([contexts[i+2], contexts[i]]) for i in range(NUM_TRIALS-1)] -# print("STIMS:", stims, "\n") -# print("DIFFS:", diffs, "\n") -# print("DIFFS 1:", diffs_1, "\n") -# print("DIFFS 2:", diffs_2, "\n") -# print("DOT PRODUCTS:", dots, "\n") -# print("DOT DIFFS 1:", dot_diffs_1, "\n") -# print("DOT DIFFS 2:", dot_diffs_2, "\n") -# print("ANGLE: ", angle, "\n") -# print("ANGLE_1: ", angle_1, "\n") -# print("ANGLE_2: ", angle_2, "\n") -# print("EUCILDEAN: ", euclidean, "\n") -# print("EUCILDEAN 1: ", euclidean_1, "\n") -# print("EUCILDEAN 2: ", euclidean_2, "\n") - -# n_back_model() diff --git a/Scripts/Models (Under Development)/N-back/N-back.py b/Scripts/Models (Under Development)/N-back/N-back.py deleted file mode 100644 index 6504493494a..00000000000 --- a/Scripts/Models (Under Development)/N-back/N-back.py +++ /dev/null @@ -1,537 +0,0 @@ -""" -This implements a model of the `N-back task `_ -described in `Beukers et al. (2022) `_. The model uses a simple implementation of episodic -(content-addressable) memory to store previous stimuli and the temporal context in which they occured, -and a feedforward neural network to evaluate whether the current stimulus is a match to the n'th preceding stimulus -(n-back level). This model is an example of proposed interactions between working memory (e.g., in neocortex) and -episodic memory e.g., in hippocampus and/or cerebellum) in the performance of tasks demanding of sequential processing -and control, and along the lines of models emerging machine learning that augment the use of recurrent neural networks -(e.g., long short-term memory mechanisms; LSTMs) for active memory and control with an external memory capable of -rapid storage and content-based retrieval, such as the Neural Turing Machine (NTN; `Graves et al., 2016 -`_), Episodic Planning Networks (EPN; `Ritter et al., 2020 -`_), and Emergent Symbols through Binding Networks (ESBN; `Webb et al., 2021 -`_). - -There are three primary methods in the script: - -* construct_model(args): - takes as arguments parameters used to construct the model; for convenience, defaults are defined below, - (under "Construction parameters") - -* train_network(args) - takes as arguments the feedforward neural network Composition (FFN_COMPOSITION) and number of epochs to train. - Note: learning_rate is set at construction (can specify using LEARNING_RATE under "Training parameters" below). - -* run_model() - takes the context drift rate to be applied on each trial and the number of trials to execute as args, as well as - reporting and animation specifications (see "Execution parameters" below). - -See "Settings for running the script" to specify whether the model is trained and/or executed when the script is run, -and whether a graphic display of the network is generated when it is constructed. - -TODO: - - from Andre - - network architecture; in particular, size of hidden layer and projection patterns to and from it - - the stim+context input vector (length 90) projects to a hidden layer (length 80); - - the task input vector (length 2) projects to a different hidden layer (length 80); - - those two hidden layers project (over fixed, nonlearnable, one-one-projections?) to a third hidden layer (length 80) that simply sums them; - - the third hidden layer projects to the length 2 output layer; - - a softmax is taken over the output layer to determine the response. - - fix: were biases trained? - - training: - - learning rate: 0.001; epoch: 1 trial per epoch of training - - fix: state_dict with weights (still needed) - - get empirical stimulus sequences (still needed) - - put N-back script (with pointer to latest version on PNL) in nback-paper repo - - fix: get rid of objective_mechanism (see "VERSION *WITHOUT* ObjectiveMechanism" under control(...) - - fix: warnings on run - - complete documentation in BeukersNbackModel.rst - - validate against nback-paper results - - after validation: - - try with STIM_SIZE = NUM_STIMS rather than 20 (as in nback-paper) - - refactor generate_stim_sequence() to use actual empirical stimulus sequences - - replace get_input_sequence and get_training_inputs with generators passed to nback_model.run() and ffn.learn - -""" - -from graph_scheduler import * - -from psyneulink import * -import numpy as np - -# Settings for running script: -TRAIN = True -RUN = True -DISPLAY_MODEL = False # show visual graphic of model - -# PARAMETERS ------------------------------------------------------------------------------------------------------- - -# Fixed (structural) parameters: -MAX_NBACK_LEVELS = 3 -NUM_STIM = 8 # number of different stimuli in stimulus set - QUESTION: WHY ISN"T THIS EQUAL TO STIM_SIZE OR VICE VERSA? -FFN_TRANSFER_FUNCTION = ReLU - -# Constructor parameters: (values are from nback-paper) -STIM_SIZE=8 # length of stimulus vector -CONTEXT_SIZE=25 # length of context vector -HIDDEN_SIZE=STIM_SIZE*4 # dimension of hidden units in ff -NBACK_LEVELS = [2,3] # Currently restricted to these -NUM_NBACK_LEVELS = len(NBACK_LEVELS) -CONTEXT_DRIFT_NOISE=0.0 # noise used by DriftOnASphereIntegrator (function of Context mech) -RANDOM_WEIGHTS_INITIALIZATION=RandomMatrix(center=0.0, range=0.1) # Matrix spec used to initialize all Projections -RETRIEVAL_SOFTMAX_TEMP=1/8 # express as gain # precision of retrieval process -RETRIEVAL_HAZARD_RATE=0.04 # rate of re=sampling of em following non-match determination in a pass through ffn -RETRIEVAL_STIM_WEIGHT=.05 # weighting of stimulus field in retrieval from em -RETRIEVAL_CONTEXT_WEIGHT = 1-RETRIEVAL_STIM_WEIGHT # weighting of context field in retrieval from em -DECISION_SOFTMAX_TEMP=1 - -# Training parameters: -NUM_EPOCHS= 6250 # nback-paper: 400,000 @ one trial per epoch = 6,250 @ 64 trials per epoch -LEARNING_RATE=0.01 # nback-paper: .001 - -# Execution parameters: -CONTEXT_DRIFT_RATE=.1 # drift rate used for DriftOnASphereIntegrator (function of Context mech) on each trial -NUM_TRIALS = 48 # number of stimuli presented in a trial sequence -REPORT_OUTPUT = ReportOutput.OFF # Sets console output during run -REPORT_PROGRESS = ReportProgress.OFF # Sets console progress bar during run -REPORT_LEARNING = ReportLearning.OFF # Sets console progress bar during training -ANIMATE = False # {UNIT:EXECUTION_SET} # Specifies whether to generate animation of execution - -# Names of Compositions and Mechanisms: -NBACK_MODEL = "N-back Model" -FFN_COMPOSITION = "WORKING MEMORY (fnn)" -FFN_STIMULUS_INPUT = "CURRENT STIMULUS" -FFN_CONTEXT_INPUT = "CURRENT CONTEXT" -FFN_STIMULUS_RETRIEVED = "RETRIEVED STIMULUS" -FFN_CONTEXT_RETRIEVED = "RETRIEVED CONTEXT" -FFN_TASK = "CURRENT TASK" -FFN_HIDDEN = "HIDDEN LAYER" -FFN_OUTPUT = "DECISION LAYER" -MODEL_STIMULUS_INPUT ='STIM' -MODEL_CONTEXT_INPUT = 'CONTEXT' -MODEL_TASK_INPUT = "TASK" -EM = "EPISODIC MEMORY (dict)" -CONTROLLER = "READ/WRITE CONTROLLER" - -# ======================================== MODEL CONSTRUCTION ========================================================= - -def construct_model(stim_size = STIM_SIZE, - context_size = CONTEXT_SIZE, - hidden_size = HIDDEN_SIZE, - num_nback_levels = NUM_NBACK_LEVELS, - context_drift_noise = CONTEXT_DRIFT_NOISE, - retrievel_softmax_temp = RETRIEVAL_SOFTMAX_TEMP, - retrieval_hazard_rate = RETRIEVAL_HAZARD_RATE, - retrieval_stimulus_weight = RETRIEVAL_STIM_WEIGHT, - retrieval_context_weight = RETRIEVAL_CONTEXT_WEIGHT, - decision_softmax_temp = DECISION_SOFTMAX_TEMP): - """Construct nback_model""" - - print(f"constructing '{FFN_COMPOSITION}'...") - - # FEED FORWARD NETWORK ----------------------------------------- - - # inputs: encoding of current stimulus and context, retrieved stimulus and retrieved context, - # output: decision: match [1,0] or non-match [0,1] - # Must be trained to detect match for specified task (1-back, 2-back, etc.) - input_current_stim = TransferMechanism(name=FFN_STIMULUS_INPUT, - size=stim_size, - function=FFN_TRANSFER_FUNCTION) - input_current_context = TransferMechanism(name=FFN_CONTEXT_INPUT, - size=context_size, - function=FFN_TRANSFER_FUNCTION) - input_retrieved_stim = TransferMechanism(name=FFN_STIMULUS_RETRIEVED, - size=stim_size, - function=FFN_TRANSFER_FUNCTION) - input_retrieved_context = TransferMechanism(name=FFN_CONTEXT_RETRIEVED, - size=context_size, - function=FFN_TRANSFER_FUNCTION) - input_task = TransferMechanism(name=FFN_TASK, - size=num_nback_levels, - function=FFN_TRANSFER_FUNCTION) - hidden = TransferMechanism(name=FFN_HIDDEN, - size=hidden_size, - function=FFN_TRANSFER_FUNCTION) - decision = ProcessingMechanism(name=FFN_OUTPUT, - size=2, function=SoftMax(output=MAX_INDICATOR, - gain=decision_softmax_temp)) - ffn = AutodiffComposition(([{input_current_stim, - input_current_context, - input_retrieved_stim, - input_retrieved_context, - input_task}, - hidden, decision], - RANDOM_WEIGHTS_INITIALIZATION, - ), - name=FFN_COMPOSITION, - learning_rate=LEARNING_RATE - ) - - # FULL MODEL (Outer Composition, including input, EM and control Mechanisms) ------------------------ - - print(f"'constructing {NBACK_MODEL}'...") - - # Stimulus Encoding: takes STIM_SIZE vector as input - stim = TransferMechanism(name=MODEL_STIMULUS_INPUT, size=stim_size) - - # Context Encoding: takes scalar as drift step for current trial - context = ProcessingMechanism(name=MODEL_CONTEXT_INPUT, - function=DriftOnASphereIntegrator( - initializer=np.random.random(context_size-1), - noise=context_drift_noise, - dimension=context_size)) - - # Task: task one-hot indicating n-back (1, 2, 3 etc.) - must correspond to what ffn has been trained to do - task = ProcessingMechanism(name=MODEL_TASK_INPUT, - size=num_nback_levels) - - # Episodic Memory: - # - entries: stimulus (field[0]) and context (field[1]); randomly initialized - # - uses Softmax to retrieve best matching input, subject to weighting of stimulus and context by STIM_WEIGHT - em = EpisodicMemoryMechanism(name=EM, - input_ports=[{NAME:"STIMULUS_FIELD", - SIZE:stim_size}, - {NAME:"CONTEXT_FIELD", - SIZE:context_size}], - function=ContentAddressableMemory( - initializer=[[[0]*stim_size, [0]*context_size]], - distance_field_weights=[retrieval_stimulus_weight, - retrieval_context_weight], - # equidistant_entries_select=NEWEST, - selection_function=SoftMax(output=MAX_INDICATOR, - gain=retrievel_softmax_temp)), - ) - - # Control Mechanism - # Ensures current stimulus and context are only encoded in EM once (at beginning of trial) - # by controlling the storage_prob parameter of em: - # - if outcome of decision signifies a match or hazard rate is realized: - # - set EM[store_prob]=1 (as prep encoding stimulus in EM on next trial) - # - this also serves to terminate trial (see nback_model.termination_processing condition) - # - if outcome of decision signifies a non-match - # - set EM[store_prob]=0 (as prep for another retrieval from EM without storage) - # - continue trial - control = ControlMechanism(name=CONTROLLER, - default_variable=[[1]], # Ensure EM[store_prob]=1 at beginning of first trial - # --------- - # VERSION *WITH* ObjectiveMechanism: - objective_mechanism=ObjectiveMechanism(name="OBJECTIVE MECHANISM", - monitor=decision, - # Outcome=1 if match, else 0 - function=lambda x: int(x[0][1]>x[0][0])), - # Set ControlSignal for EM[store_prob] - function=lambda outcome: int(bool(outcome) - or (np.random.random() > retrieval_hazard_rate)), - # --------- - # # VERSION *WITHOUT* ObjectiveMechanism: - # monitor_for_control=decision, - # # Set Evaluate outcome and set ControlSignal for EM[store_prob] - # # - outcome is received from decision as one hot in the form: [[match, no-match]] - # function=lambda outcome: int(int(outcome[0][1]>outcome[0][0]) - # or (np.random.random() > retrieval_hazard_rate)), - # --------- - control=(STORAGE_PROB, em)) - - nback_model = Composition(name=NBACK_MODEL, - nodes=[stim, context, task, ffn, em, control], - # Terminate trial if value of control is still 1 after first pass through execution - termination_processing={TimeScale.TRIAL: And(Condition(lambda: control.value), - AfterPass(0, TimeScale.TRIAL))}, - ) - # # Terminate trial if value of control is still 1 after first pass through execution - # # FIX: ALL OF THE FOLLOWING STOP AFTER ~ NUMBER OF TRIALS (?90+); SHOULD BE: NUM_TRIALS*NUM_NBACK_LEVELS + 1 - # nback_model.scheduler.add_condition(nback_model, And(Condition(lambda: control.value), AfterPass(0, TimeScale.TRIAL))) - # nback_model.scheduler.termination_conds = ({TimeScale.TRIAL: And(Condition(lambda: control.value), - # AfterPass(0, TimeScale.TRIAL))}) - # nback_model.scheduler.termination_conds.update({TimeScale.TRIAL: And(Condition(lambda: control.value), - # AfterPass(0, TimeScale.TRIAL))}) - nback_model.add_projection(MappingProjection(), stim, input_current_stim) - nback_model.add_projection(MappingProjection(), context, input_current_context) - nback_model.add_projection(MappingProjection(), task, input_task) - nback_model.add_projection(MappingProjection(), em.output_ports["RETRIEVED_STIMULUS_FIELD"], input_retrieved_stim) - nback_model.add_projection(MappingProjection(), em.output_ports["RETRIEVED_CONTEXT_FIELD"], input_retrieved_context) - nback_model.add_projection(MappingProjection(), stim, em.input_ports["STIMULUS_FIELD"]) - nback_model.add_projection(MappingProjection(), context, em.input_ports["CONTEXT_FIELD"]) - - if DISPLAY_MODEL: - nback_model.show_graph( - # show_cim=True, - # show_node_structure=ALL, - # show_dimensions=True - ) - - print(f'full model constructed') - return nback_model - -# ==========================================STIMULUS GENERATION ======================================================= -# Based on nback-paper - -def get_stim_set(num_stim=STIM_SIZE): - """Construct an array of stimuli for use an experiment""" - # For now, use one-hots - return np.eye(num_stim) - -def get_task_input(nback_level): - """Construct input to task Mechanism for a given nback_level, used by run_model() and train_network()""" - task_input = list(np.zeros_like(NBACK_LEVELS)) - task_input[nback_level-NBACK_LEVELS[0]] = 1 - return task_input - -def get_run_inputs(model, nback_level, context_drift_rate, num_trials): - """Construct set of stimulus inputs for run_model()""" - - def generate_stim_sequence(nback_level, trial_num, trial_type=0, num_stim=NUM_STIM, num_trials=NUM_TRIALS): - assert nback_level in {2,3} # At present, only 2- and 3-back levels are supported - - def gen_subseq_stim(): - A = np.random.randint(0,num_stim) - B = np.random.choice( - np.setdiff1d(np.arange(num_stim),[A]) - ) - C = np.random.choice( - np.setdiff1d(np.arange(num_stim),[A,B]) - ) - X = np.random.choice( - np.setdiff1d(np.arange(num_stim),[A,B]) - ) - return A,B,C,X - - def generate_match_no_foils_sequence(nback_level,trial_num): - # AXA (2-back) or ABXA (3-back) - seq = np.random.randint(0,num_stim,num_trials) - A,B,C,X = gen_subseq_stim() - # - if nback_level==2: - subseq = [A,X,A] - elif nback_level==3: - subseq = [A,B,X,A] - seq[trial_num-(nback_level+1):trial_num] = subseq - return seq[:trial_num] - - def generate_non_match_no_foils_sequence(nback_level,trial_num): - # AXB (2-back) or ABXC (3-back) - seq = np.random.randint(0,num_stim,num_trials) - A,B,C,X = gen_subseq_stim() - # - if nback_level==2: - subseq = [A,X,B] - elif nback_level==3: - subseq = [A,B,X,C] - seq[trial_num-(nback_level+1):trial_num] = subseq - return seq[:trial_num] - - def generate_match_with_foil_sequence(nback_level,trial_num): - # AAA (2-back) or AAXA (3-back) - seq = np.random.randint(0,num_stim,num_trials) - A,B,C,X = gen_subseq_stim() - # - if nback_level==2: - subseq = [A,A,A] - elif nback_level==3: - subseq = [A,A,X,A] - seq[trial_num-(nback_level+1):trial_num] = subseq - return seq[:trial_num] - - def generate_non_match_with_foil_sequence(nback_level,trial_num): - # XAA (2-back) or ABXB (3-back) - seq = np.random.randint(0,num_stim,num_trials) - A,B,C,X = gen_subseq_stim() - # - if nback_level==2: - subseq = [X,A,A] - elif nback_level==3: - subseq = [A,B,X,B] - seq[trial_num-(nback_level+1):trial_num] = subseq - return seq[:trial_num] - - trial_types = [generate_match_no_foils_sequence, - generate_match_with_foil_sequence, - generate_non_match_no_foils_sequence, - generate_non_match_with_foil_sequence] - stim_seq = trial_types[trial_type](nback_level,trial_num) - # ytarget = [1,1,0,0][trial_type] - # ctxt = spherical_drift(trial_num) - # return stim,ctxt,ytarget - return stim_seq - - # def stim_set_generation(nback_level, num_trials): - # stim_sequence = [] - # # for seq_int, trial in itertools.product(range(4),np.arange(5,trials)): # This generates all length sequences - # for trial_type, trial_num in itertools.product(range(4),[num_trials]): # This generates only longest seq ( - # # num_trials) - # return stim_sequence.append(generate_stim_sequence(nback_level, trial_num, trial_type=trial_type, trials=num_trials)) - - def get_input_sequence(nback_level, num_trials=NUM_TRIALS): - """Get sequence of inputs for a run""" - input_set = get_stim_set() - # Construct sequence of stimulus indices - trial_seq = generate_stim_sequence(nback_level, num_trials) - # Return list of corresponding stimulus input vectors - return [input_set[trial_seq[i]] for i in range(num_trials)] - - return {model.nodes[MODEL_STIMULUS_INPUT]: get_input_sequence(nback_level, num_trials), - model.nodes[MODEL_CONTEXT_INPUT]: [[context_drift_rate]]*num_trials, - model.nodes[MODEL_TASK_INPUT]: [get_task_input(nback_level)]*num_trials} - -def get_training_inputs(network, num_epochs, nback_levels): - """Construct set of training stimuli used by ffn.learn() in train_network() - Construct one example of each condition: - match: stim_current = stim_retrieved and context_current = context_retrieved - stim_lure: stim_current = stim_retrieved and context_current != context_retrieved - context_lure: stim_current != stim_retrieved and context_current == context_retrieved - non_lure: stim_current != stim_retrieved and context_current != context_retrieved - """ - assert is_iterable(nback_levels) and all([0", - "image/svg+xml": "\n\n\n\n\n\nN-back Model\n\nN-back Model\n\ncluster_WORKING MEMORY (fnn)\n\nWORKING MEMORY (fnn)\n\n\n\nTASK\n\nTASK\n\n\n\nCURRENT TASK\n\nCURRENT TASK\n\n\n\nTASK->CURRENT TASK\n\n\n\n\n\nCONTEXT\n\nCONTEXT\n\n\n\nCURRENT CONTEXT\n\nCURRENT CONTEXT\n\n\n\nCONTEXT->CURRENT CONTEXT\n\n\n\n\n\nEPISODIC MEMORY (dict)\n\nEPISODIC MEMORY (dict)\n\n\n\nCONTEXT->EPISODIC MEMORY (dict)\n\n\n\n\n\nSTIM\n\nSTIM\n\n\n\nCURRENT STIMULUS\n\nCURRENT STIMULUS\n\n\n\nSTIM->CURRENT STIMULUS\n\n\n\n\n\nSTIM->EPISODIC MEMORY (dict)\n\n\n\n\n\nHIDDEN LAYER\n\nHIDDEN LAYER\n\n\n\nCURRENT TASK->HIDDEN LAYER\n\n\n\n\n\nCURRENT STIMULUS->HIDDEN LAYER\n\n\n\n\n\nCURRENT CONTEXT->HIDDEN LAYER\n\n\n\n\n\nRETRIEVED CONTEXT\n\nRETRIEVED CONTEXT\n\n\n\nEPISODIC MEMORY (dict)->RETRIEVED CONTEXT\n\n\n\n\n\nRETRIEVED STIMULUS\n\nRETRIEVED STIMULUS\n\n\n\nEPISODIC MEMORY (dict)->RETRIEVED STIMULUS\n\n\n\n\n\nRETRIEVED CONTEXT->HIDDEN LAYER\n\n\n\n\n\nRETRIEVED STIMULUS->HIDDEN LAYER\n\n\n\n\n\nREAD/WRITE CONTROLLER\n\nREAD/WRITE CONTROLLER\n\n\n\nREAD/WRITE CONTROLLER->EPISODIC MEMORY (dict)\n\n\n\n\n\n\nOBJECTIVE MECHANISM\n\nOBJECTIVE MECHANISM\n\n\n\nOBJECTIVE MECHANISM->READ/WRITE CONTROLLER\n\n\n\n\n\nDECISION LAYER\n\nDECISION LAYER\n\n\n\nDECISION LAYER->OBJECTIVE MECHANISM\n\n\n\n\n\nHIDDEN LAYER->DECISION LAYER\n\n\n\n\n\n" - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "nback_model.show_graph(output_fmt='jupyter')" - ] - }, - { - "cell_type": "markdown", - "source": [ - "### Train the model:" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "ffn = nback_model.nodes['WORKING MEMORY (fnn)']\n", - "train_network(ffn, num_epochs=100)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Run the model:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "run_model(nback_model)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.7" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} \ No newline at end of file diff --git a/Scripts/Models (Under Development)/N-back/Nback.py b/Scripts/Models (Under Development)/N-back/Nback.py deleted file mode 100644 index abd8173c02a..00000000000 --- a/Scripts/Models (Under Development)/N-back/Nback.py +++ /dev/null @@ -1,507 +0,0 @@ -""" -This implements a model of the `N-back task `_ -described in `Beukers et al. (2022) `_. The model uses a simple implementation of episodic -(content-addressable) memory to store previous stimuli and the temporal context in which they occured, -and a feedforward neural network to evaluate whether the current stimulus is a match to the n'th preceding stimulus -(n-back level). This model is an example of proposed interactions between working memory (e.g., in neocortex) and -episodic memory e.g., in hippocampus and/or cerebellum) in the performance of tasks demanding of sequential processing -and control, and along the lines of models emerging machine learning that augment the use of recurrent neural networks -(e.g., long short-term memory mechanisms; LSTMs) for active memory and control with an external memory capable of -rapid storage and content-based retrieval, such as the Neural Turing Machine (NTN; `Graves et al., 2016 -`_), Episodic Planning Networks (EPN; `Ritter et al., 2020 -`_), and Emergent Symbols through Binding Networks (ESBN; `Webb et al., 2021 -`_). - -There are three primary methods in the script: - -* construct_model(args): - takes as arguments parameters used to construct the model; for convenience, defaults are defined below, - (under "Construction parameters") - -* train_network(args) - takes as arguments the feedforward neural network Composition (FFN_COMPOSITION) and number of epochs to train. - Note: learning_rate is set at construction (can specify using LEARNING_RATE under "Training parameters" below). - -* run_model() - takes the context drift rate to be applied on each trial and the number of trials to execute as args, as well as - reporting and animation specifications (see "Execution parameters" below). - -See "Settings for running the script" to specify whether the model is trained and/or executed when the script is run, -and whether a graphic display of the network is generated when it is constructed. - -TODO: - - from Andre - - network architecture; in particular, size of hidden layer and projection patterns to and from it - - the stim+context input vector (length 90) projects to a hidden layer (length 80); - - the task input vector (length 2) projects to a different hidden layer (length 80); - - those two hidden layers project (over fixed, nonlearnable, one-one-projections?) to a third hidden layer (length 80) that simply sums them; - - the third hidden layer projects to the length 2 output layer; - - a softmax is taken over the output layer to determine the response. - - fix: were biases trained? - - training: - - learning rate: 0.001; epoch: 1 trial per epoch of training - - fix: state_dict with weights (still needed) - - get empirical stimulus sequences (still needed) - - put N-back script (with pointer to latest version on PNL) in nback-paper repo - - fix: get rid of objective_mechanism (see "VERSION *WITHOUT* ObjectiveMechanism" under control(...) - - fix: warnings on run - - complete documentation in BeukersNbackModel.rst - - validate against nback-paper results - - after validation: - - try with STIM_SIZE = NUM_STIMS rather than 20 (as in nback-paper) - - refactor generate_stim_sequence() to use actual empirical stimulus sequences - - replace get_input_sequence and get_training_inputs with generators passed to nback_model.run() and ffn.learn - -""" - -from graph_scheduler import * - -from psyneulink import * -import numpy as np - -# Settings for running script: -DISPLAY_MODEL = False # show visual graphic of model - -# PARAMETERS ------------------------------------------------------------------------------------------------------- - -# Fixed (structural) parameters: -MAX_NBACK_LEVELS = 3 -NUM_STIM = 8 # number of different stimuli in stimulus set - QUESTION: WHY ISN"T THIS EQUAL TO STIM_SIZE OR VICE VERSA? -FFN_TRANSFER_FUNCTION = ReLU - -# Constructor parameters: (values are from nback-paper) -STIM_SIZE=20 # length of stimulus vector -CONTEXT_SIZE=25 # length of context vector -HIDDEN_SIZE=STIM_SIZE*4 # dimension of hidden units in ff -NBACK_LEVELS = [2,3] # Currently restricted to these -NUM_NBACK_LEVELS = len(NBACK_LEVELS) -CONTEXT_DRIFT_NOISE=0.0 # noise used by DriftOnASphereIntegrator (function of Context mech) -RANDOM_WEIGHTS_INITIALIZATION=RandomMatrix(center=0.0, range=0.1) # Matrix spec used to initialize all Projections -RETRIEVAL_SOFTMAX_TEMP=1/8 # express as gain # precision of retrieval process -RETRIEVAL_HAZARD_RATE=0.04 # rate of re=sampling of em following non-match determination in a pass through ffn -RETRIEVAL_STIM_WEIGHT=.05 # weighting of stimulus field in retrieval from em -RETRIEVAL_CONTEXT_WEIGHT = 1-RETRIEVAL_STIM_WEIGHT # weighting of context field in retrieval from em -DECISION_SOFTMAX_TEMP=1 - -# Training parameters: -NUM_EPOCHS=3 # nback-paper: 400,000 @ one trial per epoch = 2,500 @ 160 trials per epoch -LEARNING_RATE=0.01 # nback-paper: .001 - -# Execution parameters: -CONTEXT_DRIFT_RATE=.1 # drift rate used for DriftOnASphereIntegrator (function of Context mech) on each trial -NUM_TRIALS = 48 # number of stimuli presented in a trial sequence for a given nback_level during run -REPORT_OUTPUT = ReportOutput.OFF # Sets console output during run -REPORT_PROGRESS = ReportProgress.OFF # Sets console progress bar during run -REPORT_LEARNING = ReportLearning.OFF # Sets console progress bar during training -ANIMATE = False # {UNIT:EXECUTION_SET} # Specifies whether to generate animation of execution - -# Names of Compositions and Mechanisms: -NBACK_MODEL = "N-back Model" -FFN_COMPOSITION = "WORKING MEMORY (fnn)" -FFN_STIMULUS_INPUT = "CURRENT STIMULUS" -FFN_CONTEXT_INPUT = "CURRENT CONTEXT" -FFN_STIMULUS_RETRIEVED = "RETRIEVED STIMULUS" -FFN_CONTEXT_RETRIEVED = "RETRIEVED CONTEXT" -FFN_TASK = "CURRENT TASK" -FFN_HIDDEN = "HIDDEN LAYER" -FFN_OUTPUT = "DECISION LAYER" -MODEL_STIMULUS_INPUT ='STIM' -MODEL_CONTEXT_INPUT = 'CONTEXT' -MODEL_TASK_INPUT = "TASK" -EM = "EPISODIC MEMORY (dict)" -CONTROLLER = "READ/WRITE CONTROLLER" - -# ======================================== MODEL CONSTRUCTION ========================================================= - -def construct_model(stim_size = STIM_SIZE, - context_size = CONTEXT_SIZE, - hidden_size = HIDDEN_SIZE, - num_nback_levels = NUM_NBACK_LEVELS, - context_drift_noise = CONTEXT_DRIFT_NOISE, - retrievel_softmax_temp = RETRIEVAL_SOFTMAX_TEMP, - retrieval_hazard_rate = RETRIEVAL_HAZARD_RATE, - retrieval_stimulus_weight = RETRIEVAL_STIM_WEIGHT, - retrieval_context_weight = RETRIEVAL_CONTEXT_WEIGHT, - decision_softmax_temp = DECISION_SOFTMAX_TEMP): - """Construct nback_model""" - - print(f'constructing {FFN_COMPOSITION}...') - - # FEED FORWARD NETWORK ----------------------------------------- - - # inputs: encoding of current stimulus and context, retrieved stimulus and retrieved context, - # output: decIsion: match [1,0] or non-match [0,1] - # Must be trained to detect match for specified task (1-back, 2-back, etc.) - input_current_stim = TransferMechanism(name=FFN_STIMULUS_INPUT, - size=stim_size, - function=FFN_TRANSFER_FUNCTION) - input_current_context = TransferMechanism(name=FFN_CONTEXT_INPUT, - size=context_size, - function=FFN_TRANSFER_FUNCTION) - input_retrieved_stim = TransferMechanism(name=FFN_STIMULUS_RETRIEVED, - size=stim_size, - function=FFN_TRANSFER_FUNCTION) - input_retrieved_context = TransferMechanism(name=FFN_CONTEXT_RETRIEVED, - size=context_size, - function=FFN_TRANSFER_FUNCTION) - input_task = TransferMechanism(name=FFN_TASK, - size=num_nback_levels, - function=FFN_TRANSFER_FUNCTION) - hidden = TransferMechanism(name=FFN_HIDDEN, - size=hidden_size, - function=FFN_TRANSFER_FUNCTION) - decision = ProcessingMechanism(name=FFN_OUTPUT, - size=2, function=SoftMax(output=MAX_INDICATOR, - gain=decision_softmax_temp)) - ffn = AutodiffComposition(([{input_current_stim, - input_current_context, - input_retrieved_stim, - input_retrieved_context, - input_task}, - hidden, decision], - RANDOM_WEIGHTS_INITIALIZATION, - ), - name=FFN_COMPOSITION, - learning_rate=LEARNING_RATE - ) - - # FULL MODEL (Outer Composition, including input, EM and control Mechanisms) ------------------------ - - print(f'constructing {NBACK_MODEL}...') - - # Stimulus Encoding: takes STIM_SIZE vector as input - stim = TransferMechanism(name=MODEL_STIMULUS_INPUT, size=stim_size) - - # Context Encoding: takes scalar as drift step for current trial - context = ProcessingMechanism(name=MODEL_CONTEXT_INPUT, - function=DriftOnASphereIntegrator( - initializer=np.random.random(context_size-1), - noise=context_drift_noise, - dimension=context_size)) - - # Task: task one-hot indicating n-back (1, 2, 3 etc.) - must correspond to what ffn has been trained to do - task = ProcessingMechanism(name=MODEL_TASK_INPUT, - size=NUM_NBACK_LEVELS) - - # Episodic Memory: - # - entries: stimulus (field[0]) and context (field[1]); randomly initialized - # - uses Softmax to retrieve best matching input, subject to weighting of stimulus and context by STIM_WEIGHT - em = EpisodicMemoryMechanism(name=EM, - input_ports=[{NAME:"STIMULUS_FIELD", - SIZE:stim_size}, - {NAME:"CONTEXT_FIELD", - SIZE:context_size}], - function=ContentAddressableMemory( - initializer=[[[0]*stim_size, [0]*context_size]], - distance_field_weights=[retrieval_stimulus_weight, - retrieval_context_weight], - # equidistant_entries_select=NEWEST, - selection_function=SoftMax(output=MAX_INDICATOR, - gain=retrievel_softmax_temp)), - ) - - # Control Mechanism - # Ensures current stimulus and context are only encoded in EM once (at beginning of trial) - # by controlling the storage_prob parameter of em: - # - if outcome of decision signifies a match or hazard rate is realized: - # - set EM[store_prob]=1 (as prep encoding stimulus in EM on next trial) - # - this also serves to terminate trial (see nback_model.termination_processing condition) - # - if outcome of decision signifies a non-match - # - set EM[store_prob]=0 (as prep for another retrieval from EM without storage) - # - continue trial - control = ControlMechanism(name=CONTROLLER, - default_variable=[[1]], # Ensure EM[store_prob]=1 at beginning of first trial - # --------- - # VERSION *WITH* ObjectiveMechanism: - objective_mechanism=ObjectiveMechanism(name="OBJECTIVE MECHANISM", - monitor=decision, - # Outcome=1 if match, else 0 - function=lambda x: int(x[0][1]>x[0][0])), - # Set ControlSignal for EM[store_prob] - function=lambda outcome: int(bool(outcome) - or (np.random.random() > retrieval_hazard_rate)), - # --------- - # # VERSION *WITHOUT* ObjectiveMechanism: - # monitor_for_control=decision, - # # Set Evaluate outcome and set ControlSignal for EM[store_prob] - # # - outcome is received from decision as one hot in the form: [[match, no-match]] - # function=lambda outcome: int(int(outcome[0][1]>outcome[0][0]) - # or (np.random.random() > retrieval_hazard_rate)), - # --------- - control=(STORAGE_PROB, em)) - - nback_model = Composition(name=NBACK_MODEL, - nodes=[stim, context, task, ffn, em, control], - # Terminate trial if value of control is still 1 after first pass through execution - termination_processing={TimeScale.TRIAL: And(Condition(lambda: control.value), - AfterPass(0, TimeScale.TRIAL))}, - ) - # # Terminate trial if value of control is still 1 after first pass through execution - # # FIX: ALL OF THE FOLLOWING STOP AFTER ~ NUMBER OF TRIALS (?90+); SHOULD BE: NUM_TRIALS*NUM_NBACK_LEVELS + 1 - # nback_model.scheduler.add_condition(nback_model, And(Condition(lambda: control.value), AfterPass(0, TimeScale.TRIAL))) - # nback_model.scheduler.termination_conds = ({TimeScale.TRIAL: And(Condition(lambda: control.value), - # AfterPass(0, TimeScale.TRIAL))}) - # nback_model.scheduler.termination_conds.update({TimeScale.TRIAL: And(Condition(lambda: control.value), - # AfterPass(0, TimeScale.TRIAL))}) - nback_model.add_projection(MappingProjection(), stim, input_current_stim) - nback_model.add_projection(MappingProjection(), context, input_current_context) - nback_model.add_projection(MappingProjection(), task, input_task) - nback_model.add_projection(MappingProjection(), em.output_ports["RETRIEVED_STIMULUS_FIELD"], input_retrieved_stim) - nback_model.add_projection(MappingProjection(), em.output_ports["RETRIEVED_CONTEXT_FIELD"], input_retrieved_context) - nback_model.add_projection(MappingProjection(), stim, em.input_ports["STIMULUS_FIELD"]) - nback_model.add_projection(MappingProjection(), context, em.input_ports["CONTEXT_FIELD"]) - - if DISPLAY_MODEL: - nback_model.show_graph( - # show_cim=True, - # show_node_structure=ALL, - # show_dimensions=True - ) - - print(f'full model constructed') - return nback_model - -# ==========================================STIMULUS GENERATION ======================================================= -# Based on nback-paper - -def get_stim_set(num_stim=STIM_SIZE): - """Construct an array of stimuli for use an experiment""" - # For now, use one-hots - return np.eye(num_stim) - -def get_task_input(nback_level): - """Construct input to task Mechanism for a given nback_level, used by run_model() and train_network()""" - task_input = list(np.zeros_like(NBACK_LEVELS)) - task_input[nback_level-NBACK_LEVELS[0]] = 1 - return task_input - -def get_run_inputs(model, nback_level, context_drift_rate, num_trials): - """Construct set of stimulus inputs for run_model()""" - - def generate_stim_sequence(nback_level, trial_num, trial_type=0, num_stim=NUM_STIM, num_trials=NUM_TRIALS): - assert nback_level in {2,3} # At present, only 2- and 3-back levels are supported - - def gen_subseq_stim(): - A = np.random.randint(0,num_stim) - B = np.random.choice( - np.setdiff1d(np.arange(num_stim),[A]) - ) - C = np.random.choice( - np.setdiff1d(np.arange(num_stim),[A,B]) - ) - X = np.random.choice( - np.setdiff1d(np.arange(num_stim),[A,B]) - ) - return A,B,C,X - - def generate_match_no_foils_sequence(nback_level,trial_num): - # AXA (2-back) or ABXA (3-back) - seq = np.random.randint(0,num_stim,num_trials) - A,B,C,X = gen_subseq_stim() - # - if nback_level==2: - subseq = [A,X,A] - elif nback_level==3: - subseq = [A,B,X,A] - seq[trial_num-(nback_level+1):trial_num] = subseq - return seq[:trial_num] - - def generate_non_match_no_foils_sequence(nback_level,trial_num): - # AXB (2-back) or ABXC (3-back) - seq = np.random.randint(0,num_stim,num_trials) - A,B,C,X = gen_subseq_stim() - # - if nback_level==2: - subseq = [A,X,B] - elif nback_level==3: - subseq = [A,B,X,C] - seq[trial_num-(nback_level+1):trial_num] = subseq - return seq[:trial_num] - - def generate_match_with_foil_sequence(nback_level,trial_num): - # AAA (2-back) or AAXA (3-back) - seq = np.random.randint(0,num_stim,num_trials) - A,B,C,X = gen_subseq_stim() - # - if nback_level==2: - subseq = [A,A,A] - elif nback_level==3: - subseq = [A,A,X,A] - seq[trial_num-(nback_level+1):trial_num] = subseq - return seq[:trial_num] - - def generate_non_match_with_foil_sequence(nback_level,trial_num): - # XAA (2-back) or ABXB (3-back) - seq = np.random.randint(0,num_stim,num_trials) - A,B,C,X = gen_subseq_stim() - # - if nback_level==2: - subseq = [X,A,A] - elif nback_level==3: - subseq = [A,B,X,B] - seq[trial_num-(nback_level+1):trial_num] = subseq - return seq[:trial_num] - - trial_types = [generate_match_no_foils_sequence, - generate_match_with_foil_sequence, - generate_non_match_no_foils_sequence, - generate_non_match_with_foil_sequence] - stim_seq = trial_types[trial_type](nback_level,trial_num) - # ytarget = [1,1,0,0][trial_type] - # ctxt = spherical_drift(trial_num) - # return stim,ctxt,ytarget - return stim_seq - - # def stim_set_generation(nback_level, num_trials): - # stim_sequence = [] - # # for seq_int, trial in itertools.product(range(4),np.arange(5,trials)): # This generates all length sequences - # for trial_type, trial_num in itertools.product(range(4),[num_trials]): # This generates only longest seq ( - # # num_trials) - # return stim_sequence.append(generate_stim_sequence(nback_level, trial_num, trial_type=trial_type, trials=num_trials)) - - def get_input_sequence(nback_level, num_trials=NUM_TRIALS): - """Get sequence of inputs for a run""" - input_set = get_stim_set() - # Construct sequence of stimulus indices - trial_seq = generate_stim_sequence(nback_level, num_trials) - # Return list of corresponding stimulus input vectors - return [input_set[trial_seq[i]] for i in range(num_trials)] - - return {model.nodes[MODEL_STIMULUS_INPUT]: get_input_sequence(nback_level, num_trials), - model.nodes[MODEL_CONTEXT_INPUT]: [[context_drift_rate]]*num_trials, - model.nodes[MODEL_TASK_INPUT]: [get_task_input(nback_level)]*num_trials} - -def get_training_inputs(network, num_epochs, nback_levels): - """Construct set of training stimuli used by ffn.learn() in train_network() - Construct one example of each condition: - match: stim_current = stim_retrieved and context_current = context_retrieved - stim_lure: stim_current = stim_retrieved and context_current != context_retrieved - context_lure: stim_current != stim_retrieved and context_current == context_retrieved - non_lure: stim_current != stim_retrieved and context_current != context_retrieved - """ - assert is_iterable(nback_levels) and all([0Km|d?1R_CD$ta4V&&zxg)-+lMpJ$lXnza>lU>{hO{`(OUI-fp8^cydHc z?8m5+k>RP~?uqZ--@3Ne?%DtJzf6aWBsWH8%cq(u(|~l@F6v6(l)6 zOmcGdv-flGbIDIy{r_DuGWByJ-&_;oz&D~VCaoj$7XAr72Q>2d?yvkwP7@ z_mv8MDn0U7yR_xb9jUT(DpDG?^KsA`ZzpNd3TeKp|5kZRZjkn$*9je#3QLFdhqQ5p z?o&Es9c5X)OPysx7c6a*Oc{LqkhT7`Km(jm5hbnWs?mjRiyUP-n5OqYTn zEw&=0`pR}U?pS0={)-28C+|p;*jzPN$D}ky#@Bo6?8AMR?4(@ltk5fgS{N42*R%qQ z4C(xlJ;n%95+-??qq8gI7k|rk+0GqDK1rU~4ziNzdqF|#z3*y`Oh)mqKFPCA>!|jv z2=Q6zE!DR!JyaW8ov@@xXTCj;+2!gT;`vmHj%dM4vAvZMt>T|9DU)d{L9(@f;|eK~ ze2ib{cTor5dGGetf|FAF^^(kMfhA2}Tx9m;x!vAtw@5>n_Utvbs9LW*QBQT;(K}Iw zuf3L0b~I&a(HE(R)&+YRvvl-@-)gD4xWW6jv`BBfW=rN8Z>dPp8ToT(smxd|TCY?5 zKmW3hSk4DdYkrEOy|*-LiT1CNaw*XnOMhuqtj;HBtCU#Uzek&O;l7L1h%HWwt}VeC z1zKrIL-2X2lAltl0ZPnHU4#n@%euKPdil}@p3i29JOfX)rN1NTelbmA+sZ9@)Hm4?Sc^z{9zo&zTH-jJEz4`d& z4VR$cyZ0XXKMMNaBZaLo4l#=nzAc8Z-Tp9XhY8!2CTw@ctT&;zd$k>q-9oh3bT(R= ztczOgv}BzOePOR7((^zj*?5+8Svqk0Xs`}w0eg_TG_78{BtpimO^3E>&pB=4r_17F zwJp>ibFcN%!i!JuXup-XTU+GMJ+5ADMVx-(C6)K3(wI@}w?>|J*I`QwWssZX9Js3k zmS%$F?iY#6r4hIde)5pfYf@{1rHlVe$I1{<&}?b>3F-cv=E)yrLb_jNoF4->hBiV_?6*%@sSi+z7GhD+6_7?nHGKpoNwhXQ1QHk{){a|%q2mb{ z`o|I?AL;Y~DVAAFvi*|0x4tmmmR#xj#9BMFS30fNTCT+eNIBT}DYlhgbwKi6v{FZ{ zt(tRcnJyfIFl^@ov?xUj+%bgIhXSzz7-dpxEFHOa0|IHizsh^5=14=53?AaLI>Kja zW%T_68MfYp5V~z|9-G1G0y=(*1y-Wcw*F!AsXVmIr6 zR#<*FJ#J}xB=?Qc%!G3?yd3*W{jTHMhY1^{!vcb>ceF_Jp~OL)pcS!jO09J3Z)*oZ zpMPcleksN4GA;A#ud_8swp`owAJozOTH7tQKdHHRvl~*bv99O>#F&ZWen6Se^p72` zTDXZ@WQcnXY7e`sWs@(Qy+Y&%f0?v&0_%-RyR`0QJGM~$c`lj|Eyg9<@lgk*@qspo z&AJg6=C9rQQ>v|0KG+e!Z>509wby30oUaw%u~DtgKE81cI^s%4b?Vz|)-`Oc;3kD{ z_RBEwZ{z>}K53SRj%cs7OS(@Tf=@GzLw+cXD6`3{R(dCW()#0{K|~?WY}WpFPB2og zmElQF8jRy2(7XQBzaO4&eg*@{j7OlB_~|bg>ygwZ%O7bui$h*X_MYWZcUNnz5ElDO zY65je`>dBTkhTV^cWIUNLhAjb*yXyRp+g=wrADU>#4UU2#|}LK*x!;V*JJR19;Dr8 zJ(GHvIA_T<#Wy42%d}r+9~)WbOE_OFaBoJE<64_ z@bZoPwUSq^Bt-f_*q9~7S9Bx+b}Nw<>sv_7vb*6pfaw2aM?`dnX^y2;TRbnmcn0FM zMrRyW>o9~yy3xNLr=|0l{Fa|EF;^_o=6$Iqhii_DR>f$wOc`h9dM07$sqObH{kKta zzk9wgA<;ohw#$?Z-h(p;#P$!%W&H3;Z$MfowIQp&@LQWU?eMw9btX&{`ai)fb-3fl zVoVO5NpY73qUx`uWq0t&U@UkfDKuYRqb+9-DdAibAN9%~16pgfPV$ik6;h={PG5SG zqc(muanbW#bNw)DLxOc1%J03zMw;=$bDj4!Ij&QCtc%Bpm`dr_GD7ppL#f>yCj%rC zfSt9E^j(vH}au%K-u*_PpTiIZNjqZ|i)pwrPW-s(!J?ln3AGaPz(M`-)$HK6E!G(1?dg_tU3`5#||Adx?8q~*pOZViP^MokzT3$9+nL2g6o0L%c6Bo8!hfQd0lJo zYmt+;jNOt3Hd~F>wN^_})Q)%DV2n-wi@WWB8-H-G1u0!2O=<2rYw6rEKs+peKgyhi z()iyW(jOB!B+N8`kO6$oN36XrR1rgY9?OyxN*fk6PZ*$d&vk0J8;9*;OLZ?q@ zwmY71xTF!ft2>QYt_TcBbw&o0;^% zXeII!a_d7GuRn3>BJ6^Yb%agEAUs2R6V|qKk{=|~+`G<#AAf1j=QAfwoF4+fQzYOa z07?+#gSkx23MPfvtjAJsxB3pLqR)D>De)6odBGPe&0dp=BQo~Yfd+2CB;lE4-Xw?$ zF^XGI%0Xa|i#bUclPTK{65}=2J$_lPd26vxpLIC&Y2syKXw>q5Aw7pcbgKyigv=|4 zw?wgF2eIA0p2}yaWU{2vS+g)rD`D^|8A=X1N(`Lc6Ln`TxviM3*16;EikFzM(b}dB z2UdA4H?)!uyY{k{`e){vg&uBX-CtAy&~J*nm*k#4K>+BGAnfq45A2Z>+ZZiifx+^Sa8Z9>b(|=w10vqWi;uHKlCzYJ&KZ_-*oB_vhtO$ zP9+53b?CrJXDK`e9zzEYvFC*EbzN}s-Xw$3k3S|znFYZ&OQ&N7MUM4G7xtawVzd6% z9_jX67q-UETf4#8Z|77%jg&0>vS+=PP%YRjpN-l{jvAE_6yu)F88Jpk{;~XCBs_tE z{&K(LJ0&OZ6@~x-WeB9)=Lb=`CR*#TOucscnlv`&wM>f->NqxIJGn>0d;(yfNE!U- zN!Zo`d&ETF>Ra4)LR*kGI<0wWi*x|cVr2e~Hk>44o2(rdQ;4A6E$k%tV#qb^3x8%( z$@n=)e?aFb*^VJTuQ{>R7J-Taa6?L51C745^A-Wn=SGyBVe^GIu`C3w%k13BXz-ooJewef;HEHkb5dZ(aNcpdx#s67};__c5%05$~?0=Ya;D3}T2me!| zY&0dxkk<;BvP5!>&@26LPa8e1V5C$0unY|TAT?K5+eiovS{a8RoUt;lYV*{_imr8H}NJ# zR+N^m!k;$!54K}(b^M(ygc%kDmHo%GO}chSD_P*4&V}kXY$bU zcT4=VhEm81`TADN4+nm<&h3sM*gIs{WT0}XUVh^%psfODQgk9(E8ea0PCkRy-7EdU zVvA4xsuL*=#t9QwrB2E+AbnG+wQhHNd6!L3TZeBvr8;6?P1eppDTO2TC*=qX0)GZEdZvwp045`}h*sLXqFiVZ=%E!q^jSfALS2FjFQPf~Q=lU07( zd|8XGfCZEH5XmlAxXN$7{+*lF_RR9im~veWwG zGxp0wFV)Ds_d}?$-Q$rVw)E zwz-(h#+Q155)LMwN@-v5>x9~WB`uBp?q_ZDCO=b3Wtpjmr84WuXccNoNU!ByjUAvaae7*k52avo)CcN*nr)g8B)JHhVz$0)xWCdvF z8Go(6O3mn@b%z%HIr$ku`AZ7E-ybZ!yq_?}@!@j+yW~tjuf2sxtQo&mBSEIS{ls$qx@`?UUgxcqD!nVWO zv{9#GVQWIB`}7jOZHRy+iN`_A$Pr>l#@9*hBN=gDr2d*vZ-Sn_99_f5Nv_iT?gR0F zQH`LY+(IBUT8>cj z`imW6vK+5n`|lDETb@d@Z`?&0wN5XChAU*|C5C^+HGiK*Jv3wb6OscVje2}twCTX} zcMqh~fhg1BHOTCR^sC@$4;Ss$%oxu=>vV+ofm6uAKIHv^X3Ln2UBU)By-LP52k%

wkJd*v?Jp)l-ZFa9z-lf1O2K;tD7dP$rBAJ$pEBEVk z#leP*j>bNeHjGhYdD~$XU|x;csWlq3G$R^EQ<@+3_r`QcltVhS_o-&xBL^Ip@#_g5 z2_%4yKE77xy)hVS3{oV+{)}T)89DUfWC_m_Eq<8npZ>k+9kMKCUIOoaUPM5d zj0YQ1)ZD%L$KC0wfztqcDxpGaI)Hf`sMuP-4%(8sd`?TgV!?6X1nUiM-zJlnAgU?r z$g=Q^buShdk?HO#)5jl~im>q*(Eh=9Cn%Yu#d$B-`URcLS!eET{Fr`?q)4Wik-zKK zu7TtY2xQPhnwJ`msE_pAyanuZnzWRAW2Fx*DE6C=RR$vXU=;xTqP6EkWjGZE-7W*4 z-bcbjmDc0+Qm|?rN(Y$bNyqt%I^tt$!5P%izl-u`HCA`Uv;&ZBp=92`a?{`M!8Rpl z@j=vPDL?*LCqXjUrM)xgW(Rv~=NsusJ4X>SZUrsZ$#fr-Shl6zQCjO|ze*aR&}{9E zA}LlHYDw4m+dBCWL>60D*z@l_7f>9umh(05t7h$jhdGIuI;ej_9qo+^PyuBw@!Emd z8??@?(B3Or9UO+&d?rU_Xo~g*nzFueQ>}OmQGvfeBYz$_hEiv#*IUz{!702+e zVM9LuUf{MofL_Rc>||y;DdXp%w-&}R&Azf=SEL*XrkvJTy3Cut<{$0RI{zC%0xTv_ z%6U4Sv56FCBL*5S5deS5o@3TV?K&0adkQHe6A}K`@2o{nze?Z!4Bz|`D$?<@KHC0d zGclTys-qdwb{3S1s!)E;JQyH4ch zZPQ6>?H%n6d=P44;g@#rGHLx0D-B(t)sf`33dnwq_CF>rS_yzs>-j_MJ{}xJ#2hAK zilzGs;)Kei^U_9XO6HeHVl8rmXUe{5n}Jz}{NDLBhD_aTZCn4uPyTMBE;(fClEV*^ zj-)2Jz7Dzf-`6Go|KZ8?2ls<-h6Mlbot+&0ufpV*DNNiRCLK40$#!#ga^gRQ$^UhT zW6Of4Qj+d!0@3pFqY94qHTH=-Lg+^ToooudB z_KH(N>GScs_G~}su}OUlp*|1oHeio!@{`k@0PI9psZ5G@p2vip>%A$4rnvj0-%fL} zms&zS(0iCoc5_nG_+2NL_#q4)kgDpS2$S92wfe+YS$O-xf{1%zyFc1}m!Dtao9&3y zQI0{Oq}6txedy=qTK@1_`*&#z#UO$5dsFnXqZBDxeHw0Vmxj-%t&7ladIt4$uI2sq z41t*y%1s|%{{VhD(Ih5JmTs5QB_Mplx~BE758jHCIctZPe>}!7F?N@h7rxNkKPrQ6 zD>*|)4(R-5nn=%eB24o_(YbWp?NXqUF0DJsXEHg?(=mimfc(49DR!La?OQ<`t-%<`zPC@FzN zY#XERypzr-&9lOi){+oQxZ#Qm=M4!` zxOds+Xt8-fcAY2AqfNW4WXTCR_h|7;%lE-JjR=}4>9L$Ndu!Ov)W@c!kt5AZWX#VH z$w&XsP8V3fkhSU>*8i!7sx6hj(FQPHp?Q?A2ChTs(yq zuj0O$TQ%<$!R|t+{2@yYfk0`zCAG_mpb`V>AJRZCnLP{{f!TOO22QTf;#Xojm&|Mh z3JQBj>jh+MyUcy()){9pPYLlmC+NJ%B(y@Q1DB^2wOg{*YU{1fT=(D(wY}S=b#dt5 z_Ll?{zsD7`4`_HTB!7xDV-SPV9PLgeg26fv5mO-`v%`?ZCmqECE$>cY<_}LK^R|@I z_Op&9N`<5Tc_Isz=Xt_5TS^nS)fbt)fN-s&2q6Lf2m>_Iaa0PdSFxeKT6Em3Ofr3? z@e+UIieGfng#xtdD1VkY|K0J?$fi2$n;Y%Y$M?aA4t{|=TGO{Bi_JkBc3kicCgQzh zKK)yO=6s>T9M+cg(P#=zgJWa?Mx5pU6<@5S()%IeH2JyN;yC3xjxd^v22%fFIsj(A zq>f|^eOJ8NEw#=jjgIXhS@wnF?+RqFjCo7e3MK66@ao80ozYt7#qe2>JRco;gq?nH z&;jvGKgTtsO%%Ys8!P#A8wiN>m69VsnM+z;OQiZGJ%A#@c{e7Y8PbnwaN5t+C`WpY zGW>QCtX=x+)bZ4B9APUBN3Ai99P|#9JnE(GQR$%G_TULrkJjNA%&@%MBOUm(oZ1tK zHHeHIL7^hmF9M8cZoXW0z!iH@+t3OwVNRtO~Y75i) z8*pYd9b|fHJ<`05ki10AW{bV9S?N3IOi>AyJdp~A&xmwwI7WJ^z6S&bEZ-+O@+nB$ zK&B4FP`^*%(-j72{_5EJz9Po=ddNheo$ABL*0QC+pj z+%2$YjtKIOzr5g!AJE^PQvK>7H~NU(tF+&p)QhaI1gkwj6Yot~ z?h&*puu8k>b$|+x{Zs#)r=DLTrEV-KQ=1J-{yP+*eV(UG-WiFP+O0%qws342wHJxF zaD~ia+k%xQuZJ8YxL3y^wr0C= zEKYikePIdd@tWf&*)hh-;B{;bekxOlkiQO+NVBy0iK{W)0%2SiD3$4k6Y?C>eKk8( zdcaoP^QH7)=)YM0z`f#xHnYES4*eF^%jjwnQvp>@fXq2ul(8r(HYXk1sNG&hLyjPi zhGZ=Kn{_2ADJ&?F^Cns`_CetgdXHtlGc;IRHltWhll(udBIJ-C&CWV_06H8Yu}W>= zyS2OT8e*Gz7bc_bDM0Fgw-h*n%wY(TuM}T+qBW;s1TB7aVB_hq#bD>abJr!ExI*HU zoS4^A6Y8cE4OQ9@_w_F0OEjB;NS8WKpqZ8|3hBuWDcUH_)-C<*Ex+QmH_pj!xY(2; z?1F>kYs-O4s=Gp7i#|j8=_lCpzBSL4+1IlC4Wbbe; zn8odk(l#xlqlXca{nktCQW8*uTkdFMXLI(Q!lRvX%o6-zYj#U@R8O?H4dNEnVnLTZg_Y6x-_|J-K|Te zWUjvFv0G=!OQ7kIW=hds;2CG;E@p7%5axLJgv^o|AI?sF7bQm-)oC3?KI$)imF6d) zxK3?9>c0!Ku*7!gvJ)FkWq_2K9p3tPtLqH}pib_+EtNMQst^~j(x8P7WU0#uQg@Z~ zJw`D?@aCRG753A{HiIv{8OMXppT&Xe^!KJTqGgsPFbVPn=$aXf9;5=i^^X9eA!R6I z8<}ay-rkXwe7~f{PZ6daTxgI5)gOKZ7@@Pg6e;;Ezu!n1y^Ijl>6qlbL}orlk(TPn zYhZ80!mn){cl`iH;CKtiva|^UCCljGI6u^d>`+j-kmNZn@Lx_8K0GWw8ItMLMO0_(No~gB>wn}}vJ$*`Y)EX@#fQGev z^=GlUoQd8Y;i6zCM;H}Q7RPqUFBQlkZC-&L(Vg`&efJQWqy*nR3#1m&_E=(rDI0p6 zzs_s|;ODt?N#L^oG&RT1xZ~Vvu0>$>(m1MpD+P#Z%l$5Zn%kRfG)PtY2X8H3V*WU{ z;$^t}*dfzsKZIWedEy)qk#Wtmgmmmau8>}ere5-H9==0MHxM($w-Vl)cn;V|5nkd< z`tS{QblC*TNVi#<R(h$AGVO^E5x-d^mPnECUr*(zYw8xG{QQ)=|T|B%Bax=M(QF|CZm^WGHAa_ZwQz z7V<6MfU62NVWfO(mDn~ZWm%mu8JS^{|2Av!5()tz(*x!HL-RM9MADNU!MZZRXF0Su zYiy}S;f8LGZr9#$EJEU%I1?_#_YrnPR+qMUCSDG`ZsgNIqGKAH%whpq+@=h_&3=Wu zN!>N4J_W~vp>y)@l0ANyc+ZD8b%KwwHbfY5d$3jVxBEQhH0{$ysSSW(=z&?!rIk;& zxU(0Ido%ZhfB8aa;-!zLwUs{U7IHfr$Zv_(QkZy#0w2x(ity`3OWxJ|B~Ml+!zea; znbNjpQpNN$_wJ62!S`lrhJxg4vi`}yQ$r|D`E`25h7YEwVYqqvNDx1)#>@Ke8&;as z={VCBs`iu8r#iJC&I#Hi0}l?SqquBA*t;8ij^9}%hY=#YE$IZ}fD@Zeub3A)Zklr* zUK#X&Vv;P>&jNs~MO2PO5*1;Qo-*iQ1n9y8Eb~Wam@OfS4(yO$mo@h_zH5b}2U&GR z5?Lupi~Ke#6l3txy`K82h<*SeOIFq1t;=Ger4D}8>wAYUl^m6M3n6%xz^!yN27zAb z?z$V!GfH%JhYoCY!{+ct$4<0n$rl*Qw)7%C>Xyo&cUG9zIr`H?>0uXJ?6k~DsuK*P zpp;rINr^XSfs()Fk#>CDjQ0q^Td`{;e=EW!23?ftvyG0eS?o$~&ct*P^m(LWFQZ@Gy4WXR~PGVQbZ+Ry;C(+Ww?uE|X5fW1jIxtAEL+h>M z$E2UB!|Pubk$Q)D8S;WfQB!|Vd+aDDp^tLy+eJ>Q_^kQNA7rtzn$v6qPW`9Fo3!kd zn_(4BS~_v$Pbz-gI8Uc@+JjiCB1tUOxfj~Us7*S0Hi|${n*%#W@-&?#PKJ1vmNxHra9OI|FJ%5m`YVD1@|s*C^^5b)OEfcViT>S5Rfo z4pXI+(wPu!-WUaGg*FGsqG;Mqv>9ao^d0Whd>F!qQxJ;jXAma(v zrX74a7u(G)GDIcj{VRW>xqX-QHbQ>x`ucv6*-G9d5=^E~&j{CpePR6Z7b~!KoO}BE zrOe=68C$PehhFc&bk_ti#Iicn^wnf$>rVNb_>gc!Ppv<7`H|6hSV0G`1#G_q$=Rg; z6}ZmU->1!LUrCXp)W6#cD*&{RyYGpdI%;iXTvh;yso=HY0cl)z4OYozyq#+uCWkt( z{SGVUgkh90Zk}sJKz99UTVxK_FHgMc03Y=l^!}Fo^k>1Ofov=3*m1b472%d^9a*VN z2kNM!Vb{P7iAUW1`Sdd6+_IZ{2mx?ZB4y8{_c%N!O{knEAPWg}+s;!eqI|lbM-?Ig~)BtI^jX>mBa_l`|8;iP(f|Sa4cFYUCU_tv2c)Sl@ zwwPCg{WNQb<|0G@R-23;G(wETafZR13+J`Xkj?y6pA9mbBAEaH5k0?dEh|L|cU}$? zW?zx9V5WeKP)~o3<&uIrwZ$CO4bx<=6x$W;UPnS?911|RgM~j|Sg0`)eBJ3uq|W#m zR_p^;`Cz_dD~qx5`??hwxT0SRt>bBDSKr3RJ=TdculDQAYvc3+>3fc8e=7;aTE1S} zX-06Cg#e$ps#CAI5;(oF6?vS0?~BQXWhZo0r`O*RhD4$WtBHM-U6~{`$zDcuPky0- zLlyr*`!}s)L$e$bA3p;%kcj{hi#o4^^w$~8VWGCr7us=B|E48d={u!mr`jNJNMu+W z)2>La*8}$ON7@q3$jobg&PC<|c-SHkJZn`D3`fTazqMHp4sJuqcZC~uUZg+ozeA53 zr4evhGImmE?l&)$p&%CPp7pr<3>JFGb!s_vxE1~l z_Svrs7qsvVOQ8ntKnCP}*TOIc$0R2_S<2SD2KwZZxi!+aCk>otcx1@m?V6U za9T!4>BONB8c!4VO{U#=B0(!sH4|PsqRf|n*#OY{t=pfxw_%bwB7B9myx6OMfgC|i zpZ<~eS`-Si9Q1nrydleU^0#n*>9PlSMRX~;wC36?t13poJT0VLT$Z>JMvSFqF9xPG}9d0<39` zPO+fDTkO5To3Hc|$9;r!mm7t%kIqEVh0gXgCLlpYTW@YFuy+h16TM;tlI*?Qu{wf9PG&`Eo9U^Vh)h=U8xA}`t9x(#@JCC^Q!uQQEJ zBlc-poOGSXZ<{$`2mu5+IN`ICmVeT&Ao?_#?*< zlfN77`Rgg9#pLk*6n-(o0Su)=1IVTfnD^7hN78r{!riF@R|uv#J9=#JOlI7DYA@V( zOll5);Z(zb?(_?(`1D{;ymdPmhV6+8Fc|H6bwuZwWv;Lu@5dzNRGAD)qI+~YfWWCH zI~Gx~b)uYG*w}BfPym%rKdevg?zMQ-utxq~#qy?1mtu!K;gAJtS}T?lYCi!vG;AbF z-M5vl{CLdTr?npl{kC}Cw$DM+c`BTXwHJRN*$4}i_n zEX2wnT7HBAXBdrx8}eA~461ed5HYO(Quavs6#$ILn01owyLMMeOyQvYFA!z`1qZ!b zu~pl5u!U-ixt}7`)DbkV^hTJe>1v(*Zm`8uTfV?Q+hoX2t^5q&%b2FJUVCiH*uNYK z0H?X%J((_M$|?j&3}7iaH7={GRMv-5{2bOv5xhOJ8- zPe}iJdv6`tOC#$AL}@ZqNpk!XuiXgc)0?l`$7ziL)`y5wr=F9dJFU1^2c>r>zUj9j zqqOMlPFHNUexr}JUS@XET0%27523UYb8Lcp=e70QbrX^;N27IU1%9frJU4_r(5a6c zXn(sAC3Byi5Pv*upylo!NFI!pBcpJrlq}~aH_y{xtMWMToPi{Y;JQYd3oSeRv1tD%K2@Vr3{eYD_0VKO*w7j{9G}cm z=$nFpQCZk&qAPdh{m{cYPPp!1BC%8FUh)-;j>jUT7A;@-%1uW7OdjMZ$+VZ)Aj_y0o?h_# zqBZFR2%Tl5TZum6E2-+jy^G6 zR6LabOgqvcbz_J%eoL_DFc^+cmA_nfSSL1n%g`YxsK#y`UD!Ku(tmB2yXh|Sm`WJ@ z*3G)nEX#G&M~1gT&;4YnFGlpBv`q&&&JEs}JZMygF>e8F(JE>Fgg)V_d|Z3nzG1*` zp3<@(g*-!#Y43Uc;cfV##&Yq~p_@D{i(KNXf2C``=TjpWa;`r7V#U!G+W7!m|8jQ& zGOnMr&EM-Y^=GUUFNeZ7bU}0-+pkTDuV8D+o!wMM*+(%|w=tIt-UpM@Vp<<#me+Om z+w%d-ckmP_1G@4%KA1lDy_J&xn7o^9@X>&lNPCDV>$WyX){RY?Eu)Vp8a?Q(6faq_ z*itB6OCD}g8T}+BDT;Y!SbcT4e@2B+8i(DMdj9PhJTDi69E96L5 zBKfj#DQk~kb@|OYGb*35?}+4uJYOt0=9sQ+dH0wS@2hs5(22VQC7puF^_RFILu{mr zXZq_UBD6V_a?k$!W+Kf@%k>gfFnVcBB9(;ATJzji$#Wzqv^6U|219aVW!K@K1@3IKc>sh|7W_amFcoK zJIoeuvwPCc6>Y;8<;zc*aOOyu>Vm-lwn(S8$a0Z}m72YTBzYf-9+X~3vV!DqBTw<{ z_?smRk4p9aH;`5OgmQjCdKjRF=^vbe95{_b$JS4E$o%XCqlQV49^Yd5DkH92AE}b^H)dhx}`H# zswc9z)s#+f0L;i-N*7GgW8FI@-RGqElHsSPR}hP>t6rJ=!Ernv37u3tlq_SyG5N8b zx`A<0D^TaSPd#1P(ZyKnl6+^$+sDTdd}XJdrQq`cW2$aKZDyM`M{V?$;ZScg4_dm} zdgbnS$S;leCBc%2OF$I~yvtMr-jcW{wHprIo z-N-*ub;n)WW>fxQ$fF|}d}m&Mtf$_+y>%UC8zc-GtgjK|hlN)p@CI6+ zAq44gEVN~2?aJlfM$HU=FQ$<_w&psLZYnL>y1noEd)86_k86vzMM8-=3;`m9mL|jd zXhSLF+v5+l`58HgXH)pw&#$}_X@D^xmB)7|A>$b>nnY9v1yT__IA zc-O#@o4XK^b|19imczMse6JW0fc_mhYc?{tQ-55>1e3c9J^WPbMr zkZaRFOx|2j8ybmbaUt$+ry6d`>& z7@8 z5S=CryY%P!up^6SM1FAgZ<$-BRpI=id7F!LJA!xcy`klS?T6+Cy zX9Vl?qs^G3%X*!*ijpZ4bI_9fb(A4#R7Jk@itTPZv{xIr2ys&Z3?3o_-^2l$>vQzY z;?qe*QLiK`>^j|uUAhqEe+~=uTB(kli&Rp`ms*fl3CXkMtYHZ&`ri3_X9x_21_C^t zc1FT9+(sweL0>5ut-vUwT3VLBj?f>g#8=NQz0^+E`?&DoQkixM=BrFO9$C(7c;2H4 zIN0V9BU)xMyvqC{Re4CDP70loHm6shRo}ibXuhFU`#BFbgDpIvr-jR&`QWn%?Ml(c z8-z4{NFToqL`XI<^_#rRnj z^WZI+4%Uek&(}Zal}$TiwfQFRNU*MJ|0x-NiX;B$#~xYf0Y^W|xIb%HyXE|m zO=}4&QzI`#uH$N*Xe>(B)mrx8v=wD z{E0VY`iI73FM?GR67q%{@AG&5Uu>eA^lsb32)27zBCbyvFu4M&ErxzReuQ zV^!8Nf-~~i9s~T1PAT+Qnwg%){VB&TzP@2jEvjVt41{|EzcrC*pC;osgj3>0 zQ86m{R?J72_l$bF-PZi`gnuK)X&sv$q3Nr+>=hZ6pBEDV3Yo0Q`;n9`wNIes6R5v- z?IxcZOXnaQ3gyqc<;3f)+h7TVwHbX(1h?A5SM6W%t<(j25|lM1Ft1gFjwr+f_Jt!JgB!h8^2A z_X{GFYYup7wiE14GlZAMl*E(&J1$X(+j}H2T6c#A(u~B-h(fa1*x)`=;^<7 z(My(gwdhUVVP%gDbAO2%0}5gtK|n)5Mn*}*j}qX?u;ah z+HRdXMv~5@!OK#J=wS*(-dV$W!PUSO*u@Mtvx-=l@y9!vzHBxSEmMgf9D$Chka+{l z8jL*aX*Aa}DG!tp*Y^e_T!w=UC&GBqet)#W?;2XitaHyY8GChx_@_iT2n=UPYqN`w zM_`!}&05J>FIaTo;V)ufWRD?NojJbiC2a@8Oxm^ktu`M3I>=4)CNj=<2ZHV*g0F7( z(`>?^&RW4k^8nL6wWm2%U)X5;+k}TGn@A?LCLsUrCYpX|rtk=^B_(@KX;z39 zM_7#0d4w)zcb*9;gg*+?q`)Q4y6PZ3ckIs_(KJrwlXqSI^0bNA+_fM%<;*rye`S7> zt}`hcULiY!&g#HxQ_|Dnjf#;Gs;cQPr08eKcCJy-stjGAs;XKFZP9%Bv4u}CB~QMg z!!+$f{XVXLOn1zM3v{x`AGn6G&jIFs6XS7wuXWtP^^%jxN@pCILJNDu#K2mudU{(r zcPF9oG>7K5QEkx8CDu8%^z&0gFeIslb2y`dzBGTs%Ap&b(L>hCbf!!b*OL}aR-y}{ zjgJ={Sg(DIbi1vA-M;GdU3j5#Z>-qTSbyo23`A3hbuCpZ+`d3?eN3I^oDe#-^(^u7 z8rR!!KR&hHV;yZWeMz%Fi0yzNFL)f8XyLIF89$P6@j4W5J14zw!->#d=;c828_nJ% zQ|ub>%mp8arL8e$txUMZt7l*b&Qf!|P6H(~U@~bP-!EmR=x45o8LN-LeR3Z#tfmz^ z438Bsq8`oh(VjChzVw6XhR#IeoI&M0*_;XgTeIZuV+T?ZChAHfvsU7;31h}`{rSNR zB2HT}X8$fw7YAD(;Ek^Ja1Tvhd%jdIGqs9vY5!507PwXpb_fB7Jf4be0iG-(V=-4Zyj>G&Fp2% zYZ-Tx(j=~$6}xqx>1UA&Cae%%d=divv~cwh z0)@_u))|xr*xg~zb$B%2vfq1k^EFqc+-?q__82{{LlKsa`Dj6m@Ngif9NKw$vmcE1 z&$z8R}jA3;|Kp9Hx6aoP#YdxzGYb{tgl$|OdC1Zijv|uwp>cy9I{@- z^3C7c&1(p(z=Wfcy$Lx-ns?L=$D7W?JwVq|&3c!V#*qqtF7Qox#S1yPKB3HH=HPu^ z9b`pZmcFyv^~gAfiPfQ(GP{W;0BxmlXQ+NK#H67qv|=qK5!p1ta=0u_>t!Uw5EKuY zEcvow3!jPA9@=cASi&RIN`Fh2)#z!$T<7ePe6Rp}9`IRlO%3)oU0vsPb?A^gbsLH zUy-hPiIMc2(vILyM&e1ff0~r<=hVeoeYKn$BVwC@UN3%Pr>) zQpx(0V%pxhcpJcQ*?+Hfem!eL*l>8-elncX8ak}5((RP86JF!hfSY9E8R~+L7ww#mTfMcE~pJYdbW$Ahr}uatXcxgBSpk6m>k z?Yp+cS_if<$ArLJ^i$(WnWGRp%VDSXCiwBheIINjbDDQaW}Z5-O`}w`bc1HGre@?{BZxeW6!kQ7e@zdi(VOj-;~)uDYB6( zIfqPi^T^0S?Y!c=C{HpIsYIrAfER*Tcb6kU2aoO~q=KE}bn3+tllvUmL>-Y}%1;^< z{j_oZkbkrdYeUk!*ojvePzp?1t2Hm`_3qP#V5QD$owxn6#XxQ0X@M}>^9>lAK8Cho zBu-5V4W>=K)21DV_bo1_?0zD_lJAE9Q?%MgC*N$MaV6C&H(`iQqLbx#p-Xu&3zP5W zHe*M5o!D}~4a;4Kj|SDJke46DZQ;Nhe;Y!>Dx?h(a!XjCrLuQ$dHYGkk)7hWI*1kxQ!=1k|xdJ zV!zCsps*^SNNZ%&;5}^IeFkfx&`P&!?Gw`AUj*`d`FlZ1UP8n_ncDe{Rc&`L43uu> zda88X` zTU-b|S?yeV`1~SU=H5Ih<)|&wfox6Tza<-W;F~#Rsy&aWXn@yE=ySdkrzJZM8x}6t zkqGL%UShtVuvvLp|J*dLmtlY2toZ&lam(=lR-PitN1&)^x9>WF<>fzr-zdu@9rI3>C`oj z4wIpqTx+WXzZ)Y0DzVv452#cAEy~egZ42BclMnD#n%25wk6PhX7uLPd)Ym#|XRMLT zF;?mB2$L2$7wbE5Psg7FAXE`=mYOF!n5kEr(<+_wNIk-7j5H_c&$rqUN3uSxbqN~@ z-Wuw_$dwnjv6Ti;6@AoM#uM)uLI%@Wnsa}H4yH3AqWKY8kQix7Xc;;F^hr2$p&i?v zJazeIpyp3|cJ&T6!M3kHgO)m7jc~H~UGsXnCbhr|U&z2#Is~O3b%b(I=MVB+prbRm zU3ndU73|XAXrQa|_r>~^1T-6Id;z$!`j?xan_NC&cmB5Dy>C#G)L)U_ zizgQk^l{cC&vTtZp8~6am$yNx4KgB*2ke*Qvq67Dt*x8RN#kMdGgOhFy=mID{St(p zsEuo3y?lJaI!^$thV$!PjQdw5t==FDj(oP&p$XZl** zR#<}MpCmhhP;_bzQP*>ne2g$z&s*zu>EEY@(d^k;iIzIY8b$qs*Zi%Pu@4+@uim@| zEA*1q(66K*QrgeU?0p$$+~bJz5ga_=(or}48Hp08;3XwBUdM0UBwe4=UYfUB z_0pJ?qM66mTX(;)UOdn_v|5p;k{G7|bI~@wIKxtemr3r|y!X=b1taZ)K1)uXRHChc zV-tdB4?L?(8G1SBpn1cNhqP{mO{*e}QIP`Uh^=fb^$FvodHuy3+K?JeGoyIqX4tO8Szi#_-ZT|?=;?0L|r{az^yc%!p7%vN}y zNXMeJ*O_;Mw;i{3o5xCxmaUR(j}KbEdJ*b0jY3XoE^i#P9b=IHRXD&au(WN**>g$z z49qn1QKGiofKLjoB-am8^c+eUWx}xb>i!_Y5>GX28}chtIzxyC^CFn$WoxwEaD~@* z?Mc&yM;u38y8}4Nv~(ZE3Z~0V-Ft)-!OUZ|*TFOht?v}~Y7cSd8AzQb%ufRPPAxVX zbqE34o4QhMq4eo&G3)vW_qX+%OWu@Zy)`8oQl-YMW!cD zsXFgbYM$BqlKpAf8k%!jx12mNyo@Z-W2{!R*VXevqP1z)Vm*G=co`z@1Cm*O8K#Ld zDRr)w-vEJZj)mz9K{h{Mm|Q;P&f=#-m>)N-$5Gzjo)JaH=Ft#WP*r$KIdbj3DD4|Q zd^7-uo&OoG&wgk!{3HFdJlgNTqMgwZ@dvqaBf>-fcxdyxK;D{3)Gl0)b393xb4wBh zIUVCqB#R3Dw-q3_JhFnHua}8y<_)5uuEz_ZhYZQ*8L>^Qk-Py1#?&&Q~Z$GufU7G>JONUaSB~q!k*XgGu+cP2(pMQQ3JSw>;j@h0g%P52{ z=3S*f%@m(-Jl1{JzKv0NwugOcv`ic}Jig2hjo@m~@VO{UCzzNSd154&mk|4W2-TwE zC>yvrqw-?Nt*Oa8AZi8W7K4pA%S4FUj`E7-H>3vj{>S)=jKXBKP!gza)*B8o&-nJn zU|r(rS)_BeAFmT6WJKIRx%9lGCEQkbYLUklUaWu!S9Sowr0Jo$$n&3MeTOVylQFaR zv+F&Z(}-~?mM1Z1-8WEzy5*KyFY<@{)P1X3j}m*W#+^sakVLn-nGcP*f|5{!mPb6@bSslkAVR)1T%H!0%6wm&9!#20Fn! zTdVzy>w=mdR~F?-XOk?(9j$2;yvH^TbG^QBM$h>I>ddG=441M8HNi-cjM?Y~61nh# zWCg0BlUJafNjx2D_6zpc+1;eplQ!Uvf6q@3B*LE>N<}|IyxC46h zoQz_(8?~N0j!BCJJr|FJn%c$pY%4-GAYBvDV69PyQiIVMtmJ4u=vLF76^;z1c6`}XcL6NN zBD9v~HyN3p1T~TkDM=p*?`~h^YL(y>HqnAlh^%SB<+LwdEu$8J()F9s->$j(E}x@Rz}0sE18dFNCU%U$c#{CIlyRc3^C&j zSmG-zs7>rP6MSxzowizl!+s52FqQ>k8TCWdK*Xf~xbBO9TKDF#1moesG@k#ikO7#9 zp|)_DN(h%B2Xr>Onj^VZTA>21xva;-HKT5r+4R4}A9*^Kt@I9eZX4v+FRv0dSfn)| z)7=DVYGwD!ZMw;SU`9Ka<6ZlsROT-k=ZMsnr7a|D<4aca)oQ>Qfr+T#KY47i&!_Jj z9uW*#N7NTolYoMCOfBzMWN^77L8tL%O~bO8aPKTWN32DX5eSw{34r;3H>}Y*MC_X! z>#%{LYp%$`H= zeg`E+KHEJ2ZFil=Cidoo5Mw{;8swo#2H1UCp1TNz24mafET(hDNX+@Nk& ztb71HQsI|%3EzF1nhZ(M$*yxsi{^>)G!ErMwu_naTPCAm=hZi+1`cJygJ;K7n!?JsBkkK6D!UqKkU7U9oOBeV zQ=Z^YnAr*h>7}D*5m6PGfP_m^7=Sb9okBV9$oR&ZWrj^2LDMpH8UR^okRD%})?3TS z^p!{(g}4;_Y`U#BR4h9a^WccG)fMUjL-7REz7>^CAMs5k5w^jo4C(Jwu_NgbSvG6h zsy1J5Zg$10l##M*d6df21IEsvuVpkYteG{n984mydFye72Ir*jI~2DWGWB#%2>Fzb zW*PzvuZ+)HL=!eDXueTWx8zn z_u5)aYlX?Af3yDbL^n@U!p9Tcu9aS`;fS`B#MsXcbo=CVxBp*=bkpDe=WzKMeO8+O z&B}Dpe?Fcm_vf~G=^LN?e15{*=WqYHX3tOF{1<)w@b>%OWMw)#hrNF&Xw$)uk8iYx n(ucnd@9*SPzPsQ(7k%}Q_ho%xTvq00eO?^zIGXhH@1Fe+OhT^V diff --git a/Scripts/Models (Under Development)/N-back/SphericalDrift Tests.py b/Scripts/Models (Under Development)/Nback/SphericalDrift Tests.py similarity index 100% rename from Scripts/Models (Under Development)/N-back/SphericalDrift Tests.py rename to Scripts/Models (Under Development)/Nback/SphericalDrift Tests.py diff --git a/Scripts/Models (Under Development)/N-back/__init__.py b/Scripts/Models (Under Development)/Nback/__init__.py similarity index 100% rename from Scripts/Models (Under Development)/N-back/__init__.py rename to Scripts/Models (Under Development)/Nback/__init__.py diff --git a/Scripts/Models (Under Development)/Nback/nback.ipynb b/Scripts/Models (Under Development)/Nback/nback.ipynb new file mode 100644 index 00000000000..dddf6748da5 --- /dev/null +++ b/Scripts/Models (Under Development)/Nback/nback.ipynb @@ -0,0 +1,365 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "from nback import construct_model, train_network, run_model, analyze_results\n", + "from psyneulink import *" + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Model Parameters:\n", + "\n", + "##### Fixed (structural) parameters:" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 6, + "outputs": [], + "source": [ + "MAX_NBACK_LEVELS = 3\n", + "NUM_STIM = 8 # number of different stimuli in stimulus set - QUESTION: WHY ISN\"T THIS EQUAL TO STIM_SIZE OR VICE VERSA?\n", + "FFN_TRANSFER_FUNCTION = ReLU" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "markdown", + "source": [ + "##### Constructor parameters: (values are from nback-paper)" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "source": [ + "STIM_SIZE = 8 # length of stimulus vector\n", + "CONTEXT_SIZE = 25 # length of context vector\n", + "HIDDEN_SIZE = STIM_SIZE*4 # dimension of hidden units in ff\n", + "NBACK_LEVELS = [2,3] # Currently restricted to these\n", + "NUM_NBACK_LEVELS = len(NBACK_LEVELS)\n", + "CONTEXT_DRIFT_NOISE = 0.0 # noise used by DriftOnASphereIntegrator (function of Context mech)\n", + "RANDOM_WEIGHTS_INITIALIZATION=\\\n", + " RandomMatrix(center=0.0, range=0.1) # Matrix spec used to initialize all Projections\n", + "RETRIEVAL_SOFTMAX_TEMP = 1/8 # express as gain # precision of retrieval process\n", + "RETRIEVAL_HAZARD_RATE = 0.04 # rate of re=sampling of em following non-match determination in a pass through ffn\n", + "RETRIEVAL_STIM_WEIGHT = 0.05 # weighting of stimulus field in retrieval from em\n", + "RETRIEVAL_CONTEXT_WEIGHT \\\n", + " = 1-RETRIEVAL_STIM_WEIGHT # weighting of context field in retrieval from em\n", + "DECISION_SOFTMAX_TEMP=1" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + }, + "execution_count": 7, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "##### Training parameters:" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "source": [ + "NUM_EPOCHS = 6250 # nback-paper: 400,000 @ one trial per epoch = 6,250 @ 64 trials per epoch\n", + "LEARNING_RATE =0.001 # nback-paper: .001\n", + "\n", + "#### Execution parameters:\n", + "CONTEXT_DRIFT_RATE=.1 # drift rate used for DriftOnASphereIntegrator (function of Context mech) on each trial\n", + "NUM_TRIALS = 48 # number of stimuli presented in a trial sequence\n", + "REPORT_OUTPUT = ReportOutput.OFF # Sets console output during run\n", + "REPORT_PROGRESS = ReportProgress.OFF # Sets console progress bar during run\n", + "REPORT_LEARNING = ReportLearning.OFF # Sets console progress bar during training\n", + "ANIMATE = False # {UNIT:EXECUTION_SET} # Specifies whether to generate animation of execution" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "##### Names of Compositions and Mechanisms:" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "source": [ + "NBACK_MODEL = \"N-back Model\"\n", + "FFN_COMPOSITION = \"WORKING MEMORY (fnn)\"\n", + "FFN_STIMULUS_INPUT = \"CURRENT STIMULUS\"\n", + "FFN_CONTEXT_INPUT = \"CURRENT CONTEXT\"\n", + "FFN_STIMULUS_RETRIEVED = \"RETRIEVED STIMULUS\"\n", + "FFN_CONTEXT_RETRIEVED = \"RETRIEVED CONTEXT\"\n", + "FFN_TASK = \"CURRENT TASK\"\n", + "FFN_HIDDEN = \"HIDDEN LAYER\"\n", + "FFN_OUTPUT = \"DECISION LAYER\"\n", + "MODEL_STIMULUS_INPUT ='STIM'\n", + "MODEL_CONTEXT_INPUT = 'CONTEXT'\n", + "MODEL_TASK_INPUT = \"TASK\"\n", + "EM = \"EPISODIC MEMORY (dict)\"\n", + "CONTROLLER = \"READ/WRITE CONTROLLER\"" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "## Construct the model:" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "constructing 'WORKING MEMORY (fnn)'...\n", + "'constructing N-back Model'...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/jdc/PycharmProjects/PsyNeuLink/psyneulink/core/globals/utilities.py:443: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n", + " if reference is not None and (candidate == reference):\n", + "/Users/jdc/PycharmProjects/PsyNeuLink/psyneulink/core/globals/utilities.py:443: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n", + " if reference is not None and (candidate == reference):\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "full model constructed\n" + ] + } + ], + "source": [ + "clear_registry()\n", + "nback_model = construct_model(stim_size=10 # Size of stimulus input layer\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "data": { + "text/plain": "10" + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(nback_model.nodes['STIM'].variable[0])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Display the model:" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "data": { + "text/plain": "", + "image/svg+xml": "\n\n\n\n\n\nN-back Model\n\nN-back Model\n\ncluster_WORKING MEMORY (fnn)\n\nWORKING MEMORY (fnn)\n\n\n\nTASK\n\nTASK\n\n\n\nCURRENT TASK\n\nCURRENT TASK\n\n\n\nTASK->CURRENT TASK\n\n\n\n\n\nCONTEXT\n\nCONTEXT\n\n\n\nEPISODIC MEMORY (dict)\n\nEPISODIC MEMORY (dict)\n\n\n\nCONTEXT->EPISODIC MEMORY (dict)\n\n\n\n\n\nCURRENT CONTEXT\n\nCURRENT CONTEXT\n\n\n\nCONTEXT->CURRENT CONTEXT\n\n\n\n\n\nSTIM\n\nSTIM\n\n\n\nCURRENT STIMULUS\n\nCURRENT STIMULUS\n\n\n\nSTIM->CURRENT STIMULUS\n\n\n\n\n\nSTIM->EPISODIC MEMORY (dict)\n\n\n\n\n\nHIDDEN LAYER\n\nHIDDEN LAYER\n\n\n\nCURRENT STIMULUS->HIDDEN LAYER\n\n\n\n\n\nCURRENT TASK->HIDDEN LAYER\n\n\n\n\n\nRETRIEVED STIMULUS\n\nRETRIEVED STIMULUS\n\n\n\nEPISODIC MEMORY (dict)->RETRIEVED STIMULUS\n\n\n\n\n\nRETRIEVED CONTEXT\n\nRETRIEVED CONTEXT\n\n\n\nEPISODIC MEMORY (dict)->RETRIEVED CONTEXT\n\n\n\n\n\nRETRIEVED STIMULUS->HIDDEN LAYER\n\n\n\n\n\nCURRENT CONTEXT->HIDDEN LAYER\n\n\n\n\n\nRETRIEVED CONTEXT->HIDDEN LAYER\n\n\n\n\n\nREAD/WRITE CONTROLLER\n\nREAD/WRITE CONTROLLER\n\n\n\nREAD/WRITE CONTROLLER->EPISODIC MEMORY (dict)\n\n\n\n\n\n\nOBJECTIVE MECHANISM\n\nOBJECTIVE MECHANISM\n\n\n\nOBJECTIVE MECHANISM->READ/WRITE CONTROLLER\n\n\n\n\n\nDECISION LAYER\n\nDECISION LAYER\n\n\n\nDECISION LAYER->OBJECTIVE MECHANISM\n\n\n\n\n\nHIDDEN LAYER->DECISION LAYER\n\n\n\n\n\n" + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "nback_model.show_graph(output_fmt='jupyter')" + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Train the model:" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "ffn = nback_model.nodes['WORKING MEMORY (fnn)']\n", + "train_network(ffn, num_epochs=100)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Run the model:" + ] + }, + { + "cell_type": "code", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "source": [ + "results = run_model(nback_model)" + ], + "execution_count": 11, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'nback_model' is not defined", + "output_type": "error", + "traceback": [ + "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", + "\u001B[0;31mNameError\u001B[0m Traceback (most recent call last)", + "\u001B[0;32m/var/folders/_8/09rzl01902954fwz0xrgrx7h0000gp/T/ipykernel_57864/313089602.py\u001B[0m in \u001B[0;36m\u001B[0;34m\u001B[0m\n\u001B[0;32m----> 1\u001B[0;31m \u001B[0mresults\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mrun_model\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mnback_model\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 2\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;32m~/PycharmProjects/PsyNeuLink/Scripts/Models (Under Development)/N-Back/nback.py\u001B[0m in \u001B[0;36mrun_model\u001B[0;34m(model, load_weights_from, context_drift_rate, num_trials, report_output, report_progress, animate, save_results_to)\u001B[0m\n\u001B[1;32m 627\u001B[0m \u001B[0;32mif\u001B[0m \u001B[0;32mNone\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mthose\u001B[0m \u001B[0mare\u001B[0m \u001B[0mreturned\u001B[0m \u001B[0mby\u001B[0m \u001B[0mcall\u001B[0m \u001B[0mbut\u001B[0m \u001B[0;32mnot\u001B[0m \u001B[0msaved\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 628\u001B[0m \"\"\"\n\u001B[0;32m--> 629\u001B[0;31m \u001B[0mffn\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mnback_model\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mnodes\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0mFFN_COMPOSITION\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 630\u001B[0m \u001B[0mem\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mmodel\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mnodes\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0mEM\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 631\u001B[0m \u001B[0;32mif\u001B[0m \u001B[0mload_weights_from\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", + "\u001B[0;31mNameError\u001B[0m: name 'nback_model' is not defined" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Analyze the results:" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "coded_responses, stats = analyze_results(results,\n", + " num_trials=NUM_TRIALS,\n", + " nback_levels=NBACK_LEVELS)\n", + "\n" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.7" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} \ No newline at end of file diff --git a/Scripts/Models (Under Development)/Nback/nback.py b/Scripts/Models (Under Development)/Nback/nback.py new file mode 100644 index 00000000000..daf3c503956 --- /dev/null +++ b/Scripts/Models (Under Development)/Nback/nback.py @@ -0,0 +1,822 @@ +""" +This implements a model of the `Nback task `_ +described in `Beukers et al. (2022) `_. The model uses a simple implementation of episodic +(content-addressable) memory to store previous stimuli and the temporal context in which they occured, +and a feedforward neural network to evaluate whether the current stimulus is a match to the n'th preceding stimulus +(n-back level). This model is an example of proposed interactions between working memory (e.g., in neocortex) and +episodic memory e.g., in hippocampus and/or cerebellum) in the performance of tasks demanding of sequential processing +and control, and along the lines of models emerging machine learning that augment the use of recurrent neural networks +(e.g., long short-term memory mechanisms; LSTMs) for active memory and control with an external memory capable of +rapid storage and content-based retrieval, such as the Neural Turing Machine (NTN; `Graves et al., 2016 +`_), Episodic Planning Networks (EPN; `Ritter et al., 2020 +`_), and Emergent Symbols through Binding Networks (ESBN; `Webb et al., 2021 +`_). + +There are three primary methods in the script: + +* construct_model(args): + takes as arguments parameters used to construct the model; for convenience, defaults are defined below, + (under "Construction parameters") + +* train_network(args) + takes as arguments the feedforward neural network Composition (FFN_COMPOSITION) and number of epochs to train. + Note: learning_rate is set at construction (can specify using LEARNING_RATE under "Training parameters" below). + +* run_model() + takes the context drift rate to be applied on each trial and the number of trials to execute as args, as well as + reporting and animation specifications (see "Execution parameters" below). + +See "Settings for running the script" to specify whether the model is trained and/or executed when the script is run, +and whether a graphic display of the network is generated when it is constructed. + +Sequences of stimuli are constructed to match those used in the study by `Kane et al., +2007 `_ + + +TODO: + - from Andre + - network architecture; in particular, size of hidden layer and projection patterns to and from it + - the stim+context input vector (length 90) projects to a hidden layer (length 80); + - the task input vector (length 2) projects to a different hidden layer (length 80); + - those two hidden layers project (over fixed, nonlearnable, one-one-projections?) to a third hidden layer (length 80) that simply sums them; + - the third hidden layer projects to the length 2 output layer; + - a softmax is taken over the output layer to determine the response. + - fix: were biases trained? + - training: + - learning rate: 0.001; epoch: 1 trial per epoch of training + - fix: state_dict with weights (still needed) + - get empirical stimulus sequences (still needed) + - put Nback script (with pointer to latest version on PNL) in nback-paper repo + - train_network() and run_model(): refactor to take inputs and trial_types, and training_set, respectively + - fix: get rid of objective_mechanism (see "VERSION *WITHOUT* ObjectiveMechanism" under control(...) + - fix: warnings on run + - complete documentation in BeukersNbackModel.rst + - validate against nback-paper results + - after validation: + - try with STIM_SIZE = NUM_STIMS rather than 20 (as in nback-paper) + - refactor generate_stim_sequence() to use actual empirical stimulus sequences + - replace get_input_sequence and get_training_inputs with generators passed to nback_model.run() and ffn.learn + - build version that *can* maintain in WM, and uses EVC to decide which would be easier: + maintenance in WM vs. storage/retrieval from EM (and the fit to Jarrod's data) +""" + +import random +import timeit +from enum import IntEnum +import warnings + +import numpy as np +from graph_scheduler import * +from psyneulink import * + +# Settings for running script: +DISPLAY_MODEL = False # show visual graphic of model +TRAIN = True +RUN = True +ANALYZE = True # Analyze results of run +REPORT_OUTPUT = ReportOutput.OFF # Sets console output during run +REPORT_PROGRESS = ReportProgress.ON # Sets console progress bar during run +REPORT_LEARNING = ReportLearning.OFF # Sets console progress bar during training +ANIMATE = False # {UNIT:EXECUTION_SET} # Specifies whether to generate animation of execution + +#region ========================================= PARAMETERS =========================================================== + +# Fixed (structural) parameters: +MAX_NBACK_LEVELS = 3 +NUM_STIM = 8 # number of different stimuli in stimulus set - QUESTION: WHY ISN"T THIS EQUAL TO STIM_SIZE OR VICE VERSA? +FFN_TRANSFER_FUNCTION = ReLU + +# Constructor parameters: (values are from nback-paper) +STIM_SIZE=8 # length of stimulus vector +CONTEXT_SIZE=25 # length of context vector +HIDDEN_SIZE=STIM_SIZE*4 # dimension of hidden units in ff +NBACK_LEVELS = [2,3] # Currently restricted to these +NUM_NBACK_LEVELS = len(NBACK_LEVELS) +CONTEXT_DRIFT_NOISE=0.0 # noise used by DriftOnASphereIntegrator (function of Context mech) +RANDOM_WEIGHTS_INITIALIZATION=RandomMatrix(center=0.0, range=0.1) # Matrix spec used to initialize all Projections +RETRIEVAL_SOFTMAX_TEMP=1/8 # express as gain # precision of retrieval process +RETRIEVAL_HAZARD_RATE=0.04 # rate of re=sampling of em following non-match determination in a pass through ffn +RETRIEVAL_STIM_WEIGHT=.05 # weighting of stimulus field in retrieval from em +RETRIEVAL_CONTEXT_WEIGHT = 1-RETRIEVAL_STIM_WEIGHT # weighting of context field in retrieval from em +# DECISION_SOFTMAX_TEMP=1 + +# Training parameters: +NUM_EPOCHS= 6250 # nback-paper: 400,000 @ one trial per epoch = 6,250 @ 64 trials per epoch +LEARNING_RATE=0.001 # nback-paper: .001 + +# Execution parameters: +CONTEXT_DRIFT_RATE=.1 # drift rate used for DriftOnASphereIntegrator (function of Context mech) on each trial +NUM_TRIALS = 48 # number of stimuli presented in a trial sequence + +# Names of Compositions and Mechanisms: +NBACK_MODEL = "Nback Model" +FFN_COMPOSITION = "WORKING MEMORY (fnn)" +FFN_STIMULUS_INPUT = "CURRENT STIMULUS" +FFN_CONTEXT_INPUT = "CURRENT CONTEXT" +FFN_STIMULUS_RETRIEVED = "RETRIEVED STIMULUS" +FFN_CONTEXT_RETRIEVED = "RETRIEVED CONTEXT" +FFN_TASK = "CURRENT TASK" +FFN_HIDDEN = "HIDDEN LAYER" +FFN_OUTPUT = "DECISION LAYER" +MODEL_STIMULUS_INPUT ='STIM' +MODEL_CONTEXT_INPUT = 'CONTEXT' +MODEL_TASK_INPUT = "TASK" +EM = "EPISODIC MEMORY (dict)" +CONTROLLER = "READ/WRITE CONTROLLER" + +class trial_types(IntEnum): + """Trial types explicitly assigned and counter-balanced in get_run_inputs() + In notation below, "A" is always current stimulus. + Foils are only explicitly assigned to items immediately following nback item. + Subseq designated below as "not explicitly assigned" may still appear in the overall stimulus seq, + either within the subseq through random assignment, + and/or through cross-subseq relationships that are not controlled in this design + """ + MATCH_NO_FOIL = 0 # ABA (2-back) or ABCA (3-back); not explicitly assigned: ABBA + MATCH_WITH_FOIL = 1 # AAA (2-back) or AABA (3-back); not explicitly assigned: ABAA or AAAA + NO_MATCH_NO_FOIL = 2 # ABB (2-back) or BCDA (3-back); not explicitly assigned: BBCA, BCCA or BBBA + NO_MATCH_WITH_FOIL = 3 # BAA (2-back) or BACA (3-back); not explicitly assigned: BCAA or BAAA +num_trial_types = len(trial_types) +#endregion + +#region ===================================== MODEL CONSTRUCTION ======================================================= + +def construct_model(stim_size = STIM_SIZE, + context_size = CONTEXT_SIZE, + hidden_size = HIDDEN_SIZE, + num_nback_levels = NUM_NBACK_LEVELS, + context_drift_noise = CONTEXT_DRIFT_NOISE, + retrievel_softmax_temp = RETRIEVAL_SOFTMAX_TEMP, + retrieval_hazard_rate = RETRIEVAL_HAZARD_RATE, + retrieval_stimulus_weight = RETRIEVAL_STIM_WEIGHT, + retrieval_context_weight = RETRIEVAL_CONTEXT_WEIGHT, + # decision_softmax_temp = DECISION_SOFTMAX_TEMP + ): + """Construct nback_model + Arguments + --------- + context_size: int : default CONTEXT_SIZE + hidden_size: int : default HIDDEN_SIZE + num_nback_levels: int : default NUM_NBACK_LEVELS + context_drift_noise: float : default CONTEXT_DRIFT_NOISE + retrievel_softmax_temp: float : default RETRIEVAL_SOFTMAX_TEMP + retrieval_hazard_rate: float : default RETRIEVAL_HAZARD_RATE + retrieval_stimulus_weight: float : default RETRIEVAL_STIM_WEIGHT + retrieval_context_weight: float : default RETRIEVAL_CONTEXT_WEIGHT + # decision_softmax_temp: float : default DECISION_SOFTMAX_TEMP) + + Returns + ------- + Composition implementing Nback model + """ + + print(f"constructing '{FFN_COMPOSITION}'...") + + # FEED FORWARD NETWORK ----------------------------------------- + + # inputs: encoding of current stimulus and context, retrieved stimulus and retrieved context, + # output: decision: match [1,0] or non-match [0,1] + # Must be trained to detect match for specified task (1-back, 2-back, etc.) + input_current_stim = TransferMechanism(name=FFN_STIMULUS_INPUT, + size=stim_size, + function=FFN_TRANSFER_FUNCTION) + input_current_context = TransferMechanism(name=FFN_CONTEXT_INPUT, + size=context_size, + function=FFN_TRANSFER_FUNCTION) + input_retrieved_stim = TransferMechanism(name=FFN_STIMULUS_RETRIEVED, + size=stim_size, + function=FFN_TRANSFER_FUNCTION) + input_retrieved_context = TransferMechanism(name=FFN_CONTEXT_RETRIEVED, + size=context_size, + function=FFN_TRANSFER_FUNCTION) + input_task = TransferMechanism(name=FFN_TASK, + size=num_nback_levels, + function=FFN_TRANSFER_FUNCTION) + hidden = TransferMechanism(name=FFN_HIDDEN, + size=hidden_size, + function=FFN_TRANSFER_FUNCTION) + decision = ProcessingMechanism(name=FFN_OUTPUT, + size=2, + function=ReLU) + + ffn = AutodiffComposition(([{input_current_stim, + input_current_context, + input_retrieved_stim, + input_retrieved_context, + input_task}, + hidden, decision], + RANDOM_WEIGHTS_INITIALIZATION), + name=FFN_COMPOSITION, + learning_rate=LEARNING_RATE, + loss_spec=Loss.CROSS_ENTROPY + # loss_spec=Loss.MSE + ) + + # FULL MODEL (Outer Composition, including input, EM and control Mechanisms) ------------------------ + + print(f"constructing '{NBACK_MODEL}'...") + + # Stimulus Encoding: takes STIM_SIZE vector as input + stim = TransferMechanism(name=MODEL_STIMULUS_INPUT, size=stim_size) + + # Context Encoding: takes scalar as drift step for current trial + context = ProcessingMechanism(name=MODEL_CONTEXT_INPUT, + function=DriftOnASphereIntegrator( + initializer=np.random.random(context_size-1), + noise=context_drift_noise, + dimension=context_size)) + + # Task: task one-hot indicating n-back (1, 2, 3 etc.) - must correspond to what ffn has been trained to do + task = ProcessingMechanism(name=MODEL_TASK_INPUT, + size=num_nback_levels) + + # Episodic Memory: + # - entries: stimulus (field[0]) and context (field[1]); randomly initialized + # - uses Softmax to retrieve best matching input, subject to weighting of stimulus and context by STIM_WEIGHT + em = EpisodicMemoryMechanism(name=EM, + input_ports=[{NAME:"STIMULUS_FIELD", + SIZE:stim_size}, + {NAME:"CONTEXT_FIELD", + SIZE:context_size}], + function=ContentAddressableMemory( + initializer=[[[0]*stim_size, [0]*context_size]], + distance_field_weights=[retrieval_stimulus_weight, + retrieval_context_weight], + # equidistant_entries_select=NEWEST, + selection_function=SoftMax(output=MAX_INDICATOR, + gain=retrievel_softmax_temp)), + ) + + # Control Mechanism + # Ensures current stimulus and context are only encoded in EM once (at beginning of trial) + # by controlling the storage_prob parameter of em: + # - if outcome of decision signifies a match or hazard rate is realized: + # - set EM[store_prob]=1 (as prep encoding stimulus in EM on next trial) + # - this also serves to terminate trial (see nback_model.termination_processing condition) + # - if outcome of decision signifies a non-match + # - set EM[store_prob]=0 (as prep for another retrieval from EM without storage) + # - continue trial + control = ControlMechanism(name=CONTROLLER, + default_variable=[[1]], # Ensure EM[store_prob]=1 at beginning of first trial + # --------- + # VERSION *WITH* ObjectiveMechanism: + objective_mechanism=ObjectiveMechanism(name="OBJECTIVE MECHANISM", + monitor=decision, + # Outcome=1 if match, else 0 + function=lambda x: int(x[0][0]>x[0][1])), + # Set ControlSignal for EM[store_prob] + function=lambda outcome: int(bool(outcome) + or (np.random.random() > retrieval_hazard_rate)), + # --------- + # # VERSION *WITHOUT* ObjectiveMechanism: + # monitor_for_control=decision, + # # Set Evaluate outcome and set ControlSignal for EM[store_prob] + # # - outcome is received from decision as one hot in the form: [[match, no-match]] + # function=lambda outcome: int(int(outcome[0][1]>outcome[0][0]) + # or (np.random.random() > retrieval_hazard_rate)), + # --------- + control=(STORAGE_PROB, em)) + + nback_model = Composition(name=NBACK_MODEL, + nodes=[stim, context, task, ffn, em, control], + # Terminate trial if value of control is still 1 after first pass through execution + termination_processing={TimeScale.TRIAL: And(Condition(lambda: control.value), + AfterPass(0, TimeScale.TRIAL))}, + ) + # # Terminate trial if value of control is still 1 after first pass through execution + nback_model.add_projection(MappingProjection(), stim, input_current_stim) + nback_model.add_projection(MappingProjection(), context, input_current_context) + nback_model.add_projection(MappingProjection(), task, input_task) + nback_model.add_projection(MappingProjection(), em.output_ports["RETRIEVED_STIMULUS_FIELD"], input_retrieved_stim) + nback_model.add_projection(MappingProjection(), em.output_ports["RETRIEVED_CONTEXT_FIELD"], input_retrieved_context) + nback_model.add_projection(MappingProjection(), stim, em.input_ports["STIMULUS_FIELD"]) + nback_model.add_projection(MappingProjection(), context, em.input_ports["CONTEXT_FIELD"]) + + if DISPLAY_MODEL: + nback_model.show_graph( + # show_cim=True, + # show_node_structure=ALL, + # show_dimensions=True + ) + + print(f'full model constructed') + return nback_model +#endregion + +#region =====================================STIMULUS GENERATION ======================================================= + +def get_stim_set(num_stim=STIM_SIZE): + """Construct an array of unique stimuli for use in an experiment, used by train_network() and run_model()""" + # For now, use one-hots + return np.eye(num_stim) + +def get_task_input(nback_level): + """Construct input to task Mechanism for a given nback_level, used by train_network() and run_model()""" + task_input = list(np.zeros_like(NBACK_LEVELS)) + task_input[nback_level-NBACK_LEVELS[0]] = 1 + return task_input + +def get_training_inputs(network, num_epochs, nback_levels): + """Construct set of training stimuli used by ffn.learn() in train_network() + Construct one example of each condition: + match: stim_current = stim_retrieved and context_current = context_retrieved + stim_lure: stim_current = stim_retrieved and context_current != context_retrieved + context_lure: stim_current != stim_retrieved and context_current == context_retrieved + non_lure: stim_current != stim_retrieved and context_current != context_retrieved + """ + assert is_iterable(nback_levels) and all([0` specified in construction of the network. If None is specified + here, either the value specified at construction, or the default for `AutodiffComposition + ` is used. + num_epochs: int : default NUM_EPOCHS, + specifies number of training epochs (i.e., sets of minibatchs) to execute during training. + save_weights_to: Path : default None + specifies location to store weights at end of training. + + Returns + ------- + Path containing saved weights for matrices of feedforward Projections in network. + """ + print(f"constructing training set for '{network.name}'...") + if training_set == None: + training_set, minibatch_size = get_training_inputs(network=network, + num_epochs=num_epochs, + nback_levels=NBACK_LEVELS) + print(f'num training stimuli per training set (minibatch size): {minibatch_size}') + print(f'num weight updates (num_epochs): {num_epochs}') + print(f'total num trials: {num_epochs*minibatch_size}') + print(f"\ntraining '{network.name}'...") + start_time = timeit.default_timer() + network.learn(inputs=training_set, + minibatch_size=minibatch_size, + report_output=REPORT_OUTPUT, + report_progress=REPORT_PROGRESS, + # report_learning=REPORT_LEARNING, + learning_rate=learning_rate, + # execution_mode=ExecutionMode.LLVMRun + # execution_mode=ExecutionMode.Python + execution_mode=ExecutionMode.PyTorch + ) + stop_time = timeit.default_timer() + print(f"'{network.name}' trained") + training_time = stop_time-start_time + if training_time <= 60: + training_time_str = f'{int(training_time)} seconds' + else: + training_time_str = f'{int(training_time/60)} minutes {int(training_time%60)} seconds' + print(f'training time: {training_time_str} for {num_epochs} epochs') + path = network.save(filename=save_weights_to) + print(f'max weight: {np.max(nback_model.nodes[FFN_COMPOSITION].nodes[FFN_HIDDEN].efferents[0].matrix.base)}') + print(f'saved weights to: {save_weights_to}') + return path + # print(f'saved weights sample: {network.nodes[FFN_HIDDEN].path_afferents[0].matrix.base[0][:3]}...') + # network.load(path) + # print(f'loaded weights sample: {network.nodes[FFN_HIDDEN].path_afferents[0].matrix.base[0][:3]}...') + +def run_model(model, + # load_weights_from=None, + load_weights_from='ffn.wts_nep_6250_lr_001.pnl', + context_drift_rate=CONTEXT_DRIFT_RATE, + num_trials=NUM_TRIALS, + report_output=REPORT_OUTPUT, + report_progress=REPORT_PROGRESS, + animate=ANIMATE, + save_results_to=None + ): + """Run model for all nback levels with a specified context drift rate and number of trials + Arguments + -------- + load_weights_from: Path : default None + specifies file from which to load pre-trained weights for matrices of FFN_COMPOSITION. + context_drift_rate: float : CONTEXT_DRIFT_RATE + specifies drift rate as input to CONTEXT_INPUT, used by DriftOnASphere function of FFN_CONTEXT_INPUT. + num_trials: int : default 48 + number of trials (stimuli) to run. + report_output: REPORT_OUTPUT : default REPORT_OUTPUT.OFF + specifies whether to report results during execution of run (see `Report_Output` for additional details). + report_progress: REPORT_PROGRESS : default REPORT_PROGRESS.OFF + specifies whether to report progress of execution during run (see `Report_Progress` for additional details). + animate: dict or bool : default False + specifies whether to generate animation of execution (see `ShowGraph_Animation` for additional details). + save_results_to: Path : default None + specifies location to save results of the run along with trial_type_sequences for each nback level; + if None, those are returned by call but not saved. + """ + ffn = model.nodes[FFN_COMPOSITION] + em = model.nodes[EM] + if load_weights_from: + print(f"nback_model loading '{FFN_COMPOSITION}' weights from {load_weights_from}...") + ffn.load(filename=load_weights_from) + print(f'max weight: {np.max(nback_model.nodes[FFN_COMPOSITION].nodes[FFN_HIDDEN].efferents[0].matrix.base)}') + print(f"'{model.name}' executing...") + trial_type_seqs = [None] * NUM_NBACK_LEVELS + start_time = timeit.default_timer() + for i, nback_level in enumerate(NBACK_LEVELS): + # Reset episodic memory for new task using first entry (original initializer) + em.function.reset(em.memory[0]) + inputs, trial_type_seqs[i] = get_run_inputs(model, nback_level, context_drift_rate, num_trials) + model.run(inputs=inputs, + report_output=report_output, + report_progress=report_progress, + animate=animate + ) + # print("Number of entries in EM: ", len(model.nodes[EM].memory)) + stop_time = timeit.default_timer() + assert len(model.nodes[EM].memory) == NUM_TRIALS + 1 # extra one is for initializer + if REPORT_PROGRESS == ReportProgress.ON: + print('\n') + print(f"'{model.name}' done: {len(model.results)} trials executed") + execution_time = stop_time - start_time + if execution_time <= 60: + execution_time_str = f'{int(execution_time)} seconds' + else: + execution_time_str = f'{int(execution_time/60)} minutes {int(execution_time%60)} seconds' + print(f'execution time: {execution_time_str}') + results = np.array([model.results, trial_type_seqs]) + if save_results_to: + np.save(save_results_to, results) + # print(f'results: \n{model.results}') + return results +#endregion + +#region ================================= MODEL PERFORMANCE ANALYSIS =================================================== + +def analyze_results(results, num_trials=NUM_TRIALS, nback_levels=NBACK_LEVELS): + responses_and_trial_types = [None] * len(nback_levels) + stats = np.zeros((len(nback_levels),num_trial_types)) + MATCH = 'match' + NON_MATCH = 'non-match' + + for i, nback_level in enumerate(nback_levels): + # Code responses for given nback_level as 1 (match) or 0 (non-match) + relevant_responses = [int(r[0][0]) for r in results[0][i*num_trials:i*num_trials+num_trials]] + relevant_responses = [MATCH if r == 1 else NON_MATCH for r in relevant_responses] + responses_and_trial_types[i] = list(zip(relevant_responses, results[1][i])) + # x = zip(relevant_responses, results[1][i]) + for trial_type in trial_types: + # relevant_data = [[response,condition] for response,condition in x if condition == trial_type] + relevant_data = [[response,condition] for response,condition in zip(relevant_responses, results[1][i]) + if condition == trial_type] + if trial_type in {trial_types.MATCH_NO_FOIL, trial_types.MATCH_WITH_FOIL}: + # is the correct response for a match trial + stats[i][trial_type] = [d[0] for d in relevant_data + if d[0] is not None].count(MATCH) / (len(relevant_data)) + else: + # [0,1] is the correct response for a match trial + stats[i][trial_type] = [d[0] for d in relevant_data + if d[0] is not None].count(NON_MATCH) / (len(relevant_data)) + for i, nback_level in enumerate(nback_levels): + print(f"nback level {nback_level}:") + for j, performance in enumerate(stats[i]): + print(f"\t{list(trial_types)[j].name}: {performance:.1f}") + + data_dict = {k:v for k,v in zip(nback_levels, responses_and_trial_types)} + stats_dict = {} + for i, nback_level in enumerate(nback_levels): + stats_dict.update({nback_level: {trial_type.name:stat for trial_type,stat in zip (trial_types, stats[i])}}) + + return data_dict, stats_dict + + + + + + + + +def compute_dprime(hit_rate, fa_rate): + """ returns dprime and sensitivity + """ + def clamp(n, minn, maxn): + return max(min(maxn, n), minn) + # hit_rate = clamp(hit_rate, 0.01, 0.99) + # fa_rate = clamp(fa_rate, 0.01, 0.99) + + dl = np.log(hit_rate * (1 - fa_rate) / ((1 - hit_rate) * fa_rate)) + c = 0.5 * np.log((1 - hit_rate) * (1 - fa_rate) / (hit_rate * fa_rate)) + return dl, c + + +def plot_results(response_and_trial_types, stats): + hits_stderr = np.concatenate((score.mean(2).std(-1)/np.sqrt(neps))[:,(0,1)]) + correj_stderr = np.concatenate((score.mean(2).std(-1)/np.sqrt(neps))[:,(2,3)]) + d,s = compute_dprime( + np.concatenate(score.mean(2)[:,(0,1)]), + np.concatenate(score.mean(2)[:,(2,3)]) + ) + print(d.shape,s.shape) + dprime_stderr = d.std(-1)/np.sqrt(neps) + bias_stderr = s.std(-1)/np.sqrt(neps) + #%% + # 2back-target, 2back-lure, 3back-target, 3back-lure + hits = np.concatenate(acc[:,(0,1)]) + correj = np.concatenate(acc[:,(2,3)]) + dprime = np.zeros(4) + bias = np.zeros(4) + for i in range(4): + d,s = compute_dprime(hits[i], 1-correj[i]) + dprime[i]=d + bias[i]=s + + #%% + f,axar = plt.subplots(2,2,figsize=(15,8));axar=axar.reshape(-1) + cL = ['blue','darkblue','lightgreen','forestgreen'] + labL = ['2b,ctrl','2b,lure','3b,ctrl','3b,lure'] + + # correct reject + ax = axar[0] + ax.set_title('correct rejection') + ax.bar(range(4),correj,color=cL,yerr=correj_stderr) + + # hits + ax = axar[1] + ax.set_title('hits') + ax.bar(range(4),hits,color=cL,yerr=hits_stderr) + + # + ax = axar[2] + ax.set_title('dprime') + ax.bar(range(4),dprime,color=cL,yerr=dprime_stderr) + + # + ax = axar[3] + ax.set_title('bias') + ax.bar(range(4),bias,color=cL,yerr=bias_stderr) + + ## + for ax in axar[:2]: + ax.set_xticks(np.arange(4)) + ax.set_xticklabels(labL) + ax.set_ylim(0,1) + + plt.savefig('figures/EMmetrics-%s-t%i.jpg'%(mtag,tstamp)) + plt.savefig('figures/EMmetrics_yerr-%s-t%i.svg'%(mtag,tstamp)) + + + + + + + + +#endregion + + +#region ===================================== SCRIPT EXECUTION ========================================================= +# Construct, train and/or run model based on settings at top of script + +nback_model = construct_model() + +if TRAIN: + weights_filename = f'ffn.wts_nep_{NUM_EPOCHS}_lr_{str(LEARNING_RATE).split(".")[1]}.pnl' + saved_weights = train_network(nback_model.nodes[FFN_COMPOSITION], + save_weights_to=weights_filename) +if RUN: + from pathlib import Path + import os + results_filename = f'nback.results_nep_{NUM_EPOCHS}_lr_{str(LEARNING_RATE).split(".")[1]}.pnl' + results = run_model(nback_model, + # load_weights_from=Path(os.path.join(os.getcwd(),'ffn.wts_nep_1_lr_01.pnl')), + # load_weights_from=Path(os.path.join(os.getcwd(),'ffn.wts_nep_6250_lr_01.pnl')), + # load_weights_from=INITIALIZER + save_results_to= results_filename) +if ANALYZE: + coded_responses, stats = analyze_results(results, + num_trials=NUM_TRIALS, + nback_levels=NBACK_LEVELS) +#endregion \ No newline at end of file diff --git a/Scripts/Models (Under Development)/N-back/WORKING MEMORY (fnn)_matrix_wts_20stim_2500ep.pnl b/Scripts/Models (Under Development)/Nback/results/WORKING MEMORY (fnn)_matrix_wts_20stim_2500ep.pnl similarity index 100% rename from Scripts/Models (Under Development)/N-back/WORKING MEMORY (fnn)_matrix_wts_20stim_2500ep.pnl rename to Scripts/Models (Under Development)/Nback/results/WORKING MEMORY (fnn)_matrix_wts_20stim_2500ep.pnl diff --git a/Scripts/Models (Under Development)/N-back/ffn.wts_nep_1_lr_01.pnl b/Scripts/Models (Under Development)/Nback/results/ffn.wts_nep_1_lr_01.pnl similarity index 100% rename from Scripts/Models (Under Development)/N-back/ffn.wts_nep_1_lr_01.pnl rename to Scripts/Models (Under Development)/Nback/results/ffn.wts_nep_1_lr_01.pnl diff --git a/Scripts/Models (Under Development)/Nback/results/ffn.wts_nep_6250_lr_001.pnl b/Scripts/Models (Under Development)/Nback/results/ffn.wts_nep_6250_lr_001.pnl new file mode 100644 index 0000000000000000000000000000000000000000..57838e60968d33fc6f9d1438b3ef2a15c6257567 GIT binary patch literal 29039 zcmb5W=TleP)9s6hm{2h%1VlwlV8V<`FbfJ|4k*cpWK=Mbvt-F286@W<;J2##&$=(} z{Y-xMIk#?|dg`3BtM=XsE6mwFdi3a?bFPrUr4Dv>D^}S3PyaaEZLo_-ihBF*(<`6o zh?EHLgqRQ40@m6+{r~-wVgEhR&g=1QZoeIoknk?d=^F9+YbfA#VGXQ5Z`JqWrV=AIJoe&gnqD_8Hh2l+p_8v5*JT*AkckocsO zBS*s$mqf*Vj7>=Mj*3ruA+qr8vd43tc6LjSYrD*!T_z=B+hwP9aSvo#DlF~2=%gLepns)Q zY^UF9wd85`l0-{}Wbm-oX@)Flx7cKShtyrxVqMUDOPXJ&z0`lwBIY;E?~Xosqd78V zY1I*J(m9#-lzvM`H|T)WYMB;DyV%mwHFvxGd@7yXQn4*eT6k6dP5#mBq2=qX`@A>HN_#9dNt=&owIwZX z(tKL}@oVdx4lR{(Y2Et3at}DD(_u`aS#0-1WH4caOlgG`sSQ`PM!Hh9Tx=(9`9w+9 zVfih#W0I@3_`v5(%qDX(@>U8?eV2A?f1LjF(uOc;u(an!z$eX32&Zlm^}{3g}>yh9op%#8lhUa7RabS&JG!DHr0<-4{^!}SoUvb0fq z-DLiHv=nm3=y4s@+Ks`|!C&S*_j>5iX&JVpDfaS;htGw{Hc62*>F~w7(rEcHyNr|W zQBr4ZbN?ulI>rwQtT3Hl`u3rA8K*tg8O`2sUPigm*!iF4Xq~k0l;4&PORILqCvTBD zer1!cGm?2uY~L*J8@QuHD;=2rS*Z`!Ic^%2KVo}%I9Yyq?3D&?sC1J01l;mZhAi!r z^5@dBeR00Fcq!ZC;HpywXN=;n_A<$q6kaLy%;D(Ot(ZW@~51knjfjTtM4!V zCS$b}czEjg7qLA6#dVr>2%J_}J~}DG9;bFNNSl-$UMp2A{EWAwJs+j&P=qo0k(+B3 z2=2zVnzizimdX6JOJZ~7vW2Cgfl@)RdAc672Z3EYWZ{71?dI9GDC?$?PKuGSg?^eu=&WK4n@lF3YoFwuS{+W=N$!7Z{3su zucV7IFFy&%8)u%#uMN^Bww9hw^ zz8*+q|5Mvh$>goYGI&Xbj&eB;m%&Y1{1xl;Nu}7Fr6yJ8bU>%RGyQeG`se!V5V83) z<_@Vy(4G+MiZpG<{k=5VDuH%?yaX^%CI!O6WjNjI=Bj|$yp4z{LlJVOOu&=k9E$?Px|(4KFS!o zH0#)@2Tu)L{=Rbb(5V%YYx%!TzALqH+-ZBmqOP8h1S|r6+bmofW;CR2S7o_A4p+p30 zpLP-7Q~HPJN`+7GrYp<@=W9;N2A%xyTzbS7E*W6FP;#^c{02$GWrDu+y?J?|j_+8u z9EUeqGNEOMwKtR@VsWfh8SEacLqB*{jgChI5_10*c{T&njmY$ti+GC2NLQPm6ndHY zUc4B6QEdk?l2kcJzK-7~4wy>zBguTHl{cBoi=D@%F-V)ty1%cTl zf(plHmX2u6KCK0nBm!)!Ee{U61LSs#tkI`gv9Nq2(2Hd9(}aJqJ8K`9+B-~`(F_&q7FiE4u z()_bJxYCKpBaAH^Jt6(t9AaY5UM7gLfeR-woBrXJ>gWjK^pCZ9wUz~J!`R%ye_L-p zNCJ@Ayz*zf*1D0<`mx9oX?*+yBxRV(l0ODX*G0{IZaM47Hz|vT=X{KaHd|l5kZ45_bnvcUB<|?I+ecT)vZ;Au3Xc!&~M8&USxauX&um*09@tPp(`MD_N2>Vj%m@_ ztL{q)s48rf5gsWOgj=_zMf;(vY{o&hI`>Zhoss!BQe&lSgFkwsPO>oM0YFpmQfii@ zi_PQIJ)OG08aSk$(9S(4bcU;C))ipaU*T$EW3^0u1Wa{!ohUr5IY->TY7as1HQhsN zTq7_kId4;h{&6_YbvCVvA!q#hXe5EmYIoqPple3+o(pu=MT)?hlxgA1yB8Hs z%8@)flf|+OF0^b1UjKD10brvN{Fg|>o~_0LTC`0H9iSm>AhvsVwbXkLcy2cL{{x*f zninJs)&-)w3Z#z`msrJ1y7x(zn^u9go13Nlss7rxn7Sv=-hU)DUgQ{?D?ZGDH71{7 z3N2rv1sD7!=c%_CdE;Q`Eh~R-X7fuiwx=CD6xIiyi}fDqa~?gvt!0xpKk^mke=R!;paUG z&|+SawFQ4PZ+S)3*|b-B2#j3KxM_SN#aE?tsdVW09kCsTJ%393+O5VTS%Et6fKZu4 zW}MXSWXayn4LBq&4U_*%O)}8fAVatH-_vBNwXW&Z%XK=(5W6fX_WABiI+&9=>qY!) zu56Lc2s7PV#8WYbXvfSBGWAyTPRZ{S3tN;Q(E8gT0F8H`kwE!@RZM@evWQ@G8QR-}hE)%c;ZCXT6BlpLWZROQyAlG0bhyECAY0ij?xi zEp`~B%6fGX{ap9<5n|{w*>yr{QydMQ-yzwSYn~K*#4d#-kRlh$%^(W7F}y~aj}vit z^|DSmZ9H@Y`jCP19%o-`xw+Zf?b}W1y=cPY_zoFz)0PtkqsY(UrTlLYAoN;y(F52q zA3DHvXJCqvYj1I`G1shQANJ9z3nm%(e+NJrZ|{(RY{AgegtqKGhc|359f{9AEjR*^ z21<9V-7P8#Fw-yPOJ(HZNt0tv>MtfzEJItE;HpcTPLcjcVMxaE_#&5AJuy5nVm%36 zwaxtaq`P!})<4Hg03^olBxx3GC1CHuT6uWBf;7`_y-xz&4KnKsG^FQ~ly6>+D+X|3 z7L%z!qh-qs?zf#@DwzRVb`CLSOVrw%GUpJ-OKtKmQ7SGQfdm1?BuU0rZQE)v7o+tt zG85=yZq~}wqX(r3ppi4;;*LGlg+2Pq(rVsOxr`r|7$K2i{*?6qW*~{l(0Sken3o&A z=-6$0qj6Cb%LDHLuwrdVwGxm;%@mQFsnITJpDP4g^Uh3m%QBsKDp`1@o>$1+W^GA8 z-4W`@r3R2bmzHFn%P1$jrG6LFsJ0w5?>#aTM_U&m%|EwJGw%V)!(i2 zdikxCU)-C)RN5ZJ!!@S-YmjC;#~5<0DM!ogCC?f1>_KF&mfA#n(Dly*alvY;sO>x# z8^&k@GN+InIryKOmLJwiDYz%27D=e}BG2kzxKwgw`Ujys0@oBWW1EbeI{o5Xl9}k4 z*RV@w=!r!|wSaKXk+~-?O`g-aa6sL=4r4ydw2~>R^CWtpRC6yt5|79#_G2ModWeGHJXhwf3ay zkVnT4$GBq?!1rVeH@E0qB5M4iw-$wCd*iY$$qv<09y^YqK9ZDXwH6Ole-_&&s}9^agy2K0 z1ZrNi7Cj+oo2{!_C(Wp_1C-=4?7Iol?E9<`TPKd}`@m$%OeB23kC~Faa}ip$D5)eq zu0SEk>K$iv+Ft9HFWS6Q7hDcV_8zG595m9pIYu*x1H%+!_kUp1F#`zycUti}S#4;a z=il|GBOMAlCA%c&Ie|8L$>byx21gTK7}Db}Gur2;zt z%@>&U!)jk_H;dM#uv(|(aa#TaWD>)Hl10@DUr{_(ACTV(7G#5a{0_V6yu&77X7kf| zd(c`;tn!9EuG``Mk-HFzPT#w%_0q6|jP7*et<-;Bl(?(4;I*_qNjCF8A4|Lp`FWa( zD03|(u-3eal8NUB41LP5c5kP< z)3%JVVV(PWT%QRJTP!JBa(k&xMH%iM1P+~)!L7#$**Kj3@7>XN1gOo&P7Az#*c(ls zpXd%hWG9;+d!|Z{3sqO10j?CKMCig=Z@{%|dwQCbKH-r%6?q}>#7U6waif>CMZAVL zU7q7|Dm)xS#+pn$b%{`DrbYqywPCstczux<{<)x3=donXH?iH4p4a|rb1~unU7Gf9 z-mr}s(iD(#nT_Xvv*u;(!9XBq*IqNNoKtQ%t8t$(Rm+B$`|md_hI)(dUG3c!4I$dx z_k7jAgd{v_9e<&%8>QP*x-Gv5C0?E*oJuX4KOMRdMeH^48<{_?6F>GMZNP>$-@J78 z6Rs#GU0mLjDixn28Bn)Y;>P#V_QcY`YrEam)bt1hok?Hi=eYy#*tGGa^BK+7E_hA< zZr2K(zUsXgcNxCgWOr^GDf?~UHpzKGYQ}XdwRc7p1SDyICyyfvT@T_WwEe^!cC)9lP3mV@wn<#K2?$2z8!Nm{(* zH4wLb{U$X(z*%LeB{TPRAw-(Lo0O(C-e0uD0Tynz676;vxUK;yd$mU@+_y+^q$M?G zz!7AfN~(eF@-M~f8cub_$9(!d($35ReAYzztS$>Z%OU?QBaIdp01m2g%&yr^) z%D=Q7SGlsvy7pr=y0QpMwK4YUBT&}9*XGZVZHX_oFWzka=A`8=#~_0_lb8-k=aPFz zAb&XUl>FR_OVb8f|p8z-RmF=T(l*BV}#>|qqP<9&&xvMV-qp6H#F;lgD=F9uAO`Jr?uE@ zy)45X+*Q1YEok~?GIx|7hOsn1P|Mda>y~?D7|DC66=6F7tPvc=5jtSc*vZ7tq#21> zOhsI6;(IPirr&GxRtTT6X;5lIGzalSULuNf553uC$f4st!8t__YXHztMb0HJgPYN` zmZP&5QC!a0v#bN(r0-7Bd6Tj4%FlZ;yic=@{^<0DWZFXzo*C8+X9SSf}taIMc+IpK< zCN&(qwDZc#9oT`2!C$))85S{x@td?L#B)3I1Do~7mtd}cCUZT0bssEM011#;VyDw9 ziKlM<;${{W`j18lbsef;R(m$?T8%^2-KIR~qI@Hst((&1Li4Wtu9CH1#cIC8{#cWZ zrF!4?U28vKCMW%u;;&;7T+Be~@H-#_@X!NEVCd@pt)CDJw#)X~LP%xt&OuB5`7YP2 zvtC93>NRqj>${k9@N&w=<1_kp!M6KQC4}hXVn!?ZCj=J z*3y6zpSQiDctlhVUX6)mEZyjRlx8Nph;lb&bUhK?q!S;pP(M}36wTe=GPT(_Ec+BD zC_A#nfCV{P|5U1DxX}!@?A-dLmnGXYOM8*{qn|bRrXfoxpq4e6Y#q7EFy&_eUK*F# z6>#{|qp>>epxLq9)A~sv<zCjBaTo_Ci5A6__>79S!D7Z%|;f98w!oZch())KeJ80U8a z(teG9@_OdOY6ATkvSY_SZ!AIUW7)iDi$f&4S%HB;GW*zY(5ZFW2v^nL0c=JOhWTm6VZilC8Y7TqX91+$ zi%)opJ|FH{sUwJ&o-5akk)`OtZq46`A+hmFz)_-|YYxCzPZKoTB|>K+xMqZCY#`C< z@Fyz>8l;4bmhyDV@r^d5CL*C~D2ztdL)$&%EHHX0{A~DJA{UB_k}(g!-f_ZITT<%_ z`m!GEWpsX@0|8T&6q->p)AnV{sBSY*hEn3a12@eE%B*!i7Q_C1NDMPk&G(g`Jin8k zthMi25Vo+zvAnhs;gPQ;o^;-=oseD!ovNP@4ZI{1A$xg62WoP{+KN$Y;w3*0iu2N$ zlf1-)BEoS~!ul)UHXveNUzIU!jQ?QL8Y!*jT@dJOBsbcS?}-5`>9ATdlAQM13+xn9 z&Gks}s&5+&A->;EUd)YLA8T!b{+cMKFKD*q_59c;ZHmFiI_K<=PHY(fwtV+l!Mn#G z$fqBfTtD6XTw=tQzV8eXUkDYU8%A$ZSJ^f*ACgeY)y?)kZ%Eu*wdU#3RLsKVKYTTJ z!z}^{*4KF*J0`WkpdZT7d1A+&BBR1sNwU7iq1@Z-3<+rOQE6UoVh$*4c9{G;2f0@p z`jeI?hjI1@deIc0c`?aSw47I!h|SGQ2M*}(Pezr?P?%1q{ctgl-r{-v3~wbaosM`* zY5IQYCJMvXzR-~yi|x!jzoY(_w4eNCb6N`&{yb%>eb#gDFd4f_oj`#zEYpm-Cg8TY z2%wf~&-ZjsIQ1qfszS&784axZ`}kh4Tc8D`E6*s+bF|-X%=B+35VQq=tPV>{kIQW2 z`lA5Lc1Y{(-=~mEvdFC@z&ieIbVgU~cGI#qyRTzQ`0o|dsdYAKtLBu9u<~nXl=h4? zrAjxtA(<9ss8rn94NQ8iM~^fQ&08Ihyp)bx7gD!|b9sZ~@!rSnr zE#R4!UDw9-La9tsN7@{u;RE0RT6cNgoVLVMVbEoEBz7{mjff2QyiVl&B%o(3m-YLl z^2~m-jyW8wf9%%+#8E5Y0T;Au->b!t8V=mAg>UG`S=hF5t)*=qMsgn~9adn-Hb#Zd zZ&>^vnBes+3zFTXeJcTAo)x%^EZ@Ww+D&ruIec!5_CJz&v0c$AXkr@f%jdr~7g`}^ z#jKN{HTt$5PYkdeSAoNF^%utdjq~z@QPjgOxHH#&5qCL|Ox6p@`wF|Jg6j!@J`+v% zC|xT*@!av9T5cskypMR@teMa`sf*O;*lswo;TJLlVe(b zga{^azeg+h?7` zukaipGPInO^@m97u@`_q8z?1vX;T z+n@DPdd15aerw{FHJYDlS~8UfiASMyxvW2 z_$3Rw4d@`~KkN0cWWfDaEbr>1&)kf7C}b~z4&3EWGbZA=vRty0aeVJf9%A$NBkgxf z840J*a#{|hwRtYVjBEK}w^j#YYi=<<8jK=`j~KBp?Ou=8#|9DdYTF2bv|^qyxK0Lx zUL07>v=G9Kmqj-xj5;DzlG-9Oj-QC?BBjdeJ0=y|B{#(|4Cv1Wv#cc$er?w0BbSdX zS&oB~WnwL;>(_-vSul+$dN5QgY-AmH>FO)}Zl(iQEI;>aR_Ia_U(wi%NnTf$dd0RP znf)7=0j1H01v^%-##CwtOl)AL9;&=QV@cWW9vEC^z+&hqiH7 z-q-uz-5<*&rNH~g=Vh2>UhWv9KEi`0*TPnFlDRBkLj<8gC}*4X#=Iw5YVBMmoxxh{ z2Opq51}QE_4?|q{UM!OlpI|d?!eRoO&uh)zL>^YG!+V~Z<`hxtrhTF0wqGWLS_gM& z*RyNrFcTV8Qh9(xk&ECbw${GbdSTrqtsu`-`rpQ9)!)Eph-5EIK8sS1((&hbzX`7n zTN|%mvxf_2gys(a?bg{8lVhiNbu-a8pN#p+flxdVSr-e@*o?7)LE^utLm!lNEVHPP z{X=G!CUH;B)wfdQPRxy1hY0op-**Tlla(^QY5zl6FsoF=lk;Wz)A^tQUR%ptyLBK` z%E+~lem*|q%ZD+M{ zrB2)h34>-EQ`Sp%@5~ZzzpZuOA*kv-Yp^hwEV`(*Qt@6%KzM09#!EAOp_BufbyCXx z$Rll*6P9D?D@N-U@H@;HyiJEzI8U6=l(DD+g){}uaTqEk*s-2>vadIJ;s`sMb^pKF9UZ7 z#0+aW#_Btu74!_Pr&8(zv(`f8<<^b!=;SKRTWLI=r2X%rW%#q^Sg$?opq*Y&Zm=^% zh$5%&e>XFJ;O(e?PoRlQtpI>kycaIIf)f@^FEFSYnNKxu=U(kWqW>t31#P^qJa zgb=g^NM~Bf8#i6~1lkEn`L_Wssy~Tr=(U0mEFqw$7!6Z;#v1mogeN$6+LE$E%;gW! z@EeoJ;J2OY({>5Y{jEPw1z(Cm4N;hf$C7keW9F&#QsH;*n2+f%x3b&H^A;t1rRAM| z;L?>?rFp}953bRXM~Ev{-)ooatRET1=cJCO9|EPl);``_6#PVcneH;FIQ~@gP!L7v zy3H7B>?7`AL_KS?{@#99+_zu5&bZ2GG+s*9A=kCizn5se?5=f(Vf0DrKf3DpB@;t8 zn>f(lEL-8Ez93_U9ne_j_G|9Q+Z*!N_v2ANqd#(w>I7?XDJWp=3b$LjHCl=+PtASd zN8+9jwgPqbLinD8W9PXv?uyjE0TgLEnyP=Ee13#=Z87^~-tJs1t8SO~Nq@SQKjvYin4Gi+_8*yB zqKgT>U7?AtDT!{8kMI2V!T0}dti5;t=IsYT5AOZ%g`oR?h2U-zf*zrXdrSzfFd?}2 zKOq?4WM{`ri}6ElB!$O>^?vQvmn+b+Z5BdcBJ9?(ND}l;3_B?cZa}oddboLSz_z^< znxxOU|ALkbbrnSSL|wIosU6gZS;zx=F@g?2~M3 z`z9VxtJ6VRiqj})`^l`6&^U}O4Y6klg~!PecilaFcLhrsS}e(#aZf|6A!gvRGK*x_s^&_)eSNa{l4}So-do0^cKz zzGk@=U07ro=sG*(ILXe`@{{EI9!F{OHC7~g51!%%<87Lz$l7ryT9aJXu796|Xzz6< z?SE4WoqgT8{CNjYG~B#ujB+)$e>h^TkS#w^ss2GvrmHK3~uQkJ8B7w zU`262^PyR=X#IvkyJ+IGd!aTy0tcejrR}xX#-jUmFcIpr}qz~Y3oIH z;Aw^TO1wx-^ZFpa9MG~2P*&*4qZ<>x!OHIGk`z)Hq@8x&Cz|1ONo&=R{MIT z_xtT-WF<`5DwXS#bvlKGLLzNO8@7N4WF4W2c_vN}=4XD~`YIFO2`1apwKufl&XYwu zSMP8~hWxT4dI8776Un zLZ-p~ZLAD6z)p0iiJ<0m=@#4pLUO;ECQftm* zt!6XB@1*hSy~udf~SQgR5C`LNPUiO~NXf@RBSsE$A2ordd7vKy5G-)B%Zp0Ago zwUT#>c59Gd?BNJ_XUa#tYaxMV0-I5zSdTefgNSdurKt%Xt|N?0V-yvj&)VO4FXjX`5>3Pw3JE zARMmK%1!a_oe{CqRtzOymSjG_^}M<6+V(GxRxsq0w8KFx!z`A;k0^=QEhb$iF~Gr< z9w$v5j=1f8xxEH##-vYlAOWp3X~9)_Qg@5dHz3)hz}}U&)d33HU(%kCOfvb$ zYLBJ={GTCPnw9`Ksair&_-`F#$jYh|uaiGv+RiK55N$7ZZ~X9{7?T!l(Y{2Ta)E>| zA(`q8`F^CYdmE)W#9OCvE8=C>wtLUwv^WN;qm8=lSSpiEfdJXT7rqJBx5-G%*}a~x z5e@WBM|esn<)>ly;GJ6u&)P$%0UkHT+@}JLGK=LG{hU@uGH6nhS*9swFA3fQqkS7+ z>L2&_5M)58yD-vdAwNRYG(>4 z{4(tSk_T6T9aD>qxu1aWI^k4epZt`kHB7hTwL$v@rJkS~`c7zV03y9wejayIBTXH! zkCW;T)LXvm9{a-y>~7}vUmOBiqXj|0hgF)}+PMYL)f-o3I7^Y+iU?oS@C0(H_w+NZ zvRq6Itl1uK25B9>zea}~c@x&FIHz;a{(?!3C2Cs|cRrY`p}#`ot_6ON=S8!76>0tM zC=RztZ=#l*xJ(Nr!`kJvmrV&&e@rGBt$Orj{2_)?Em`4Oe2-u+w3bL&;>R%J)?hR{ zK3x}F+4#<5W;8eKnjgdMv^MdB3>i-+mf6`{pSGP#>t4Z!tN26Jn$=%TamD8T5N^t0 zLgYI52$2#VuH!$*zIlL>EiF>?U2NAvanSH>evM?$I`m%JkA6X@Gu|vm_CaC1R&F1b zyiGW-c0Z!f&H0&(er68&)}tNhvixJF!snuFt#^GRUHZ?ka6l?&4@gUm$v11dR^pqM7g< z!b|Hl!& zBmXI5dN_v1G)fT;72(&YAoqBGeWPqcpvB#~oF+m!RNC>GDdg_j7^DmL&>A-*ILgDc zeqmB9-pQ`b)2!IAq^+4NqF)(BZ@VsI$53GOh73>X%p)*3@g~je`ZP&?;PXv;VZG@N z{9Z_D^Tb#tUgz$l?t$pWDVzG8rGF#N<*_gUt<+D54cannf_kHa9#H^&2K4r8$NGIh z6F}tsCMAbS+vT&7UXokZg_hK{D0^CRym51VUjd60R|)8j9~Zoi_}L2HYtP zrwOb6#b%7-DjoBDCI!jXl0Yep-x+Ew^ynTq&OUrn`!B|_{cSHM%=}=vo$=?1^C$6} z?b4yGJM0-Qq2=s*5qEVig%12^r1ZEQ-)UC3vNqBPoz#Y<#;={*PB`(qVxtkXEAB*8 z^kxLV?Tgqx=_u1>?H}WmelzSNmAhGW)A=8_&7HK6O-Hpq9H%b)0EAyPOY)&zpSr`q~qP9dPzUFh96U5SNs=Jdn*4@x< z=%$e5d*tPoLuu?hV`rq~ZN1HckrP$GG}*L8*_qE1|E`H&eG~QCVUE^K1$vRqSbDP7 zr%L7uF&lzgkb}~(nbVMghI=`Th2z2WsoHfHZf zwDidwm&3+vG|gwS3-oNPy-!!2P-^gcvd9}5TA>YhX|Y1<1Dov0I17fNx27*Ow$w3m zm1Zt`cX`otOntZf{3`7$-sAEDy&lAA9_pEw97X__iTQ?=mF}VthpR2Q=A*N zw9r$l*x-Rf=!*MEC2%ZQ6K(}NY(xq+YY9GLgF+xFw4AimY;s0&8bXPxem&N;Y3?c+ zNqFrKx7!W_9GhlH4F!>fKAu_4abB$tqxQs;1u;Z)-Ak?*)`E=(u4_{|kA|UK8ZZ1C4ko<^D#R!cj!P!6J? zwRP*^UFXl3$=<>r6L$}xwvZ!Lc3jtv$EQN9mnYGgsN`zEcz8u*RLEjPa&uDHSz^7} zy7gd%6hyNri+cuXtbg6Pjj6bOTIq&WYw-`vn9tiAK9SCW7h{?Npxx(ncJZEmgb+B) zB)af&xOUugQdW8HF}&&z9(z4D8ni=wb(bz*ehN#ZuRW)O{+zqDj>Hq06}#v!T4ATZ z?UH6s=K7wTneBwcXiG&9oq21&>jQS)FXh+mw96d8w81i6_<`Mw-}w*6LK~vsqrN3S zSRx|KGo|k7d2^lDLB{6h6YF+Y@&LNxf&9flG_4)|s7C036h8_IbSFv$get_PT7;QH&f& zvu<3*wXj(8(=U(&GDSut<@V2R$}P zKPY7T?fp0D_U-o3Rv)Q;g_AY+oD34DeefK3eETWz(s#{9^>JAI4LVcC$cgYJN&k;d zU2`!|ICW35)_@0&@!8mCdm)jAT!wMYkev>`+Rqp}S!O)?UMBXRH$NU@3cdGe3Gsp- zuj!jh{r-0(m+)xK_apMi4i}^`99>j)g2+O%WggK?_f2L7=(tg4@KZQ{#XeRJnnp|4 zGmuKnN__i3MGjoMc!r>8#L?8=9N;BO8#K4 z`;+ImkC962A<2G#yIV|#Og|v~57<)xG|!MVTaFO|KNmsKven@>s>|ewn4nMa$m~9X zTXL^R*ButF%u>wccJ>9!RG`r5)9SO3SQX@vM}l30c4)auh8_kK&N{dUUDkQlv?Ox5 zCKtT72bex=W3&#iwEL5#Sax!nUE4L!5I2?dI*_473D801H_P#%bX?ucvJImj3Qk-B z66%bkfmynw_m=Z^Za|8btk;?~#I^ZN>udVpPqhuAsq^*Sbt>@(X9{<*_))2I7ma6< zr8jc7j)R3NVK-ahS*=*kbB%$@b?68H?u#|op)@(Z5yi5Z;VFl;!ubI$Iz{|S-U)t{ zPX;Qbd+))FTVrJMBzUkb{hmg>+3ste{px9&C#6#d5RPV(^Y`?1a0gGV#A-jaa+~~z%i(_Tu?Dkr?PjVKXvmUpZ1D~I&P7FAF_BH@4@a_I(fa&~ zjxi9BQU=xKtk(?mh{^rg4nJrhm2o@a?B*w>Nn0N;X4$bfh?I65!&Xzv?%UJnUNk^0 z3M|hxcDSR;`m|}Y&P&!})jX&pPSWS=Y*N`q?Q@p;gW#^q3WoniS86T+mt#ry#>Y&< zL_Q#YA=4{#K842{QY+HNEwEMJcLURcSYo>wV>hBTleOiYS&I1k#WLHn2ZchI@nSl9 zCZ*o@&m|(Ef^_cfLmA#{lxUKfl^6-e#;_8}+A3que1U+g=WeaJPO5_z1_K_`_C&%j zX0(|HSID**Qk+ifc$6|Xj|L4he4eCz+WclMz_xC`qCG0>p7QUHC{bjTLh*Thv# z9Glj;x`ltsKVw+i+52FmLT8qt>^^@!_7wIgNaL0|GIIqp;G-tS8pCUSwK9-|$ZOgX zZ-5Ls#Qehd%VTizi@?G~-w*4YlZ>t7QL7yNA4M_Uf*S= zemO>}m#<4VIs3D+oi}XNX;Y-28vC@wL%WY7A(BW3J#ch!*kbtpk~b&q#GGuwIzc)UKjZ_U>kddYATd`ZE$ zbVHJic;9d}W14fN@Rz3RrU~{(Izu^i^@M}9CDMFbO5S)FU`WjqMpu7m&7wte623@- z|Bu%ICNM!K<=1wQKo%)n@BEmn@i)t6TKfjyWz&u4%N5%Bh|;jfjA^%yUXk7u4m(*N zb!w^hCsRAxfInLg?6vCw?ZHr~O5ZBYHx1V|B}vxG^T*8u@Okgb6jJPznE;8mMkc-Y z5}It7&ew_~$m|k->PmiBLIIUIN-jg>9 zd6+hF+6*CDn~bdRk}lWVOk&G#gK(q}psD!(7|Nf4k#83&WM%S4cw(vA~%zJ0{8AehiUE`yweGw%iM8Si8v zZ5f1n<%-lk{&vu?i?A%@#e!TDS;C-;BO8I=T=$uSZJ9Tu;za6ollBKZbZ)bQyD>cp z@GrUn`FR7wtR%T3I+QMuaZBD>&ACA&{e<>+to&-umeiQszz^7Y?xii?0AZGPKhV5g z`;pW~R=!3``~&$6<QwO6vp0+)77j6#PyvrFG0(fG{dWoIImXyy-MS3R{Y`je z@&%J`r}I>-Jxh$~9t6enFpj5LA6CX4isyyKgyWFx4xo*W`=XN2uz#}aqX-PPalT^WxawT{LFDS5oz zKy5-wwR*b@u3U`EB3?O-)@Wq(ITr7%J0}YXTm5;$B;V&aA$_|{Ht@VD{jYE7jLC7q zTF++kzo%IGh`F@XW&c})RIQ+@w(ZmQr;BXbqaAiq|An~bcN?VKja%n|F>zD2fg_~6 zXb8~S$PU^a<_Gn(TbJ#29F)JQW+t__*PHHEp9xdjg@YKbnVY*TH?ly9^A=)?ske}# z3ZF{;XC^?hOr(m)a3w5_QyR{Hh{KN~=JoP>lRMgMOlt30@EHUSQRV{aYU?635y! zX8M_p>(Y8MMaP5g0Ri@sJ*Ickbrs^%+9i03Z(?16`aD)@+0%9V&0slU`YlC^&Lncc z{;C}+bC8acb>oBQrASe++DQ+CDUz|@#|kn_qGp*&RBY`+|$nXI@XNG|!1d&zf;JBNw)+<>F+^taGqW(Pb-tABgX+^!?M=^0G9 zkN|KsSB4Ml=ap>C?9qbYbncin2)%Srr&a|8AX`80((cPH>G%WYY9u_UT{s%RlR9N# z6&sp3lWL$jBoiM#!MG%4z<&D;9GeFQY|o*HOmJOr72|Wy2r+Lfw zlw1HZ#qG_@4WFJ0QY0cWg`Uz+c)6KG62ciDUms7}O%Nz*=C%Ii<6XENV6|u)xZn=!q7m!*Ab?9eZMuE#(&9c=^6YhdC~Uc&F{i zrrI&sCezr8UfLwRZ=Ff9H0Rn5#H%?a$&)1~@vu}qeQnYU)SDA5Ic`RF*D$}XFNe8= z(klloH$iaJKQqxJf9=?ut`+uVG#-+BfN4zbBbnJY(uK@i251K825A0=NNs*^u;=nt zG0p(&M9tg!fN;q)bw!7DcNxTy8x7+WvjDk3NDyw1aMAf2@eCmAf}w3TPu%}NKrNPE z*tSdu*ZU|-i(*^GkxM!}Er(e{g=k87kB)N#E0_iGF)#Vl748Ys0oOQ`PKOyh_H6Kp zF{h5FIOtYNCinEgNH+Z^f45`VB0fyP6x-#`ljos2$HwQvD_XSU0o%MR$8aKPAA=oQ zyn+q-gEIO!nbI`!%4PCNu6fYu?}{>moqv+CZ1Fpz4#@Z`^X5DI)%L=jV5e`=W@k<{ zf>~po)tbMPt6P+P!R(BtAxM7cjAMCwiR7hG82m)3jG%X*iB=wU5ZYA{lT=GK^y$q0A*AX+e-+;>}_6KDUTo~ zY>~AOc+%6cTnLoWLo)TBGvt{vv;#&hC4?(E8}jEBW0+y>nnzQ3+UY23*(N?3r2PCB zDLO%F{{ejeI`1jTb_Dh(9A*+ffM zV<*4q0Z{0)GLO4Du>r*6gD4$wmBG|g2Kj5j2!?GVDbu(f0$43NE3<^D0YLSS4dw*l zhzXySTKinu_j+p<9UHCQcJxdv274MQqq{u}i{s``>vsF<1l-DoO>InsMtT?NJNvYn z^`FcL*bO1dH`Q!Xw&sf!$G6IsYuoijy_ma2>tlBT8I17fgN#0Un`Ep*Y_*3+d-7Ex zXjI4c>fnl1I8PQ4>L-K(xxKh8eVRzD)e zRJ(LB{IurIqW1<|uH)#=ynl=qzioE5iF6D4aI+Aq2oC18>@ z_5#Vir{f~KD=12=Wbd=8@Ll%J^p(W5n*5kyG;=2|@LqrctAss%A7U9G$MN~Z6T z4>EXEDbZ33K3PLI<_-EvxmMxroNAob0jP6s^U*y^dTwdPrLg-@xb4E{11|{8E=ZgM z+?e#=My5&aJ)=K;rPhH$f&pJ!ukE{hECd)Gkw>3Tu4EOa4g?aiVtvM|D3RQPlv77J7K>Ew?6kjgpz4B`d9wK6D+&^;mWQZf% zebj8dh=3kThuxi^!`4JI79S@WI3wdR@PkciZtwEazpqFGq@(dv6aTITN>{%fOwe+m zhu37m<@BZVa0GhC?dm<9{1RjAx&%{l8dDoEHbvHZDLx^S?l7n+*7=+#r-{u+u80FC zXDnav1G>qDz3G>JT*WQu)U1rX?+~VxF#S4Z`vBF(} z*GXaQa-!O)3Ps{pTIfV4NF@(^wehP`s)rIvDd^&tb>x}1j^Fx*-%zO3h=j^7Xp;PY z;WFKEbLykTj0`vxa(LyOk_sWFgEw^C$2*?7?!X?I*Zx=JYnzljAQ?=qKsjc@+Mu;t z`_6$0YikS>ACvJl&0FFKbDwCm5vB< zI=xIbjkR-njcj;Hi=x>bf-{lPSFSmcyz*d-5FN97F5|}x2X9DtEA#Q&7Ex5KjnaR^ zd={nKy6^wvu?(faDmEq84eIc5tlVpTlcLwRHxe<7+&zkls6r||c|3YQ_eHE^9YAem z+#t9Hc~~vYpPl3swmwF)KZMKpu+pRspHMpN7HNjBnjq;$VW(1SpXQ&8db!NRQ;Aew zp_SBu%dM>kj$wfso(QKMJG;XNyHV~~%iUddl=J$iN#7(MG`x3@0mcFC^m(l_&W75g z%?*D}?1tIE|2yg{Q`t@LGH^JVrqe&PER2`iE}Xt|MFv(G_}@D6mPlaX{2IJHwR^V= z2Y~Qc?LYtF1juQ$Zjtu2GQ{N-cBznNp9q#=AzhZXl{9KU~>IlLQu!#)o|6uXOx)g$vDY zSOm=9pzy_+wP>r`ay3h^O$@^CvRlxaGFVNg3z7&PnwXGa)HD7vV3R z*MXSLQ_I&;dz)=!9L6h!MzT$tt;gar)^WA>&{rb$kWtA~q1(*pl=F{IJ)-H+T5!Qs zMDzm&L7@4*{LEuUi3SdKc50_oZe9!LWpT#{8^dc>pJmge{0#k=zKJ-S;Kx}kn|}?& zk|Z5>E@U6rLl6efL{bXU70|kn1jfN~WV=p10?J03W=jJk)^gU)I4w1uuO;cP2xH5U zldrW>TXs^}+V0EGZK)|9CO$NSN4Go5&j7P|%N#*&Wx@;EZsdCH4JCK?P$e6;7k}VO zYiprLV5{x7WcZFWts}i{!w@~b!Y3Hn%zTWoRcX_6M&>8KKDhV-YdThC@1_3{2_($g z@Z^WiNr6eT`b&ym>F+Ng)G{tkW|cxlx0$7hzZ<0Lv2qCe8?WWFIp5i%#ZAFHMY2XH>-*0ajm?VL?h8Glnq`E0Gi<9me(;lkICO@2nD>@4EdJ9Jak)i?8gP| zekOw!8Yc(WEd=7nZfkFBa6qay7+bprlOVbdJ%RcOno@hIOK~zkmqt2A8Pfg2IL(rY zZ&K|Aw1$|*UcM@@`Uq_Fi?+j!+dHWX(=Ad#^A&P)!H+2O&B))Mw+AGF3=a?OzpA}% zX8h8^_sX?+(<5wsfB@hFKsvt+P}zc#o=Cm7a>UEh(&Zj}8C1&+;NXn~`MH`DXl5BJ zKc8&Z#<&OYBM|D^##$bIgG6t$SI#Fi@n}lovB<|k|LeVS|7T}@j{nzYxf5oy+{w^H z-~Vyu=hS~T%e~>u&keuC%~FtLqW9mWbHvF65efT%i}5dCR%;h%pwP;AM0W)Tq1*EH z?=l@uh_qE`owZkncDcDS>TkCx+?exY8@4z7$ zt%H4{ylhe@E_>Ufp5YhTxg?W0`Auk9(&*zRO&*U7n|~&amj)klP>wJRDkm$Y_9za= zxj(gv5{hpxabl=GYQt5nN-!?c&L~HkP-dS$^ym-(&)4oFjc^s;clxYFhhiDHv(8`E zDVhk2iFF$8=x~t{T1|{AgKrc=(<6tBXE=YlS7txnG1vXzJE>6XoGC5QdXi_S8Q!Fm zKbF7iD{WeTSBucb2%uj-?5=S$d^osf+?_AUmYwbSE!KuS)1z^Y5N_4ymSohO)OmMJJ$|@$#KAro6wWOmanE^jOWb;xRt^?AiRJn8a zf*%;qvLq{+38rmf)Qrp?2k&`kBh9d0UeTJ*xPaQn3YUf#k(<_&3Iin9n-5k%9+>Jj zl*so7&YHwdKeu}MrE9!bv*KAGzj%N|d+E`US6`ioP2-m=X>tSexNP;hWTI&y1+<`l ze)5&$)x4&HJO{FeSIIl- z|EI2Ve~Ri#;`k_wF6amt4GKaO6$mv7K@r8kRX_zsGl+;91ymqD7SuruK2Sjv3>eHv zkcR>pAmEFL3Mzt3)#=*W+8_6){de}$lf)8hBZY`_@45H%>HhZPboXIboxp#i-#dFK zU3(K!a@2lkGt{G)X*<+KK<-N0cen!Y*QIm9bhEvN#^gexU$1$zO}a{js&Y$STj6=j+1Engk}y~YGsWNWn@U*IeKB&LKAc_F z-EJA)$dELv3H?3}V@Az`cH*&K^J0E2R0nEl1UcIh* zIWxIoeO_I00GFa2RtK4$ZN->qfmaZ;&!o5DoCd_b)*4rVZI!5KqKbPwjZZ{_5%s$QAQW>iW(l26OPewP36sUT>J5kFv6OVc%N%Jl@Z zNgkjj-kC94Cs4Ow`c@V)%-v{R0@5i+?_G;atTt4$Tb`ywgA;LyDbiUg?5JB{(zMGo zgp#X-=V68`ud(^tiFEi17f>y{F%HO*r|G40|FKiy02aNwBt9N95$6VO>ww5a3}L~4ZO)I6cavY6#y*;_@m+@uelm4|~n65-#waOC>2!8jE=9Qg%i< zK;^sNviOB&@{81zgUWc}tQMBjj|F~9w29Uh`)WCC#Q2sX~i;Y_0lRHue11V7sENyJ)&Q2Q!OgR(g0JlaWYi0^Fr-VF5{3sIBehX1q*n z!>ooZ-GoZvB1+?qkufx4+R9nqDHZ1)BJ0KK2!k|B;n9W+xQwbp6nr^KfYfs`MwlLr zF{4hKXUq=F9+WA0bv&?6-wsXC9UI{!ZHa$BAy%Pv?tnK1wP(KJ1zY`k?N$I`pf{1# zJeCs2TXvAv2X7%Lo%v^w5lDcCU$OFAy#Tm=ZtaobAk!Sm11C9u*n1?^N;KF1>jT8c{TAuv5^Nj?vvZeSuo|I^$d;s5}-yQV#)@ndA|!|rcqVreVHy~BuygYkdWWZ z*xsCjNWPE%xhKz~WPrFS@o2OyttB2vkG@Xy$LEUOnc_VG)&ZYR^7=UWd8*! z9kn|gl7KX~ZvhwFsp{-(E^u$Dai<+WnD^Jt&aC6XJ~}omdq-C8zFgP;JveOZpRUf= z>guEyxHHt%S?QeHQ0|JZD~t?`o~!ZPO;`GpWZFBA8XNA4m93#TU@k zcN)j$>}S`<_2@8sz+y__u4a9;YX>dP2@BWEzH8uS&b zubUYcu;H~z;^oHnJW`-62OmzSrkXJX-A8NA@hf1<41{GWugnb>ti}!iE75VC*B-gx zK)if1Vj3L4VLijMjrT?-N-vw4GC_1IY`aa<+!7w13LN&B+cuh#61R^a0N3y(pu&Fk z>t!}!+k~BeE7}#Xdk)6NV{07^;2pMFgond32Gc6RTaHB|`e53>I)#qZZi)CE^QGV5 z`8Ug;wJ9cC-_A_3e5QL_p?22XXIz&4TjL8mh{9{C+tXqN5I2kY9bgnL6vmM-Mw%prP zB)1p1Gi%&g$1kkUch3L&KmYx&`2U`@+4MXAFOLZfu{k~*6Zm~a^?A5Dp<2&COI8XmQmoL5k_iO(L Dw62j4 literal 0 HcmV?d00001 diff --git a/Scripts/Models (Under Development)/Nback/results/ffn.wts_nep_6250_lr_01.pnl b/Scripts/Models (Under Development)/Nback/results/ffn.wts_nep_6250_lr_01.pnl new file mode 100644 index 0000000000000000000000000000000000000000..81c92a32c597bc081309c1372b620eb3e834e1c4 GIT binary patch literal 29103 zcmb5X=TlbQ((VglLYYgCaRga#nSJ zJb%TYK-dYCX@*ea|pP_tjTl-DAwTZiG55bZ}U@)Zu^n&ngEuhuGKg z$xq%T9!iK!i#_=2+4G=_E)LQEzyD+{_?+snCn}87uf@K4^&}TTMMm#@?%>X4thF#ciu%Y?-EH_nME=7V$Uq9{j(7?-*@YBB$Q9-HKdN9YEP)s z{MF8-t_}+eyYheYpZR};I;5q##T6$nib`GK;2{4l>bKomr9X5+>}N8mKjcrS&aKxh z8Ay`G*T<5zL~NfWK5x^bW!zdP^{FIhnpVI!bHBzHm>f zbSy;6rO(oODSmKu)15>Psn)J{^2g0ZJ1za;dc|9M9i&-o?b2c$lJP|!g;8W^^~*%WHR!C-^(R_ zo3trp8yDLp`dAI!4UM*d&ZPLY`U!~xDl!Fe)u-HqaQ%W4%J!Mqv zD6N)K`K>ujGe_(<+UTy$t8^Hn*^;;N;i=ue(qzfETU<0-|M^S3*o$6Z$&uvu(r)Q5 zEj*zG`Zrm6EEg@;?z0cPw9t}nZIl8g`BOS@udVHxeZ^T?w1e^3QsNoN?=$jCro{HX zCOzx7OSWWLTh^}B@i;z~S=*!!oBxUcGdNrFwMgq;%K(>bwLUGn8Oo?f7Wzrg;q%XQ z9OG13+^<(EPX+>ne$Hrr$_=`8aI1Fk*Ps2`&wv3i5b4s{8$MEa7%bYDDO<|}v{a@6 ztgSzFC<Q{qD0-=0^r`E3G`gZ!|htkUTjZ)1oomv{9`I(D^`+OwQ z=5AoEufo`!Zc5Ib8``Wh)-x$zvO~+5Zj<#zd$vB{Rt2EX{w&>6@P-&{);zISKfiPF zBF31pn8KexY0o6uxcwQKj#dCwMmm;1&eR^fG%0qd=ei48w@kj{ zBz`K4mpZ9m<`=2cQvdwI32n3V_cr-=9S=7XB*1F$V>&)-kWp(7Q=f`l`9gnNpWd8x z*ZjcMjHgvwpMsS>Db}CXsz4$jcb6+{Ve4f0vz4#HjP<030RMSp)kZC{v>|Bo#hudm zim%(9Zf?5G(7uZuKy)-MT#i2swt8Z<>a8|n{YES3#M%{J!N%_q9x`-WyS2~+*2d%@ zoqCpNCZ&}!yW(z8_!0(O5g+v7?j|l_dqAZX#B<{#gAC5^zbb?J@3=W*_vb~Ru26mu z%y_~nblDJ`KQZK=(a-(K4KM*mqdbmyW@t;g|TPr7$+fsMvEsNRxs{hp>H zQnP1)A2!bN;zpHg&3IR4eAb*GwXvtI6_R%&$p^qe<9}pktIkTV_G2AeW?yQJ^m}QM z37KM@d7)E;cOG$LYwn(-(tF3>M4?uk<@(bJ*aT8(W%S>!OX&$x$rmkNBK={pA1t@X zmG~mhNq>s{oKv!IO4&iKYA<)W!7vP;z%Bb_Wc5`q?w-rtC%mOld(Zkyvqdarzux?L z9o%R8NTp=I+^$p0{GhDy#|j3tex8|X_v*KG*JIo?>)c7{U-)jD)>%RNEgt;ltP?nT z`4QN1NJp0v%l7Rv#6!Kzq)D%#!aVIGjhvPiv9~YfqV0=x7@OOYi&=V}KYS~*)+mJ#y|O5 zVFi;YZLpmkM|jwdf!qCe-q$(ngO+H$e=H$iBlZnvEdcFR!Ct6 zdB)c9B!3+vqDJ47xbJAm3bpUfL%J6c&?j{Xx45T$=Z#KV(kkUJr=6+)NUd#w+P)Sf z{lmwb!B~M}tN|@MDb0b~v8WDe(JP3oF9=N#2;EF-xf_H~EA|365>(eQa!3{9`Fk6$ zSEikmZpiY@EBU2=Zjl3KJeO(P(~szkVtX$tW_%}6 z9`ELZ3aEEVKTeunVNefMj7b+{u6f{4vSj2gLH>(N8vYq)0y!B{spW|lTwJR^lCO|0 z8Rt?d_i~XwcOr;z*MV3q2*gwf0q^w>e8RC{g_WUWJAhd)en6i@;>=q49kd@-Mm5yp z@&*~aB6XIQy!rwIlyR-EmOHSmMr!n@3Q4*8y?$NKV2TXUz49P1hQM7X^kHWNJdciF z2|?JqrSbw0s$vqjlF{Xbosx>xnrj&WFq1+)GbEtR$M)!tHBmZj>5TvBvs$*wq>fst zS|&fwa$1$O?y(kl+##3#w4AkCzX$jrjR+%{@9RUzAb;C(%{nPL(tJn1u5tF;41((q zUP!^j3lJtT1(sFwB{^0 zsaG^TnWIqKFFiS;<(AB5+VN%}Rm2TD6e>W?^UTJryEf??{doSn@qMM6+&^2qu6v#maa}qV!O7=oHCdgyUyX zQXtPmvVHjbFIQwNy$iu(vlgvis9EvcVh}>fTPqb71xuB*FZE3{S!(iX7^i3X@EL8| zAj9V*XS>u|()Mug2g!CdmCaS|e7#{py!lixsVj%?W~d?{&O);ObCcvy z;jJg6B2pS4lNA^Ce~|7+^G2PL+aPXWlVsjb4bZhOWOG6mOicHFvw zx%@KZv&dQML&><`B;Q($@d~~6QFI~rB(7mOy5-b}F4MS)U4jW&iOOmSyQ@dheKHK}<9Lamaj6L*{ufZz46x3<6sz1nSs zYjFRYd78Yl#frQrl;_ zxrxumjln){6lN;|SDlpNn80yow*sSX{qM!8*MG(7 zHWR1Y!&7&dINf04bmxEK^#9uB;IL6klcD_IX;d=$V>1(|!?<%KOcI`!T|c;NN311N zujGfn?@LqN=TVO~eOyZrketentOiRHCth1$WcVyRHjI5?t5sS_6&URN#JVAq`fUM6 zmvC}`bjo)raF=Eowg|Dp2yNN-36R*JZyW|6bxKCF>QBjD@sU{3cI%~N2gHJsT5_$u zBk5J-DtAMfSLJ7t^dF;Q;M{7Ncttj;IT&WzAiuO>rQ|!Bbn@>KV9h_bja$dZMN^6z zS{bpNh{gPcu)!!*7G00QF!Xi@s;<&nEDh0&qXE8w_*>=sJNbS9w;R;_Q+s*P#Q?P5 za=7Qu87x$$c^3SEXqVi(pEE2SkDy4d9lV(=jq3 z-PTR*O^jc{l^`A|3)kVbG7A1FJcrQ$?U7OvNRCW6uidK;lHI!vCHC-JftI~Amn~1- zv_Kn{O9eChek@MrzMPgW>%y`CDb+tq;RJh$r>|Bmy9wXeGW9wpQ^qz*rKO!83G#l( zq~EXu82=NF&>Nv_DdE6e%}Yo-ic{P8tyM=i5NqG?W4pChS{6omyKobrF%ysJYj>g| zwg);IcA78QV-UFgL}Cq@j{$fblKibY<7nu~Puef-hR;ek#NKzz>ppX>rQVJCfHO)1 z5I2-fj}n&cz=Y-U&B}~e{8AgQNVa82Ps+7t19B?giji3-9X}961QW2|H)+lhlpfRi zkK<APuo)s=}A?3S`9X|Rq<)kE<8H6Cc&TD})b?b?nj zTP!AYd)4O!k1Z|O;Qg8|NS`z^hMG9k5doHMYy%VZZK3-xxr3mvrOEfP{FNV0udNI) zRfT$Mw%*FRWwxzD|2|dyO$G0e}==;J=g0+LU3V=Odk&r8#~;D$+gg0~dK9A%`}0~?(Fqe>~-mw^qbmoKK#%Xu#!k`Ca6m3 z0BQe$oszQ^7uah_R=+94s<7oROBy`Y-YB!XjfQK~*)>S27Kp%%raAoS1;VP$3|~fq zLx{`T7tqhPh`3u(&W3LS39&-}pdEZc`XgE$<2bJ>06SNY#3z4^)+BOm9eii1z(V;R#jq!D zp~tmloiuMFR)8ZpbJ9b82R~qd{Nbs6TVE_Up97@+&N2EOHCO`wqesWw&S16<8h!HT z17tx53NXAuB-MFRvHhBNI^E(->8DJ8#aY959g>r!n)DA!ZimHeqYfa1va!=2w;lXJ zU#R+=6g)}%WY9XgG4k0lM{~arY@GWx@+#5Fy-!P_kz%bB6VDE4VWjk~Fm>3YC8wG4 zUgOHReXEJpVL$3gijQ(grtu4Pq3I`m)waheT80o}e*dK5At7VKXau}=Jx3=E1MpHf zCdDaJ{%pk_brzST)|H~2+l3NupV#qYXK3hgvEqydqn(B@Dxk1rs<5L}cv(leFv^`UzocyQZ&=x$e;3zi?&Mn+wJjLXaz zi$E#uI+ID!l=aBkL~m*j3_1au+p$n=wG^+x0#z1Yw%^s3WfyNS)IMfg8@cP90Wd1+ zyLQEsnfd|S5-B+i{QBLc#JZKLEi{AM_e1uVbS}&(F=n2}3#r@dIIZ2;u+0Mn`X(ZYXl@z%kg;5AMsUr-4Pg3X->g! zkXw;v8J3EWfpzZZ2%iq?9AlhWD#agR)m?j)e$-*mL__QA7RFu&Pqg7xZaVZyDsOCh zjc0lZfWZyY;cQ@2tGRJ^gAQ;2AwtW0Hqs~4a8`9jfRf068G_z*5?_{TE@zsa2YJ@* ziW3w9P$!q(;6wXT@Vo5buF!a+BsruRV}4tA;m$Mb)cqZxrYMelN_|!-70Ft5-c0Y= zy=}}i_W`rCZ~1dpwhzw8wO&M%7_uI}H}S%VI<*Kl;Pgkr?lj$ew-@zD{h=u_vFSP24=YvW`eDIbrAw`uV-oS6e^AdG;p` z{p0@WU8E&rn{P_(J9l$tB{m+syOHFBYF(P_t3Tf`d7~8yV%|s@(F`ZjwbWl;TAs2Y z(+Y<&f4n?1pEu%zv_&T!$pQr=M(N%qt&xvOw)Bg1SijP{xJQ?or4hsl>mY^$$rF_u zKWp1EAv z`I!tHySCrjhbZYe7yl86q(9|q`7|bt1@Q00d7W5Cn8QprGZqsOjoS1^BoZXOA8;>suw7pFlj-0dAbMNT>a+ck_AtH+QKzEPt003O7PI zd*t#_+(v2Gf7cr!8_P`!C|JbC+k_B+Hg6+#}JZj z7Q(uLZJU89=Zv2S^B^dU3N$MzKbD-{Xb|EqdAs$yS%j*w^!x6$ep_A{hBn1vxzua_ z`FZ*sBU~oQ1GK))-*0qO>eguAe&eiVuBJDcC+)AO3Frv+z^#AN8764y!xK~SxElY$ zHMOun(Mw{LKV|L+yfw;2Df=v)lScoeC%tf*EtMXU6L(g#jS2gGbq??KC(uEWfi)Pr zL0X@gKrN8Ji?!yVRJ^ys-s+!|kB*b#G)oFG^f$j%M$~qAtx;;1CQz)q@ zMZ2(1(P>1+4@;W1$lrT1bqCYG#0Z_JnSq28*3%=}qb)KMD#+C>E9KjZgyRM%`qM`T zKB32E=M78=ek$1I4H1t3Q~| zewVdpWvq0AmA8bTPLLUFmagXMs4*-#SblFJOGwdTs69IX)c2WcszS3@s-1#VrSc+) zc}Z8OX*v|MSh8mIm-*tTd->z*3C0RmyuVh5_;GHR^;(CVq{mxo6OALz9dW)0D5|-N zj0CypWTY7<<1SA;ru7VqgBq`D{;H460k+uqYSTR}_b^q`bhPGfzmyDO?ZYy2`ID9$ zn|JOvsidb8N74$B(V_1sfK^L2U%TeSep`qYkf}M}Ib> zp1eVB9C-)h>bJAtZ|=OAe6@5YYWDUN1G%R=wDFR5+?fYR$F5iMH;q6kgc7Odc0-^v zFBo;9Oz-lNri-T3TPtnpq=anw_L#{P@Owd^E1aBd+N^zAwFFlcnoyA$k6Tik0J2Pf zZBXX+9$A}WsQQ}x0DEN~@K+JPd|mkdE+0q8IB&IinKo`YV*-Fne|;=%tLUV&GVtJ~ zjxT1E$WI{pM@KgW2AM(MMh#aSAc85Y_=z>_w^Ff_fFKsuER?*62%?jT-lC}J#5sMD z(A74n#5xm{N`L3mau?S8nYNf4K|C+$%&%YP%k5FU`z|3M*UdB4fVB8Jz0&VwXwp}o z48iqn@Rmu&Q}>?`bLHU_HORx7#{^|11t=yF!ZTP}wER3gazpA9PszYRlQ)h^w-dwo z%_OK&FKJJB#;05O)V_V>mHb}Ig^|(I_X3V-owtdD?wBym4mxZMyGA=*xIS$!vg~^q zSpI^+NPI9cbF6mI5AyR~; zR3|>t+4$8aNTeVv4!l(Qo0e7k3&NkWW>&Eoz2d(Zx@0>rZL#)Ao6qsI>&EYINLZ86 zU*@YsX5Y_2v;4Py9z1gw0pwp9_c3C4`uz~8v>?&M zqvo!^A{}fsK*Az;D+z}DeGv&b>HxJ7NL;A~>@ED(LD}En`PxGAeV1Q{KWzX5kkgXG zl9%Rau=V8_#N3Nzy8t`j8*@DsOV5ywAHJGhm7>iC(|a6cDCO=M_$5G!Pe@_xIq+#a z%WU`+?O6*GY}}yzK8HD_)!L+`((1HZrf`S7Ry!6;Uiv%SZhNoE*oCyJH{drr2ny3> zEY!@$^`Haa(}&?qJDrixJJ2DfjdNZ~*XIR+Zq$D|zCdTzt_wzKpcw4+@OCrqVWyfT zRqNJ4=)dWQVzQj4gg_IM0A;i;9vqSW^(fa>+Wcwdz0Dj&DlNZ9SX6^6GnijK`*uk0 zLuBL!{k16Yd9<-bC0e#x%ibrM6P`%Li4!E$E`zbore3GX0i5e+YNiuB27b<_^r83W}a^^?gex zQ=EfguC7jo`7wR9e}U4f^LYipQsy43HMszY0`FbH}r-SKO;Ey1jgQj@SlYHs7^xW$`g z!WEO6Y*m{4QtJE`&@Wr>{zZN+$|Q%<(S5gc$u9W9_IZ}}MJoubS?el7uRoZ&#n3XW zI75qqK5H=-X))Vv3Y|U~z4LAnrXO7ep&5P9ee}t@^<-JtQYto*#A|gb5FN!alPPAA56V__Y!X(Yn7DJfYDBqV{3fJlR z5jIQBP00)2YeM(h7M+E(r_Vr1te%fP)`^waf!mxXiLWu$t7&PLz2o2YGO(RV+KDKM zB9@lgcbLG^df!kj%{V@9qSF1|q*<9Z{&kLWS|@#H0l2-#fB%|eZto~Xpw1R;OmwA9 zOsQ1mV+0|-{b9ZHaNw@fH#pZ+-UCv2U08^wag}{l8)9C>5_P8f96bgnOu@NG4u0>wBB-xuu z1+Ol{luhv`@tsW?>o7Yy3XT{HJ-QO}PDb7uoZSN>g-(Pz!4ZeNBR}iFEwh5r7lQ5o zKBC-!RHWqrsdwTAN1IaYE5h>Ig>{d>gL{nT`12bmv4Y;7NYmPn^U>SPAp50r4ff>H z-Y+EgwEhg>JbT|o)JCK81v6c^jqNZucLXc-4@=G3et@En``K%xIZewX>!l$n{pS&a z5Y3a`kJdvS-m~yDcxbgkK3q)S#wa01NFZ0n?jHy>-NXEg%sh)sQpKV^8U$f^OlG(c zU_LIDw6d~e515-sMR<_*p$C6!g%zrWtN4m82GmV|)&VZ-a(G~5>cmBz-G~4jK8-g2 z>@0m7Q{fPM!&bcU6JJyquLNn?@x9^ZcIfHLlD+R-w2|RcF&nkc6(>NWYdrQ_2ifn( z<=GoZr00>yNN!eU@-gj>UKvBG*?}&aC6w4jK#Te0V)^R#`hwVaD~j*Tq8QPwf4!vL zRoLgnly)Bhf^AohnoQZXG!TlSS#FAX>-uY zd|(pewI}ZkZTLyv!B^-Z>|rIRD&{XW*5H4mrwQFE`hDaE8Z_e^ZPx1t!8JPp%a(7F zqCH#}V%mA@9)h-G-Ys)dy*}MU9nFA3?cA%Qf%utic3-!Eg+HXY={U2KX87ZKKwMz5 zlUBIIxoEyKM{Q?#)e#HQ6{O-G#{^8*^n!a#;l+Lu(|3$jZV_*lyH~)vzXgVij?ktA zDLHJWLm%+lIR=xnMYF7>T7FO_cig5Kp@S~l$jTL0O*qIz=1Jo~C&L6uXWjN6ie`Br zA5I*5#-*E#tF_1H(!0H666itZ0CkOyQ_Kz-Z9{UKlo!d~f*q0nYzi=n6m%#IGO(2ybsCjv>!S<&*3PW|pLkF<1?{(H)` zn(oUGii7WMD|N*REY|P}t&BHpVFy<6V#6@f6LpyhmWCt2Y;AML@7V|M6K8Df`-@C` zL`<6~n)3F%CjXpZ8he@cAJt#?-QG~LJ0OQUSQ9gejNN8O5)a+Xv@$L_5^YA*{RDC8 zfWF~0K?TU> zB&|(y)?b%iTV#Uz=St%;nCu#G{jm=2(Y8mXh-kbM5SK6?_RTUGq(!IGvE4Ntd*PDt zh$-0try~8-EIyEdnq}fb<`TfuWNixFjSQ_-y1$muLisu&#b*s-*(B){fcl&=S!Nb_ z#e}aI+Ln>C@KW8I>*!9Q}Ig*9S+;fX!7-2auIIM$dm!(AulDTflfq3Zp7k=cRY;E2APAh%Q zB#ukZF;K^j9)o}FUbN&|+&M!an*V6KmioT^NEko*^obTou5$b>PyelFWjc^+HlkVA z`dl+qa(tb(X$wcN!`qC>FHmzHFbYK0o_G$;2mXTzq*RPi@_r$L!A^Rt6(4ngx|3W| zBY&4+gJ}nIu`8OhKhV1M{JHVbTZb$^9SfK5A&VFU3(Zv&Cly*xwANYsZyq)~s((w1S*ab+->w48 ziDR8T`~l@|7BaOwPzG+DkzCET4mn}`YN#L&8jyxlQqN}X5W{3cRB6*%`~hbcTghoU zBVFXML7K+#{>N5p+YS@l598k=mF6%p+$)1SB9<-%;SfT={$+^D>N+Ca>#YBK*neyY6b=CJz2%{#bHX@~H1jd|R*R)BD_^fTjk!g`g-; zIPk$NSCh%sX@S<>HZrQ>{Sv8vjqgo7kIKvwrkw9)vhfqn>yFLHFy-9`*Czp_fcZW5 z@Vj<;alK;DL0<9zpWjTKM-L#;+U|}(s%3qDtK^)Hg;)0_GO1xOKiYxhq-F!M zqK(2R#Bu)<^4Fljpg*kibtf%r9E^aO4!(a^GF@a#L+4UfzO zq>IJHUD(_#oSu|l0Ze?Fg2MVFV}a|W?kxox_5J`Oi zolx#8vzrV)L(Y2XcMmoqFpYLC@sPY-mmp1gE@N7TV$Icj7Q=cXFN4UALsG&_vR-|L zwQGz9Kk<8kLn6a`DLH4~XtfjGfo`fk>~aO0zbw|)_k?QA^JEL{ zSEtj9GRfzqP*1@=y0~3{@&}wSp6Yyxg2gmLI$uXiF{#2@PLlBUIj_`9vv7Gxjg0@wi8EVHLAz5V-Ws0fn7VzG^>Jl6u2!crq-f1ELmS%ldii>3Uvu^3%EvC-I1zWVi8?+Ac3e(()Ftv&$tol*^j!XNDyI+~>X{(swk|a>NPEH1vtM1}px6 z*4|k$zeOO5!zLcyUd4oUV!hU!d*!Mu-M47@spA{p7`%@YCoBPOx@^b|`P>4BFG+@g zZ9lZ~@TN1GYg+!J(8G`GN~ANY)j&9pAygkSqi3=39U6O>&x6)BzYrAegp4Lzr*_^X zr?tW0b=KjF(GD{0mufj6Qu?5PHqdN%r%1oQhAU@pAzQWclqVckLIlu(X$Lh$@J#k; z=3pmeVC>PO^Uq$$*qNKSYK)~)>*59CYyAD?^>EYv71Uwlt4%R7L&pgY9B`BZM6>TD z0&EsE;F86ACC?M6R~vGQlRD(&c;I=HofDR)p5>Vpv(s{t(M@lsEzlhxmWAqi*ki?Wo&lcGT?;Pxbj9JL(SnXGdKM zJL(qd@Zqxz(cbKL)sO91j&NrKyPl?H&hsOPU{P-3JQJQhBwz9TEi_5B)2cWVUNv#6 zB5#u_=#o#J_h0CX(kZrB{og}ED}qP>H|PA6rt}bws<$p|(V`8cHnz}19Bc2tVS}dW z-85~K^3RvF#S~-Hu1}>VV=1}I){^bJ0n+cQTy>8&?$)ulS51B)FZXMw_QQDqz<&FM zP7eUeW~fntFgdYv73mMOOP}8(t{XYQzVeXW#?c5Z63<3yTld8+82Ms9jnV8a?l_Q5 z&LewdD*321v!OQbe82&yeufIleyojVk6kOR5YsXXU`pTi+%tA5KdXHY6H?Gh3;fvB zEY%MQLQ}AghiRF6ngaE5nY{SdFYml&g_9{*)@WAxw@crr7`ytR$nvfN*LHkzF^=Ez&~<;!PdBtE z!v{jk$MUF`MNVYEah-l+9f733d+z0aglZQnjvte;2AK`!nvtnzD3w0Sf#8(~gZIUo zfZ7oxW9y`Ag()+&^QKN83#D8mUuci7R9xJ%opDj@o!_6J-P??a!J-qyMUAq}o2dJB zR63r$e?c8px|H!wuq)V{{oIo^n2rZT-EZqLl|c^MpeJ?uowYeqs!oz6a#$oyzz%H( z_j|LHahESf90=sn4>UlZvYN{7al^|G!lZ+d4?Ke-Z9euLqx4#A5=GX&rKe+-o13LX z%8aTj*z0?)Ec1IZAZ_vS$o*usJk20It8*jnU-|=H`<^}z|{wl(airchLJEXw_ ztJr2!!SDA3g?(G-@N>D{+OmJ%Fa*Ts-~pmAq;6*V%lEqI(buC zwt^Ot{r5|s)PBD8Ad8rfy`)^rju9dDTb=aN) zyBEl4{PmDfHeR`C&m*?Ik%%Z)n%p;QoeaC+_PXsabSyf|JgiYn7HHP&$LUxCvAgJ) z_Pu?otf*&4NzH}pt_Fu>!!BuEfz1l%kLug4BTKY?E#SYK5TLd9mzsEYf{kYx&ERcA zJhXpZc@{Rwl-ruKXcfdzk3mW-A9{e*+H;za_+@F?qQyG8jH&^-z$1pMQ5;#I72WaP zHTLSWWe7)OSDgNRftDJ-qJ{?JH*TW(Y_KlL(B0JgvE1(r^jEp_1ZzBEZ(pQ&D-StQ(37%}|D0eNnARoY)VO2=weB6|3i*Yl1gko??G1P=o?l=aalujsVgcT3b?d zb%EJzJPtnW#DvSswalAY*vqc%JRcQjO7Y<1+INzjm>p(nQoWlr-yw4^z!&upTb@!r z0XCy3@?X=bqGrx9eMN3MpwpS)#&lmfPlgWdm3%Mo>ES4U<9Ff;>YI`6=TT0%8|TqI z<&tcq)BHNPlzgDk$HO`|q`;)}@zB#3fhuJPS)WzLjXnqHN!@{bDwRY4d$HZ6`0_rP zTH|gcV!&DbLCl)nlW6{CkXoR1cX5S%RtBE?=;vP#?KN18EV#G8XMvGUHmk!m(cq1k}(-4NQ_Z4OVR?7~(Sj32NoY6w$ zwfdemhcJp#wqNng2RXO+HFWa+ql-4*@#B+|)UMvH-(6#wY*z-orfCA{pSO_bryRu%hKOHf(U`_lN9bdZ|lP2$g zmut=*Gp4w9&1sn*`TCEQB+Un|?xGjX^sk?a(65KrN1N7fTat7n@1&&rLXcPS4B| z#@Fn6N+yyui;#KyM)Q(C8EQff(2{KSMj`P>kBq%tr33sxKJvLiBB_c#^x6V0B_G~{ z*&5P@);!T(fH_H=76Bkxolc}k4TOk> z++`Cc zCo|8Q`<;>MGlx^Pg5RYhoh&luFI~yj!dp7)iA?-!Kp;O@06tIHOqjn8osY@{=0WS zFUf>(9JfTI++^1aw`+|HNXbC>GEEMuyC1&|EEVxX&m(`C^6*yh&<%S3?%)=^@Fu%M z?CWtE5FoFZ7&bzIosjQ1n>NfH^hiYn^q81>gFaTX=99nwZPMcH+q60fnu65^q$)~V z;8_`s+$?`xmT|b3{JrzyV7SScry;}P`|P~^3C3)lIpI!r{>gc)Opv|0Txic_W5NPU zrr+&=MrsxsYQBE>=J5#feXF%Bb|b{qeGQTcK1&Qnb!xywePW!B2>K0I6G3m9a(K?pv^&RuCjBFER6LFzdK~ zD`5Eu#P$zc(FN(QDCQZ;txC4*Os`4`O)(OsNd7)vGp~@zFtpM$xM#Uc=@0AUIaWf~ zC1SdbVLEY9dV*nRUysB0NHlreix&FGaUJr#L>0^m`_j}`I7iX8h_;X`+OY!p@fU_( z1SkLAK+sS$D38AO-IKDnMAD|`Q81D=80RA_%I{E}oWae?j*Wd%vhFrnnLfQXpVaD= zVOHAW3*?L3!)DM=aAn;A*8^fIiYo0ryqcVYlH@|IZ+OB+$Ci5Sri2ndVnOUjYS~KbbtW3T9nSk|U=U54Yl{}@{9fH>%w#ZJoA;A-cvgPpf@KMY z=w!y_;AS{vilpVd9DzTz1Q9ol!M;Mnc^;$+!tEPm#^wUz_cu#pcgRZPPh$||qgD{_ zDq6H}6-k6NB0L9lb0_~sd0tob8C8ed%8 zkELwGL9OSZJd-97y3LVKj%w2@%TFiv`Xm7|7I!s2Lw{V6;**$}{t%llR=y=a^s)~A zg@k2Zi;$<-KNOTe2%2m1&lmlf5F8Utk;PurA+>Lw`ed@f#aOj(dny|#tyrr2GW5|5 zmMdas*f2KEO{FMpv!V=n=PV#)nBh)6jlY^fg-uXATE2spHH4gP$scb=w*>jR!Vg$C zUSvR}rslElJ@p}*{F+y#XmT0Y5flHAGM4fjF4ImpisUE%9?4gC9VhtgJ8obvmxh-a zEMc}9Bupfl_|7D7o4n6!%TtE&$5W;sg?Sp?g007_NHwoe;0L-tpFOnYlEL=dv$yqE zxPFZ=TLQbZFcZy?MFkTt&CV!SMVt!B3!=CM+BC@?$-~Mwy&LjvO&3yD&>b9Hq!N`5^SRT&&$ZJdP z`LpTiuDLsoDB7k2nUBo(_`XB?qqG^X1D{Z7h+wXN!sq>LikRusF5H@h(-K$wJ|)F3 zjTCV3CC2|ahNYX`jz3NAjU#>V=WitJ@J1}6ol}SEn@s_FR`BS=hi-r7My`FHkFF(-fnSnxOaSjLvfaq=bY1+?L<>KQGo{Yj2Wt$xsvux!yV-Cu8oU zd%~D?-KQ9{>cmnLouiU{^#+zLe_S=Kms{0bvIX?-nm0dhO2+)G?V- z9pszr_QbloT*gv;lIBmW4dP*iD@)={C1B9_&UL<)8fE?vnrY5rdHH+u2)IHN8B$6< zuKgd@v1}^#4qt3+_ASCrTK^V-0k2Hb`;zR>ux>Vy09V+1o<4njJ$2sU-?e_d8}Bio zU1G}KOX#g|q3H+>B%TZ{JAy zSs6UH(2p!q&gkVQ{a-Hg)5H>)DO-M~Lkb1n#COO1qcb7QWY$mnFtgdSP!`1S%%1Zi zmI!;jn9Ve46M_<+(5b?pUk|Xa{hsOFOasa>BYkWydOZIEFpItCLOfIb8BQ)`H&2`Y zRWQ^CYp_eVmB3|xkd{3mguM}RttW!#jXH}YGY=_u1t(co)}gg|mgFLUJAcR1<*or4 z>0lXE1h1cW9inrEe{r-VSLO=hC`0< zj0^ZgS<*~r?zNt)3@d5X-Ieh&nqa)abNcTtzM;zH;|xu|YW?N@j5K>z=c2G*?^@(1 z*yxivN-@ZY{&~JMK3ues&>|r|IHHX*x1Q#D8U6BgQWA?6)*LG}gK=k5bL#}{sstVn zd*jbsxJTIGP-&Wf&ZX-zHyDZ1n)jT-qa@n!;60sKcCVh-M)p_TGyB~tx9Q{wt*7CN zcs~IGf3nUs9Yk8*MIcnI!p;a6OLBOhLX^zm2Eykgc5TwR6sUegY^M;Jc(5DBH^fk( zZMTuPJmH3*k&z2J_3(Cd2p{t4gD)Gq`tZ#>S259^C zZAOJj{!L}0Q_p6YKv_uKpk-f(hXG4@Zma8@cI_h@GT@%f)2|VCKb6lItmIN(0Em@zL7;f`?w^3mKm1cY)5aQ3%Qta@KGbT5AN^{U*Jn-h~F1L$u z(1LyUF@Q`*O~bR+V{byDkm7`=Br3EcfYRUsUt@%95+?I6aPX5~hrk#KhsXN47dIR{ z5`Ws{sQz;j?z~xKSi(Wu9;V;a27qRJJ)stE1U9Y4y6iS^l8&9wY`?YX z;d%MNb`1GzvEdQ6{j+$1VaaFV3lUOnSLTy$c9^Trm&35Hy@D(J`mD8Ij2YhTqD(ok z*9bg6+8~qfWN;g~z`nck{qjUKnc>u4T6m3soW#Z5Xx~X2FP=OO9N~zgSGM5$dfIup z&(XuIsirC^tV8lHZ02~L-(*3|T5{l+jxF$ejj={~ek%cWVlO|j?l{-%ou{-d(8I|m~anI*sWninHxlX1-H?c|K=0}tSBL|`k=H_~d`1Mm^&QLj&} zKwjdSeda0OSzqafE$oBxYw4Q+czTf2C$!@Qju|3@(#>gSyo3HPkFc*tC+HIB@NUi_ zb31D9CX!rlDld}JIr(~))4SNWOyfS|A@wltg)+Qgo*m70T^oV_?Lb~=a~3lAV}_|S zANOn1R)QUqQA7>SLSQqBxw=+fl#XZ%@&phB=EVYY(>BM|!Wb;6E;5xsR zyvOe{Z_{~Tt++|BeWulJyNtdO138bNliK?zWRdy`SKwgeLI zi)@9$y&M%s=byz_PH-D1C}S5g}- zKbJv!Hff^>OXU)F>}ew>rf8|R{(S)z+sJ@2;%xfEK?D7_GMTjOfQ);v5nyiB68yjm zKISuZnUxMSYM=q!vfT|qG`fuUbXqdA?Y1wr_-5!`hMq~|#kbf8>BU|1p^g}203u); zPK|uUX#`>MUN-vb@Oi^fymc~S$35Wj&kB?23`G7jp|b~}{L!XsTKoxbIB3ac`F``N z{;@VclA5%m43%lEUrPHbSI3r^&hi-WVB4X2ObWCv*S`1%k{67$X7mLQcy-<-&?>F2 zH{7KnfUL^8{3aRjaKcIK6a6Uz((Xe-ZivwzUZMZlrp#mmZZfUXd5O?Pwr4P_ZnW_RdZh!k7^K4%_C=W;FlU=kg@HT`{` zf@Mx;X>uHv+E6SyhEPe|XI`4nD1&TKWa_Q!wFM?@Ifx_5NBDRFm=aU|<{ks9jC^xgOZCWHFyY%0BKW@ebD(QT=cmr3O-Ky2n z`5EhgH&6e^GDTa2?T#Pc?AKnhNBv5_7YBfDFDYff-H$Zgq&?}u7Y(JE_!ws!aE!qr z$9&(vkjlfTGh6!CY>0ONp`+F!wY`oetsz%k^rB&_?0rPbuaaN;wdXinRm|hhEh`VQ zq0Xe1t}_%Ab61eixjgJvdhRY82Jy|?vj=(1*ALVA@RYu{mYy*cgl04DhAR-(PmF<^ zLFa5Rh+zjE@ZTV9PLCl_LR#}B>#5oNItWW|T6ItxFUHfs(f*Ip!jk|bw6uFy? z`sK7Xu3@$DK(6$c%#QEMh-N^=i?*YvM^e}Rh45d;JN*5aEFY4R8c$<{0 zwOY?0yFp0PT_#X+6mzsn^8zDQ-KC5r@NGZ_HFSzsBf9A<%G&6Xd}ojAGUZ0H;pvlV z-w4FVIC?-E*(&<*{YJB+f(?B-?7UM4k&<7~7vnONsCg>^*sXV-kmheEzoWr1M-HO@ zx!2lso+o5stdtjMvhu2>IAi|pksGAy7&*UXmATix0}C&(-2>p+DV~|DPV<3Oi+ce9 zESckM>Z;BhY?m=X-=ZMA+rD4!h@IGIk?$eZrJ% zAiK`S;M9X=)7zo&)Wd11N8+Nc{`Z^S{_9PT|7Wk>_x-OuZ%56Zw`1X{$N$I6@lX6` z&szpBxZi`4!;ceGo#37K49kzrPfwd<$Mcn1a0PD`3c*jY!z+=ET1qF8+~Wy7snfap z6lLr2(;Pb$)8~u%na#YX`UaUC1o#rX1!S=A$BBG3;}Kt{o9te{?&ko}%s6 zL;l@IKo-Ckwhko6?Bo$-bH9r+uv33Jk+SID^EyHuz7%s!Spee|q~?XwJUltd7U#R5 zvO<2XHOQ`HyM;7tWeP>y`v<>dY5zJJhStJ+XFr_g8>IiZj`&NFFR}>%p>+hv5QnwH zno!Z2=Bh}}<(=*aSVw@Y$)j!*1(9b0jo!SePhBLH|c>EF{Aq?D;Z z<1>)T)Oxrok%cOEoefc}%ZPuN$5*P9eH?6clF>6Ri+J;um7>`PkK*WTQ)%Yj1ps4j zA!u-5Em)Wyu%0`-)H>e1p`9sEKQ;D1yiQ!^O(2Z?7rAaEF!C~afk@CnE#1BC8iq7c zXCA+p+p`#V>er1+Hs1#myoLC?j9xjq#aPQ)D8IMUC-67#(cqCz9Yg56GT{#Gmb)V} z3I1H`;-Zv`%tN8QyyqG%PJ^R)*%{xpZ==l; zh-vYew>$Pg%(fF;MW3YdA}F#Kk_z8_xMGMPkwSPdS%z;S3v8^L^GU}ZP`ps7-q5Vi zkN3P^?U!;W4fB38z=T79OGRWrq!|jeLo+(JT4#fiq>{IEI28#=-r8fV`THH?-v8tN1^1WxnR>aW z#y!ShV?S#@>sf2g@BGfW=9){b1dw6|F?PgzWR9&L_-dHt85Z`YAYthID|rij2}Ui_ zJdCp>NHNE_mNBnmXQ8gWDr&wm>2re7Y?UdZ)6XfEes+sm8#8-^`Y+5!1H-7I0Bxa5 zcX+lg>lH0Q`V^R{!z_82S3edmQ4S-WHF-P}zysyZH9fjbk0Lned)nqO4>ORYH!Lwj z99s~v3~uJ4yFGE?Nyc%S5JbCqeJapQ5mT^S*e=e<3V;M&APrvUZ3h(bq-BM{vlVCo zYJWA$ysWAKA9yDlebh9!S;hlHlD9Q6Hf%e;y8spEC67Y^N->DtVmlvUc5ka`r%F1{ zHeVvAiZ_R?-Ym^dr)yy3{;tjbT+j{|-5`!F%4gIKbyfPyXhax-30F)E{YGw`{ws@f zt~1dX#J&^ivD2ytiN%e~MkfFvwmot#i1gPe%o29|S^BpsBoRA2Xit@iN^GoaAq=zK z3~DMpKI5;eWjx(dtrwkHY9^1!NFq~(ux3EJhhs0xkd@*rlkRBlOEB9dv*Ejg3Zj|m z!)9XNx-dQkF~gmm%=YjkiG<}n^ZVbpH@syKcLQm`Xu>pV!Kf(biRRi#4~lVACg)#c@nCw;U?igb(S^ncl;Xw`TGrh(;T8t2C>Q zJOV@-QWq1EB*PH^nyZju{jEl>Q(u~z)7Pv}?_rQ$m01_hpZg+Kn!^*7wgN0aLmmTb8kKOc`Rt)-}B@*o9Vu&2N84tvEwn3*kZ17nESZ*Q`CRJ#$zDOWKrsh ze3gk8Im_z~2dYReIaVi4J2rUj4Thxk`AD-jLrUd(-K{;dwoa3Kd!;Kb)S%wnt9sA5 zRg7@Ge*VGDwl#L&c^U86Cl7t`z6J?i?c|PQk!I?uicfqx1r@i~60Nu_!%StPXBKF< zT;A1I#h~gSG~3u{NFDk#$m|ZI4ob(_-OSDEou*+2D;4O)I!0mCe_u9(5+c_Yf^l7o z%>-?KjWBl)Nb3qVJvr-s`L+V_#9sMIyM0xy1wn$Qd~X^PlIid;lhs{;0QQ-#4`Z00 z3TpMPUb@!@YdB!Gz}hfBPEYFSut}L11T`>t!BNfP#04lNv95AbjC5* zyXK|nh;-7Dg+2i04Fmn-*`v0r`w)Fsfs7tB^*(C5X_79zC7e2=?4L0pAdo;H6sWbD z#&BP^41XlrwKH!UFRkcdi?tM0dbB5&hf!@L8C7~xQE~P`lLAOtzh7X|eYh4o3ASvF zsw4|~fw1Ubo{P_e%CPn(*Q0&mKP>t^l!@B4;z*Y3z{TQF-em_5J(R&v?9Q8zOjU$K zI3>;0fE^QJ1lJ%S%}Bc=jTi+h3dUJ$+EPLkV`Oa2dFd#^n1aXhei8ik&62fPWfBd$ z%)phHGEYvzFnt+*c}M8gVQFEnkEUeOwlIynAJc0z{nM3EuvOaK88+6G$pbfSf|{QB zU=~v&7g>yxfzT)c^yxaL@b)E4N;2Zw9vXn{R==?`SYE~wW7v*^M;)@E{R$5C3F5TL zP$~AC8u^G{8{~o{0;-pe&I^LBvcba8+)*|?C0JAnMxtc$XJXQ3?Vot`fwQTg@?uxq9AsUG?&pc|Z_Fd}z?9Q?8skFxx9XWoDYwLyR8Q zvL6ISk`#(4$2mG}^q3+Fvz)6cv!uhuY32FBJ!HS{fJPDHC`U zew2Wa*A@5Wwu~#9Z8NkyH+VwrSN+PA2jE8g1B;|j&?E@<#8lbyUirkb2V#dTK8z2C$ z_+J~T!V}n`(h@5$Ryo0Uk3_T71V^uZ|(0m4HJ$2uE z+VqBe!wkNpmEpy?b|NP_M3j-@G7X7yU>0yKZmaxx<4PGj~ zaFNpk(~?8KI&1KYg9ktU5wlot^3U#{^7NEG{kCPg>C5Lc4gc7I0Da?&udh!y{JQlY zKiU6_mSsH9r`8|4Y|HfW`u5{RK_1_IzI?tv{PpRz_TxMM;v@8nb_@0Od%OO$SoEku STc)R;R7e|li(dcX+W!Kyx5|V7 literal 0 HcmV?d00001 diff --git a/Scripts/Models (Under Development)/N-back/nback.results_nep_1_lr_01.pnl.npy b/Scripts/Models (Under Development)/Nback/results/nback.results_nep_1_lr_01.pnl.npy similarity index 100% rename from Scripts/Models (Under Development)/N-back/nback.results_nep_1_lr_01.pnl.npy rename to Scripts/Models (Under Development)/Nback/results/nback.results_nep_1_lr_01.pnl.npy diff --git a/Scripts/Models (Under Development)/Nback/results/nback.results_nep_6250_lr_001.pnl.npy b/Scripts/Models (Under Development)/Nback/results/nback.results_nep_6250_lr_001.pnl.npy new file mode 100644 index 0000000000000000000000000000000000000000..271d26badc311564b8fdbe7bc1b70b4ce3e6e1b6 GIT binary patch literal 7232 zcmbuEdwkFJ7sq#T9k~=Dsa4q2Tq5_&R+!6Xc0 z>mC)URCJ-Mbk&t=-E{fI@AcZwXP@tWkMDm!d-OWTUg!Dzez)%!TA6gqpdlkdL#Bkx zh)U1MOOvQ>kx_+%qhcbX#^s2Ul z9}z|2Y1#Rca|_y}JOUaSvsNoKiqTo>Db)`rG5 z@~l1VrIoEA&Nfb6)Vm%YvCGaG?B8#(-v^Jmgy zJ=jJ%dlk8>J?NvdD6bR z4V=%lakeeFb{=;l!8M6R>Pbs^W)*lx;oFXxW8Jb!7|Z2$3k zINOt4FOQ2Q7ia8^%H34PtzHl}Ij!+@pX=>xA98&?E}mRJWBV&NAo%kf=xhSHM2{Op zF3H%z$_)w54RzL#8|HC0le@*(Ta~-568Geh_K%iy%ZjzbogG1Lq{odSH`>@_<;GOv zDn6fMoxPo0ipQmrOEWfIxs2f4IA_O`%k;Pjn%lE;(zgD3=$U zbI#_Io8oa($xSo1K)LC`xf#wDlAGyqv&hXhwn(`-W!%ONy-Pnoaijmb&vkYlx%nPf zOm2a(CCV+V^mqsNt=~GYL0YU`m^D`-pOn2IsaryMx?Lk9&;V9Br23EdqO&iNJLqwT$h~ar zVdY*4{&=rC`x?0;9``!AH;jE#xue0kW6r)s?ro2Ihupiy9#`%}aPB>4-zWEh$9+id zBV#{S?qqQ86K7A6`_$t;Blo$nUnuuwaPBK-Pm}xFj zlKaWnpOq^O&i&%-ujGF7xZlbBVeC2O{tV9jFPpR;SwIBAZgtOf5>){;A9QJ6-ekt%D& zBIq-_7LAkZkd(#nPGzmRL)He{`_DIS)@J?knaEGiTrjbram33wj zD0}^ycRFV!j*68oXqy2ro4^m})Sp=2EqjAy?Nm+m1scZmu$Od8>NkFPBkwsA1AT&;rkdzJPoyvxAhioXO z5rb6OFcyKbi%ypwn&A4x%FSq;+=8U+R^F-XHtvw|kKRV`_YF@rl0~4b;&V0%jg!$x z%944fvN7Bt8;fb=cBDSD6c#~csc4*}At_7eoysz}LpBc6$atj6GFed8F?wygk@2_t zzXB$paWWA}*(BbnEQ>p2lQE5CBUP5eg0iJm#^!eDlK`6@X$5gXZo_KXq-$#QdYn_l}+ak*$hl0g-DgnWI^b7yDgHBCipI$uNXnM+PG!rvL$(6b$el=)-Nk~k34d1Weq#SDU$zpBle>|W zt>T@^R&$4J4W^N`NR{2gg0h{7QIo<_X8W>rXq?=Or0hQ4scb!W$TnaaxgV*rjVvhp zCM&*ejTVc1*#l^tY(i4@An#PRnLA`#FpWHfRN2EUC_A=2smF$G0pIGjqH(efN!cU3 zQ`w{3A={2=WCv1ZJ6TY6c2~z8;SK%2Y{_G2oIH-CY!~lTwwpU-doYdcMXGEc3(CG+ zR_(5X2NwGu<`Za~Jc*?2Dc-5*}Y+2Yj9Vh{nlJNXmZZoytnNL-q@%kzbK2`;7%Y7jHK)m-l?nscgPxI z8fk=7*`+Lk${M3_(gaD_WxP{aQ|^#mj%lPBQe{`LpsarGQ_~|B2Yf@h5{;8cBxP6e zPGwhfhwK_mBh8U2Yr%rD)OQlAEI2jD|D3f%<0J}6Sv2od)`~l1*J2vE4ym#j7L@(_ zNzWJG?d<>ao3uvbqz#g?w!BkWJMNHOk7=YmQe_=j1pS=dfW}EjBxRj=r?Sr6A?t!^ zq$^To-B?ieRlALU&Pto(f0*6TIO&0;tS9eO){8r2v6x2UkSe>81!c8YY?{{P5%Fa= zp>fh1Nm(D>sjM$|$l@`L^h2twKMTrkJ=XBKS?fmmvH@tE3`A0vz&n*Ca))dXrjaD1 z$_BHbtox2GZTpu@^<_iQI2nqh%C1!W%%>=(NA_;_D-8yYA4 zH)+a7@J?kTxkEMz)5vI~%92@7RxRWCh>w2{_=P?OjgzrR%5LYK%2K#PmWpX44XLtp z7L;W+`6s?Z;W+=9WuS2~4oTT~-l;5;J7g0ujZ8$UY!VB~)*n0eZTOvYFf=n}um)Hd19pEGWC~X#R+9*~$KgIR}lCxk$?9@lIv) zxkFZrX=DLXWhE>qYn$-(PX`AE{J~`*8Yhd8lr83+%9e14Y$>LZJCG_{#)7gfm-Vmu zVNt*fupEt(6-dhND;Qnr?ND!Yd} zWa}`E+>2D%eJm*3KlNnIy~}cZ*?KfiHXtdxpLZ(T$Q`l=FpX?Ns_a1)lx@0vOr5xn z0l#f;M&o1)lCp<*r?Q8+L$(#u$Tp4?R)uJ=9DsLEv%GdD6%BU zC@P(FQ0YVmlS(C}luCs>JfF{R*U#_n_1g2-!>iAAb$_n+`@VOuIh3UJ?U8<4Y)nDS z*o2(O$ZSdI&^TdS&xFLr2?O)QN#=+Fd6E<1!zr1=Mn>@X$lT2Q2>!o$>%HBYm>Zlal35r(KqA?BBStzImF?nfOvP3Fh@6Vn#-_$h zwpF^sX4$x@wyLq!ny1F5R+(&1)59EBm>-F^r}wK86BE<3ojqgvw5j$?W8<5r#`TNE zW8>^uX=!QQPvVCksh+ddv!-X*8qG6oP5gIuZcMKKEa&9nDb6*vmOjOKxfRxd?QCsY z>lj-%uqN5_|80$NwqDlsF6Ydx-{i9omK1wcNMwT#+(s_SQYT_LqeNJ=@D!L#}tg^&!{S*jtpl)#qkion2$m z-Y((u^>g+%a<>Oue{y#io1xsDK39J6#?tcGks&w0*}KSP23!`oY-4kji}+mIP2H|+ zT0Ax620A;4TyDS(CO5>`p~?;Oxfa_G&)U{`miL26FH1fn!?VMk9YHQH;PS~?V@0`< zKDRUZ+ZJEX9_MXbwtMH%g2A43&W<8i5O8;s8*OZ%a$|h1%WXCH)IKpKe4S&R9Y=0_ zz)c``kFgV#yVvJx^eJfAzp#gQ?2m%fR$FpCd!MtD$Q1?LWO7rCovPe4pX+sO*|8n> zj|rb|y0bIL%?!9%>P4)1FnSJgT_9j+&rJ#TRQ)| zC!S67N>dike{jhl&pzzzBjn}>+yZh7ja{VNVxN2A-Ihh8mq)_axy0E=$t?}IW#pC{ z`M~Fj{@#vayyLOsoW<%H|?W?OCnpNKF>aN z_A_#y2iz`l<;Lz-?hBur-FtX>dZ(!8+vDt)UA?1Gax!~)2FzxTo z{z2|=!2LW90q`xZ~vhW9$j#PWs%^>T}xEOA5b_ zNesSuJBdYtmMWMkkT~vfQ>7}Vk!r{albpta-%Ka0tY2mO;V9YZXq=pZr0h)IReaNz zcQkc~8Yk_Mly%^p z$~tm~ED6)dbx4(UVnNx?___I){+Q=&Ui#hEt@jS`#6#ocdL(6?d8e{u?vQoCG}0BR zvKv@X)@^;C{ml;)cwPI|NS#wvJV`<0q#Kg58+oU)o47;P9n(lEQe`)@pzPg08tr=W zg7C|e^g!bz4M|x~-l;5|J7m2ujTofLdb6PH&6P=YP7WO(egOKQanct_*)6~Hjg`G4`Uj61gWz5 zEGS!;+-`rj7ji?{0yIt*A}L$MJC!Zw4%re+Bab3gwv+{B`!<(staE-*C|icc$#NuR zkMT}rk8_7?1*VaeNR>Uog0dr3ckUigBkGl{LgSb`ND6(KvYyN!jzf zQ`si&kZr~^@&Zz2FS4L4rP|Qf9)2zAOVCSboNPf-_A>8O_6m2%Ud1%>8d7Div!E>a z<{M1eipI$sNXp*ioyy+g4%yq7Mz$eU_700mW$&VK@*a}1?YvXj``jV>0Mp2aNR@rW zg0e4{m(ANy9{#CSK1SnY2a>X#yi?gH+#&lE)5vE?m3_{l(pR<%jgxXDWxIK&vM;zp zwg=P5mq?X;#iCN#*JzyVMN;++?^O0JcgViOG_nt=vhP_`D%+37$qz`%e&n6X4seI; zCrl$hBUN^gMYL>l`nqd3^&IcXFKC?nilpoi?^O01cgTLnH1Y>hWrtZr%WA)#ylBen z){{TcIQa`n+26cV*%9uL9mO@J?kXxkDC% zzs52Wi&R+^tpAd|w6bQ8rANnj5{Je~RU~EAc&D<{xI=b2rjav{Dm#-!^ea2{D_Ikt#cf1!Z6F85dV}S%0s_+gls`99QJYxoDi!LQ-}f?^ITs zJ7je*jnqY|?0goKHQdy|)-jtO<9>nqnGhhE&-lETUzTFM8+qUAyrCxD<_(1SDn6d8e`# z+#zd;Y2-4b$`V;b%g%hfUfzn|dwFs>8YiuglwH9)m0iglva2wSv_`6|4U1^mp*F<} z8sI~7kd$4=JC$|f4w;8(Zq7s;nmq%4Wu#sNF0v>icFo8YjJwlo{TstT%Va`d}LAi&WVyEGUa_P+sTI zs_-xIzA!7L~rwB50fpL{c`0cPh)}4%uK#BSVlX8_J?m*)TLth9fB(!8?`Zafd7) z(}+c?OjuMZ8;Qn=LsB-1cPcC34%yw9Mn)r5R>*>~WgnEq-&8u#YdWOIL&pYZc`^o# zld(w3#_>*N@wQRyqY7mbtqkd#g0oyv;1LpB-H$P}c?rn0D1HVuuF z=}5|E@J?kjxkEM!)5!ful@+t7RQ3QGC$o{1&EcKO=5mLu1k=caNR>UrqEgvBG)^8y zQuYY%R5qVGWD78jEJUho5sPTq(l6I1CCeyJ7Nc>p1WDPWyi?gy?vO3RG_oA2vd36N z%TE15e;kdI6-dff@=j$>aEELarjb&l%ARCFS<&bXM@wR|!fz<6(KvYuN!c3SsqAU) zkgdfu@(faC>sVC!%AQ5zWIZw~Gm;@?WHMv}M_O8Hoc}Wx|6+xIVDf)fN#)Oqy`rsn zq~fMxuXw!TCi?McDi%++vEo_Md?nF}<5+##IDML!Q+26uNWI#AqQ3dM>PcZxfMyBD~;-+%9&S+gg} zvvbbcIXlTpCZnPN4TA%L@cE{xvD1Y3e_s-vkPu`LXjDm7Fa+9V}Vc zS*gfB9L;S%KmUDu`XECW z*H>N@F>MLM?YG??lP^Lmzag5Nq0i1G%uV)N<=2|_wNtwdMZh1uQNEUNV-o?)V<%rr z9O=enIq@-%VAA$)w+!DdAJ3Q9E{s18PJ%jKuE>&pefU4#Z#iAOa{}J5U#~7I{hnI= zyZrn*p54|WRiSF3VbUL5pKZyIZbZ=~(RaC+}A_rjN6J(_ynyT!raI+>l)I+Bw%z97Ty zO1BN}?~&;JtY7}}GQN)NIIN43OhrylN`E|`o{Q6I8@l054#56NujaJI6f9AS_|oXy zbSX#H#xg0laZVmXlFzX{)E=!_TI?xUuuo<@#St%5)(0V6A zmIw!DL3V;RnNs}vcYFe+2~_2Dm)obx&Yqw%KtjUf?lzC>gHo`pwVJcF8*o->%NnI1 zX-A?)gm2AD^DTP{e12{$`Y~4m&f2V~xLs?1Q?vlRm;cgg-~IU$0{Tayi15ai8k6Df z&P~;B!{HJU*Ba8V0#r6O(DOgb3xC|)xp*FKESKB8aa>@2f7fX} zF9A254@i66C5cYKH88m#2&x=cMV~U!{Ja@Ps^@DZE4<)HP#57sdZ$8^P1ATR;i0KR zGPJbkGn`YJ3iJ@QtSd}|cX&77T46p4Wa3~7au=0i7%}MI*CRt)0r7+`R{njX^Lt9|DBs+;)9|r zT0r=eqwTMbJ*r>TvZA^mua0zx0&;~|)6>6>m%jJ$Kn+db%k(yv$Ww~VZ3pspArnM+ z7Yse-^>3{6tX&ioTt7|HjET}4C2R`z_l#!moPwa)it&ePZz5-zdi9eLYSlE2;8^Q+5S2%XGrTW9$ zl({+yO;lmXc;Cgn%Tiksp|3_~p?IPuPBu@`6C4}3!?@P5ZHfJ=PoQfo+zho{W`LyWLZ z^|Gos&P)Zn{y_WFmo8q$^=Bz=3SVh9KAR>F&G1*p=xeoE3^U;cA{uH>7MfvG&k&-L zmKnJNtT?_(1Lart6!#P~#imf5Lal@JtXI_<3tPiKI|nURV}^QJbDlZnr^nResD{e8 z6g)gr3-|3%_fdmBl#WyBQJKFXhzZXl*54}PZrrvt<+c+#{+aUy5A8V{zrMHm<0W3i zGWH+0%_3@$cwS#)QMQ_6)~qyQGfsK3s5!=J%F*hG^@<=*C;)vq(+aa|X7h@8j~uBg zyJb8cgs|3?4mWC_5>KJW*I!JFr$`YQV<)Q~U}(A|TYu5}V{)k_vQt^QuhhEj?{Fev z3uczOPEX*KK~kIflFoP_sp+YxEDIx_=wXR1V&~eXAZfxZcp}Id5Dn(tG0xU4J-=7s z1oIr==_zlh*cvJvYJESR*ZBk4LIqKCE(fu6G3b{B1otl%Xqnr``a>(b^4t;xM?NVN1 z{iG54_L>qn4ezyez3Opxm%EQG`@;2wCgD$}r8dKHVguhlYL?x6Q|D%R*4LY?gbOjF^3(tH@XH${u~{4b#c1m&?`La6%o;xoHwpstKV2G{9PJe zEhY0M0B(OA|xQn$UHdP9Yc8ngY-FBLc9@!1Q1_iSI*t~g=eSOoMGQ09|;4SV~ zzAoV9aCY2aM|x%TEKXYZpxeLwp0&p@4t7MG zYN28)>9Q?;7oe~$Hy;}x)ids7Ks6Qxj_d|m9YsvtF%%{xjVG5fPWDGif%zrK@87Q7 z8a9|`Wk5h?!5HA49%Rkn?C#>P16ldaH^`vFtHI>fL82uujilUs+sR%_CR)0+wUwT+c2&&RUU| z2rg9^{F7+-lJ!p@%b{lMkMAv)LdPHkSD)`~hBKEHQgwSdyDig4SuRN>eg@Y=yN;w- zaker>eofTsaK8;AL^LLl6*YYVD<5Fk|5(c;_0-WgyN};_e4KZP=xP^6Je|D^%LQmM z#~jbmV5(Z}cJ5B2wZvKu#Re2**cNHt-Iii?kJca=XW%MJ*)!^$Q#NzzTWDc_(Y75W z6NMcDDS4S0giaJ&lIDz{Lw?uQxMe1#y6xB`qjK+6$Tc>x4I&I>!o2#YsAJ8!b4?{i zi>usReN80xQ$>{QNr54~3^B8S7ll4m!0C%v3Hl;I$z&90j`H9K*dX&=!j-kXl$6AO z=C}62a@|xNa4PCoLF`J9~_~JAlzycPO)(vn!G6K8t&M%hOjom_y`(9BCY$g^7rO+2tQc zifsPHG;Hg6QMtxg`Jq_)nsb^QW4q8%LYE@InX8NU9e2-H^RT?qqnw~8nSzXmKQ~6V zWS&LN7Xv#o1)aMuDb~f$65t*ElbcVnN(#Hp}&W*x_F1=P(i=OUFo{s!!? zxs)8qok>eRvb}?n3lu~DLKxokfWUxwSVslv42PcM`T5To{p*mfoT_%B_%-K^_&@fw z;=)P-*JiFH0!TUk@6b40aL@Y&qP9XNbRy?Y&6tmc%I={e%I$r$;caUrX=Zy(JKGn8jJ|CqsLT`BX%i9WP8Tt<3M~AsUkr z=-neGwN76JzQx)eX|y18qm~Z-vFG?Y1UuNQ{AgT=gz2en-MeVsP}MUoFe(E0T3*KB zUPdD`Z-=I6zdo*{v-Y!4zm%VvYxy3z~}SBhxfIIr)`Yec%*SMv$xY1kNEKQ@DHM!1oH0z7T+mYK-Dph)%yJugX`aC#KIOtk(y9$eV8)nicMoAX2 zww9~P@5?t72WXlsn`%S@48FArswGdzfWc2IYT9RZnqbXp&iaCsaAF(HvPtj)TMHGJ z+sUd#ev_8WCgx(+y!c_N6Lr-&(=doiP+p5|o1BuWC&O%G$0PKMkwL1L^Q- zxWsF!P4VUhuHht)*%cfnMu&g$UzoHnzeR#ZxEJ>K81G#(&KL-K=YOOXgCS9Qg2u-@ zvPs{Gd(q{Og}-Df8(t7gN&F4C0uo04eg1%$z+zSXtjA52yltWEe8^&Q7P~j&%Nw1xp8{?hV`_@9>TRDDg z5#ODXHrBw6&(<9`PteAVkKSE8zGP*~PU^*+^+{zS;ryPO)rDBu=Bwt&P~4Q3wCW*! zhE`c{6p(4weCv$f(c<|x8H@(uW0Of(S>n3(0+QmFuZ$A3w0HMt+Gye4mt++B>vdt_ z#Yv1h?38PoHPukkztU)E@6L@c)MM|MNSfxER^63u*0~SGbJPTV$IalDmel1T{Vy=k zp@ev_`Rw0o7|h=z$Q3e(hG}lboV^XJm+=8_s8o9toDsnk}?vN*f}0?WlfQOE80Z$Umg_smR&f9111 zlaupFzM0?YxTmISm75AyI4olbpe_U$G%)Dzc5LGs=eIar$THv`$rTLw0#tYNtmMrn z<>d7?NNX^vM)9Jg8saVEnN0l^pm?2!Mox_)?wdRg?4PSLxiUSj+G*JN{d4akS&k=S)vDLz!^-87f?1|Fr*5{ z#syIRPHV;qFuK~!@g!sQ*6RJ@u0ws0CD|dYgS|7@q%%x-Fl*MnzFF1Et2prC#-Ki> zLrrt=&tx%~4%sZ=xr&yndV|yOA~WLNDF?<=K)Gi^v8PluAKhMGXo)_p0Z>zTuCv$j zL!?Ps;I&PWfmmCv;I}4Vc6Y4lOW|@)$DfJ$jvLK3X#yZ%XOK~Qn0KK+7f~BFSX%}{ zlzT#Q&!RSdI?htpTrP8bSWjpvn>pLn(u$|7C9(SM#5|+?!hp`ya8kXVQ@wuPnb2>G z&=YGZ>b{l4o;d4|nSEY;#g3rFxkqZ0YrgW3PZRU-ktatT*DZHld`7M6Z<%M0K*#XY zFdp1GWQ_VG@<5yHnBg%!!on}}O~a|oJ_7Qa*1eD^(3 zvN}=vywSk`i9NTR4%yO{SIsQGU4qqvW16Wq?bv+B7@eK4F7MuL7Xe?V@9M`cNA4gc z!!NIa^Zs09GCWz%C7UGjO&owVoxNj#k;6ElLeo0$(rV+dMSsFt&c`p4vo+qq3xnLIzhD1kGFM;w8YXm42zS1* zQI-fyq#6XuMRu20l z&Fm6sXp_ctV^aN+Q~ffn;o?Cw&80Bhv;Ctu-2dfA`0L+xVC2R#x8-d_I*YE?r!b<& z9N!xmp)Vk1+V!JfIW9phihH-lN*&7hZw-fVwYtB#-?Q3Q_yY|vYZPQ1TW)Pj!k1tF zLIl`Bg#PHgyzOPv*?qm`%pYZ*M{L`B{o%+Tdhp1C=!F9fS9|2OOD&|Iy{;}3`fqR5 zQgtg-_2`R3???8J9peU=XV`J7UnV%uBe`kmEiZr@WIWw=ljkHBO7FiKRYwz>c@`|n zE^Q@|7A(#+ttCgFxhr1@R%N-uZN#lF!qbni<+DZ;LQ9ffzm@=7Uzi8D`xQ$nT?|ZGTC4be9ZoKWv=zpJTwRNQ_(U> z7C<4Sh^XFY3K?^KqjURpNio;{=1MvzZ9=O@>ezri7;06wW0H9d)CyyED_+!lBY!5| zN+W7i-rD0w%D6JpBE(h5?ZU2HZM|ijS?;bB zVupTwKDRM!sMxp6`_Ow4yhEJ&(Xk}@qHj7T0V4hMX+^j~dyixfqQhUE{kR;lvfj@U`P#E-R zwF$J^{4q7vZMNo{H4n1;ml0IwhbWffn{^zAyj`jmrk*OJvY>$}rK-Gq1$)#0nvB|i;#CXU;$QU8lY82mOj}jhKpCyNbeahW(Izczm zIY!OHN0|RtJ)K*DqufW#+OF&dL07m4n{WJQZObCHQ)Z;H;h(2HtIEe?`mHD$m&u+3 zBRl6j6GeK-dUt&nY%j=y0B%5TvF^qUERX%oD8%=6AJU4uRMG0vZro(vXGJ4WHiOYOb zc6DY~#KFOej1|K(h8c4AZ#qP+I;{Aymv)wb+3UtcTleQ}kv&^CrKts*4zI?(+ zNsIx9Ri_&Q#8BwrCB3r3s?-&sBq3!b!!8P*Oq0F8 z?z$V@(CaSGa^CPAJ2xGw>x1mxEBjks!9={y+7}Rfkye!)ozUwIwD#f`1RXnA>n=k7 zVKyIM2M%8V9G?LJUqCDoYZ4J4jHoxB=>3y}{XY)&`W3d*6F1x|iqjLP)6*xJ^gn(8 z0iW=1R8cZyg;(Kx=2#}>z8$sg?Rd%&H+m+i#A6jz4POC^aV==Y$JLdSSiOeW! zJhqZ)XvArQ6Dr=4$HrvgOhvk!KNai+)_;25AifPpcJh%z{PE#sd>n?BB_KN7^nqYB zAB+cHL+j)ux>^zfpmoZcm0iKa9$IyUCdh@}S}p~kagRA$qC+@}2>uZ%B=)_OV(##7 z<$2LAs>9jGdGn7U6CO9V)m0?fF-ZUx;CDXF7{vuVf(n9g9GSGxNuL{r@+y>d6sXt% zhV83oZwvlu4_Nz*(i)xCbqnAn2;Npx3sbzHLh8rFa#+9 z0W&@08IG`#RaDo7|fCR>t*YTpdI}3j$(-kB83^aWP1i&R~cdGvX z!*SOB?A@%u-K-F}q%aI1C`Hy9^@)XN&ow^+(SUboFW^ru)PHpF zCmrzxd_Zkjc1tT>cz~gKSTgEyXr~U8Cd&{fODmdoOP05s86hr8ik}_9F-sH>Bf~C9 zVwDV=+mMM%#s?}w0w*9}vhjf)kU+&yzzO$DHVkmV_HHciR-Wn+6aw|K0)5B9@sbVo zvOg8zSc;RWT+U@I^M6v*#mM}_0TG_7$NVd*gYgM}sqQ#n?;5AY$-aY01jvX2{b?huA{wl zzyJ>wvO6X{HtE27R(FjAch`t79S8B2aKM02K%g|)cX6_yU;wQ&nIF>vGcpX`y9INRh%h04#-D^=6r<+; zkW(U!*Uw+Kl>BGa;-u-4X%C$93e$@Ong7UCvD)&Egk@el`D(9#yPT=o-xQuRx}AZc z0JC0G`34+d8JFE!1s?)~WY4j~Ds5?EdCOd~Ac-Q?fG+{C%-wb?3NX*mT3h8cBXb1< zys=))d?C2Q{233J?`)~9@*xG5mDp(bd{PH(RQrsRG8+|-V5P|Vvu?{6UKHtOc^8iq zCUj!~*S0N{UD?yvO*o&1n&mBMk!2hrZcGB~$>sfcjIdGZ3Ibw+b_@1rtpzFCs3b>3 z#!8Z8y%(FBKh=_~wTQ$4=8qdGUE)*xXT>0aoZ5Ph;Rs~N44;;juTDrGAgmcRw9Vk1 z=(&afhSL`*Uw8a1`)mPtFc~4;=o_;*~Sw3fCiu9yz2iG*MSdo`0NmVpeyeGYE9VQxx$0Z&F)-b z@2>w>p8r4dG8H9>ek~l3ufO^u z21t>OxqYqx)Dyk{Lka( z)yR-PsW_ic7TLZ*jksufh5d6bxl0jF0(|C@m}~CjU4sFib4mj4<;N=b#7c$_+>2~V z(R5L>+uNrr0`#+RJ~a^B5rn6M3KVB$Vxe9}iC;!Z?ovL-tDK@F9Dgv?XS)R1h*{y) zmrfKXt$!Aa|2JIcNRV}g0XRQB8U<}()!m_i_%izJtu6%z)CdKP17Rt{0F2^fzmHA` z?ik*d`i9f+YpdH;p45U68;!{<_L{LSj-Q6BqAC^up>jK3@J-o{sWqW5B&Jr8&_73-p2l z%Y;=wtwcUJFw3~RTe2#@E%8jnj|PuO_@$9 z5Bb-0i2_V5-nJZNgNqO;fJc7yDeW5(&93_z2l^#+l(D1fzu8}(CkAdOOr3=AUyFJx zSv0rL-|ETU8~*$rPlZhC^dl}_Y439GOw+U^>=#^i@l)NIc__47-8V8}Rb1;pb1_@B zAS@r@f#h`DLk%98w_?lZO(SR!TJ!zxwRt2cT2EQZD_BQ?fuX+)JNFW1_&{;D{_%p- zo|8$c!GTuR|;O+`eRl$k+mfsz9{Z(()_$yP(N(pW*9bsZ%y)#~Jv8kZD^3!+% z)g6Uz$tH>HiOk+1&7Y)0bM8up#MgA$H`hWGkXHMt_a5Mf$gk60V9XmD+v{^7-#&qt z?%#twPu$zFzGJ>BI$h=!^Qv6NJu7fZF!-}WBciLz1T$tojW z&}_qzV@jLS`44sZrJeSZUCI%kEst+^8J3f665C!uX<@(RfnD|*HyjM zvgoZdT15SrUeO@N^H%e0j5Uuyq45mUuQz2;mL7&uv5Ff=gG{zWavnB%s)%eDr7biP zZF4Q)P|lv}(e4^=&2e28DXB$!|8kE-P)1qWZi^EkONNCUleT+FfVI87@bORURxo3I zSJ`7C4nbuNHfU!o8F+3wkUb&IH1{0qsWfi}$H+veW)a?4)`}MimQ$^*TJp$qrHTRB zmCe54wZXm8N$Ck4)FAQ7z$`VO?Fq7&yp`U1PU}-$MPY}S3`r%NF`xEC0{lRm!wQDRJxk5@Y~XCNmWaL3sm3&lv~(=@Wr9}iX#&+= z=nIO$(%&U54U^NB7Dwa${Wf*oFm#)F4jO936dVdXN_@FXKeHYlb#|pKSM}q9w$M0y zLw!{a&}MGuCc!lGC2qBx>f+i`Wfw=Dg4)7e6Azqa!nvI4GLOv=s(U2Dehs!FQhn!z zpjvoxybhJXsl8sQNldSI{!wfiW!U>NumonWCF3^ZAj8v<2A83VxSogjZ;M5*-6nn* zIgE1!;p+1@0SKKDJ(|&O80@+mqsCZY5GqCXtV`fx_EZ|>-|MBed}r$NTBv{~49}AS z4~dmvzR33Im6i;?M83?|CAZZoxPESs*A=`OFUYc%3$)it^)1|F1#Zvr1nJsQKfG_% zC6fM!fxVn+@C@1mj3cIb8q$X@ZTtlzXU1F~{|Td*TD4q?BPs%hL@-zB?c8e{kH|0` zN{(eXZG2~az5@Y4mIJKArman5iO5-3S6Df+p}-u{{F?2!5Zo2{r5fd3ac#V27mZXO z`H--QV6_d?#{E)cwr!dgUjL-`Ft-|^8cazARE(?04n%8JM6<|@?zN=JJOldo%)EAL z{)##?NIQt0cpK7E_}EL&a&au#lSm**fP6j<*7Zyj8ve-Keb(`AI+C>mWT{+7Q zc_iQQ<{TwIwJ^5bLLOK}8=*9q6*h_X{Rfe`)HtReO)+Kbg%6!0i{EmKMW*LgRj(^) zSY|0vWY`O#R=*)4R{sKy!&pdmV=O(Api3SDjIjiC4TmN?)RIq9K`Nznx=YjNyt``* z=b=U(9a^(66eHG)fk5&PrA#?CG>wSWfQbkA1;0C$n0wV0WP|}>%0UKJmVtp%6l3TR z*Pzvf*d}pb;l)!COPC}?tBDl*3Z&=1Jm{y^diB8F5)*r{#d6$DqxFZei@ z`VG&yE5*&Za$zw2UZ{=-x>TO^ML===VtA717AO3jo+8<)K1-%4deIs~_|}1-^hy|l zCTqn|Zao6w1e|N>t~Si6kdIVKYIpXEI(TLB%Nvsch$`~MlODn?iPDNsd z`C{ejL4z=6Fe^@A`0Z$Q6IN)MrpA){O}3d)^u7(lN?K4PnbuG^5Zn=dsQ)}x3y&CC z#lxF37mWG4V47w%EfkI*cZj@;(Z9_Ob(e+HiggPqJx2}6TgskLPl7%Xs1FkhCF7LL z`90R77qYVv3?tDc$-Nc)hflUq?4JMF)Lab1v@aVXHUmK%PN}PlGY^G;A^(&E%Ew4* zVU3$zZK${l7LiUWA0CcF(ny-3$g+Td0YXMZ6Q@E68N}iU??NlJZAY@KHn3YE7Vdhe zWbMxyjiw*fH&eaz(BP^DGk8Usf`EG5otzJ8Bt9wf*_BQ*{c9t@;Ep2KQpO_w+M?V` z7#(fsx=@coe6Ml>_(9e*Y!D;|cg%pS6VFT>bz zcG()yv`shhtHLVdZ5b!j(UubWwpS0E8Z5FwGO>!woD`U{sIzbjtioV+VbGpN5g+t! za4x1(n3x(*Eo_OijzpZ@Oe7uwCET~I#Xo2SmYQRk7JR5rw#Wnh{Vt#PWN8I%-Dg|_ zds>aa256mt=J?Wf)GtOZWCN74pXN8+8RjDw8FC;*Yf5yYLIqRL3g1}v>7Docji!DZ z%(n?ia}hSR;SH}IQD3o~&eoSBH2C=K=>l@!SU=^cVL?P<>_f&EV;8y!y% z15*(k-*{Jiq8$0kP(yIl9wh@OLTy`WlUpnb!86&M?N0?)EcAZ-H{-hR*aG@jHfZrF)3h&?MzrO@Y?r_-30j+%g)#C2ZA3o z4W`{i7!+dYBM}9J^Y{u5pNoWL5vt^Vr!{0q%9cdTx(RP)IX<@rWOvg z*dM=CXntx&NI@m9{isXFC*zNc;(X&oqMxLg4WYJJs)qRoO$8SeoO}3pXuAdf#@0SV zr||26c)km(;`$9Yj>!YD$|SRMSg&q!Y?MLQ0fP5Fk0WCIdv@5}B7NE}|?{J-RQ@$AhYtKdh6(CsS zs;EptS^qm3h=1^cLyM*m_PL9UKB}<&gjYB48$`12Bpl!rseAhyS|hqC+9h7VXHxG= zV!R%{=Tl~@a7KS8u?w`b`l&kEgtlErgwX*bSGEg2jXrlwAF71A_>oPMaKwR`K{oG2 z$vYwRQeW}EWF9u0UDe#3WwQA5bM+q!Ns|-dkwkvK2UTZ8x1jMIh;dpI*4u6N)a2MhAtua6rmUV z{Ie+D8m*jo>&f~x5wQokTufd29N%_OP6TWaqX|_5(~pk%RmOELG8vi$V9(fC9HzEjru3cdWH2CArdj<1B`|A$JXL+ z(LbUjX6@%TW!*pdf@h987-9d^;iOX}^NyJeB;Xi`hQXsyhK!B7XYjLtF6g2!5Ldn0 z>&IOrtR|Tu;7obc{WTSVd}TMO@-q?DY9D5Rz{9zu-Bl9RV>JY)RtrjE?g|@*O-xKf zoBzo=dQe_Q*>oBqqkm&52Ob)JXwVqL!10co9mWQZF;=?YG23bZrs3qi5_u+C2ergt z1kPdTW3aw2dI^9q1#Q+3k)Bjw=I3cbBRFu;pbXh+T0hWO;Z$UN#WSSUV`oS{HE1r5 z9Ioe&2MFtfg!CPdR>qtaLl|f`7%wT8joFTgrmY^?U1V1vy>kjTEQB*V#K?v&(JzpP z$VNUxbxtS<&FQlhfBzQ?W}U$t5*WfSE9t^#v6JOCohO4gu*52NgmV86CFpON|P!x8i*cfCEsRhfXWt)=2rQ3sKG29ya6iskk zr@tb6>9^(Idct7VglQTr>B@`Zh?4#CTO(N!f-Mxy1MfaQ0v~xUQhX?dJlSS~1Ih-6 zE@Wm2p3>Zx*i;N-CxnsQg*d^X0V`XXdn6<@3?1qYw9c^I48zjC^Pa)~jneNA#!^n4 z8QFl!Q}%96LW#lXANJ?q!O^9#h|ntXw!M5+gJp>#&JaEsd<?%1XP^IF z01(1rz;g>(>?C6PODh5&vHgeG+SX`u-hp{t-TLuJoMPf{a zj@0swnM}}j;H3o|Y>!fHV<5=T4&eP{>JXu%EaRynLitjPY%n3Sj_rnd2#0QbIxw&U zNjnr8T|RKa&YqBy1~8kUgkUrI5<+fv4XDv3XBgbC$4Zv}+t-miF6oQ654o$kVob>K z4yn#Q4$Yxk2 zY=iYEx!Wjv_4xd@*q~QdA755E)D_-^Ez7=!sL@xzi6SQJ>%zVMv>PvM}$OaO&vL z8Q&ti2>H+@lb@*=YNrio?U?e!Nu3Ho8xsj*q<9&c?}k9`g`kC72NM27cZJzgPcbIH z<)UCj`+KIQ4WofL&dz&CBV2P99H^hYJ-`Va*{2CXarH*y1}OpEjApzEl{cyU<1pl^ z1TxqrP~QhDH}t9BUZYEGJ?K(AZvUUP#=^a6^EYV4)12ltP@S~He5)j(k<1=;fcbz^8(aH;-9w&#>~UQ=c{dxr9|%PQ%XL&%kp^F7x)_*@E7i552N%2Q9jS?X9En`cz}`}IPB1I| zG_3KM5Duwfk(qC^J=DXC`qydB8&6qJ6L~2n5B(5>Mg>RqL5QkJszy(@F5yCy7N2MW7wHx0s}m^ja3AC!s8lt#|?7@gY>YzSP||+1j#yDMLL)a1mW>Ek#hQT+Fa`JePg?>SP{*0=i^DJ^WcO zq!p1wC7rB$SSwy}@m0_ynir$NQ8w!((;cg%!#)7@s=~6QQfzMko_wFUmqy}n$DI$$ zZ##@`Hqt%(EEztczHNS)Jv+X-%gVNKG+^8MN7l%MzA@s zu1AxBSyL&i`Xr3`P`P5){k`0|yZ0#Lpv_$b(r??tT<|GqB}uVs782=4<@QHY zaqU3+N!?seJqd%|U|{W4t(evt$qIcHW?wRY6fobypcXZ19b=6=*P|EpR2dgO!hEec z1-Nz>>#t}He=?s6(u)Hn6pgzA;8?rI+;AH<7~ zP1Eu`XIu=e&0R`!g{(4kG<+g%*?DB5gyX70;ERt|S7Q*Y#WVfg>!TMUqF8l-IZFEwAnx~LcZP=wz!*{4hNCjNuG_Db6J! zsHG4XP|uoc1?VoJ;Kht0iObFrJRM*s&|`d@2k~wlG2z9*e$<qjG%t_J|-CmuyVc z4xuik)y{Gfd{fk8`oVNv561xig@;pIr6&`oQ9iQOkK}>YF-~uVr?0k=91@oyRFnL< za&Ta8X$8vDv_}F9ZOcJ61z!1K({08}F1GZd_ze$W+0ltSKBffwjtNw6dVvjaHoYdyZQ}_#IFHLb?`w#O)A!- z=z9?(!qdLtPM#Wii?(UdL))mPCTb!+o@+b2xk@Q)h2ud-J%W-W^T5}qm4zrJ`cj0= z6Fr!W*uxBR1~3u%kE(kwCdVy^I}`~mi-9r%wf(^jbch5}2=Q2rY;2g!0_?;Mlf-LY z{KPa70?t0+JQsMTK^kPKo4451?SZm1|M1Ooa zyCrg5+||Wx$h`0aIAJaa5-G8E5|&UQ@*@Zd>hcB%Wne;Z3HuxF*yQyP&rY5t?2`0^ zw8>Cc=&zGHS}Q6Y{(~}s09~x8h~Kw6&9*Bf3b6yUf;#2cV$Nzq;$MCB5$6{z)O4_I zsDkl)@s25)L65Ss$-`Jyo9r{fL-r~enbcG>g;|FD?2JPX3&gQS8%91qtM^I~iieiy zs~KzCE>yx=!w#svBYhu|Q(e(BdKGNwcSJw6(wRl%o21A3dm?5)(AO@lC!<%qHtIl< z$|mjxKO>BS%nN&^6l;N2xaDT92~1B&N$_iq?8$D&)j3-p9yk0sj!is}fCB zO42yajl0)$+BYPNX_LBa$`Y)vWj)3cHTuq$5;@Nztej2*ofot!EKDg)NQUw?> zSJZ*h)y!Fs%0r&HlrHE7pS4s<$ZtzcMlPpvm=cLdM&q6%1?OfdZ zihuun`xPl;l=3HX;Rde#HLui?Nj{VjKCETPxKqO9wcUzWoXuw1Kc=T&km6^AMbBQ0 zDSy3i-7N9Y>YW`2o06T`b*We%*C?THP$w0N$f$H7zzvX$MAWaD_aDA5gfHK_=@^Wj z@ccAHf?#_#jKn__4l^e%kJsgUu zo=w#qdwz0sCA((Fo|7c(QMr97@#kCqhreYn`qLHc-VkCc$ak3+I0X$XaH=jmjd2hO zT-0t)GPH#mO9*nW2fdohFtNRdNBVac&aC_kj*PpUVfz;vN%6;^u5ZhLcC4JNEOoc4 z=H>cv{ie&9wRdaXl7RIm3N#h&y#7U}!2d(pHvmbxE$g;zO&in3v~AnAZQHhOd)hXq z?e1yYw)J}Ui+k@mFZMn!ZdAk{RS{Kd{rP8p`DL!PGV9j+b$eLY6~SX|s!iWQqe2WFCH=7-0NZCr;r=S}YSM)xNWWXh%Vl@>}35*9+;Ig&OJ zCdGFJ2cN}{vZ`J2>)TS5?GXw#3pEj%9CMhIW-z-kcxukzoNV;ORF8#wLG~kFFjFF= z>iYiG{QY7>==5!atrrCp!2uEaz(Wdy7pK%8QW&tNOobzmgy+*PCYJj8Ss9;8EKPBv zYR>k*aW~hP#;S`@H*weVr7h_xFB{Joi;=I+{}h(Bg_y9(i;0(|NW5#RB=E1yAx}D$ z3)nSsu~2robvU2axdgk-JUgxHk_TQz!^1#F6;5)V?U5$4dJ!w`mz_$H z)nUFBWYqD_cIw{0PadH;A7mbn={|2L2odBxP#L5*h!9n9HTq6mUs(ekyzeG1c=<6J zEb4ppP0tx?xWRATDpt<@;t_uEt{-WJW{CRUfrpRqAV;nlZqPCPHp#>@97j~a^43lcwS=?y zMFUO6yumER;xm8J8-#Sr1}Kgcd5aWDZ9Z4$up>NpiZZuJ+aXd_$8qvpmH&6~;`R)GR7sYc zP$P~!Yu++mMv-(7dxrPFO#Evb;I%XEutjY;k zX;KS^fclXY%}>j^WZAR;!*jAd63v5Y!O;fe&mTPW#@~?J=w;8zc4%8EDfL+fJ(%E} z5-*%4NF#4SZ#%NmDN@#+t?WZ4*ErYLHnZ}8f>BF(`G$+N3V*e_=kq8Q8WpCH=Dtdr z4^csgbCr-SN=tgjl(8~3fG0xcG zATsejU?o)Z3fWuVWOxX>$>$*N&G}CQrc8J7 zB(Wy8u>(yp{qqLA+0cymFS~k09=Z{7%vs&RV%Yt7tS;xnm`65EvStJb1ucUhQoGURK}N z7d#++qfBjL&h#7my0nZZ{%o;gpP`9g;jN85r96jL2f2?K?ja8j=Ib<#`8V-8?DO;a z&fi|-+bJrqXQDeIZ%F(*s5|z%k1dr~lZ(SEn_%x;t4qQH$BzTFrg1jXMCnqf7mbRR zpi@q7Ul}bXsHL~$8(a$TRvmMpl0GR2qD9{P~C5IKBCb^MKxFA-qR5) zI>;Wzr!tDRwO-tAsa~?hi%yy~*6FRsRHxHQLdw}Xd-fy36vQTrkAsP+f5;2HxO^30 z50xNQDB!FrugNroU&^FLZxU&;JuiB^d`}HJllIXMfl_ALx8~>^2`@U`RZNM|~(FE=5dc*bNSc8AS;9B2|pZxO6x=u||3_JdvBi%8%@N@dA zu0Q!bt?v-2ACKm92iQmabf&xv$mgZ($#Lb=^!$UV@yPs*WWAxe zQFDyC;mw{EJPqw+Kfz47N#w2KWG2(3{YoZ|=LYLz{9%XP!R?`BiMC{!7+U8kBvF>O zAM`!ubVo}e`^6Ye$nrD4HbTCu^x46Yq<{Oq`hn#w zxkP!eR0N@e{WkM;?qnPc9QH5oPuTL9Vcg3koFxUAT@VjH4Q|(5rpiD$Owmn zvx>6mh=hFyd76p%DKV}zT$=l-;vos(q~+^Wu^2)h_9Wb)gN|>^4C0{$Ma47pNssE; zo&DbTX@!rd`iBhkIvK`Y4&XGRK3e3ZxM8cJ`1p-hsnQW1Dpg9@6C;(oo+b77F*h;q z(!_ckU6AIFR*R%4sl|jr?nFHt=?N%<-;#XjIi9Web7b;mFwUrp))~)^p zT|Ci(v$yd#6I{ldS&25(Y_AuEp2_y?TF9i zHuYdxXGU`mrU-pY^g1Yb6B_M$8+(&WIf9n1Vl6f4&~rI#lnLfM?;yEi%w~?F){6A3 z?zm=GS0YQh+(@|&$P?a95%lkW>@X@MyY+N&HQBGRe?^*kd0m8FbJ12g?%Hh=cR9f1 zeH33*YCF_c!2tig*j_dJ9(zi$e&9Zw!1gecn4+d@y#swVm1ONF*&*>>dcIm#r)~V| zY%WyY;>FQe!T<2ux7xh!Nq4fp+MH6n+3DpGcU9Tp9A{5m{ll;@B5}O}DnHWEEVi>; zca-KlB+@0GlQF0JBFOi0Rrve}$DF1n>g8rwZt`1BYg8L!heWenj!^II+V-1VO-d0Q zN&SfR@yv4!Y?Y(iG^8IE#$5)hSvtjX*J?#GLwUVM7WI*5-r)AHl7K4WdEze0qphgL zlKHe{Sfz3?d@~29>0U}_lHD`3vO~#VZt;$@sV)H=Rq6Pk3usxuMcDAYSs3#A6}ws zF)s-2homXp8?WM1P~+ov3FzkP^!K;Kz1J(kh0Fe_#ZDLYyNIYx$Z%MAp0}~h@{o^{ z&tqI%n~O^l6Q#WGC8+H$r`0N-?u(lVEA;bdc@N!hr=RW)tM7T8Zbz7so0oXi+_};* zCN2wyaGx)igh(Ge-cQf#9&BBPx}&lA!sMTai!XpH;9j5I-k-DY+mN4+Pl!-_z)_gz z6NIDbC9?NEt)Fk8H=mbZD`-4{-}2N$mt#JMQc5zWSbQ#5Vs7E>*Ogtmy?=TOMfp7Y z@Vu{nzFmHJiCSEiBjD3y zHNOiv_;@%yZ0Gy^{jRnZPaSjK#ikPX6Y8YZXS951+bL5k7Y`5fibWaW8%=dy$q97@ z!DK)5J}AY`_Xb&;pM)oSG`R&CLoyTR?*@-%1D};`imuJAf?B;otzH&RawR^1PF~*W zK56I1zXmEtEnU021&w;G20{2QdFEXuZdXgXomjd*9`0BH!yk8ZfTP2kA5^cQaIq>_ zN>mUzRWx^6zq?k+w!81wyMuRjF~x_g_|7U-?@m1zg~Q@fQu8Sx>MC_GRfVen{8m%j z4<+-jAC;h028m6wIj_4x)q41=IY!1@GK@4Ojh#A1rg9h4+Ng@}>Up)tK|m&qDJP{9^w2pLfKV(AJe?sh_q{Go@EI(rG3r6j#9e)N-n}210=EB9xMIrB{7WSUxrE zB{jT=jNQC??ng9(cX(B_m?+x3dfP534wO88-%Opt!h;)Yn~U#WP|3KSpPMq9fGDR^ zY~67@mAjcr7OQ)X4|RBk%gAj39U#*ddle=$65MW~486PJI15OSA$(f{2Q#*Q9{g16 za05ld;|@is908eUNr`nqUYqY=a$~9H-8huMa4?5bwV}>CBWWq~vKY12@vDT?}-UTGHxkeU0r{wGI0V&Yjt8%Wv;P?s!#N+gns?&(I)3fjFu>adQu>AeWm|Lc@dcCeMl*-G z!wS^XC3W8Mz-h9_ze>**RxB0BEG-q{_<@o_#bk;&Pa zj4|fRsWsQivB}{wZc2H(q?8CFFwrfBqX2PJ!roPq;AeChUW!L(&(;4{XIHbU4R7AP z+}bO2oDwOTfybkr-rtBr$W{ek5qCUtgIqrl(hA9o>rzzH(57$FhEoO0(3}=Gv~a$k ziN1C$zn?s2Tx+%(2T(a?c&bOajp^v)q0YR(z^ncvc{(q1KhY;aGV;bR&-fY@%Rvymcb1`JT?&fLrXwe3Q9ygDm*5L zwLB;EaJ9-HV`O(+OTAJBfn`Rdb`7s z8iiXFvWyFM1(ebTccas4ru^%$QJgP2rn2h4DjF$X4;n{S7LrvT)Py#0KAD?HLxD)4 zShR%Xm6n^e@gF_W38{KL%~TH#<<&KgEdXe$ECpEvaZ~fnjSLSq_Z*J3#mtv2&`TOD zdtwOglBBbYE_?V}pXi*JKh-jqT_)(H^wmD7Bi#*5MLTo;wt~Ekk=g{$rr1Tg0C$8HAk?+eB(sZ zyPP0W?vlJV;FUb&cIsGg z(E=*R-1#(q$Lcsl&s?U5ZwY`*kPyU`B9*2+xEB}Pymd@*0blt&j`RHnH=V;A_||o= zf^YFTfXfO#B-^I@Tj?E{)LhdzQsa2{`yF>o$>Pk zw@^V~rcl#&iH=~VE73^nXMyS1{Y(;%Y0KFaM=ex&vEeM84}rvW($c=g08;fxEr*?o zT1CzYP74eL?;V7^bA*ta>-(`B$k|wxR`)FB1_4C?Vu11DHbUB-)a~%k-Qq0-NN|vg zmVI1I!{IRAd4LdIjldq_yS44PTic+yy1c#XDz(td5M-%dEpp6MKdY9~tV6)I*Nue% zKo8e|p}1Gp06|mKhlBVWkSl-Mj^Nzgv&yz}rALF6{dQSe@OT{`AoG#f5EeGVl=00vH8GxSoH2wAb|ecC2j_i6#MxXM51MQ={l~INPyxg$Jt4zm;OO9Jg@9h zMwA4fTC5SaZm+r_=(4&*9lz3^xrm~W-cKQA*(ETtGx?e8JpVrB*hEM`sCH3Jd~g0X zm3Yi~^StTY8a}<8_{rRwRk{3(EZGEu7UN@XhE)etoD^ad`Rn4v>>9z}Bal#7D9*}2 zkAJ{DZ++I_Zrp$e1{UN+wB6p0ZEw=&5YV_!W{FUi(Tgw7anICvc{pQX)*F^YfWc#|&kIJ5$$q!0OH+Si72ZaZXMS&*N zBM5urckcC zxbW-zva+6y|F4zlUjCOBj>f;VAg=juTF|`&t9krO3mz(ev~bobs!5ylmljfOzqGL7 z*ZE~-Jsba73t#u@@!wjQZ2qH#F0cphf6)S<2Y8Ss!A?SMY@DrV3mCHXyoY#0Zcnqb zu8bL{^EWuo4gfe9?3PywX96rg;QAP*(SP&Cwp*DVTRkEf9JX8X(5M1E?u-$S_JTfMo-$SobYnvK> z&=eXnf(%mZIBq1RigU6D0tbW3OzPpfEIOfR% z$*msImfhW+=7?eVUpm%rkWM<_^U+h4dz(~z-U5_EA6-s;!=oQsdmfdwmqJYm@;%8( zEA<IX-9y7XA@;>=YyB;)3Kdr~lPG5V!^k#fXXz_)&B=sTyW zy*fF*;abDOSBqy3g>`ynPJ@$<9E=c`%nU#u?1#8-_cI4@A8qse)G+B8x>i9F7rE6# zekUHw%XtE9%(F90Auoy#pKiJ)q~0Ff(&lxcr9{%ZsT>SN{GO^u3!NAY9< zDwc+@RUI(^x<*>)L0r~bfd&Qpf%$e%ehl-`hV!4IldbAr9TQ^PDUY%?@pCZg3KQX1 zI^C;95u~p6F#;D7P8L;-=x(rxHVhcFMr|~w+|vrP&8dQA&XxPlhIhmsQQtVyKd~sH zE|Ev?zvBXc(BT457$_12ji5)g_MdnQfbhR~>x$!xx0WjZjkf>@VSVHN;w|*DKfI-D z7}t_({TFZX)_w8T*YKQQyd~W9A9+icsAu@Ud5cK(4{zmw)870GZ}oqh^Pi|hxQ47r zd(u}YT0`tgeOPnqmd=rSA6YA!y3l;(CaNBOCS>cT9d^s1S_CG^xy*?=Jgt(O^{i#L z)3+LTHNBi*;bM?=aO-H~uHe}Z1Z&{pK2_r+wAz*1k4v{`s zgK{0J$BC|D1D^B`>(BENTC_UbOET3Ro-aOsvPrQ(%&=`(0DwO^P+n@p!d$k22cZs*2rpW;9Kv(Lh8wr4!<7*4p_$6}T~b>nU1PGtHw z!c15h&IqRHzM;WEKks4H`o!kgf6m1t_cKr#n7wMNwz=5T_Zp1Z(!N+{Am2&I_%s|s zuQGAXkr3|anV3gjBjfteg&t4d^KidE#LNuo+?RB7dA*$Tv~d#)E{Yg#d%Wx(i7o!H zWE~)4@LJ-~_z492Q@JAS+^325UCf51JAjPfB~KHB#O;?_<9G?D2QA zy%OWytjMQOMN03F_iu$y~y%8HDxQG#j;ONkJd+I`{&wpTCdQ^_d~2&1jV zLszl1aYhVMSi{LaeR1E;@ZO<7SY)}(s=JCaqVN1xBpSF>MKpYrzCm~IZ z@d~L*fx_Lsa~82ZLnFK8fUon&8pCgs(AOfuR3-#;mh<<(ot)eh6YKeZGwH* zcFm5;^W5ERDms~}OIuIjVhunNX)Jpx}=s~ByXMuILh40td14b*TEJ} z#paCC6a)zvhf}px>D8MH(ddd)tn06~h+t?`7Wsqa&7^lfNfvl-J2zAvHFX)$Q+VBi zZvxAkPV^QE8+`gvYG)kl6>V+MZif@VOIbFjS7^~`rXAHZ#VFO#Ix}rRGv7iMkO0Gc zqDh4|PL)mPqPp5)HwJ~F9LA-uqdh}?VRSf&(Jf^`;@O-S_IH6&h+;k4`>7vo!g<`Q z=OZyLkR?XSXo2Fd0#!!0i)Y*pq}N8q0;|>*L|y^?#t+Gj@%GuC3JbWjcK+_T;#v!( z*Nx;9JtOoV8J&CEQ%yLgXCQz)JexhVIt6`Rh?gl=cFVz#X~ctyAYnoyi5&yw#$N}Z z?ACs*@Lb9n_L?-j|M02YS@+%X+c_7ww)Qw2wl#L(Yz}MW%<|06n`=*4fF#NOI)BX9 zFy7~I7qpG;;I+M^bIrOYdPedZEZ1Ozo+SmzcA1wm1LLvO=b)bRTc8G<&py*>Si-2L zYBjtLs64kKJQpgu5mi+W-XF{Zfx>R64$1?82w%$2mkZ*%t)5>+?nPs6Fp~zsbPYJ* z13os=WDR&87PWgb(0Lt9!gv%zLaKx#ECoK1z|Z9eC=w(}(lfrVpN2tm#tSXJq|dw; z*)9dtkZvt7g^X+EZ9=4O=oF1`rwUksDyLrA2&n-oO)Ck>f)1@fRo1W;_3kJVe_4s1 zojA}e3WRDVk{>if4yaQ#D3jy?v|sQbxoRdmQKp_<`G33vD(SCrc7S3GnY+4-9etue z&`KUaI@%6mTPg3b%vKIOP)Uj$aNbV|VTHgOHi{1GG^eg?&mGh>UY7)^Ck2l!Vh77P zY{pc!D}aeA87^fQTa!Z&^h|@p~%W}qYz@h8jdM7VG<{8@bP+k@*UHS!2!Av z3!u?YKrE&2?jZa6cGPy{A5?Mlc!BR7yVN$1cZ*yKL3hW&*a+^1qW%v3gG1=kx=L$D zNgP!@LLbV1#bJ;DDJC|1*#OQM1vYz}FC!=rTvljL5YEJ3Pt|hv8xU-W^w#tU!8S~Y zblY~m3*|KR812X!D??bT;9@QiKmB~VRkiN-TKm2{WLC%cmYyHG@@|Q3onwEV?dKUV zHv2O2)k(U*LDjSkh~#DUtRdcb-q-rCU|i}2^HK!H z|Bsmv;LC7ilz(|@ip~JvwMd>g z!aY<8su8Fnp9(4;Dz*h>O`l@{*j&_KGaL&;iYnmiw#k7|M;bw&XhWN)W+MW@&PCC# zyNa^*I~x3l309EF@84A(pzE5Str8yntr9c<=jJ)A^ZRPE{>SufpU4@&s(vXK6wbC6 zLmGP|Q$e!6B@)1gx6#1y>-#U4 z3w%ZH7KVJcu=;br4oi5*14Zr1Am{p_dzhPdj(1LdvEg2W%-X>wZ_9n4R!{Ay-Aa0z zhUQRalvX)P#5^#=1GNKG-?l#+Qft<`adJe)q`{fZ ze%Sb=%_M(ean>|1sg2qU1buz>f2!vZAf#_}C<@U#=2LxnKe{Xqnj$F<7|K4U!jru` zaY;ezdSS=V;iAFoMxT6JuI-++d^}m!@*emAPw;G4Z;|l#qI_d(!jYtzcS69FZB@A) zFlbSM9>O(PERyz<&z2V-NHq{Q=&)ln-Z7muPl+(PeA{zG7ZdP2$t7!NL2UoL`op<1^3aiSpiVO5!p?ict_W^~vB&Hk+kSK@B@+TG$rm6u7PU?$Z8_ zyrSV{obCYT8J_vR6>C;8K4h9@Xt-krJjAZ{qn{29`pgj9{)s;yfnu~jD%dj|;@wHY z&HGy(<_(HGC)R2*52;14{N!3~XO)}F$6rx;KjWP)8#DmGpIH3wqx6e-fWQAMC&vF} zlx}5n!ix5>p|kxxYW{oh0arssQj>WrCF2hxb$1hG6%{$C;)%768n;#jwEOPrED%wB zQlgH8V0Z-VA@0Myr|Vqm`l=X-__BI%I5G7?z+J`0IhW9S;u2j+0%dHbD52sruO5xX(?1gxxWoV9y{*PK?E$q66t*K**+MJMSexjJ)qjyP}f7IjRY zj$Pj|x3Hi$dhf7E^K)`eB^EmZeGI@v0l zb$ld?(`4mp56v_;Ji~yN_XZu~o_Gs?q$aCp$#Q^~hG9jyC@)yocW>#xjfipJ!b&+`ix7))Fr z;^rfS!h)p$qV0_Vz{J18a^rH5+h80rq9`HH@K-E>{49EjKE{CZ3;~8$01|QYBOD8c zb}9EQxgjv)m{jPMWkW4&lVyjAQqo{T5%AgsgTa~2)mL!iW~OB|Q$!ox9kg7F7z})6 zCO&g(D}qCInrGpiLQ)!ODWB;<2ZsJ&sU$(j159=CX4KEmNzJ8Uo?gn01e;#?779-E z(Na(r6-s0Z_ddUTUmxijxzkdrG)WQTmFf~)^+wVb8c6T9cdt+_e@oeB6m|UVxx^n( zEn&ZA0h|J}j{w@5)-&Jq$y{LpH7~qy&pJG(Xm`z|&~Pjckq1fso|{eUgL4M5#5OIUXD!chA+myN$W$!X->Ty@}1o~#)WYb+2&_OZFU zkT>N``y(z3;oFj-Mr-<_xpY?LDSgn#cLGchWGV5{XE`tn;Q%#i%W(TT%;+eiy;Ry= zqj>+|Lp64f2XaG-X1Gb?X=P9^10ZvO#hhk|k}`rHL-|i!0VS=!O*>+7TYir#tL2Vg zljnAP3=;==7hJ+TXeD;~jF+%cCU=R9KY89}f-`ov7Pw6edm1l9q1hwG(F1y7&F-ikLjDnJ{otj`5TChxJ!Rl@uSR#cco5YMygk`l5 zdo2}SMlGCRLaag>m_%mrw#jnZMdBjJ zsxw#CNgANCgG+7`Cx98 zVb$k>&<$M(@!HlUt@V_z9kQV`#5rGI&WrgOU7`wH)j?V{qhgC4r)p&5M=A;#leR)4 z8buGcX_*Ve63Qh4JYLPRzwi6f1&{NJ&T8|hLbt=G=i?j7kR=<7aXmjw!yqPX(aEi- z8)GXj5FF$R>d+*BRrKBTO@d|kEb17y<<~eXg)X)tJ$0?9YR;c8$IYyoItL-A=tBqJ zu`_%vTG4C@?-9%)S`H|pvj7)4k2CfUmr#l*qrH~oFA=K9H|#*LGjtHEj$ty491NJh z6)b6j4IY%DWE99+R^=~Yr2dG7Q}+&+l55Lf3i)vlN0)I2_W?oHRAa9hh&gKf{FaMD zxZB+upW5Croj zvL%q+3BiZF|94KYns3ZcWhc0mzmWz5;k?Hx@vVL!A{J2{5J@%JC)AO*fU=Zj2g*r< zmQ}^q4V(6231_qQXsDDYE!zDS9ix|_Jg^BVTqaNpZMsPheWR|ivXc2>*8Z8;lZ;`3 zmycE5nlRX#l6FcP2kSzm-5-xGw0}=mY3@aQX?ey7Q`6a|yn}rlkUJ3kRvxAzl^H5o zdGrHy^GAulVvg8$J7mbQ1(+wl7ntP+r(+I#^ZC%J_hDWIXXC5*>(s+U{U#swGXT~Yn&oazR(GY9X863l z`=X4DM?1!zy^h}xXNH3vWKcWE3TVrw1yJgcg}HKx&fN9)6*teL4R8s>s#mN1R+k8$ z2P1p?+H8Cj2Df5y!EOB;W(W!EfYe=P6hs0Yg7eLs)%A?`bu@dqX#8qBjr8PfgE!d}clCxDn>&;R9M1b*G3KG|U1;j?apx zIfB8Q!aKep;f}^a%{K)&W<#VniQ06-qUp(a#7I#Mdwa*^>l}f^pNGUm*~mxO2pnkV5a#bJn6%_$`71+22vkCd1%m z8O8JRo{*m#UZ1A(@*-xm1@HCv;#)8^0PB15#+yd46KAK3>jBFTg2{gml_y^Y zC@b-VvGo1^Kv&~eS%ja49@_^R2mv=tM~$!0>MOVpgcJlljQUL!(rBz`4+w=T#t&L{ zfE$cDwlT?Oq|_G^f2)uTVF1EU5*}gr4!}<`55!&wQoadqq^J<2F#D%LE(l6(j31Jr z6?_3dpp_5$H&}!weZkzp+0yQ{{9^e~-iXbkjE#JnN!1P+|FVtmwF9eY48C(R^d3$C ztF_JHt=GxHI%dcnS;d1QDg;k9wirj{urAlr>dX(e$&pqcjt{FS77h$$pk` zm2SKuClQ5c9?AJ_{i%G8_ylDI&NQ(?RUArRHxhX@=mCo3U`T9#%gPesu<6`o%Fc;p z&WzoX%+(6i7-0f#)b(V++Hxdiy`i{zwESD=sKH_8zoMjwS_ZnjlHGnkBZ0f1={~`; z{#IIJ{B6xhD+>S|@J27iTZje2axg(B7__DQi5eqsH3!bJtengnKjx2!o!AzB{ELJ~ zeU0pjv>G0aeui&eY${ucBHu#EhGZS4Tm2&H^#KHwxHHl{f=`8)uXY>`_0G(LA9z?h zfC!ab@V#-$o=?iE$%fAHM%S7A8#e~L>c_!&pH_v5IUy=^K01?sf{!4_zU zPek(b$8@X~LX~_>U=_XOm}d8&LIU{5EYE&qiHcfcD^^Sc;MM`MBgG}!>xxi;R_$8q z9%t*ld>=Qm155TP3cP84uv{1ms58G?FF^rrj_sjR=;=#2`~?5AGOfh}(j5W?0PrWW z{ri}A59}XeUZ%f~c}@R*Wx7$(%4UTY;UineE{rkkJ5*w`xMTt|9|wV4IQtTbl#(G* zLYvdsFwHsrfX`-HVbF}T&3YGW^yT%s!_@jSQ|8E$Jv(L}7DO?7jx2}qPBAOj8CW&} zMIKnBK1KSMKjbt=&B8aPq2JWwS8R04AL(nIq0{%l&1GYGytgW0 z>F)4ssa2EgGCOlpY9<$%+uQ}MCDVq~PhHeh1hb|OMe-F}Ak;{2yJ%5+YXLJOU_{?QYz&N1G&9s;mK_3&dGK@8Lq-oPS`%o-Y1^Of zpCR9EKw=OgB(xJIFdb*|$6B_}a@Z;d4;WGBWzUNh!I z*w1be*&sOOUx@P6*V@NLd+KJZ=XMozZlc~#ksUM%-Z}4TWjXUZRglITkriH6CC8i5$B-qXWLAwKYB; zEUiynQ1RX+$7}CrB`;}1VkNv(0j>IC;+MTu>u*AS zJeHzesi137qdh;%DX`@tNJ#F={rv+h>i4gDz94b?MPA?SAP-#RKyEj9+8ugY0_N0y z0Kx+*Dn;uK0MBwm{(GWywHGIDqP(sV8H6G3eUCGuSbs^+?qG9jM#(t`@Z%L`YC7^C z91DS5ff5+skQ%Yk1nINO#`8lCPUc%`>AKoUz7FZE!{M-~+Zry0PXUd>RR(A;f z(C2Z1i4;`j- z1SEx34@kO2QUL?17hbR!^m6e81T_1{9XTW+fgh$1XZxXbPv=3m@3lHq9^B7;x3LqP@^ZDJL-KBM}eb<8V-$nK|}c6Z|&|Oc-a0 zxkOf>04UD!E0UGn-IYJc;lcozs`2cQ^1Dsrfuxl1tjhdunPXp;!PY}%o2H|MX&7H# z4!Cw5B~|%>L(ML}JG`)gUfoeb7P~yRdt?f_guOwH9pBkLXZ|vyHvM%^>uUoTQ=0!` zrb=hZ9web!UX!o)i_ZZf9VD9U2m&s!U1ZhHIvazq#4*36!!K!Ub~Gj}r^gAEd%A?P z*Z?$UUWgE!xe|a+70SYIRu=yjozV+TGOAJB_n12K1A`K+)`#3KFVD-nbz8hK?|FX6P=CJ-D=2ePPRhqZI zkZrd`lv25-*17xd2cADZD-(zMd3AGc8bhY#}auUNpR$7L#b~)#_IeDj`qg`r0}+K}|T0u#Kqm zkj?-?qGlAJQ}2fKz@I%C9*XEmkDR=c#)^rELbImkLYAp0tA@TQ^k>Y#^T?pZnZ@V` zz`fN>W$$3cFNMe!&ADpSvH|`+k7-TJ1JgbN?fA;Y*6&nKlCmo$zk4SO1#w0k`#{v< zUCr3sb{2cP*ao`X_FooQ(E35ky5!Q%*P9zx!!OiTgtsf%^pu-|G8$8ng+4R%2RVP9 zJcPXU-g-`6&bhaF>YOM@8)Lrq=LE#;KR)Wp0bPmTFC|xbnOLLoXht+3VBX<{ruPvm zVYa6Hn78Zcl)}$i!U)u-V41cGv1W7R;nd4iMk+S7B$}iEiK-R0-DM8L>VG>5E_E#3 z`_blPa4*SBp9O8+y826A?yWElW$CajPA`hdI9=b6Pd=P3t)jDjv7U43`pvo5@!1nU zNt$oJPa35wTeU0yK1}uaUCMY(qE~?! zd~u*fB_651@dym!d-crL-14DmY6cd$c`SLQkff5$`@6`-C=?98#EAR!z#BN$a~uV3 z&Nj$z5AY73bD@Vz?Fko~p}W8Nr>hBs@i#;Jcf`=1Srz%BRs`&$~cV&tvCyzmV1z%wrauBWfvxPyj(_ohOaqVIBj}YLn|%&O9@dnhYNf{ zju4aiRQ~8rkj+@GI}%Dhl=7WcR30O*w`f6rr*X&CNggTb1yb7X%!J^eLF>d+UZOQBSE9V1&R$~Of+TX0 zEev|wCLtDkhdMs#)zlc=CEXBu>5w5FXrmIlmsweYtisY1zZH`rXEv z=bY#TGuLT@&JQ-HavS$eM{}n_hNv;GH~t(Xa{TZ!E3#igUeR3gON7!1{bC-HXdG&z@kAr76w?BH@1_?aDHtSAD|vcWkKtEV5EJsaj>6FS}(!703w4T611By}v^BE6mN2?CYiMM(CeQUrri%=Rss@-NGy zgQRr;GVvh^A3W0$=QJhEmU93{u!vgpZTVR3en0QO{LrepD6&P!d=?C0yocLdLEtY3 zCCgJ{=3)eWtqAtAMNO{Wjx0;-4#L9Q%m^0coHJMA?>enL$4>yKt+9;_w$*C@385ZM z#0UW6f+1e4C96CiH_oF{t^)8;L(RY`R1JJwO{gtTs;D@RYy9$QU+(HtF2a@Xi_!l$ zrN4Jq$6t=@@7)#M|H@tcKUw;y$MFE88FHlpk#1?wxqy-o3Elip0}*et6oQY}$v`yW zN0_DE0dbUCV(u zm9|@Ob5l#H_Zfv;j=;Qsite+}?t>?(RzTiQ0HKmC<#5OU4n}{W5&hP{?3cjV**gS<{~{+-7isxJ1SA}8B&Kq zm%{}r_-M{WY|$bu+146{ubpHx^LUd6!>fY!t>&vf$nQwSGiX@ggY3pf-V~|i zt*j9JiE00bv$u?@b6v7Vf#B}$1b26LcL@^Q-3glD?k>SSxCICyK!D)x?k*t&2omnQ z$liVW^tb8J=lP6G0RiyscUiSA|WM;kBbxi!XbryZ-oDT(;TW{Zom^g%PU& z$ym&u@5=xwl9vNK9j#KGq(7KOgeQ+vBOG^*L)h7oLXwa)WhfbTH453jD}Ix9p13_t zfTSOGW@0a%C^1IGXA_+WGJ|}u@y^TL+|G&irK&YiyBz*BCZU3eO+WU6xo=FuLZ}qm zdsTOzW%WG;8LK5vNDV69-ahq|hXUgf)`-aEo6z^;dz|;lk)M4JLPj|oLl2=ChuI;G zr8yD<_b4_U$F0l-2sFxGdLlvMzLwZy72Qv|2BTnlfuR<}i^yW~BO_!mB<-zoibC0Y z33d5^(J+o_oL_L8^QFFV9k4~DIG+v0IiZl|E|*2yuVUmC6 z)v6=kh91ANiqV~5DOJwrL@gfaHpas$Le;+g)2TO2sBU_INZlmGi&e>KyBK`8QSZ=7 z`w%B8g$sfR=+yK4jcD=dd~8)*KHI((hs(#a;WIZp=~<{^=-hi;+RC>w$1Tz5Q2aRM z@p1IB$V>U6wVaPeCD{v3k-gf+zJq=G^w@=1F?V)q*Knn@dGR%f)%~$v{LSia7?e(k zR&BzhdVI@Le^{M_r61SoOVP|>0uhwErmncPb)t$M&#;&Qrvo=Y?EN08EB;6CgC9yw z-SKC=?Z$;GWq;bAEh$ySx|Mjm2{+tVXMmDO_>9y$JP~jDr&DJvOv!_N50ot=}e8&)E+*=n?sX>TB@yNHw_DEr(3h&f*+!{frU|{X#8=@8KnH(WSJj zB{MKIaU9!S%e`ADupF}d!FH`pbXjj%DiWfZlV$Ivd@$Qu;AhI@@Qrbk=N;Z6C#xQp zjad7H&$WOW^n+HMhG@+oi4W!4VT-{-Pcr=OervWm6ss8bgImC7p~ z%DQ<^R$sW;Wf4Z`s21*X87^6&vHeRF2 z8UeTgEpli{;6@>~AU4M7T{1DpeTG+R+KrAcqU4H`k)%xfmfBF3rFCd+W$R^|{I)h~ zX;M$SD|%^b9^PCJ*cF6-LF7KQac}MctLcVs9Y%n+FlI!}QS1<&{iv3-B*8;j^& zbJX5;q)+3y#GPqBP)Z zq$I%W`T(!1`N5!OAv%INKg4<7E%mQ;wCk_P85;?v zFRt1Fo%#;^`BBBZf^Zl4$F-bj%_^W%kNt4+v~Y4Z-J%8R)Su_|pU?98Gkf9Bv>{EJ zwt)JM`2eWz7}n^L$=p?a0I!eJq8FDHt`wC=Dwve%^~c^@fq1@3G#zO0BsBE1wtQYxLV?hy7RbxPa z|IUJZubVvRXf%i-R;Pt=cm$)7mltsVO_#uAU)b2-T-Va!H-oE!$~??b9w&&|l`=@_ zk(ITTi1(We3!N9OvMpl`{a-QaM_&vDzv=V68_`O))x!@N*-c*fz-Z9W3kdL!FX{Pg z)LVsxApT-@?KB1Vz5)G0>3gUfEAUS6AxW)Zt`(;^2*h451-J`33GwH)E-f zeef>iz-c+OiAPMNwKyS(tz~_A)prWUj3yOrOTyp)vwtCVV+5Ez{n3hASgm(#Z(c&U zh(ykAMbSn;cFz4qH>C3(LE{VUJK9C8S}&t)_xMR?vP}5{U;RkRmU5jPAB7L5t+yBZ z>-->QKX3`8YXO)YlnVdM?9tEm>d&lYJ(DCLW-sKy^xxCwk)bcfA|q>|`k=KqqtQ?? zom0iZ-i}g<#x`H{o|TJjJhkd#?HX4G0&?=$t!NjV@a{*e^Qh-Bs9875Uezx&jDM51 zO=9IS^tHvsV|!SUqSr=TJ(&*MwL&%HhHASIw`tYqWf$!(Pui=Bf2VfRT$7(lQbHI! z=4x(#sD0OV>;*m0t^bSKUz3BV{awr7)ZQ`M$`Mi=P_hq4jNp};2vGaiQFuF~*@(<6 z99Y6QtCvt?Fkjd5x+cB2%49T8@nKm$osXFYj#Hn3OX)*XLBhD=8hy1;M@S*40n05? zQx%M$O7%Cmk}Y$*zhr8zUN}uD+DlYBNxuxopVj)OHtX@?R*{)f3r8LN?KwMl7Xf<9dsT=% z!6lAcr$r~#?7Ty34R#KZ$ncny?+ax6rgRO?Fq`Cojygd6FF3l2>WmFg~SamLf)XaueomC@=A&d(p7229MkOU6wTnIQo%Lh+`Zb@ ziS{`jISMl6kJ|D+vI-R>sy_^GqC09Dx2gAA`rZ6$2&1Vy=F}AT7E>PF|D~fi8~e27 zJcW7IT?^X~Z{%5u7M2v=nyNCM8v>0Lb|@pfje|kX4M!>Mpa(pPxPpwq2|S>}cl?_Q zUp}RnV@#_A{?dqW}ca&e~mz!4K=asJGgMuyjcd(U*+4eW}QBpQ$&Te*0 zu>*{UJV6ADuUHK@>ct2q-+&1wz^>>jZ;O_GBk^HcJET|p^1Vy@Bjq1FL5*J-i{Gu-c?|BxsHeACd+T72QE@fWHvJ?rmQ?D+)IX*xBHVq zi|jV40@AAD*2?wY*qy_>;t#uvcVi-vmb69xN5O-7`o!+Wh!({dy*o%BveUHuhkc>0HE%?6 zFVd0Ky3xJJiBT^>Gbh@M{)pm&F@?$RRwR4}l4SxsT`0K0?W>7RkC=2>?Ck7w()OTcXX)X3teawYLtlbPzM`<>RaZG^1y(p!URt@9Nt#2K$6=XlDTK-3HP+oqrgc{uJ`L^ z+dL<8eK@ZA6updKKzKK`KO|bX+)u%KyZ0-54JXIsLCuX#$t9*?ow=)D>kjP$J{Z-E ztX&~KuXD|B=dE#RD;7Uex*$gl>*&vTY56at+C|!Gd50yaJjY*zo$_D0d35vG7b{GB zm01>sZA-)NT3O;a7_HnQD2QxLY$=;v83~$Wbf+xaIn9)j{H~s?wytpG5MeO;Ejiy( z6h2ZY_xA?P{Z<>x*%r zs?Pypa(LHl*($_Ed_-8JjJ$nOwm|t5AHPNte%G!PrR;d0?ZHVPg8(L#kvJ9(p7yAF&j< zH6t6Lx!KFpm5>IVKIKtWXiz2I!rKRbiMqoP2ocy+R&#ptdf->}3uMmeZ_M9nuD%~@ zVW0hYJaaMp#D=I|u}ZUU&2D7rt@gE)*I$Za7kvRob4F{XM1}(puGQV{9Va6 zqp@Ut$@yWjsvt~D?mX&~T4R=V5#$v_&o3A=@i0AvZAfJAg^d@xxClSf8xy8v#qCo( z40R7BrZKPBRrO+)Vt0QI;PDT6^fY<)oD$V9qtPqQ9_1R8-j!S{mTF1sPJLYNXSt+J zg(#3&3pd*>8*7jr~KwCKfrD;x`@m(mTE)Y6zE!tX~qg|8bV`>AtBqLvMt#)o2= zq3DiXjsS_0h(padYPXeFn3}t63(~=A_N(2iO1LelJ-AC5JU2Z-HK-iAfa5M-+iDr` z{xF@5)a~9tpU)QhtRNQ0*Yq~c4c(M-4TTOO+e4Y)89cM!ir-R%cSgyb80TmVYNiBJ zh<#xYnQGy!dq1EY0oGLgG)=cogpf}mu{VCJgdyYCA<9jjjC8r2X9?}+_v5`YA2~xl z_%?@Y0_U$}2(OD2$fWk;6+xwfb_stdXFgtiVQS}0xQbRh=yS$(v=j|tIC%&~Y9|DF zEG>>{LWVFS+4b~Xy@!`U z2O_W*)9f9a%K8N^gCDv=OtkFV3_+d;3M~ecN3~5cg73+h>Tl<*K+|+#&@?^Z`jicr zrpFk~QS$-Q^bQws*a+;r;M6}oI{u#?U9rUCH}>eC9$m3A1@6~p{+1X8@-N-pQuv!8 z6X*hM3@#Q7_%}51`qJdwEiGBlY=*56_iZ1zhLmezPd$B1UomKFw+q_dk$^C3eoGKj z3Se;%GW|1V!9UC5|JgXbBiSCr;{9uP>e)y^W*?p3vkO2IAQ8R$fMB=sI@uUg&tqmV zaY9^=dAt1#Gs_I_$cVX~ySeBt=h2gTts_L4ts$Ff_Z&GerMjlvfbIKu_G?57Kz*kx z125F+Eie4wg?b^YpZ)3DObT(MD0P;H@BCBru7ZkJGDFa{X#KY`mf`8J-W05IW3`tL zFE^5Dx+D_vJS-dhke(#=mZwd)OK4G8=JZhD21NE3cC*Q9-UqbqldQs(bI0ZZ`Tg7- zsYh)GLO1Xz{Yf?W*LMqTU;&5r0v`%1Oi>8gk!p|i6OR?%bfjU^ku0?GX$d`^K=JWpi(Inf0t;8ylTcsaWf*XHh zUr4@C)9;P6MyuKd@dzLAJfm!~322thBe}!k>JuRXl!$ihqmEz$EbYVdTX)K`3lm8x zC6Nll$9Xk<8ntIc=YB{lLbnrtLG4!zH$2we2=D5;XsWjAUl;FRj~h1*l<4DFRNESt zM|@%K0Le~N8it!1crcb?OnE2Ufa}uDiS!ObRY#4K_3=G}uzrAoR#R&dd9d6DQL#pE zr?PCc;VVL8N_%-cagv{<)VH{FZ8(w1(iid>=>GA23gd#o&G?TjRN_txTbBJ~Itu98 z7>K5`n1cg6HA-}?jJ-;x%q8lrKe9anp5$;`U&JdJfN4Ro`Oi_y^la4f{PV=m# z4O_>lzmz4oa&d#Tl{{FuX$SS9s!AyY8CXe&O|w-({tHtIjfXRRZog;;Qm*fm9>-fg zqpfpuXX|2AsZ&mnXJqO7E(#~S7%el0#&?SAcm-@WUud+P4pp?4;omSLU-vPxk%c;` z$=MPDK^t5GEP*63yh9OnvsN@7k)Xo}*&49JN9u{P;%RT=!`r%d5fJ>v7b8~J)mQ*) zK8+o0e;5R7lwnhA>e z!R-CJfNe$rx@bBWV$u>8I=XQ3H_5ta1XR7IQRm+_jp@HA7$4G<@=Ep4&`T2-+9rKi zdN=F+mQMpBw&l|zs`w+fI175G%R?=oDtc~@i(OrX|GptQ*hv&l7@kj}|# z;%x5Sb}HqBFU!x>__NxZirc4IsUwA!0^AKf7yLr0&INLRw0}ydX=_MuNwofu*9k^~ zu-#(Z3Y|ABWjCki*RovUuaCwLizIa|QGA|37Xv>hdCF_#Ln5=s6{J$+g0kYiA1fZ$ zt=?zNXdKxmWx^~no^1`u?U?wHmo{upVh01H8*zKaDKv~1|D_u#zm|D-Ff{^UTarNA z`z>%A({ymAeS-^M5|7Z!za_ElPXm(mwYzSaFu~h@BI;^2Z;DIQj5CLhF}qn4eh^>66FjYr{nO zJI_i>_G^w37PeP3>9}M?VXdTU_q(_Qsu2zdwmGq#;FEv^0EPKKCx93L>i;}4<^AUb zct$mfNwNnL0D$d1b$xQOCK}ttk`oD3bgLPaVm7+2bttg}0{#qcKwPawtMK<6Rh*e* zgLE|Mziz)<3wpp8qB(@r8{R;Ypr2Z0Y1zKW=kLJnh=h=L>t)axA?Om=*$&cBzSy)A zj8E1fm#NJD>4piJoTAU>rIFpeL}REz#n^J^I?YW z#bEMFj#ZM4>jHy#`AbeMYY#d1@9$l6vOn4!SDkHqw!=SvWG!MSa!=md_5 zE&6NcLym=zbZ!}&O11x<+-+DjV#fv`M2Jzym5 z%wct}a(0y#PrT4s3TkTMYz*}hlesQw|eYVY86uUHflvUqye=M$Jk zw0?G;@yP1U)8uirB`Sz_D zrh#Hl!-AIwR)`7Gkpvo@HA7nEj2>#DlN3j_n9k^i;`kx7;!RauSYOTAIvTq5Ubepv z_H+Q873wi~X!?W#{Og+mU+~}gEth316}aAS)0fL=s3Xl(pEmNwdSN^JKCZ#B=%ot{ zT8AP}sNzlX)$+;Bo{5&e&Vx${EktSp*6|)iR_-v|6<7q1HbddOjY{Snl;&)pb$Xe! z>fC-YM~E2{kkrI@wJeWaiD>>Pt)n}3#5ugO+qvc6m6+%p6`g5WLhRH*1L^vHvx9@I z_Qlx@9JAwwfpL@?;cG#F zm8A0=_X|7ay`&K^&*u8Gfd>hBk|E`E#}%=Zw0RR+W>>JynCb3wx7g@v0Q=Vlo>?nq zy1>e4-D~^|HDF#ZHBrhyf@*2CBU!um>p{+jx3tV$#k%zgoxMvWl?VWIhV>_5CL|>{ z#qd{k-bZy2aOET)Z&($o56I=nk_ilNipk7gpN+V$k%;&j^A#-c{4G0r6Q@e z;iJ&hIel+q3!KdBVAyo`>|lwug_k#^%f0;GZJlF0=xo~Am$N436Lj$*)%eR~t-HK? z8APRni027Ib66&qMd9AD7*5%eL4(IV&6m0}($n!&RI90EVW$8o01EVfP62`E(3$N& zTWE<%(gTWW%>K0p+E}2dE^=m4;`>)sP29&4N%^FWP0k|xI^~a4tPhSo=}mR_-@6|h zaok8MP^#U5SQTyL8bNZUXChf#iW=a7XPBUDFygF%`jMQ3q9!;H)o}f;=qtJ?{?3Qe zLejhs>xo^3o0zN83yEyjt@g2Dw$!deDZ_~j;sdK5*Z4)`ffzxKv^l8fbojA_;P<^8 zqEm6)=ZG4uF-F3M!6lC`C{;;nVfipVYO3vb9qZXn3LE0}A#}!HrAI7&(nxIsyBp2Kblx3|57Bil z!{WF1wEy`5BvrUn6mW<*t6p-ge3fh=qUvMClV_mM z7TJMFd0CEYqoY_vn=X{s(-v_n%41JIKZdQYu!!0DF{bb2YtH@R^asoD#%i#o`z+#t zEnBn|GSXUT*j zRnAhpQwO=ixVtq(4gtMwS|u!E55F>mN+M5-sb(VX<#r?pi|f$cNJQph0v?TDoJ*QE znt>tP_DmtpFfaYS*(W~Tk^3ljDo(j=0*AA z8{#$B(T?V|u2x_qyM^{C%%~y?uE|niS*tS47#A4HQi4XZynl~meYUVc+tai~f3~O9 zfd^5tsb9iK+iIy=Ea*}q`jQTOv61@sCY=1={BmZCvMALyEU23Y<5LaB$FH36e!vcX znT-}x2g?m2Ny&V68o$r3!e4&taqSN7g|cY)`%Mq8+zcPnGdpT~889zgYe zdGU2yN|IOgWt19dv+8DwNWjB$9`#=Oh9e4?uMBd=3^a4AU_tMlm~72=B<%oL=DZp)d>3c`J}c?lpkYI>`2nVESJNVk`6k3 z{u5?kv>TaR?=b2b%_71lTK13KE_gGscXdIqoCUzL*Nqwg%ij^r8mkw;_`U^v>W@=% zzC*A1LUGF;acwOfqXd1rIAiYBtG)oK4W+PV8z|Ypb5xj}RACf5nlmZI&^jV9&_p>? z$KXj!at{p(|3(i#zXTxsK|%h{;r~1?KeLeVOlbmC&HpZTpi|R&A#&mKK^Wl7w{_t5 z(4-z$q)--ydA$DSxkk%1fX>&<-TdKh<>73=alH`0FMptfqA&(ZEDGmUa zu>f3lgamLoPEBy=Z(P>>3zxSOyOIE0mi!e7!sR#smk|J5-g^_97<6*VDI*C<4ZWtu zN4BoXktx9n;Iib@HSIlQ#j;c_3A5P2qez1ltd8`E4W%T(q$1P?`8;%bg+l|FKtp&f zTRyBHoG0-o?PD?hg3>y<28=R7V{8g=+DGPq@r#f-(OSj4v8~E^GI< z_p;x)8@#$i;ST!*rLXxSu&OvmU_qRn4d61J&VX9ZM0siH``6Xgtwt{fIq7Q`xq@g~ z*Q=&VWNm+R{8KacL$n9S0X1`$Lt_0Ja?*NwG-YavQl3kMT;en$ z=Lf3EWCfCO)hVeSQiXQ-tA<`oOM}SV6{#(cO(lP80QAmGhGB7zIxVKzcdF{jm zJZj+yL_t+>=gs#R{%YUky?hFKs9Fd-RD~)OM2zTi7>Xe^-B{olCca+vaGOKzgo-j=_Lyf z=p_rc=+?`!9un@QkG~#>1VqSB0;fpE0{NK1-GeAZf_O)Y*Nbx**v{{smR?=f!A;-` z3j%2prOUDEjMW6(VU>x9TOfupOJZFzO?eO-xEh<9zJKGhx3Y&W6E9wOlLoy@?Zjw# zlp@4X7-{}s9Eg@qd!5u!l`5oF8-}GH=IfWJGRxF`v`n1(Yhh8l$^Te=qsoKp+Lqq4 z(o_la!SZp&Z^iPuH}VVU)yjp`XNOU#LD`Zz|KI4o3~>Pvl}KQcTqf-chpjtWKey!O zrL0)msOds#G_0pvXr1T4m63s`sO&(=j51CX=D-h>%%<71PbKpe1msAol+zEYh^#3W zFGUR@%?s!jch|>04g>>uo7ODQqm}ItP|3_t3o4nT8Dx}LOZ0^a912yFyAt+1>B^6O zOE9j;4W$^+1aAhh(^0DdE(4 zZ66yqHm8#o*&h1bF}{xqq>?Sj*hpI+!V;A#i+JTc)opdoA&k3xK}iF&DC ziS=KZ4Dofk{@&YdIhR$Xo_7I1Bh}5?4Q<6rLpeF=^}>cq-A{Z~n>iQwW^e$e{QY+C z!kvhkNd#$(%j}dYs*0A=z`Ck4{8^!$Y)0cUSAS1aQtOVK#gBien4=4Df1BVJ9`qA< zL4%Mvmsu?7@mX3UvTVs!f!`H<=mW}~ReNLWw7d9mcC*Pg?H`gD7)#6mC%B}t4&GzO3Gu>x|Hb80qcVreIGp$W9>lrV} zmv8UFm8#p(sA9kqTzo_J=%?*LY_J&9w1gu+E7-QToVNNU1E`lbaDf%Zsovy8bt;F- zsVxuTb3|2nKxj^N{Yz+$MVTdf5}IZHO=$KMb#qX37*PO(=Bb4p0sIiliC!~*G^J{) zEh{&n9Oci$z@;12DjG4erWuQXqp}3|dv=q`Ntk-@O3%oyA;4QoL$lb8Cf*AY3+c|{)inlZ-IjOjh z?w0<%PUbh<2X3(Bh47QUY1jU$!+xy#m~kUtplsIGu-5oU>-HBkr=dn_5a;nDuXyY` z-@J3yl}|6IOW?sk=dUhW9qU+!OLaVBb`+4J89vkY0FpF>g!Y>2cDNRo)w$fo|zFolLP-RHiO4?Nr_A6 zoj~Zdgt11O3{>mBVry&sA@=z$)+CAH6@bkHPuNV}@s2<;$EcF>Q;3y)rrvIKtFN6h zb6*rqtwh>z;LIoNVRa=Txt|Qoqq8q$BPato245*li;BGJq70($rTr9Dk7YjTvP8(O zt+bm;pTSsS{lPRHDCVzm$_D~er^--tWP9l- zB{3!y;Wh+-viXCMb$x*b_Kw z0O0)b37o(FPjLQT`>^o@&Z%PnIEPpV1^st$ru-Y6UEF*Bfb&lfIOmA+yvYN=ndvM7 z0B6BG0G##Grb`lr;W|g^lyAyyDC@pc(YEoMpcAIleXZ|?(_m7OO^nPBosq~4Jb1xZ zmQ_X)oYWln;LF-;owZ5bGT|lt-NN6&X13vml{Of|EuBoXRaDPG&^y99Z#ZYWdG9A` zkCxMqy5>)t!gz}Lq}{c+kO{>Ul;}e>3qS9cP!`(iuI2hvY_nBI-EQyLL3J~?N?mmd zEuw0Ge@R?k>9rb~jHvW7R<&p26?^>%2$+}ue>Xa2U30eBlVdNqL)oIq$ z>l-EjoeosQrpx1mqFv?vx8+|bI=)_QxZnDlV4 znd82S4>VDDCH!J^oVjxS+YjlO>D%4>ROiB-N&~v!%@;1}DUGJ*g{8pUdjk@UcFIaV zI_NzaNuMFC1z#+zaXS0dT(L3Ko#GU&S+MOQ)Snfzh3lKmA2VsdikTj5n+lzsl-SJW zfo(W|#bh+Vr5akqgn)WL!k7^3Z+(pVEw;(HsA@{3?zynYMtDi+(X|uzuHT?Go$6Fc z<(suSo%A+zW0H&mf@*`^fg8E)*ucb-vfN%C@E8OHJSeRHS-}u`R#|>#?)QH!lQ;DT zpCbNgU*n&YW#rqx>*VWm$)`G*fe>+$ycfQ7LM&c2x@3u5nV0G zO0ZrSNK&eZkF3R3w<+E1Dmz?(5B_NQozR&QeqY4sv}kEF;~nH+ExWkB=)1F{d|_{Z zmB)x2PjcGp;_v19O6iSX$-)0_(TJHxNr{lNd8Xyes|6uYCA{B8<%F7(29g$u3d@0v zken;f04A@8)HBEWuqNyThNLu}9G~%hIMX`J6tv~rwpEtz>-&Ue1lrwd*oV=?K72p_ zd!6hWo~Re(czb_#;0+mZ#{V<__Y>m*|&1fZBRH0$enEsubr>hF5{c$Bk)*VVz5 z+d@~OU$nJy#GwF6^ab_6_Al=2$qKyfUK{GPi=~vBs({lE*yFbjmM93)Ev}Rl6`ZGNW=iq0 z@zLbcjZ5ms#>)0nx^Ff}-o*GDD9*Sg^Ox=TH_3+#p=htRke4b3g%OJu#=(h8d{r=I zU_(AkB~yD1XOS68U_6gq!tBz+c9qpfAp<+dvz;A2orsCHb>wsluD$(SoU&_cgs&pl#k!C)Z>Ue0M3;X0)1m7xJzP1X1S?3EuhX&WN zgVhyJ_SNN{k#z|kk#Wh2kQ>uVk2Onq<8t|Qb50$nFFYH_1W>&Hb0*lMfdBJu2KzJ9 zzh^M|#K0b8hXNd(%g3dOhirT4FUzIq`~_>_3e~NFK;K6YQ!Pgh}Ud|Z96O-f1yEf^wF7~0O#^7jH#Iar@vMqR+WGBx4)%~4e&E=fp zDuJJ~e;+QUGkyMIw~r*(|E_y-JAIh9Q zgABZ+uNx8BXWbtip^{!m>K0o=>s0j0!BZmf{UI!2I+GO{cZB244w_VFUj_-OSrTz# z(zaL1; zfF7)GQ$=z&>Ug2Cq@pE)<)(?I(N)_FaddueC*rQ;!fpKSGa`DGi%#UIL#yellmrt^ zayWw*a_+V3bqyTy#nbE<$@VVxja(ifZ)`DNxg0atAz(m|qsK74C}%isjo3<+Oc%o$ z&J1Otmp8X-Jrzb^stLv?MS+o=uJ|%My3P)|&A)mXC7P+d&hz%OExP6Ea*GjDEXE%A zena3bB2`;;#A=mDJ44S)*N5ThJ#>Q}c-R3I+vd+=II_tn_8YHPuNB%f5itI>v3dN&6-BO}T)axj>|3(%yT=F8Vq{ zdG^KBs3UNWms&^m7Z$52(63ic%&UgQX-8B33csN=$w3mU2$)6-N8oWDFcpNCM z#%v=QZlaqzcj7cZq0tc2tO{YCm3UTXoZ7|iA~&a%CXKP5Twa;E2iBf3ohfsuStwm2 z)LcX8d7<41o>-1F+FRPC+Ob90`m|J=ZM651FTS#4(BGrSm*!9GaC=dY?Ix%#05VKZ^}l_$^mbIMMOHJN5Td{i(QNVY2J#D;BLm3N z2;6B6nfGB~pYiDIyXQnfTxA!|1doZN&k9BcsWUOL ziBmsWUx-tq$E%U9k4M1j&rj&9TPcOFo1k;)-fSVk>M<1mE3zI z&b0CjQjHT^{Vfh%BdU#|g3Zb}^$hc)UJk-_(ZVVfQ=Q9bZGi&I^d)NcB4E|x5jL!D zyqssf%v@lA&>61vATc{T!I=`?AaRRtp?WxK?1?3W|1WDQG@%ZWoI{YKNu&^`qRO03 zy*1_eK?vfTo6GK1voK}m|HHRAh2Xp=sDL8_rCkr%7h78f5lLr7!L#jFTRsSwf9zxM zZ1?Xa6E_^M$7}z$&{&V>PSu`X{t5QcF)aImB- z*m%elur2w0ua%6Y0%f2HsmbyI_@7$Jy79=XMewI;^2n2el4CRxu)T>Hd`i#b4#4)t zQg)pIWP4*N+h+W??Tx-L`%l33ruND9rmALKQf*E(V02sIrYnOFE zzd^?#8Mv7YJ`@9_ACulwv|L1^TIo0h@AQdxC>bxbP5)xTg#=1@2xMXNZ#^5-Jog&=!A>{jRzUdQj-bkD zY?JoC!n#lSm&?lfPv8pnYZ5O%&=1tcz~?eAZkf!4U$vQ|eIHwvlqk_87p;$J_9wO* z8q;PO?o{2#fO?lS#OtG!5zqTsNiv=XOnE>)JFn)oBW3K;zT2mgZ-Ke<(Jha?4=*^7 zII7$g^#VdRsjji$t0f7unydUgKSW%~k9j%mBJ?--4t2%NOP@{?7E*KkHZd8>DxIAT zuc|UewY9|^Q&kAlQJ?!Ch08&Fy!VD8bPChvnN_3Ess7h@)Xp=i?|6G@@Y4_xm?1jT z99N>eE_ufa&d39hep4|D@bWgDoc$PTWrwqK<82zrke2!|mEs|%d)`1x}iGiTcP#n#T(T1Zof@6fLk#vh` zzvQ-<8)8zzTzf(78*PL+Jc==jOmP}zI7)@a$Z=dpj!8Mr#LphBI#;90N?~;zJfxWl z^F+PJzu9k&`QAAiU)JnE@*z`jsu*hQA(DxN9hLgKy&nE}mrr28u1$ zM_3XH0|Z#Q?bFcn2PP@5P5J_FE{|iQ*f|B&(@%8`{y(nnA3=>P4_z9AC}4{MD$4)4 zT*u^jMnh!(&vv3t^p2F5WKbVR4eRRM&?T(TDNqupWWz|S3WDM(##ax5wfbJev|RD+ zNy{UD?_B#p@}1vV2EzUwO7p8YON#*Z26kdcf62YnLp(FDNPqF=`HZ-5&yp$Tu6=&l z?JNC-_vkpS%|h(3g%|rcHhzU?Y%~SRT&UGoSihG>4GD)9qTKj8`8)sO?!zy}jWgHdf*apV+TiVjPOz_- zqJ@6AA|WOw*x%R?ZTh4a^o8=Cz)%?ag;N54Yz>YnYuGxXvcN1_lbLuL5s&&I9`tR9 zEj(c%a`3I)Bigv%CaQqHt1AYqT&!fHJo zcRLCuy3^@%(@V4!?pMTDus0#Lw5hMVfP2AR6rg1XY%T%|QW;9YRk$>Ty?#M zp)qGhUIx1l#Qh4H`o(5$Uohy6m@63CW{^4pr^;IcPuFkfIj0}*>A{)qYcPcmGt!qa zdQIs=gB&9)7G8#4j(`WLTY7?hrt&;aZr_SF4pjW*uYLM43UhXz_S!oc6$7@JeVmiu zt}m3b67T&_M#dyOX9FrF^ZZ_$hLt0j;;pEexv zjbvy&gvzo5I&iIeJ;=JQ<2K`})O5e@aWqawSx8?R|^haIr|0ZSn5cT)n~9A)AN?rQwtJc*_Qf zsVs!A0_!4LpMjz6PuRzUX0QO zAHIY0#X&*feNW%;;c+mRunK0>MFoGs(7cDjk(EeSmBA3A5@H5(HLn78#`f@ABxRJw zrz>k>O&#kytq9Wc<(B5a$+2yn$@;H_-Cx~3i48x+tz#rIixbI6WGH+>9<>wuNv&7z zoua02z3pBqKx=_-KuW9l=-L@QlscG}h)I_kp6}MFEL*K7`c0Op1ZcF>QkV#@U zelECUbU4yYzQ7|HGPE;RcO=Yt{>&3E2nB^Qo|6YtZ8hU0p3t9CEZt*C?h&6<;Pzn3u3jP~E z7AZ3Zj&_eqz-RZ!C=cr$_^v9lpX^v~S9~SxroZ@I(2qshC^6XlEcpz#(}@G3Phk8ZAd^Hqj#mf-N{IXy4 z#8|OcE;XHsvWvH)rQ7|xA6LeGLUWW)&iwb(@dKJ!NpJd?Wd{Vd172t&E>P*+XOq5M zp%M(5o`Wp^=02}<2T?q0&j}@`p-mQ0sgqFpRh5-^Kvr91vp+85ETDz7r`-bAaHQnm zvAJ}>qn4-EW` zm2c>YdXABy8VzjI=!MvLKM&439OC^nTOuGh_dmsb?6HLjM-i?Z-0<>upXu5I?5#L8dM$L&r|w`EjaFYNPF^3BG{Lh)!_ZBgGS zx_jmhK(lG}^W(56$T^lWg5f_z_#_Y)I2;DGg4C~|2WlLZJxm9dl=aI6JQxXMGJo02fIBaD20O8ZS$!r7(Vy)CIk z{YV->TV{_5-72Ci==PfJKoOh6q!O9%Lj&B>K+##}JVMQR`CL5iz&D!wtype1>;t7o zl|pfqs+2ZhoCL(nA5g!QOc|)_ z+zC$hieeDZ@eJQjXh|UUSIR+$R*m~ffpMj#mW{d;ZwcZV?(K#|pxl4Vh47VNlYuO9 zot~9BrJ2gA^kOs($Il?lTAWCOG@y+skgp7Od!<#*iY#_nxZKk^<5;xPCNuLl+OiBA zymBm{p!{a#|K9%pQv8wS{~tT_zlc9-Hq!oo^qgZU-}Ic(68>7xnfM!4%WvwBH2l%e zi9c*+Px?M22Y$7CkERq`7;dUE-er3^L{V{$fTbURMK4;(y2KhD+!v2}O=#-x#pXFF zU+qB=>tk15yurtEZjYY`I)CZuY!;bC5(62oao}o@^V7vXvbj5a3$8PLKRhQ(cA2T6 zT%8qiN&vyF)Qeh7uk%`Xm9Ad=#t(fD%dZ+W7HT|H5v_dcP5aTGM$-iX1db-}iYjU- z?iGlHTo|DJhy}-(d$uAx`xUm^L|QVJVrq0c>xu|IC=4V769H;jpCV^tiWCY#;k{5V z8(-6WCEyM>i3F3P*6J^eMAJ%nb0WQ@Vpz&Xf+DxHJJ|rOfJ0?dcu}+OE*j!4Dxnqt zLFa;=_2)23V7}+Q-aSo2a1Nced%sH|(3^OPkzvaa(-D`-Ml_A@IPcHE7 z!+jxS5fl~F3)te{#GwLuX=hZo9rIj0?JP|ePn*u97QDIViZ+F!;2%xcN;*$HmrG`2 z#%JgeFJWD!!GkDkn{oj6IC0%d9z4EOtk`isbW9d*k(y|4mBd;D1Gm_&5NGdAkC15F zr)OIh@qYY-D;8wT0Ic=^5PqC}6Mo!%_fC9D#_{MnZYO%0U@W5pPO+hr&l#X9aV0j9 zII&9ohwx)on$^9zkEnU2>3Z@Aj$Ih1+{Zwt_oGP>sgu^%YkX1hLS3^Ys*`0u26cxy z9<9)r+oRrPN_c%sA=rth`u0(5}L@E2lc(AZKAs~lfB97#Wq2x}KwEmQTpee5?; z31;TVVp5`=`ip3;pXtCZk8ChS&i!rfv~fICoqn8YXX_c?S!p=a!em=HC z5A(9x>Gz-@A&&{7Q^SK(%lAy1$jxe`fhF_%)(I$!p*}K%00f;Akv$7{W?^iVs1@MDr=O^R)kl{_5UL??gzjzF^LvR!E&20? z#=ZYXX!b;X8HHT--kU%xR>ALUxn__@%M>%kMKx4Sf!qD5X!jdl^q zpK}jh^HrVKD!nMeKmi0vZ))!TM{~zF;U|O#nTsi)F2phjA2*&5t48`2JAm*LIXn`! zY-Y%}4IEYMU3PK=ylQW#eAu*J6~iAucwE=rITYV52Fz%TLdnHHsf$c@e1|&Y&zEx6 z{jiFvI-20oZMC}EfwZcm?x%HD7@~qEAVvl5lzNtEnju~YM*^#)5Kc9p*%~3I32u@} zFsn4r0p9>;L8o^jOoRsipp{T>bdfi>A>pzC0^sB{Ct*T%2=j+~QzV;l5&P?p%dRH| zzB|EqKNyW%ImQ1QCujK1<@Rzjq%z|I^y)u9e9G2`LjV4C5gbKtuE5s!gL@;&uocmqV$-H0 z@jQiuN!KrWB%@eyrw<@JzuEM^56|I%8_53cEI!L$&?EoC+B?*itykFJm=6jdFqUwq zV(xVvV!r51N|>LOK;KUX5h79A)*B1QfG{|z^uDfbO1sB3wITT38cD^NjHjgF0d7%p zElWoI>?!6&G*wZu?%THc!=i2}D_z33zzsT4V~hqEe|JZ{1mCcW#X34%i`ujz{pNrY z3KLBSq(wbFMlTQT!T0FqVqkd1j_msb=X(ka#q$`dB49Cy!9QMVcA0DMYTt^hAQ;PugW* zf)#x2}NP^2%S{=4Artsh!WVpD5Ljo29=978()Y)}&L8+~*dMZqAW3*JFtcB+gpofWy9-pj#usrkNWgthb2 zMSgn~&4K1K6D*3{lA&x)G5Ke~MBs^TO}I$9470 z2Q5e&kixFVELob7>#KZNk`v}GqLuCv=0m_B|xBd4Hee_o(JC?s-mHvf8Hz@xS z*Lb@%Fv5Q(r&2S`=?gLGB#d%@tLhgfA{9;Rt^7B68CVk0pXcdml{P2@4_i8CnQi-* z4`+C?sz+dH;hTK^EXu2l&SSgE+kAk#(tQiKP0-8t3O)i{IZkEb$Mr#*SmBJ45Y2A6 zM37K)ZMY0XJ0yXyT}h}CmUT-ks#(go>DNM9O{L-YLipC_owC-IL)YK0Z_gl!QsVtb ziwnU;Xj{@{p*I6R@>wU+lKCmpwTwzVvs=#Jl!7{bDEdYRCW7kid|PUpZM(6ysJ&h( zeAOR5YY{CiO##*p)7CMWhh6K~lEru5;mm4w_0UeVaM$rM$mOD0Ad4UaumTFmOq7$jY_q!lw!lQ7(&HWHbQcJD>gU~b{1k*$-FPQ3v#(%Hd$X)u68*^ zZW?qdj>o(jd}xLxoFrBLEYJHmNN?h|sO*)i03^y_mlLBS`tNEv5A z?+O%Zrx_Pi+4Jr~mRH*vo{hEuW(d5`3uyI$lux;4*i%m{n`Oz`v%80POF$SyFxXZ1)aLXE z)nEGBC>BUjIma{koRN@1D-7!`WCCeTSa1%y^g7eVDy`L;Qqh)6fyJ5DTT2Pl)G^vL z5MR#^#`YdJjUSG754yxB)>dC%oE|RR8eY^p?<8zqPJ&iHJsGv&oHmxVr5U-+^*-ty zk9Expu)kD{x9gsubA+vPAhunfmpXILJo}TL{xxkbs!u)b#=1tUYECb9Rnz@)WURL+ZFkqs zGmEwP0e8LXr7nDRBW+*Jic`x<^oCrAkL~EaOOVAT4iL$td9Qt!>ihfgy8ZRK*W0II z$=kG-t1^~H_HW)Rgby9#sjv5!26h+f>YpuR&*>V|wAV)(?@rcTWz0lqCdR-_DX~Hv z!y$5WX?$%7sh50m;pYyd&Kl+)#ws&tnY*sTE${C1MCKWmQu4d{<-Lp>YhE=P`br;) zfy$c_wZz3#wF|+~;nhElh($AqYhu*y`T9r;$1BIV#Qf8A z2J;<|CO)e5I|M(gyGy-n?ZbEd>!X)$EbCh5`x9J{mC>iFIVnuqV8mdek_wUiZlH+^7?erQ2vD0+z?8%ClQc?iDH=6P;(!8JJ&FbEPF1pz%jUc5u*8$B;JD? z-$Hj9pJ^$GMCrjzPeK;~I~byv94sc;V-$td$=q7z@WqAAIZpj_DxR=UUj3M^IiQof z-T@+vX=c2<9Vn@v-X;FsZ2KYaD=kx?f(z|+1ASab94pq`DT;SWz7U@#ckU>@ZDHjF zArA>p^%IH!NIh@TmSXH7}jkAtzwHi+w@od;P*s2s3_IMsFv>C69!N{Z39f4z@%lk-_LXJ+U%~r&| z;bL>GF`OjEL{b+Z=OXyI%b5h3C7$DDcgSF3WC-r zm^LkoU@j*<9y;iRWYiIb4+3kZ%-=8;Oa#Kk)x*-#Vo|rey;|;<%bL6+*3-KlcDKB@ z@2}kO24Bygx03FwFa)~7JsHTbIi0cwT&Xvrq%#(*VATTT!^*r@tvH-(&QUEu_{tZe z0PZGL(e6=}6Ut1r(%c zH9;h8KPVmQ{@u|Ju;!)o7$EO0q5Bdq;y;r&`s+6zc=62abB$Uvj?)^@Po3)m6NR(Z zo!W}zGs=CEA#C)Y25)?`BWx$v=1wEHF4L?9lt{4lj^3Qc{xO`{PF@KyTIh(9Ezp9ke8%{yElUaAWj_#FSa5ci5v$Ds@F=nB`E1JfjtDe1tu z$_Gh2b@8^DNC}hTZJSPOH>yOM?)*OX*C>R{omu!>0H1-}vp@HtjGJ8FFzO|K34vyX=B3jL71Fp zDV&{W!KLs$_i9?C4|LxdUS*I zI0&Y0{lok-cXZt|)=!p(iLXXLqmVs2M3grS`$Fk(a;ltX`o7QeM9w)sx&=)G%0Xie z)hvVHu%Abvx>xZ_fcnV4(E#Q>>zJhy!c^e}7$QatNZGDRW5)|1`_)IfD5D8+2W`ALG;H%sJE8 z?3%X(xnF*8j9-z`Bh5?PP`q{~<+%#V*q!5X3aT&`v_^4@eU2J5`P9dnY0B#;kU3-@ zS8%i20xSQa3V!I-)^89SSim`ig}MeH9C!Z}B@CpKGQh^xzOF@!(WZh;H^!JDb)Mmq z6<{bpx4Q>-zlQODkeTw+O6w$#Z3gfX;IDu7$>9=c zvZ7z_u!HvIlhSe_-31Bqzh2>uP8pw`yE6lLgIMLiw|&4i6@cvU2T4Xe2dp&n~MKBZ*)k?eZA6Jg+UXWggXJ9AP-1W_>@h58fUc=dldEGn(b4Wz{7@TDp53iPCnRxZ?P$8D7yv;_|B|h-fu@R z_)ZZ0F>pyVlA|c&D}6e4c0_X%1AA#7arji(QXT^kD-kF#5k zy4ogI9}Bc&EV1a-j|kJek6Ib!wIOlZ?a*6H_L%NkuAFOgsad7>in3u(YjbJn)b$GM zqHi4DQ;lHZa5vs}Q6WgfZ{irxc{Sr{)%4I>14{bo(x9pLdVi?=6J&}@pr=DJieI{F z;nT=0q^gl`9SxJ&c3IhS+tkl7`rf<`8N)+X(ZUp_<+v@$QZ%|IWfRGDiO0t$eM%lb z=2xArxE@%LzNWL%38n^&15KM$LMl)+O+Sxoib$A#8@{1A)l}!8um6geG0RQBJwGpL zL{1{X*s8UQYk$4lsN!+ZlhM<@65+TCaR8UXfl7aoKi1F-MB_Ze2+gr2JCY`?*^=|l zbYrObfbOj18)$O(DaEeSy;o~g{z{DGWV2xY>Dk_?(7fD3+7T+z-G;ej5D?95jPs{* zf9l0jvf@~SQ!uqH-ejbG-&m*e(&AhZDfv?H^l6Be25hy$`lL85{76p3;Hzwm=#Z;! zlF@B?v0a~!(zv+e-k3NcnQb9{2n4>n%+X>btWg`A0g7+__oQcCTdOr5Kz#c zdH&?-WJToh=}K>bI#&7Ze5B+e14&Zj=dz#oOGUp55LC7cja8P1Hj$Qt|GU#s>1|s9 zEN{%L3*cPziu8|FmH=5>+f-QI+KVvH3XjGqzK02pP7`ntefI%698_@%0?n#geCxIB zlZk3WcxY*(ENptXMvW0k->W*R{Ij>jG0){I6}(~2v@9f#y-qdIU^vjqDbqC%)*&9x z$rwSM$NNtEP-KNr6> zgCFOIv`$3xBP8}SmO)B_yOg|9=uur_e%StR8O{rn3@A7n|2#={Oi%Kf zhaeYNzl^9qeKwG1y$!)AyDsa~^PPJh0W^Qrx>xWS%f`5S5V0>cG%`GFZA-EZ)Q57# zYuY;Ld_Ru{$h_=`gFM+UfAyvpZ-!Y|Or@H++osFAA>jLqMHN)3Q7n1=pj3g3XL zVb;@yYCr~fBRZi3Tnmb7B|CHBVJb2}!bHUt;8PFc7YpFto9)K2IMWrIG3i0Ek`+;M z-2edG&dlEexVLSp005^MG2axvxi3k{z7rDPrWWTx3->7jXWN%p1S%k zN9)2W+g#P}K@{|=lJOh^&%^Il6@^JV||s`9g#jX zStM32tD1m0DTa5`M^-M+>PJ{p7RY9wL(XL(|B2$DqpaN*+rJ2NSCL4{WX0cSygsuC8tv9<=rf3eK-;C|a!m z2kxXr5CBQhHoTRDl&lcUtL?jr6xy6yR`Wv9F5680Qb@ZEM~Yy3zWu|L^c>lNQy^IRmC

4!ZCXH#*B4VM)8vyc@9ZstCnDrD_5w)Jqp@NmiRXiL07UVH^!#nV-EWsHD3H?;B(2yzLgR#W#|T3QG$Ctki#i@XJ09zv1X zop+Pr0+Ikxt6*06C4=iPlU-il|1GddLUIO$J-J9?3X#+F;Gn(~E%KOrZGk0{A(1&= znI`R9?L@J?>wnnoFT3i>>I&C;{&D-`nIOp)cZ}BHys?1XP>0b*&LK15$t#-xW6_*F z8TF!--D9pfIhYIPTytUI^czoiZOs?ScD_2&wM=gL>Yy#?+s}X{LQ;K^;MS>Gv`f(` zg|l2j|8j4&y}Km3jeJ!9vPhjvn$Qbvsxa*u9c_1=Lc-wkwre|y#K1P*`**JGd~EnO z=ScWAh-2HUtM`U3?IZ+ym)5i(9M(W0FJpb^yuJ3PjDlNuY;dG7E+U}(sf2SX4u(3{ z=z|-0!0ns2EH3UI0{c;LYyS!%#+%c)Av=Mi_sc1bS#axiYf|Z|m8M)N2BeE)E42K6 zJA(cEp^-^{9l@mvZ%6R)+YvnX+YwB58cgU5g#tK&p-=!vFktmRJA(bdh$VkNg4O-s zj^N|#5W@ecBY2_hA+)MR=4ACbhotmC`Xg>|D0{Z)7d8~lkMDTON{v)h8jQ4l%PEO! zyHj0=VR^OILueKLrNqK%47On%m9su9dA^{p-h{tR$B_mU^&UMLajggoWD!aFk55$+ z)P2wXTLDN>(92Eekm@c2#O`0KBfu>+|G*-kcF^t|)!hi*_;H*G{mywP^6B3oe`d8I z)w0&skJG*Xx^_xc+y!^9{tC!a*Q zXsLa%&>8K)wY}r@q5P0m2|yBrOV;bOR!voMuqSz@zMi;%r3n5h&7{*4cKxc^Qw+03 zQQz8D~5_~!7#hnk)ld+z}UP;FBAn@MV<-p0hBZyi+_W=)SFof&|SaaE;cs+?%o8y!ClDT;I3AP z^nHnc7e;i6|2j%GVD&$PyVRRSC@;Um-Lcv?xEq@;eINCEMd<$=?h*_!oCcTbJ1^f` z*L+wR$Rd~n6iAHmx-?f-E9UgptTjHjP?-7|z<*N5B zOd9XOi**91!346tZ<KY4*rbO0$w7Y;es0E%UZf*ze6Kb| zYxjZ~b?aR3u~VI6L{n;F=e=3R30%NHw9=^UZfp-ly)wjD^ysP?+Dp3X{D+^NfoCFBiI5B1-JXP>~TM%uFP>u|PQ(ge^4^ z5h6+$RgJ{GAHPjXGE-AeN^)~WkP4&%p$`N3g*E@gmnXLt_!L7bQwM-Ew6VmGyM1h} zrhtdJdf}qQ;W87;+ysI~KZe|S=}pi`zZhn#auGDQf}g#I81A+}Yl1X7^#Gf1rhuVA zkrX!`&LVbf420^RL)yetCC1Z@a>@;tNR5;HqDW>XphT_`!YE3G9IHS!O`;Mq=}&F! zBVaG=bEY$EkzKz_#-qfY@VQj72&H7IL^No!MAVc!;S7Q@k?+SS(-48{9_2Jg4My;| zP_<+x7fRS56-}Y*D31DO?C4S=HUsxJw=uJPyKZwY0E( zowhO3-G3^EQVcI){>knK8 zTa{;gs10WN{2aSznKQ`fy0^he9^p@wZPV*YXHv@IlDejLBx{Ep8c7qEz1dQGjFd@m z*cOZ0kgiS+wBI4S>dQjmml++8LG~H@>gz6|RLqfx5;9>np9Vy5BP`7@N)(P+{G1xI zym%+bm7x;>@@QNPB8&WEH;Z$A|Mo)dXaXBt|4IcsNiqcq-I8kEXq@7LMtaycq?{J& zB-0TV`SBe6^mm|`hM5ko2Ya9GMshzvmkJ$%sZl2%^=Uw>)~d=VkPR+Dm+NdTcxlYC znLXDOZShl|Xkyu+%xn<;2|9?i|1-tJtj&ksM>uem6UI&pbf=|_6Q8Rc4$whfgT z+e9L4l<*1~bHBEXFutF2R3=Pyu+Nq9+5r`fv-crB<(Ycr%apzL7G;%6GzQdYg2_MB zu-r}o9tIc0=si}p{20Azl!H`I<(gM5w;OFXmYD^{0 z7$DM;&>Wo4dvd-msVkLq}v5F z2FP$h{{SJq1)(oy9sRCoT$Jr%qeF;zB?$umy|N7=xK4j}FCXQ_=S$xpB6-ppb}%xs z;K&7c3uPs{W(UkRpEZQT0p1Fs8S8kqiLJwCCi(9Xu9W9H`m4!BfCue3EDn(4Iv8$l zO1hohW}~o40s<}HCgd}7ajp>T4Di4W8 zOASLU2k@EF{Qu%Jp^N|YnNoq$$Gp<6fhQh-&ybP5`HYgVsuI9w-k!Yq%Z`p&L-$lJh5c58f0F(2%TH+A%wsxC>>sPd2!p9;iMb z5AKJ$x71sIH_lmCJS;QW&E#dhiD|#f9 z%I~$n2s&ZCTzQDpgpGbH(|ERg3OMC9Q|kesG1Fg(o6Y|w~J{Eq`cFak!V&B{6AMZ)Qb zu+fSV*=&d4)PNtTj$o;M*1*)MP}TYp*;r0s_tm|Vc#0<^wRs@NLj zMB>1*t!f1SkRzIl2uQW2?Q3Haao!k(&ncqV-GxmbX#f0hrCj{{j7HsBb3w>~zO||m zR3kC;LpcyRGmE(^3d-vnBgg}lvXDW^sl@%FbfI2O8Z5>k`9sjj9y7);Y3gF70#uEX^3p$*MeV4JpY&Dd^DhT8 zrGZMoqNRaw&d+CpxMq`h{Z_0zp%Y}WKvUnITmTxNWIlh}M^gcjB=dPI@uyO}mH5f9 z6M*ocUEG62tisYbZbwgSzJDSBjB9|<(B1#J=^-umo|3B|AFqs>tmv(ms9+}zMqlA! zw^s?yMhB43DBM9KE6PL}k59+`+fL%!HfanO;}q#$e7U6FKi<6}KNgRsWmzYD)&L9d~sDNCjf-I1zS7F$+ zxLe_HA1=X%4FoNL#Yg9q2N{w-Q?}1nRPnb6LrLvf!xK3Pco@81vv{?%*L>Y!r7R3K zJDhjE(wwdq&6*G+cj*xtN0j;vY=C(ZD`|CN*#We9oNr1ap~%iSnaRAcXNj@5hJ7?z zg@%PLH!1~Y_M2>wIaxw}-k6pN_J0`-0% z58M2~4DS8_vEd1_*_cO+^a=7|AASt^n6^I*6|p7Y9|!P}`EMB=AW;T^kWZTeU+v|& zZi9Um4Dh84qFmkUmx3XP`xi*KDQ|->YW@*rw82KGTEE8}&U4k}Y4^E@4Ei2iA z_ox9swg>i7tt}&e$k~_Uq*=7qaIHmPwStGuB$GyK8@tyHZwEx9;p1UmzJ zE*x|ECrl;_)O;qT-D+GG&Fa9~MoW|)d<0DHkqCDN%dHxKv?sKQE&OBIH7?$OTAgJA zm3jFkk5T@lM!!tMF5oXAF$YwVgFGG%sOF;-YLx#}S}XY44*hQLS1C5Y^{ppX%qR~= zuj={v&0ya#L;Dtf<)t2di!J(Ve~-uZ6!mrA@EFRk*g_y4;QAjFTM!K+XkVaxwwRJp zkyyeKQ$b~N2<{wNu?Kf(eAc_>-L#%~Y^ba#52ZS(iz^LD=a{gA|D|G_N6|8`yU)?) zR6F!+mz8`X)lsNd&r)LENU$nj`)^o!ziP@c=N0nq?xS=;ZkNsSpMYkGny?Y!zZ@&_ z3gq2UWgrI16`y;qxg0o5ilCF^!=g-sLjHZ(8L3bP^w2p+f~F_sW8QPU!WjPlu7|?q zi|rP`ms@ofZ}PoKL9{6s!E3V~@m1jNpKX+#G9F)l%^!Ac8tdM=fHGW_mv*`OU6bvL zR0HQzz7{Tz+{KA2Hgy6b`$JcYk#k2CbP_KLC_XBz%YqZ6QY#~U2nPi_FICitq2o^l z5v3lBRBZiTk%*CyWD|Tx-za+E7kKN+U7SEWrB$8U7DTXSWye_ETkK&oPGc+Q(%$10 z6&-1(j4%bmxp7jLqtT^BM<}G2(WT)Ktob zNONBp9(~-3X<=%7T8^_cx`y7p952Q8KY~tNlu)iR ze8ItW8Q}CU9gv%jGjGYQ7qT|;OQ1U4%};d9Me(M-JO?+VzH908$kZd?)SaKc$el4Y*j)CHkP#w^`ekf_4rq@PRrx*UAzCJJ2Q13!E(2XLCISc_0z zfA4hs-}6c!f5`KIIEA;ZvUh4*Vl-Xu2i?qCH)syppgTL4-6T0Lj|n6#W=`#&?m5LL z1w_@>Ug$B5uY0?AKG@bThwEl`x!kY8W#b3N6cvf;?<6E@aq`Q+a5bwuu$XBy<4_GTHCsWK^NZZ~ba{A9N!P`Qj0H z=OhdZTssB6+5}kU-Vy`E3|W zCVXYUHq#(K0=Sz=7Ie#$5r&gf92imxu#8dQq-G_IM1T*@1N@r3z=uow+j7H95Dt)( zPu$6JBEKOfJ!#M__fo)CV)$TtB>3PjMZkx4lXGT*KfX7nd5dU)X=O?-B)3cK8YScc zuj9SoR%1ly<4al+6wx~0<}pC|dr6ebrF{qPXT^h#0^hOrsax`dUtkS(Fz`qgd>n-Z zf~oBN3hloZ8dq=O66?e^&=!V8`1}!rW)tfyzG#XHalcQ;aA(te%)L^2XY)uDzXU9z zyH~6qm7e|LGaD!(27wD^D{zRq&;6hl$g5z6=P3BsNWpP_bjrG@7J!JUyTZceRxg5x zwd5xYI(>=K&p@SIn|2b)s1mmmP>to3q$MlHbY8p=P)GZf_NLNKR4vsnWdx>^1e%@Jc4EJQ49 zW48N?3nxb>Qer0a7r*mFW0tjnL}McC!QRva+GV<`tv=?(g0|0V8m(ywJwgV<^oj{Z z;F-WAN#gZE8|5d0|(U>hW|;*$&U5xoShj40l0*Nk%RH%CSR#!oM4UTN$Xko-d^V@FxZ z+t1A2{MQle8_Zto&GGXC6nJA(Gs{>J4m>wbtdJ#5dBKAQ2;rH$pnlLmvgZ~9vOjh) zKMJ`|1QI2ieqwHa>?i7$W*hx;oB!xEq5{;RBnK_StHD53>|MK*o08F@49*TZlSaz4 zuADg)VQEv5+mo(fp_{vmPodjWCS2;tqOdzTM#b^p^!-cLGLKGWFcyC3gi4q#fDgQ- zNVPVlO#;93k55t-*9-gkwL}0RTkGQdPa!)#Jr$6Zz0o10<%kF?e6n}qp>(t!cHZdP zTB88E_Ar&S93NRa6&D*E#;Q%wBX7_m{6LD)DB0sG4WWOGaU-8~Fxr6B+qMMy3sJT0 z8#;q~E);1wyx&vAQqNGsN&BRs?EgxhIcuSI`7h*|6?*`AhR+#5p1Di^@5wXn^#5D( z47K5;bDNdN+kDLS1mbbos(D@Ikeme8B31jd?)dgY7Z>@MXG=i5I}`N&00I2S3d#4W zxE>2mda2!H_ieWGvD3E^o*Hf4eP_xR=;bGD2E8`sF`dvP-G9ViEe3+^a)TzUid?4X z=1bLpp4E4aSxkh+qa9_F1V0}Z$HwND>&A+_>8qUX%3f(FVml@>i;~cnWR<)WD4sI* zx_F&Wc*~2);@DxC;$jA~l95UIb*eh~GqK1`9-mVH6^^3#`1bl1t{Gt9rjg~cF+5Q( z&HYYz&tOXyU#fn->%kv^qG+i%GSa%EkHeFfSuPct)Qb{svnpm{yVxdbV|C$;DXv2* z$tvjJC`&MgviaTBXpI|K%x79caX{&`;*eim_vHD!I!jpAjknD}e3w-GZLsglKj$5` zSbqnW{GR40F~pPhK}{gZ;Pf3(la^kMu5@9!&qHEYq*d3sua-%Z=e2hfJyqwjrSlzH zDlQf4X3J!pyF-^uA$*7Xj0b7F00_K;R0&7uv+I==mMzk&QH|E%IY*YW4f?vcH`Ui5 z0;1b3HG3x1QpV7>Z=qR^h1{5TLk;geu5s&4Zh-0d?8|q+w1ewBO<<1Ts=lez+J4a{ zPy1OczBt~NpYm)x7`b0SFlFlGtwWq)!Aj55v)0S>?S!pTinj7W=OQ?0n&rnvdx)xB zq&xPWjcU8=!Ybd%h00C16#A*^YvSpE9IhTiQ(0Bm>d`T?lLtkOl4nqE;0C6VHEMPq z8?onoHm1NC*MhSTBkorYk(d+nW0l3A_^?7T5zAEms!Q+Xmmxqp1tqU&ImjK%mGK``|kIdqivFrq2 z#CY(pWQ3Q_;mH=HMMMG$5De{9)by{W8rfIAwSwoMM!-jH2EezVM!tKH-$0%rzrn5g zu&f6N%k?V;eNV#pd7r>`tM8y=UU(1KehF&SBSb)1RzXZ)-^VP|Oy}kw$fsC5RfRNP zg>)DqT;YpU>XNH4h#KIQNh!*ejC33#{9YanAguXTHh1s#OEK5*1$^qMBwq}N>Wyel zE(HwY4^g5uR7b8)De`2GnKNx7SpwG`qrTp9wlu;Gxj9l;!yy8V8ah!(%)RA~KXCw-+$f|0-7cJu!EP~WjmyJEX)r0o7>6BWnB z-Ts!mkPn>1HcO95&dGRlbpIMW<|||2`ZLwjI@ro!q0f2J<*8jwqz!?!6J6B@!o8`v zJ%6qbXTpqiG`zPAJ6tNO5 z`cHkc7w|E;6rl(|M3H}e$BT>(d!HGQ*z77J(ilN0hr5q`GPacrwVS!FN_pU!Yvw9O z%sRPsX|ZSJX>kH7<=y*efI2>2;XOUp9(s<=)U_X=Q7*67F(`7p2CM~S!kyE=i})mT z&P@NgQSID%5~TVeZh+uvb5>Jo>QEgJ-DDX$W!#i(p%d7LI;tkV#oyX7m(Jr3J-v<7 zv4QkCDq{@tTQ*P%>xNj@wSR`Uudn9Y?( zH%a;*gl540Yt@{8QdVwFRIUNBk)SE33g(aC_-mD*x3e9q_z?@HFJpTX6D z`Pp2R$CPf9x^!Ws@dvwF-ma=++{WqDFy>*G-?v2n4?KkCC)Ngdux;))g3@i>akoubUsc4?TJb>>UiNRQ9aV@B%$|4$x}g8iQo6 z1S}A^k@`rFT8Ek$(28P}A6;zq5DWFidr!=|VGvm^dQbcq%$lOWMR7UPDRzCSx#(Gr zPVcDhG_%ygp6)+AJoawShxIVP(`djxt}qdEa^(}4baObi9*2AkG$ZQ-Ur2~L=!qnC zEoY2rB!=;rK;yDQHi>w*g8OKIdHzjpjVxlP5tnEDdC?xf)Ygoxg@3ztq+Bd=8M5UF z+~nCL3!kA`lm5jh0>`$`8tiEBVg+j6mqh@CA4A9*jF0mJkm5lj!FoiqDM#$0-AQxg zLWHRDK!j-I42{nhog0u6oQGPCiV9nj^s%8QCSDE@EWV2fYCDpH9%wG(jVm3{&m;&- z5r&QF&B*dw;=FSa_exbj2zIluVAiCALi!o@8x!{E0uJJ1HIvO>F8B1E@2ZvWF`*6e z%`6_|7Y%XyE^qAG9$ObGI_V7tol(QswpqJ4AW}`8fZPkpoCqtH>{e2k7U}E%zo$huD+Z-@R=I8yV zw=CG10gN=f&5-h2}md8$ub*pcW`yZYVs~7|# zozkX8t}v?Bx*iL7f4ot{WJUzM)4jl8lXhR;TIn)<)#>z2VXICzbY2}=3>eECi{ci1 zj>@cUA`079Xl0Fdt*!$z7yqmbc^UMdQM3YGeZ!1~{s>OHF*Cz?h1bJxCRmI%p>D+8;>s=?KJW}Q-T%rAa~NQ71h0W8fdemgbpu8Q@O!|7!Y z;hiG(n3EX$+6v#Hr3VV%@zHaSA{d}BC&}93fC*+Xh)K=a^QC|$F;d&UjQBzv;mm&l z$r#MAHKA}3IiEv6#R;noau&hBTLpYuUZ#9;K+{D8l9b+av2J?w>ir$x|K2Q}UFid6 z8RY>i-BS&BR@vseZB_k?Fl1|CO?kE@e2S0{5YpzUud^HeSQ!OY$5hvNNJ^4Mzx>koWd|hjI%C)VjZTlxuc11*t3I=Mo!n8M9TuB7=zfybB!8vxI;|){ z(>8I6n}@QIChz_{{>QXPC%24EbTYt8)Ja|dfUx_Qlub15f4PbO|1%-5JQlDe^y<6_ zKcxX&+8Tty>CwE1wd5rd5D4SE2u<25S+(V+^*b!8Ot8jpsJ_A9!pr+c%2rOpsy}WI zUqSEldZiE~M(^!L?M--R(E|m_Kx;XyxS(=UNFIQr!G6U6Wh}@<1F^soY>*wZ`Z+Q{ z2Ub9esZ=;(`+bE%>Rb?u;u@l>4$0@2-~Is(IVQUeG96}~+ zI>YqMyT$EpYiuET%lW#0<6ggc{Bq|&uX(-BMPY_2CNj)Y(9zB9#SR+{F3xpuCXd@r zDZ~W(`);a|Klo)tKwi9EO$TEX3E=WDliu+Q5-X4$@Qrx7#!;-Bk@6x2#E(onFY?of z)6%+0wy+AuN&X5(xGbl^7|G;iOg7?r=wJknU9+0+n7J7|c|9yX5Ao}Y^-*82r8wZ! z?Unbffx3f~6|t!ae#uNbfSn#{OeL{%29_jo4rs_UyO=lXq-PK0il7=_pV^zD4Q(Vd ztiu!ui3$_(Tm2=Wm`(r*HHZI~gp%auBhfa3;gPpX+sMs^mjaq6Y~=qv#q7T4BVhAN z`PnCJ^Gf&`z+inB-kv=z_^W-wMhuW-W_>@7`~dV%bYAoR#}J0i>m1g{XPR3z<1>aq zn+5pzKPYP_DO27_C%9l9^Fll39_MD5WvMfU$6`kKv(% zM=Lrer?E-PK9K+)xTGk?3JhO)l&*(wX;ic)t8!R$8fC#g|LzeIf%SyyN&s<9c~88O z<@Y4WkVRc>PAiHltg(Uw8hp;t4S^8|M)$}?F%T=Q4~;7OuJRvX8c=2x`ecKiUVU%Z zU@y_todSnf;?%8o&lg+842{h*aVUHr@j;k7_Wkfu?Oo0R;w~7i-8E)Zz8;qPBO^1+z9rPHCR`cqoQnHg!hk6&qEj)BZZvV*vkSYbzx@d? zjtj~KZI7E6z{3OeLMXMV`U}fPY>#&_Xka(|eqll8d#xxcBTzVKwei*9j0W7tz$~t* zk!lM^Nj~63Nup2LnGH8fR>1b!u`BMhWE?waO~zm+4<;vI%E*T9-+U#BT)3V1Hafx> z(vMzU!4@bKs`8`85e0*GAGDjyPNE`tV~c$|cKR0)JZt$&#fVN=r=*e6w}}wocvq-3 zYvJ#nB*4zYAL{O*RFPDb9+L5NKf-QmLM=cSuS$mzo(L8ctA6J8RsMSmwr~(K1%uLO zcWBsG#Lv==*jz}hvL8F2aY2evBoJxgASJ^`K&3B1|9>=lC>RO|`adgul@mWJeHSCc zzWxuY*Uydso|`f0`QIu>9+Im!Jzd{+-tdW1>fA(aEw#|XFz|aFe;`T8e4+`eh{xE9 zD`h5Qr`Iq+X}i}^0!PYQsdVOt2#FaTq4K1E31CTqe1D}-U}&|cXhNESDS2%M7>Lop z85P+KG*$9>2H3|NugtJ`A{j$)z;#@|K5=^0Pwax+qi>H4=BfIFL^T}VJAa^0{; zvsXZj9@cF?SY%s{{O^d8V!-&%4Ik}Ak31$Fa!$e3FS_G9d<WW`ISa7_pOXtH6j1ymQR)67BVlcUoKmsaFcg5Lq*TE6AGt_4OsPN((2ZI; zItwtAmPX}ccmD@@EAjNEZO>oRXnC}>GP*mlwclUgZGvr0MmyMDO|{TMF!UxJ^$y01 zOr3)KW#Ud$YeXc( zvDh9i(Nq9?l3e#bX5eq+Ze2xK1Y-6_LRBAaDU9rhm4UdCdEqM-^rd3#JdQ^ad5^>3 zOG+KhN}#?;7Ge=xD;SkRG{qaEU&BA9(B>|aS^`xrDwj%J!8e0CYECR+Sj$C88|qa` zwI^#oblvMm{;GVSmDw9h*oPXYHknw;FxOO=VV#qpf;7$^M&VE;n8+@wV3Y>*Om&x| z2IL+EE_h1dubMZSL8)l;98}8S^$C=MCje0D2%dl%hDgdXu(iL0P{ezg%x?j89iy4IZ0IlGiQddCd-jPj)@y6|6@T*^hcoH1+~NBlYGPsC1BWlE_$ z2>l*6-*EKkVQ(aKV=3l%Ln!8k6pj8DN{l9zOBfEHl~Nsul#8x^mITcy=8p!hnB03Z zhSebHe;K5tjKGzTLoK1@7A})wm#34+z*IIqWQB9*Vxk-BpD^cjayJhMfQ1&MbK?UI ziaV-9k_8O{bh_>5eP#&Tnwg-Ki%6xwf_k8Nl?e&9THHs48I^Ya4~$^i2`q&Cg%Rez zFj6K0z(^s@Ul>`?1YiU+whZwTBa#0ZBMTu)k)IeL@BYLH`hYz1|0s;e8@UH78vR%K zE`07-!UTuE!CsxzIHXV;Z{6}?Wjc;dOXP{;z2eS5KR0)&lwJHGy%qK!JR|M2Q5yGLETK9>N%)b!wF2$Fu z>_KE8x~qQl+(s3E!?(XHsz8D615U<$naqK#z?m|-0A4Zz(!*OWAOD2y0f*y8nCV=Y z35T6C2l&!sQ80L8Ai8{T9-&N(mVIJh3<$r8Y5Vz9zpb3fuOQMVQnJw;bBz3%$K;A8 z&SLr?&sM}IjL{px zEy=(qelx*BPgK}_4KBjyiMPduhDj(15sV6J(RL8Voamv8G{(bb(1&XTVpdTbEx-V} zs8y#T6qO+5~c^F!e|g|K30mZ43l@$LPk#y9e^jVm!tdE6T_gBFflvYl>-q0pOA= zQ?zmkF@gz8{YArB$09L`3Am&p%q-bRDddCV1&3)e3CR%JZ`BoaDMI2ZctdxnBTV~B zePnj#hH05wY)5`GJwgCG><#;*92Sm+A>)fK(31>Ga2-NDk}OIv04v00|1VZji)f_~ zCjFRw%2b8A;_nm2%uB#4{;i5vp>sFz1of}!RPMr`z2QpLWFX05dTgIVI!@XH`4l?| z7ASd$GfH*rzWt}VsT0&vomxQjNg<~Y5PhOe&vzgAA5=0$N;r3SKf**iNlUAf`SJGp zU#^WCWO|XM`DX#IlfK9@lUIs#eEi08{$NoltU65QfhC905fjFD)FY<1I2q&`31J@T zA+qeEd?}6XhIu2_vX+HMrkTde5Pb)fQsTGTE$!P=wk zSk~F&){Z&Et>Y(@-MI*Ve?XM`NZ}CfS24y!xa0xQ$J%-$hDMaE!v)hL(_s}!c2V_E z@p1b0++E@m6X$Pw;I}5EY-7#@fm^l|Sw*-{^NY{Ze|OH2>F7HcwhqEnVg6)sYEza_ za9Cg(oIm+oFQPez2H-MDZVqx+oDXgQ%yc%|4r9Jt-kf7JAuU%=VnUA1{FkFSzIhAX zFL6c)--qSEr1*RbAbE3p_(!piVu_|OYn8Q1f|Vx$qC(>m;iKW7g$zk!a={9W^9A!? z<)u>blm2yiosbrE!M9763lY$Cm^v?9u3#SIDaD&EVeqWPDhxO%R)LW#M1k>_qWNE# z>6bI-kVr_I1Wo|_K83m06Q%y!QhD<}rABu+OkqnXd5FD4)S-BQ0gRpMM^Idpq4JGb zCQ`BfmJ@=T2{Yw*$G9UmZjxT64q@cvM*V!028qAla#52`~&Mw64PWV50_e5K9WEYSSbwfWRKd6BT@m zDQ=Qo->T39%wOyqpQ1ql_Yv}Πs2X|4v@H=o>owPINw2A*zpqNkai^U*w(CgmRT+V}v2BUSbf+-F} zHO5%(nZ2FCM?d`1Sb4f|o*iad#QQ5%*E-GOLw52S>sa{~8esCKm6m5JBh4mJhykzb zpY@_R8Vibj2A3F_@GW|qxTJ|MnXfI?-H7vSB*AAd536fDjor6Nvq4CT)X6}KsUpE+230$Uk&&Ie7J`^3G1JRKht-K{qTsbI!^Syyg7XE z3ExXKNuF6z7<7-Bif`%2;>sG{`F?R(cFdBdZtTXip~1P4#B*s^=TsD>eIQz2304m% ziRcRD<*u-HE_ZspUpucqd=f?_Ym{zw2d(x(e5$y{-vh!oc0(nVb%o9|7p?h1=F`kE zxmhU#9&NCaAIE%#9vp!qi^S@>0Ri_)pzHFZj^hu+t zlh+oO2CVO$QbWWKZ<5wj+HonaSbTREe{f&pqZKJ1k!rWKKDm1|Jtd$ut^=l@a!LN- z@#qS}nX#krIagu~AjF_ClODBiUN0)7xlR^l!2K1K%ss~}T?furulaM;GWdj(zFwOF zc7xZGfgRTF{G+KwXPu@=m}Q(7nzSFU(CWS!In{f5cdn8|S=7J170=Z3dbP;P zywCAgSHIs~JfL?|H#G~_ZoXyz_qf`n?28K@U0Pi0Sza5J?56PZOP{vYb&{US*;wan z$ma~E4w(zf+20$N;%lni#CTLTRZU~UbxzI?FZI0dR}i$SZf~{@@h86zIXZnF8#cIz znho8aUN&x?Xr5X>%9OKFjELv!Fy7cSf==-H{A9v`~b&sCa3 z3A^6S?%eDgx~1ZePif)9X-6<>AsD<}tIqWtTC@TZZj&vt9_wUl38U z>040D-wbqVLF)ph1F2-;deB@5%;pH1=ZVLm*zbQto+ah zFc)--?Y%>WB;muw`&#mKDLh+uunS_N){SCQzQstUsK&$4=C>=!I%0?#1LA?f&fM;a zu6R0ND5&5tyLDM`HHh}fP-afbX%_Kfn@o^K7GxK#amiB$*&y!u2k+;cWkIrut!FhL zpwDT_e?MngA>5|b+3PtP8PZr6Z=`Z8;b+v|w(7 z*`J0yu^!t;*pDVW@;B}4QaD>-ARxZ%*_mRlZpH8Ly~ysII=wiYo+v)lxw&n;FP`(H z88>^p9CvuEnQb~+xbGZ2J?T_i@HW1b$y5c5?CKeMe(20z$hw8LdCFMlmcMxTx`%#>+S$T~g96B8Ka>YqC1dVJi5ZCl0;uGmSk3 zy`;Kd&>pNfk9q0rZxM~9nR^KThFnz4Neiq8QGPU+krVzQ4Z3)8F#zNw34&``OovgOCQaI+WpCQGg~+I8(@276M03y72X>83mho(&1za`Go7F}pl2qT z3^!h{RNLBYw$rOa>}N&+$9aTI>wwvDL1Kr))XfY@$3|3yxKirPn-rQ|j>9*zR_w>O z+TI8@LBYog`S82OniGLA%f~l`cME(lk#b52%ytlMrpu6GtbKthDL~LI{zSBk`eQOBAec^<oOF9#lrl2XjThvQ4-n{ z2KFJ8fm+h1>flcSVnI;jpF!c5p->mag8DlFv;Yf_4=^cnV3c(4A~t96GdUfjh*#_k zWfG(KpAXLfVc!QFt{xeEfw6KZ#wTM1_4w)xvGtwN+H!W$oicM#Q~9y3{CNFD#zVzs z-AgOw?a5Z(nbu?Oc-phAmc$369OZl_Eut~n<=pwUEZ?i6>xAj!XuUbT*#$lGa@8~M zt=Y+YvtyQq$J3(eQuDx(zB^Uzwt4?ji(GHHguqor3+9&qm?14S#>2tlmC1v?DJJ)+ zy;J_%Mbm{C7;A2J%+rAMQDy$4ef_IE=FQ%)8+DLaCg)`WTR9O-jbD{? zw}KC~D-Br*Q4QG-Rg+R;StHl*yA?%~iWpxTFo-6l(6)cG!8$k6d`{j^J_8G|CY3X! z$z_e$$p3&Q(k7K37C+HJ9s(t!0I$yEv9?iPvgfdw^YJ3ZuD?j+gNyo)9xOAWTJ3gr z$p5A)RALl~V-zJ8XNvqY;(Q&yfDV%*wx~TBejcv|DEo!AhyFcJv-8!S0t=sG1H3Rb zSmI!kv6DUd$Q-aw;K#bN^7KC zu$HvPsn$28XF8WZ6}nYzIBO)kpu05B6n5(cL~X0jpoR|QzExf+yZgC+Ava&$DlzTM zOXn)Q>rU{4e%>0@_!x605_#8jyIlIXamPv2J}k9w-R0NWumCPh@KrkO5NHhBBrSR; z#&hMCC;Q;ur;rdY_YYIZEFMudFDW2WJruV$5S=d#abU83rB;Z4KT+0rF8453?)50E zzYHbS{dtp||GIz?%=Yt!)xl61?FX1%=D$5A%PKNGho5my4W-Lk_`t{Dkd{>lelQ17 zb`N550TE~*krSxPTD6jF1`fh30*-C45{&SP&_(Xp{{_}W)&*6r{JC^>Q;_ zN{+(%cjaE~C*_RF3z73O?u>Lx zwSe)fFB)|eB&ap^v%JgY1M#Xb!t~x*gxFY^JNIn_JLi>gZOO40S2@`e_sx07rpk#{yU01 z$R(B~|3c9pg}+ebK@LDs$zLei&H|u_7VMA0CyFHhGm8Ew{Gj~{MO>dKD!!{t{vU** zX#k28&}*$PeYeczZqTJK#@LydFTQx^>!DVqR3sX(Z#GkSb=XwqId<4Yk`5}fw>46@ zeX}}OmF=ulMpArRyek=ciBkbvC}`B>{@P4JK3IAEtEHCk~v5cB4&M(IP z80)5DeKS^(uAa`}x;Ty3mkWNDdIz*ox)EEnub4Hb>82g74xN_E+(yeduGlQM0C>xh zS>JBhn_aZT{sgw5WW<}$AGIpALX~WT)+pQmqWK`}&lm>*;cMmTyj?ffQsM0_Xsc%? zUh{Kfpn?RS#vx8%xstNBtkj#N@0EN0}X-i7__WQFx09v<#2k?OoGKxl3BuPs~eDZTpXp$_C68TWIznjDtewKA#$sAi%d!L*^56H2u-&s;xmOXS>XraB)-yHvb0L%3NvA|!$I z*C*M0ZSIZV`Ikmx`_yQPFDx>qjW+$9ZW>ovn{$^V3!P0nLFBKpIO z+c6+9FP~u^*2^&wlh6`@958WY`K>ju1EPu%Qfb6cDZ9<5wYSh^WeiTS?Tab;3V z8_?VHL)^Yt}=ptLNxS)jd1GOf%J69sn_Me*8BIo zXdU||pqZQ!Gs7&pb0#9zuKIT4e#X*xYb2N9n5}Y_uYv09M|IK_g;9c_f{}_-*7Q zJ(BeU;?wcfGb8yG4hIDE>E8amcw=>OjFwE#H3pAN8-D+~}Y=shqH5U3mw z5YV~ysq85S(#NE(9zi9jvaA`7SFMIwbdJf-U%+Ui8&sBFGljyhguyGKVG74 zO>b*%b5Jx4d6+5RBC zZnO>b^c5&;d{!rQ|0J(FDJw2{PRu9CY{#$m_4-Ab1Vzr!4o%PSf;sJ)H?j6#xNv#R zTlUw_*T1Z-T`vf6w|AvQ_+*!q%_2u-?Z~?LLm)THJ9@d9<)0_V_Tdt^hrzs?=8x*~ z7Hqbi!3%5?=c;xnv4dln9K$m+bMSXeS~y zp|)giYUP^8JvgfIx$^Y2+S1O#!7Hi1(q6b_#w`yj?)%2tAzD$+sF+SFh&dpr0i+7F z)kD&h!`-k3@$+(|Bt#VWRcPagAfF}dD~qWq>HW;rD^2ray9VX^08`p`H45*(xgJ*R zHAUR3al7l)sPzS&Y1dIK^+?W5h~=nrSD1+fs)gAs7H2M;zD_&Hr33SgAPr4%q73^O zok>OxFNQ&B(K&~@W*ub1-=^@8=PUCXcn}|~MOfFC`4{nA=}vP#7$Z1Zh9QWNEDVhh zQXtE8GUwmw1rVF=YnaTkm5Aso4+DZp#~FP#ifKBVtNWfZ?wmlsO3J3*E6>@X-HGf} zlIdTDAj#h<1$@I*QgLA^!)%Jy)?`#JoH8j|yYKdLPe~f*QfHj7H6Ee&OgA`U3$Do1 zf@xv}XXulFlGmC;R-yO73@lDJwq?z$7sfBgF{loEzmZays&~t^s70yhVbl*_==lR1 z50SyO{TgUq2$u1*;wXx}Xupv59!JWfADGe8s2%ctlPA7oxgjnfD<8Ju9%Dinni{6S zTE&K>b|ITr*;)=vQkIl&aPhEy$s4U<=AA98=0a4Ow57@}kySQs>ndKhcO|hHJun`Q3Kq{oAR$N{#3W zs%5_pXjhx9)|O&s4=G(~Y7ygt>33N?ku$X2MrKr;@@uv8xspcIR(TQ?EP^oHUx@sQ zyvufXTk)bwc18{{&GHRxam+x92OMr2DD~EUD=`C95XA|7XkZDHEs-S# zNg}rv7TigJLhoi`EJZ5%!yh4A=4-$`uop{(MzC%wRQeOqr4+xdRyGDiH6&% zUt7a3kg9n?vZtQ5@nUsZp^W2sccecz&0|fz=;Tc!5E}BlJL3|%$~9n-)v_#7)fE4( zgym^{6yua<6rK@kOZ9Wri7u9W;IC|OELHO*4?m*@OPD3p^_{tJFZ~%z!iBNfNHUI2 z+>-3Y(m2~uL4@X-$$0j-=WJsDQkF|F?@q@g+c}W@aHNU--7^<#Q5szGysgrGW`smL zcM_>-5#vO#$UIblTwC1D2IvKEveGzumLl`dw=tfDEX{nCa*Wvp9o9B+$e)xVJEkg$ z^6oQa@~Rg15-$!Q6Z*2laKkE%>8w^UoBMKCJjfQj4^UzysRquf&ah(8yF(s;g2%vYGE)+tTwATC}Ia`dY8yCa&yUNRcDwnC=i`O zx;{I63Ty1)+f+043!Fei`c`;QqV=osseDM*b?U&ON>Ts%UmzlkYw`UfLbKekX<+;D zB~7*&2boJ^+g`O!iY|x7kvI+VG8t)uUPc8j!h5%QiM3g!x`dRq{_$IR+>~g{&PA!$ z`xtmSG{UjC`uSfqSKqn3?I0|jTiZURlb$qC~ zeSPmm#LX1QR>i+{O@N^GX;ezL$>-pbWDeth4fLr)moMlzj9ZVMJGLPQi=UiWYHxp- z;~@sZVVZlF*aXSd)0qf>W(LHMQU{_|et6!m4V>RExYXreTi-Y_s9f4ij_hwZYGt&H zY#4LUoD2H3q|CLiQ!lksElztNBc32)e7tyjI{k@qjtO^;VXYC=cn8rVw)xW92QHzA zl~7}X)Y9?zC+b-x?D>Lq?&0%9ex2?&=r=g?qvQc=ZuWpy+^w-;QnPHi%7!KI=d+{k zR>Ct^zRqTOpX!jh1Zp3YRF+9S<-lubtLNm6`z>P}UsajZto%ZDS);BNM?BRL{;mAJ z^@c}tom9hPCoy8FyUnP}3tG~$)l){r&af1jdkQ2ctURtfv;eSMx6!CPCSH6-FBA!z z;VJkP8sAIgD0-t3Hhe}-su4ZjnO5$wxTuZ3eCwj3gI(73oAm4AI9S@;J3w7ll)QXJ zkxhb}(XcC1WewcriSW_+b(L1LyW*xz;}MFuvI7_0#SF1!(4Fu;ie1-W%X`qpc+*dM7gFHO-~r4u|5I zTM#$IB;&T>7djAlUguxR3zPcf*tpoE^6-8A8GJ&-NY^y3w)mq=mR#yISF!N8o8%k` zx%}|vM+mK0zD@-v7$Lxx7gqhe1g|-mBg3vsFk3YdA9`;n6HIgczp;5da7-Oa?A~$j zZ%zT-^y!5kN6S{A0;jzw?ZwAFaNsxXzYFAHhRV2vwYRsXt|8=epRliZ``>5%mhC4n z-R&j(ch#_%KRDH}Vza)87e;~{;d!3=K?aD42ZH&%55}#i zn|Udva_GSo_7X%8J!6^PI(q#0w9q0%G8mYkfPg;ju)nv^q{08mLSy)Q3(fTJEwo0Z zHS0CHuU-$T)8OC*e(B-y4T3?ySeTBGF;2iu*6_l5WKCyfiNu1*&N?wZxTx*oNrlPW zX9IDigmJ;a^PG=Vietx)AeZxxMdkU;+~JGSFNhYOffsuTWF|7|<`{`Rmm4K{|yM zr;j(_2X%4Ob2#sDyH9!u)bK?!7+}fgO&Zqz@UPf=wPN>?%Du$gre^6uMp>g>n%`Na zMfS|YfD6|R@066+3*ZdWTpML}?F-HOh<4xD(FA)>1#!)qdh(hfUkun7VH;sKKd0># zC4b|ozyzv0A!=gBWOo2;?zR_&ge8F&GP=s7h&*OJr*rIi=ZT@Qgj0+i;RfvOCJfG! zn!+fqOdwDD7CQHZ|3QJU#Uq_$!_CQAP>Ejk1})mj<%P#dQX9&=b&W(p{uQMVWx2jh zGjAG`(CA1##y?N?0JCR>(+CyY?LBAGk$XC2v_@vXT${|ZQLfBJSzHsG*Zu1|5N!lP zEJLJ8sGXcrozO$Fe`->A0A0cOUGi*=|6AViH-JdcxC&Vcp9nY}C#MW4X zK!sd1DCND4VDwuRl^o>N1k1175dbxgt2l7V4%d&nV}c}Vi-nmQu~ERH?0T>d+gmHl z+K)gL*S7)LpSAeY&RV5 z-EtutX{wT0L=nTG3Od|tz0QdKJ#`d7gi*YtdW3GcUi2iIwn2AYnUP8J-Jrm>t`NF3 za3I56iKGka39qQ~5?H*Z5Brco5z{I{Brr%y&XE>_zZ(Rdra1pYgI${$S8jJq`P{hm@Ay4e1U%7D0=S(X-|R5H#bA%? z`64=?kx-!P0P!d@QpzpPuagh$PAwdRXDBW1RKYnue~>KXk;HrpWh+$ zPYtKsIrmP(R(Q@|sZS;xJ!nRod^q&3JzKf?sd(^S z7oc;0hEhUT9$ycu`!Fq*NyV{ru$#7?7lm#s*iL~~E~0EI>Q0+O3Z1)k63TI;{A6&s z&-o?<&BikbT}?!bwtmYvh>EO3g&QGW$q8phmd#{h%7p3QjjJwyZCx)A)lB}2+gre3 zW0qxA8HmmdOr2Qc_}J}9)=nI>#O~fV9^p+o9>uh5wV@Dv)e7&vp1KV!*E08eq>ky` z!oA1fabQg#c+A|o@Zgyu^!AD>(zx}liwj$*HS!bcuj8}F*Ng!(8q>jhI^Ry;?2^1^ zbv1fZjsQ`$vYI@-!CrgFB+w|bWyq*N4?ayh>x{@j84RIo*Dtimg~>j~-XNFC%Q@XQ zDKaRp1vHqPH9x>0`xIipBn17&4QlSh>6m9ofQw{z?xNLdL*67qVIqGimR$ntr|(eX zRlQ$Zd%rxkyQG(Ml1qUe@{E9_wcL}k2F4hIBurDJ)1(EwIgRnOK#w)sjxvpJ^;AaV zN(&NISvOMPZ`~iT#7=(srj|w)V5`>v5==9iU>1PM4Mj9XAg{)mBK<|ZLhFmSB5L{$ zsT!!uOMPuw;@H?*ZsW^eE6TT0kWM53yPH3?)8Ff@aaZIjB)a-Tx_;UszRDqD^O~dYL6Gzm1V+Z{9 zsEAh`+?%_d-L@Og;e&AVvWztA9?{ZKNH;hx)&=~JIiQ+NYyI@;Wvho%coV=-^xiYS zlCNC~g|+L04a2|?Lws#@?Bl{D(bI&%I2?DD-Xm;Qg;F!W#JCy}?y8vmJ(|sCHC51A zg-|7lZRjgJf_E+x+mPn%AJW8xujLaIwnt_=kxUlZll6@K3?kUaHn`f=nk)R94x>-Z zoBIRzg`C*4&n0#meX~4EB~}?8rxmY6AN#){lFxmUj0n)dBUaFYG1iEJ(IZPgmMzo+ zdl8-^Hw?6Jj6sgoj3dtp{b zx^VZBo#G|H8{bY1Zob6u#JKGEm!hgBqzuJDh(JvV`?0t|zNeqU}K{cO&k}&v+$Ss62BSWR8=U1glJ}$N3Hh z9N;gB#~Q|XLm?51TcQJNlQ~YU${g*#s$U{nT7bZG*CfYKA1`~=8CBWo=-!2>Wlxk@=_+SBPBBB zLM?_CNt=?7al(i!=nT0&(!CU>KT|W~YK6VDaq?%u@q|KaO{paQ#&~TcZ(8DuQ+3o= zn{Qs$FO`u651M47nAJWroiz3nm+EdhD!(AxzrKYa(~FCdT4!PBM8g*w^@Qgm#8`%! z3)?A}DaTw5yh{=6cNt!g9I?i;v)zKamhka~8mX#xDmHBpSW|i^YKU%}9uep3qGh~C z;oTIYE;L9*%Aqz$`&Fa~@TPM8DPF_d6{RFCN2}dmIw~I3(wJsf`opUvi$lzzo;+Cf zG%{B=TZqLQdhw$~dvpX?f@++`^JGW(SO~^~eV1jG#IXEx{Z0}H) zpZPFC89w5)x<0PSItza^MpCw|`nBNPyWS84KN{o3K&&VZ4FjKxzk4~NPN2IEy8NcS zzQ<>o{Wh}hQow>Gs-sVTjYzn_L<0Avf8}doHtWdsJkG{gqHUqpO%<3_RZ>!Mh|kQT zQi9&i?dcubZ~tUjXCx*cKEGeGL#ZbJIDV1cJp`u@%Ec%0F>R%I?{TDirj^t%F#qaTC|14577$iZdP?~Ua zK>L4nph3Jqo!Yia7*91O)|huFv?7SvARcMKbxInOYW>;sEzPNs1H3e4dwiOr5E(>A zc*7f|-~LI+maq5H>v5&%(GG#h6Ur3jJMb}it&+mLUK1H!2Iv@2;le!=P@j?tX(D?3 zZ<(Sb%2pNRSj&?y&{R-Tc#oRuoi{0Y|ciIIQ&MWX^?N^A3^7rKYMjGzijd;a28irS@4sMn~ zOQK9AHcOf)p4pBiPlgE*=j6#mEG^ld-zFESZQ49_LGp3TwD8-k>>!G!zH2!~#b0*Dm9Sc^@=H5m_)9vYmP} zDUBBxCPo#rt8hc4mz^`vnJTNQ&>Z%Vu8|tKIM`cH2v6IOm^E(;%Hub*u)4)l8^`MC z^(zk*9VKS^zA6FU9U&>C+3<8y(WAXRGmPsSB2XgDTn_n+d5uIOPcOE3F*mZ*7pUh+ z!zdbVU>phrIlYwymCHn{?vVD;&7?)~2Y2_8-|1j;mxw~|6-8+(sfcAkRpc>Fir+|% z9ekc)3A~%^o2(Hzy`|rkQzc>Ad{^ItPe$s6NYiJRS$RFwRh}K^&}Y8GHp;Sd3WWY# z39hzRR*`Y6i6XqnJMTZFWHYiv&vL#pyX(-s=sB3lqE%}ca-aU=Yq#QTG^nFcw(Jk* za=r)h;UQ^*m3%3TX`|~iW(}L=kB73`g{fQ)k*E@gdGTqDPt*yy6S_|wtM_DA6n$=SAev_BF|0TKMpLoBXE$0s#x@AtL5R-U zwz>xA8GM5;RW@$TZ%Hua^5HQF0UwqTRiy^X=G+ZvEQeCOHOj zhgAwCAFVU4{!zCxl6a2fzQ2^*M@MmVRo)ZQ)MM5wY!}Zj6-+|)$EbOuy(tkqOc&?fSIh%6mSS z6~t?$rv-jpriGdL><$NKXIPIV45U~3&^GvKsn5_G#K|rb7c)mrC~8G84L_KxLERJL zT@<*7^D~_S##4;^7e>{iqx|vuGb95&1{ht@i5ZL3L(%S~%+gUJ#)vTB^C$IV$yw3A z3D(mj2jih4XJBX)ItuiZm#jtWt5Ib^!jV08!t<`7@c!`+q6Lo3`+~=??L&Gcrojwis!$q3T&G>|f^hD1-*aLXH(s2O!L0bC-{hI^HT|(G&qaILLfT+p z&z?DxUiaMD=6FhtB4zL+b|$03s#`}38-ANKV*lH~wise(Waw?f zaoE%`ya}mtU!r}YZTw1s`TgNYug#~hPdGg^|KG9~``-)uKg37>+x3b7Mfr5SG-a^R zVJv5Ignb*Zel)kk3**ZX?AH{E`6U;a$M{@M$I;4nIEU#!3=|AA9*#!@3wHTGgmLuJ zOzyCTR7u_XwMx=-Hv!m-3Yuz0(zqAEUI+3r-iaqFa2|W$;-SR;On*tdZ(W=uIBS^V zBll=bHex5vCWpFOm0{9&y=9h20B4yNjoR09S04}s#NN(fU6gDLREtrs3&@H5EJoaqHNJf zUCyD#H-Se(M7>MFHQ#sXQeTXYbTSev#2H)PN6*2NiUfW$w}tiIlDw4i_lvSi*!{HD z{P3`Jt)U@yQY-6Qqy3WO;PV@o$3V4sVG?vHCS46&o_S^QdfLvZj57JmLD(au1@!a%C|LY zk|h?4k!do+45rHJr^n!NDAUT<6`?FWL;bn8tiP4Q);N(y)%-=^YD+FFTFRJ{iN{I( zwR(|`ZmyTw!RC51scQ5mQ z-@QuO-CplrirFUinGxcAaj$nTzQ9}z60239FTCFd67eqJk+NzZRthINTYce@mkc$^egKowQ=G!Vp2&Dr zn78s}g^oV5L-zC!RdTUN3!zHw-_jDO`eslR6R$a8%WI=dc`vF6-`MXA%vj>F$YnYV_c^3R(6sj0ujf;k$wQs^p|QFR)?N8rT1eTJ*EX%er&SdhnDmRzG#am#9A zu=%p8n#-4h-#=xt4@+pal`+v!Xxr7~wnQy}$;|&AUZubr#Ue^ku)sy`NQ_Qxd&mu< zc|5UG;>)}9a5^M`Ibw>E;FYBBG#zeOMq!^N4o7aoE`QdJ*%G2cHhWNatIrkpR~03i zBh$cBB9oKH$S9E1B8SgwO+91?wWQU}BtiC8)`l@)@rf!LSc4=l4_rL3yP>G1FTjF| ztnh~j^4w9wq^b$^9PPw5zs!w>2$Bz~?^822uC~E|Z5&OtZ4t9|SJZaJlqEq_S>S?? zWENeA`MLquv|2EHi(@QQ`Xx|&%0-SdOJM-Y<0>Pb>eH5NX_LRA*vc+F@O3my?Hnc& zy+CA$X`WG)ZV@64ivx}HpfJ?Ga(?hD7n+hY$!7GGV#`IPTE5( z4NEumn9|jnqcT%PW;s3`GJNaVooQiQVre}5CXm5tY20z@YXX&OW4}8YeEf0{p2WJh zDN+Qo>sqX$Ucbr17SWv4Y14Mfq95Lq_7l(9>cf#;1G~GuJDxBMEwN+HgLi|hW+R+4 z^*dfR7(4o{q}JZMP7?)?0{bi9$@foU@{rw`WMhmVb_2W{2gD6Kkz|?q;9T^8jvGEE zQd?93$us#USf#K7Qd@SJ9$mxm%IYRli&VD?dR+FAVW1o#qcjs^El?BgZ4h$1o=8-7 zL4k%1=&p5PIbr12Hf$i|UV5PL>PFTvy(l%q)MF-{_Fo9_NQU)YtJUdHC1j)?`jGqN zw~^3OsU4tb2n4ZoAIL!5;2Au3!WE_&UEeWMr#>>vG39wgTX_)<5z#l3>X4)7FJ$G{e~QPIMnTppB)QzzK`44}f8Uyfk&e%j?}ELEYg#!D8Fa#bGK} z(z%$dhuy=|vM_{zQ8Ho~t0FJew>hUhbgMG2Y2zn^-wkZ_)WCF^>|mdw39+eh-*VO6 zq`lW#p;Qy>+}?I}@OXNg84X-?Pm2K-9f=X%pY~g=X@!LB^vttFzY!nGi-KiPLNWG8CJAv zzPw0tGYEztI1}DlmQGdq23%7@)=ZIF8d+ z#?dqCQ?ODj`?U6+E;D6E{&Yg-1IDd@<-VMrBK!F}dOtZiK9v6Kso*&}@?W9AL)a30 z?8AuRbdE_{j)1{YG{J#=rWP#%f46%B@w7ksJ?2Dk9^8@}xs@c{QJByia7@9lP0RXz zlTfkrcoElfFYOF4su^N{A5X<*xmYZDgb<|}Yra!$tTQj-^ac+Sz? zM{diV14NU&6n)Ipv-q_y>rZe$Ox{sg^we-;)*XGjS|8dcWJ2dbi0xo4VuC6iKKP_- zh$o81;u9pgjoYw0rWJrIGlG#1c(E1}D+mXM&$JYAsX=OE<_BjZ0Yy5wQ%G+^bt9y@ z9}QUbQge^Ioa>e!%A8(<(i)7ZzBqj)K6`h94cF3fhFVN}=QIG$R#>eaY? zZpyfr9+urE-~3ME$4>0K2%Ga_#fZvPj9F1(YSb#_5obFTP#7p0FHW+NzEQd0N8T$F z$4z~U`Y2hG80n*dii%byRxuNheC~nZVkpu1fO@BS&y*HvN!`Bd zQ_vqGJGqD2o*_;ai(=nb2?japlm9Mf`ic`jup&uZ!Bg5#UTJ{X$8sO!D&5KVOu>sb z=CmL1{P7c55=Jr8tr$b_lhrqGqeYfU!7&)Y*_?FB{(H3=Fz!0{KJ(V9$V+!oBv5}`fij8?MF|v=Hm)?WJnq1b zg?|fUvJa(<=K~aIjH*NIbfaIRO0T)SWzj*74^FrKB`s0@>c=1`VRaaIF!EUpV3z2okaOWwm-I6CEK5zJY`fEUV)heMll6u(4a zTP*j2J>RsfFww3?FyVGQmoKApLI5tx0_&TE5Or5@*weOhFw@S5= z?)K#_)<0ews@gq?+VgRH113l2PBL6{`@jUY&^8~Tr}<wV3ugIl9}aeZ4e8nuP;e?^|0@8f-26jr)kp>v)IOJcsIL6$TQ=4Z3)r zG|ScAF^?mnDBk1&c8tttClNd?p$o9%Gn_k(Chx#7CSisfoWkLrAZRs7*Dg`!u%OPi zQ*-|bG$|X-YLOcoVomb+CR3p3;@oZa4cf(|UG&_A5mM9C91tvRy zP~A;dlTr{%{ITD>`TI+Y!eJAeM5FvnEdOJ>v&(VXWa{Ch8*e_LC_4LbW<(!GL6uAt z@_=RKypM8c1ykIYtjIW~uU#2P?w#E@Jab0zeeX<%tSFGo$HEQ3h;mZ!)dkYAEy?Ir z2qN>l(siYkw(My~MkDS8%3NmfaG$W9ag}i2%Q6#q%{rgNNV$g47{Q!i~T%Oy0 zJ+JgXz6hd3d$PAvvn<&u4fUSqUrj%XGsg&O(oxk>VNBh2&bNKExi8H78ubteJ%Zdb z_9DnM`T4J!fgJ>xQm*H?veGZ6k3B}w|ibcSymVa0*YDbC+M~hEx?s}-&EeA zFf*refr_vaMy^p+`;4Q{o12;-mKrQnAn1Ok=E|)gS)u+4sg|M^ldc9K&4KOKAqvX> z&K-Ki^&`Qk0IY!Ga1w`&+=txGe9+!x+1%nrgr`}A?QH7E;EJxu{Gd>>-Xy5w{U9O} zLeXR&Gm($g1_X9`_+!Me7d2YiT4WFc0ZsM=1$xpUf z^!0+oHhaXlPDdVxT@Da-*b|{gl3mhwW|T%lP5b3GP6Nq|G{tX=T6~viFdw=syltfI z92-Ueg?>n0sTcnZcG%|K`BI!Wa!GFxR$$Y0-5BE5Ley8=k=*s?&?Qql0CHHuNqDTH3z<6lh3Oc}nevv7kp}h7bP` z3O1_$9b5Ys-b;5!()0(NX1*d)@5c9`a7gKWBhLLW*Mq)3NKuWIK1FS-D{Bx6e6BDK z8Ng3Lr}P?fozfgTR=X>X!qbO@n_#l5c5=YUe%jDO>_gaJ_lA~>x1i$$T}P^klv)xb zYHt2O05J}v=g0u3^7ZWf^QpA?ovFn98q-%fzTfHLQ*vW;=Hak$RF?@k1MmD_*kaP79BgRh~ARO+Uf$uG+4dPH`u z&PxjiuU*sOb+n7S>1&)E!fT(Gp%!AZRdNZ}Tw7S+7s-h=e5NaD9g- z#Tj4qVPquFC@=B-V^tOTZS6ae8WRpieGj2%wCO2q5bi4x8K39cXKL(laGhRwara9On|&QL3)yJd0rtL zIO*T?l!f*%s`TAgAddGf(2Jw4_L20*v=qrNejoo%#t;M}u8?*#f7I4$h1=zmK%OaW z+2}iqpH}93bwwc=JG`8o%)4Mx>yj$F2avE?kkq<5Z6jWG`npJU2WJNlM0tX2V5c%; zFfIDB5MIq3)5D-TOugndUvP$0*J<;HB}YB?P^CCkeo8`cXD1qDS7X#?yQLh(EMtT< zE_+G!erxnIDF$bnn$@bqfnWKC)ihnNe1n7(Kwvc_piVw(RMihNOKqPN)m)<&7)iKF zuw{t+1&Hdh^}AhL4O~*$x!FwU3$d?DjTZ?x(=T3?bVNwqC${R$VTSzg>{p}0Puc!N zdG?xz+lUukIv3Sqc&B-?>6pA27*9WEhpuI7e}FYddyv0hz{(P2;*zsYh^SU3!0}7k zCtOstpR+ihvMY0U zlPRz>2**S0&pr1VW9)<-Lr|{CX**mu_bMOjIS$3kMClT&&$Tg) zaSjPgPq-hSyFw2VW$CC>xAKpbQ38VH_ZCu_u*4*pVv_Va)7re>A=wR^qh^0(`a! z%LlPJ{AM{WT%SXCJsw=~-%3DU#*peiu z*0LF0rvVqUcUHm0<+&<4(8^k0ic21!D{uCvv#5-3q9hiQ>Q6Q6%FSlI;&zb|6z!Lj z3Sz89P6=5U2^N#y!_PX_T^|@#f2flq)I0zV{^*O?4jyhbHRpj;n}29xNNS#?TTNW( z8rMZ^(nPO33{y-KqJ$*zbJqlglV)u?d!!KQSvF6JW3exF5zIUIvieyx4!u@BEv9!^ zXFmv+3mbtWB0=)708&qbXxuy!+{7!`M@$>(_95zkGH6yWrI14|l)C^C1V$ocz zY^2&bgTFAri`=J4KUmHLlMN*8paxmPrl2EwUE8$7mCL(vo;F`8DR~S#>`W3!7cBTIZiOoT%&pQ~Y|M7< zK--e@YfgnikmnirD7O|T$OO#kLu4!kv#c&8sW59LgK|GX%-$K`08em{l;pP z*EdBt8AedU?bK+WNgHa`z=~mdPaQ{dV>xDjd964Ms2NK(9}EpD;i;7VGS?%1KJD`|?=EP` zfnlD2Vd`?wovL%H!-KPUkVzNu32~)eM+u)fyN3x2el>Qb#qH-l;RsfCau59 zS|b*Cu|`{Jvs7CAN`!Im&J}z(!+R@IMjjlqtDAM8SML@tPq)wMjtd8G@=ly?sqm|3 zY##>ya#h|Zu%zR-F{Tilp8_w>i);Jq&!#W#paI#V+1%JXD^XEC^j=F=O6L{;Rys(#gh4G=zQd-Vb~87L65<8 z;EHXCi_gC?$Ce3>ApFWvTB4D)&i4&gJDe%R909xg6oHqjc^YGGNv`n=!-wS)##|qA zI7bKfI8jEdklMh3ms;dE*$IlkeN=UdggLB?kp&jobed)h3ER{T2Agpv8YI7G7U+wo z9`iY3lLB~dg4XAUDZKC#VT5l}J(O+1f-VMf$KxZPvGSnfVTI<642N%9$Z>)! z87ox~cyP$Rh<$JU`PD`~O3KUo$qj`Vx9RR-UD9q^_>(Im+q%Tz>=)PhpU*^^-allO z@HjHXzgvA>=x1mOiO;|3ZL}gV&2Mt3hC&rpvS1mb9p5hYKUJI7KW%p8uwmliXX1$* z&Z-fx<%Ary2L!#H&f3ZO92W1a;cq^|rbHjv3Z3Gzw2iGKIIib@Xqh68PihZ3)he`I zzowl|^Rn#wR;~jr#}3SjG6_5WlNAEQ_G_;CU{1B5`YK$D5l2IU2}@F4Tyqn6$Mjr# zSlM%mbSZ8;D{YZE=fX_-ZKZAX^Q^iGw4pJf3~2!Jpzv*{swzo1QYz)gbLw3aJq_e7 zwj}o66vKq*AOge-D*;wbW^<~zj_vt+?9jH=L&%T+QoWduQkLcD3Bz+!OVAzgtRHjHclFjy{I5Njn!y>B91!O7M^jvWVb|6@ z!8bj)zF@g(HqgeYL8$H(vhrQqKY9e|-I-Ss6ngP1Y5S1utV*BtW^EkXhCV`GJA4QS ztYOYXOU3Z9O|l<(kc&YW93_&k3oZp2qKe15}6x8~-QvrF`(!OuJ! zN)DC>FA<<^7p^0C=gKX(`^p1id2)sxklX`%4th!;-xVlpuExg_KTStOHw297o{K6% ztrkusxGqD1KpY-}?00;nLoHkqxWNI;2c!4FHmf4(guZ;Vw!t^jF$M)-DRo@5qn1Gf zyW%VB6&98{dEYW6_^2HgRVjkgo4x}-DAhuvsC~&n;ew7*xa?249a*6mSnIlYP|7>d z>rn((qR=_wUw852ccX+}pt~Laa{Az?7Z~Vk%o!5qlNAou(26v~q>1fIV35~};=g@} zCe`yDsZ@?fTJWowe_(%v(e>H60u$|U2?_xURCQn><_ULwLTx2*>R5yRmp$Fbacx&g z)G|Z1_a46KV%V+bM*0lj6x3DL{a9H+dp|70@Ry51IS9bGD5Lh&+ovuB64%zNJ}j}# zR%otbr#^ze9syr)y0uuqc<}X){?A9i@3*sr{~bg5FL7zMjO_zm8zjnnrCIMdovwfn z9MA+!+ssiW8K`*HD^|7_)%7oNDSY;C;?gM}ID0

?!n~!~V|4Vavo@VFxQ4h$ zeJRF1qz|@5vh}wlTiJC+Za00N`o7U&LyUei%+whK$SAl(&}%BoLQgS7C;D6Td<-N) zqeBmxx7}@(S`O@!y`x@3T+*4KFQB~0g!p$AUrua}1gXCSEDTO*J%_>PMokXdtBH*t_# z$JGH?>a9HP^(u{D<&;;@gUZ6?K4kfbGw?KinU$>I#Ji2)Y%vH!8T&C#)$oS3m5*sJ zmJtD2v3dy;hX8GV`q{P90AysRn^U|2v{CV2Vr1*&W z9#eQj9)Px6>HZ2Kc4v@D#{4)C*h^Q};?$B+sVyF(scZ&_E>DrgF`uAucR0Ji7`oW^}6L~>Ri;2B0k{OkGor57iE7@ zHWXGMoJ+o@#e=f;hqOCwY$4siUAW^w4wfkIH}snK1@(6J`ljK?F~+D6lA_)j7T^5J zC-V8!sb*XJp%Tdo>E3G>DHNv@1;jiX9UmG5R$dJ!K0Vc^54+U3evPkh{gr{n^~sMS z^w*MEXFL~?{ATcVSbZHqZQv&8SEJc3v+)ARZQ+yU3sHi-5k+V2ET4n@VZ=<8j7NMFEZSl6OWE zmlxtyvXhhN=^NgGvQPJsL~i1x zgvl_F`z?jwwf@<*^d<9^VDNWF4<+s-7sQJ`Fm zbgmU)4x)%V*0$t{-Pt`3jMG4+kOxC?ITz0vA*g3=$#P{-VH0$#q-53*kFdUaBHdG@ zTw3}~WagUZ^{eG_bI2+R*y6o@oBsK$^;`0r`8P1^ZziDswZ$VDHYE5`K3*=j*8uor zy>0Oh09!n&>Q`+k{QuLI@**fIymf~5V8JemS~jj3UfH?+TdGr-NeotfJ8h_AS>=DD z-+KN4^!v?Q^3AW`vMW3HT>&3YF;=$fywWsArnXTQ8rMg+iyIW2_gf=iJ}|MDl&1%J zr%Yaga7nRu?ejfukklIlwDo4+FnObwZ|syfL(WPNVBzqK$vyNFe)QO&=AsEJ&Bp~& zz{H7_mW)xyNY?DHPI^~44ngAxM8|tsTY1{Hawe?BjXh@EC@TtkGapyUFm&pnz(Qoc z&~9t|1-u{2VU63+`DfE=ISe?RF=+I45x09D{Mz5qZ|fe->c-~SPhTo@lV1Lfe)nCt z5*&wEQ~=O#3RUHHL=Q1;%;Ov5X<%7P)*5r8!y1`nfVT8*O!vCYPF$zHRMx_Dp%3Wc zfw!d`L#t2mC^G<#%W7}F`$6+mI?5Cm%TGaq@1y~^&LO$0*~!hl(~f@j*N63@TC8Vn zwdp90b?q;??6^XOMAFLBl=MGG>QNGjd(hbjhu^d%pO2$>EYqa{klUe2Hr$JTJ=O1I z0G;*;I%g62oxmd#bXCaY6#25zA0tmf!A6!aL^bG{ z;1)u&s^u2AQ(Z5|kR#t2k#MlTnG9QPN*U=}Yo0toeoN}|NQT;KD0>a@l-p5$SdjT@ zExn<0qS6#Lq5PGyzP0phqe-Q8DaTgdJ`C7OR%Lq+mb2RcO{&&ocM_o3+IJ z8&32$|MmY5ozt1<>IQB!8Uvtnuw#pzpZ}YjvkW%PC;O%&8O&)rGUUj%CE=J|fP8ok z{Oq~6FlCw8xN&a!b0Ixd$cxyyol13>vYm6Fzf69)-`!C2BYB13cXjrIaXAE8Wvl)* zWosx8Ve)gvER{}EQrp2;$=*#Su2kV>mm*CPRTzM^^jC5Yu)phkdM=^2c#3YToq1ku z&gfU*o!n#B6(yzwtnWmBg8AmRyS2Tp@7{H(O68Cz{IR|>Se?+=;JJ98O$tm3daI8u z@Z|+~I8-U=P6)l*osFHp$~@T^f#^$?>BMMk&J)!l@X7M(Zq4d~sB^0yVb$3^+VTP6 z>|cA*X=j2V_2-8hMc6Buq$6+%Uq4)t4XxKjZ?vfqz>|K7X;6T>i9{WL18=T1x~e8SOBL zF<483B>^q`tg<`VTQ$2^2s;nWPAY2J=O>{f0BcF_87>N1`vBCX+Vu-gIMQs*xHN;b{i`Q>yK__TZQ_CQF%=}?3C0~Z0b%!yVi>0KOVQzK4 zDYqXyWPkl1ttICF9SZw*?wbu@E!mS-b>)CAO^xNbkqQ0CjjZf9Q8^?oGIQ+1d%9+=gn4co}erKr3I!XNa&nkxu(? z&TG{6#i)^}=B8Zjk-e86suQeJ;q-Hm<1|b{HIf^Rt*PMshSs|&H>FOau>ApeQMRmj znqIUU)X-mQbV0=kqdrfD0>mB{i^--qGi2Zro8ExKepx&*q)$oTnmQ!bZc+$yW3$nK9=HzeO zw^}rl@dJSSMi*PDd*!~9w?B7I;C&bS!eCtXYx9-+UcAlBy3t}Yk~04n_ibUhJ*rCh z%6-eF|Iqq3_YDZfP?+oo9fSpN-vj{en@g1$zSdDxseZ?#Hy$s;KkIj?pcBk)PC%ZLYx$k3(+s52N zA^`V|4B);2r3=7)QyT-gZ-YO$ZyQDc_f1|5_?02-s}BI&w?9C+UFQFT`xZA-d*!}C z?)3x%W3YmW0o-@c8~089C-*&Z?)}nX2aNH?eeZ;C1Gw)H!;>=Zu8V;cvUz{`XgtWj zaNi3oUJ4s{Cua(UmoCx+&l&0Dm~lAF0Kk`#*BuEdM)7^tE{Xoj&>5EaZtoFOO~|4`RL$kcB{I zUGaJStXjDW_ zw@}-gSouQSQ-u9asR&~JBovD%I}S?y&C>%02aD%DMm5~hiXr5YZN}whOVlXU=Sa)h zcg4L*OR!1wOkmo)z=#Znl7=a$EAJa@+c<+;;v;xrKW2{g-l^84>ZS+(rYGTfcuPw;NU@k>|1u zH^`T%{n(n$0PY(hid3V@&kN z7}W%=+o{fwiJ%WNHeKdbTDN9k`JqQ0`P8o1&4=-y%GIiNp+6i|;8tt-a3Em z3w2sirbjab&c}}Pb@Qj^@L%W3#k6(i{)-O%oI!FCZQR;$oc1UsbEB&HNn$ zuq3Eaq%&;1#G~Ro->`3=qVN^a^_-wL?0d4OTSqJJ4AqBpyY`hm!TX874hs7F6#@ip3Nq3jFGmprY} zqk(9ia|_ITSr8-gVu^)j;D7pqJt0mzJ z$-S7EtBmo)5lxMA%qka9UBrqoUtA2S=cTnsVK0R!kFbXm84V3r+eBj7z0edKp~r`& zRQ71ctXJVk$d<8o!~JNc z@HhvJ*#gQG4Ci?nqe?;ki!K=rA>2pEbK;jX(mOqh;^=`8F&k2a!C1}0AW@bpAh0-4 z5`@PRs+FG(b;`W~AZ!+Bftl;v%d(h*RL{wk1jdtaX%fi{QBB_hmPGPq2k(*$=m?{= zqs2Zc;c>r8w_P`=%?BOY|B-HMT_pY^-I_oAMY>J;mvjsCCf!#4DcvHsx-GdnDu9EA z<-tqto}yM~+GtD)+ci+Eu2n*6ux1{ zcOUQmjtXytNz4B2hhq5tAFmyptnv9rAucD^pK?bFV&dNBizHT(kDEB|6tV8KqVsr{ z@xlHE5>nzM?s*H+oc7h-Yq|2_axmd2`&{5IpTHlrAQVG$%-Y<*m!?T zM^!6tNId^zm?V%GcI^ z$`@sm+!W+Dd&pKkyDz_LhqP?(aslP5S57xbn*q?UiPHh#$q>aG06fVmDo%x%^9MW` zg9^xUP)O-G0l<@?dsn9m{Yy5)lV?m)ySs8HUKH_=9|Rb=)4UV~tv!&m5wcI49)zvR{2ux0JMwrd-U<&nfqq5+PYkaM6IYA}WwP=BPL#G9ky!YP{Bjm4{ zn?F7?M$kDETJd`V$`{i$?Q3F0qK+>hG2)N%1r8X?z9mL1j8^`weDPzI{iS^A`fB;P zcLbp$?R5=Df4n?9tAcT~M(QBa%2bz2QB+4oNW&W;lE)R_arbUvO!R@4@eBxtRQ^1# z^y=K!1}^^L+;TYg?ZZAt|JS?M%o_cxAwR%Z-|%|(k^tPj@bKU6US|K_-RnGU5BT}(f^K`& zHW$EJg4R#e=!m6Kgy1^cs}RxAZr<^}BAcb^ZJL1&^v8qo4oElVZYtDSM$%d_y27X( zwgOm7%6V}BYiallU@cY3zgkNl%&C(J-mIlR6C=d`u$BPjOZ%_o>xO3XO}bUls(3A5 zo^R+Rz*?dOpp#I4K__o^v<#g3Ukv_cEp7AcU*TE4NLre>6zjt{F3QF;{=0nXnNy(A z{jGeN{#W^WwU&~4)?TGs*?WU%Y?-6+W=%+y0n`)Wwk;C_c9BOX4Fx&>{!zck=1*(M4(pKR=yjTcv~492 z=`BiP?epjo5a$|gAYt)e){>H7Qse>UC32<3_W8$g))=0xmunJ%;#{j5e5k_EyioL* z9e7R7GwuBbK4Dco|9E~%pBnX&z^)kNOEM$h6}4wl8{7C7*D+22{k(ed)IapA$<=zC zG1wOl{mvwNQ>IL-3+yY3M4e43h5(Ohxas^}u5;f{C=i`I4N4u#%+8&$NP92cVMsAh z!@+MTRI)A;&c=wPmD^2z@3I+&3?v!SAtUc&M7HKVx_0QR+PD(hVTJAX=E>4MASZK> zFz7w9yv%8qF$EfJePHOZ5tP#=pur-X`N}~`vCE>&{cB=`S&`WWl+ww|LYL-Pm5mc} zL>a2pox|Q^)fZG*{7R#Zdxum~5PEg~bQ);1(#Vp-7*^{3P$*Cz z2jDq+z4)RZgpS1^4S`qY3De@#){Q=_kr6BWBSDHp=P%WlCb}o0aZ1BxNoG!>iDdm5 zR!PA1sa6BWk9F{vTfY)bxfs?JcUa`UW*Zv-&wif|!){F7K43HV0()sdk6;}P1F|Hi_%aZ{jo{G?OeV{?yOeR#^rn7k#$ zq$%Bd@KIu9a6(?V7#SMiQM|Nc%n?G6Tuerz8D*tBvcepQ!=mpkv_~SRTch6{dxUg6 zADusOjYKUr$JRFnt@69qPxLH z&{py72K-l!#6i|H?hd7D>N4L3S^TXUMB-Sm&G`cZF%B4yP56R90v))ZAaSVQ0Pm^~ zOh)~tqCE^=o8%`_4+&JMJsqN?Y;;C3$<=-|OxkzWSNR?n7ri=(5=4^ zc;LOAdp+;lA$2*}hjZ6*)2SFvAyy``I|^*JfIdTIXL*xFi4>w%=*dsrfhMqi@U}_% zh^=>OX|e9rJSpY=W7h9mGw*xU2P=spKvEm}AI=i>o3q3>_wLnMs$buS8!v#vtj>S9 zoL0~9rrsz&bz_B7S5!OKtn5&>4F-hW%$mGX2{IH)0Y@g^s@P6B(=U3P63YVe6K$f5 z?YecmBZO`~l@|dO)M6~&?=6|v7Q-7#qGh(<1{I*R$mPmem}{;AeR|++D94Z#q_`R5 z3OMGtbhH5kwGl~<&xsytzgM>nTbpr1s{Oj%jU#s_+3&(RQ?iURMqx)@uzk!N7rgP2 zHmB`)q69c!<)cDyBv?OrPs~BZH;CaZ7-`N{6)`Ot6q9`wb&ErO0uQ^XX?Bl7+ zZ6`Y+Qr;OA3!Jl;E4l|wf-xJRuil8H4jKSTV3iUEKndgAe?SRmH%9kozigQ(hf`mn zgkb=bz}31eTUQC}GuH6+qtV9~-3UBDP8-IzE1gS)pvlVY9!X(~f?o_xR?U8E4BZG% z2vj7#mb;?tKY};-8+y#Tq82rOY?^}{sFuE9r#!w@Y1R*)RlNai_v^pz3^Yof^BM+w$jB*>L2z|clP_@O{U9Et!h!#zl z?vmDcan_fAZaW?3fjMSI?qC9wy!8W8R}4}h{%9s8;*8EsxUG|UQF z+;UNl4nl>N7mvxXA$EeSfv;O6>_aq)5>!yk@DW(vy=(D+kr@g{n#O~;A>J_Yv8fbYf>BHKP0vNr|-V` zhrU}voY{ZNYL=Vk?>mo~HS)J)s=xN#f9yO={=dHamlvS#*8h9oUH$LATdQ^buYEV{ zAANU;&MKnTipl?}?*{wJ*a-g5zFXuU`tDa{36K`>rYzm;xV`q>dVlM?r}Hm&Y8_hh zYpa_T7{Hs%3kDOl54pSEl%;aO*a-b!%F;iMjsDtqJFgJ+D!ACussMsMx*HMc>ebcK zK0ara6q0pr?K;J=MtCQEjA^xPUNoE?z&*!jw)>1*C>ZL44u9p!DDkH9M8O2<;__l z{QTxD=}WTnRe5u>t?+&>?Ga2v@(5;E%5-pNU^_|EOtWhW;k&V$!_syHXiI&v%*0-^ zMI3$@oL+$

sr7r6}qCpe7^NNPN_ErnlM zMlVM`+)iIAP)iZRBci4BhGw!#S8Pzt$qz?VQ3h}>70bCOEjM4*t$vjlzHsGHn)39Y zDazn6w4PQ2#+}&qWd2z_eASfJVC%WI=1+u@Li~*`3BS>Jn5v^%a^VB8WKM2A(@a-m zHxH}=|#o8dFs*~kWrfTFaH(pDCeXbEXgPqKXO;KTc7b>GVI za9$zRjj>5V|CldQU^kbFNX)2Ij^`rPG9Vr-uyrnFEuuEVQNMZu?F&H`L z^AlQcL5iQG6RziXVtuMVEutiofJ(0E9<*BB4tZJ>T4G#%;R*fjIYFY7I?+IO6LWHr zCsyzs0GAY~j*e0fem!KN@(<}KHZqfI(;fW|i&af&lm^6`vDZ5<-zgbC$rtBro^Knl zjjn}Mh~NHoqzFsb083Q(4AlNk)hyE#T#I>6`Ucq`2SHZMm3oakG-W~-sLl@gHrL&T zvzy^i!Wv|-sHzlCB2t_!C~^OuAIYfWJ!dx%Dh(x%QgF^`kbI1crxT1Mb^s`woib5r zMmL!?993i7v97f}1oA;)5Zak2e>EiY4t|g*pFs9PU#S-_$e-HM9(@83lYq2+`>VF3 zG8Hd6`Q4qA4=apb!Ac!{CmLPm-T2+FEABM6>-(TIsAEDpjzBlU#bhwXIodC9;-WhP zEEFIfwjp?uU}um^-Zqz9>tAZ}{8Ouj??#%=nv&*)dJx-W9pqph{Io%+OmevyLwA6c zPr*V2S~4`wD0HZZk|B5iwH4w$vQDlm6+l~hJ%9gPYj4edljFhi8kaqal6mJjl`#DBz5AbG7t8+89Jj;Xl2zdC;#R~2g^e2s3mm<6K;O9Jx zTm5#Dz?RjOB;5;xx>Ht+rY^8nR=)kR)foHnle#Aj{_y9omr%8VWv|aGX|H~6D9d>+ zsw>Ko8w9t~>|}t&bNeK1q_$=$FA?`1@Vrv{t~kA2I=z#8Tb+?kjcX3VDL%Zs zBK&*FepK>8(Vqq^veqh-S95mE`zRS+W&(@$Keo>fUNEGu5)Fo;Ght3;`~pH1)a!3E4*FOjRVX;GQ@^b56d zK=w!T@T&Y0Yrf{)qRpyH8y%d2Nq4l_;pljHGtT;$f%-y|g>i z)~b1}68J;*@nTUC@zMG#`$2oOL*XrDl;B!@umi`>8EZKE#S@qEtf}vu!}a#Lq$HVH zf7u6I9B#=Fvnz-&D4}FDx3AkK2`H$Q_)g-1&<=@wEH{@#=BBs(@D=d9k|#^nPjbk? zG}WswO`HK!*{0I=6d%0MG=w=baXGw$l&OCKkKjHCHR(nnx zWvfNWjcP@^7fs6ZyD?Ak!B3Di6Qd4bEV=Qnkx-C9HN?=6?iGD<+Z_@gH*C+TI?lN^ujZg2DSOyF z4yfZ+chaiT=VGdcqevsc!qdj-`Fw%jIA_KYLN|t1igw$81&)phJ7_}+p1lfL-e*q1 zq=Ag6m-v3r9lV5v5MsdBz^%>7>-J7(5bPTB1b!n@uuSawyeDWzq;gXp-v%3HB`W^{DT4%0Sl1ys&V@sdYZZ^j}C!I*i|vmg|5ZsHY%u7Do{L}gD}U!ZOR zd4Cm!5iY=me;#yR4xGY-JrrI7N~`XRQfsI}g!NRQVR@wcQ7+!{xlHSrSB`I8fWfuhL zSZYN+U`6N_E}>wABlBFCKT4ROQz6~`_=ga_ea8PPPnz^RQywbX>Sl3Y0{Hvp=HyB8 zMTgZI66z-jTX^;~?B;&}|8=^>Hj`AY%8USajcgpdtqT_M>~Pzz&7%X)1gfh}#$QA_ zd3pumc=5iar@aj=OfP{WJI7{O*iU!s(j#uu%a^_4p8>q|&Nkh^fq7Pf@DxqR-A`OkP6Y}>@$|6|nt||moG~8v&^mx7 z8xL5>x+3cJI3RrRh7^ff@gDr~F0EAN21$T5EusYQlgbE3>o0@K8~Rc+wEAQ{pW1`o4PP0o@3`$O;`(8v8GXTZ>IYTAug1jN z2N6ow|9-j;mf%rYE%UX>=k<-H^>lKu_>re>qblNQM(;&!uIV!0-1k=(9LKSHbvk^# zoZptfI~R4z@Qy87*I~W0&$=B8cvf`Q%Z|tr(!vXmoS|qA-wA1skZYNvQEJ6fk~N3F zpALlaaKc{EwitX*VGeo)A}k6mgE(^zn7Y7ZOf&NWxzKneACxW1*9cQKYRJq}>XGv| zH`z224;(4U(nKsf|ML54|JgDl!TU*5-Js;$;f}yv3UBV9?v7XB0WR!EsJUI0D4w`z ztl%$9Q3k^KS#3PRys=D_G!V7oG8%URlKxVHlB%D>-RH9%$9Nhh zH;R){b>iThA{NZK+PU-T&Ts0>D3$T>B3_#=ewz?BiW9w7*s!f%{vT~`8J6X?w&4oW zU6RsBmvnb`H`3i99nyk?ba%HX9nvD*BHc*0bnNFvXRhfw_L_TtKl%SY++&RUIsKvVeb&<*=JpKa#$nfc04n4S0c zn{|FT;5|onp5@oR|4{fLsMT5De7Qe#NAT2uK?k}!@Gd#sE#9~Ga1Qp=dpwzRh`Fm* zQ2(_XMAhSpl3GmZt_BN|qts_}0P5%--;)WdtM}T%u&kd>arpinP?Z6>n{*zfJUM zMaa-Cgnc_QZV%paa_U^DWwI|OTUwjpb8A%CJbC|Ef#=sZ&y>ZG^K@}C2-J0z*f9X8ue&{CKLm*w6=@Ij5rV0;|Q@gT6u)w$Y&wG?te39&r0HzIx7Sk!a4eA+ zp)2Od8GrogSn`(1K1W(7%)J>_u*R<_?yNyVpIe0U%&*K_ATN5?fZrNnv!24eAu@U| z-n_jpdwU=K2;^AOT^O1ootNY6W!c^Z97{O21VE3Gqh$}!W5nrHcjcgTj*2UD5Gu6{ zt%@64sco?NszsMb_bY-aNj&$a9{?+PaB-WYv7r^1Ta(ElL8QY+EDgz`xo)iYvMOHc z^y2Y+NgwQ=paMM#+Tuxz9>roQ>9=mn`H@(K$K{v_I)?Ejqc2X+isG>F`YM<-5}3-E zXyhLc(qnv7r<^!{pM@Ae$P4EC3QO#@MAD}ZUpXkZHL|YZ+hsbt1+kS(udF~tMx(~y zK#|e+)rTUZq`UATzI5Nnjr@iLl0czs%;&1HdlWw$OInH6;JV?!{E8}G`Hld+9Cc`h z)?8E4K>rJ^qTQC$pc%JU7@>yX;m^cYeYbvFzJTk4N`;Wqswa+vin+pVv^`G+) z4o7_RBJ&j^B|-$6Xv6EU1cRQY6Qco=icU^;M1dfjxuumoX}h#H*qwsHDNn^dNUMU~DqF12w^ zo{$hgDix-ZT|baY#sI0bx3zoi#GmeJed(;W@zL97dR(+e&JV7Xi+%Rc1Uhu0KF8aL z-u5%y3P%9W4C`}U7E7U~QTKBmMW}f-c@@FJK0?oDi`#XIZAZjvZUpXCOBtEZGnkgU zk4K+=@i58^KBjZ>LojN)6gC#`4j4q@Tp&X^U&29j7UnkB25Za1H;y;YE~HrJ?}kLp z+W1bYMtqf80wDonm7uslh9$&CRKT!ghRZv-xs!SxK2B^9_F!1rxKIXPI#NdqKR9+h z*AR7;@_}?<;%nyaafpFWao|PJXIP>rdF}I5?It;V`8Ga}BmYl^B_!u2@mUAk7uW%s zfMLn@F~c{IVdR- z+%ZEtW)`f`J{zv19hR1SmC-XmoqIr)zU<6L*W5nDy>(q0JO3rB#Qwh}VE?+ii`i%z z1nuseF+hy5Cw3>`Mr;Jinxr2X*3(sxkB#l-NsLUB^yA-y?1*pNVJ?Qu50!J3xrqv_4ZSUk|!L|59H z*AJ)?+f#)~%_3RWb2n*CT+@qIDytasy`_d~)PS`JZu?B;7EO9K+FSw|55)gKl`KJ) zir-PC=m%8k>W8J`0aZGY`VnU;MC9%_bF~_9Wm>&_V?RBQ=vzlg(_3y| z89nwis#yVzqI4HCN%NbI{{Wg%cv`nGnaqqyBzB$V2 z0-?^R*E>MX_YYKw-6mB5`Nr%4RVoCbN-_o-E)xn&4$PCzE5)VdvLIAx#yA_su5jt; zl&%Bk{Ia#E)K{GfbX(@cXsvs9ms32@DfLA?F6Mtnm41okvHv12_X}D3U(p82j}*4L zf6a!j{m6#q{5>1WX=ZA+*g*kgL%*TBL61(TByQ5zHpY4?-3yv)_d*7{g26aWcuKkXjSU4?-65-c8vFEn22^Q*fiAsr6%&JVSqG;Zi`Z za@u{z^mGKqy1@*z(7l&4OJwvQqy+%T8KL{YOo|_9>(0;hE>r2U#ob{!* z>HB=#$UWh0?9q}|t{L1y=02ZKzE{NORe;ph8hU`0Aa2KBM zZ*fD2_q})K%TD(XBTJY|RZ5Br{nt4K@eas9mBV+?{ZIlPv&B|IJw8p^z83|ju(-1O z$wSL{mcovy1&F-wfO7rh)Xs*1NhH9>;vJQdmg(SE9&RO5o}&z}_^l76-YGUe-QuOk zp!4&|?ck1r5BPU-7Sy3~7gH)0dE$!Hk`(>UQYo6kGMpv}ET7^&=nQ{|s_CcZg6R+} zcVU$wGt=n5Tro|lfZXn#D(Ond55W=U$Rdn{{TU9W0>Yv3Yqje@IJCCmQb+5>pHJD$Zur+pt=F#S>H?qRITmABl` zT~@0C3$3=LBhwl4qzdqBr-QNs*ajo}wbu)gp-w95-QP|Dof!2pT!U17ueDOm)ydk@ z1KSXaD3tUs!=dcI$ddiy{F0kXq_5xzyC&!$<`d4e23<1M$xQ*@A3rcg)=y6{x)l1;$ddVo zk;MxOWMq;5&B)^X!^rYjDwq8)MwSfle=xEbBmaLfvQ+*LMwVutG|4=kxPe;;ZzCpg z^^8ZzPhg7P?OKAi)OuaIKep7oEyl-_{#$i0B))3OvZVtss=`wkpBt$V#@F)sZ=q0I zH8{2FUgNit-D{Of$IZpO_D0(asq0ntQg@TQ@NoRZ^*13x+hEIr!bTluQQkQA6tD$_QSF2+gbRggGk1NbFFDA5A34LUA0)}J2OhB$z22$hOa`@uE*MYScKgN!UMYi!^CWE)IrS5yCcIP}-!>wk?|{f%vqRC-_=z}Wxavki&pKiP%? zodwn=akJci$2J@{628QIBh%0T5{fnOK>ST8W_a|6P^=2;hfwU%PoWqa0}&4*CYV&| zj>DtC;xajE2J+r?89H^xd~%ZM31|!rv?PJUy7z9w3(H#ggh)&ExoI`8z>U7{*i3}3B~^ZVH<$)^(VH0 z^B>p-(D?d|ZHTq&J-r2lVrst&#R&NS5Q-Ubkl&5}!8RxZLa}`EXGZ@a6x#~~apf0j zclItyK)0+qEPJwSiJET)!4I|px_NE$-;S?egsgsXSo=R4Uj+YZd|4#u%H_7BE7N14 z#JYI#kt!*+7e6RjF6nWfDavFZ*~vQW3FRuzX(DTx4nZ18ztn%fkZf;v+ph z<4h~-k}wjrKbf?*giOz-$s{^&r z^gMe@*F)8}j?ipTXI||&yk7!pp~L%YX_(Jal)5pU>#Yc}E*7;tX6wBe#@w#2v|X$Y zpF(7#AWs%)>2BsKjpVmu1j>_v;D+P5|4_1gC!KtDiT{U^C2)?eH{v%Xi{4`MDmYZQ z6(wWkpufl}0G6-Hmtilm+8q>E>{nj}&YCuRCx%;%OW1eF!{6{97M8?!eX#gsqTX*B z?pj|pA+!ZSomfO>0a;WvvX9> zm#MFw&y4)*5S<`5yZ40=Zl~l(V4aI~-okEYo}Z4YvgobgEXKBWPlbGyG9rUtjBd0} za|`Y60>E+$84y^`DWz8p(qZya#@WZ~9N9w1WUD27d)>NgN!|DSvfKL%#>?GKRvk@U zeFy6D=HaE`yi*9`zmKn9vJLG2TgK{dY=c*{SH!Q-2F1Ui4P=TBLKfnXE7V)+KM7eR z9)v8j3)Yr3e-pAS(9Hiy$TA7{hmggagx&RWJz&Ex$KTUvI5nzls2>ZOH&?+tGrE$E zYr$iWgq|^1qa)pE5c{fqn>!)mMM;wmAY|#D*Sge##b*tPEQg11dhoH#06vyIi=BK? zkdFoaKYT2*%TbMsW~x=$nf8vLRN*MadNr{L|K83zZi zXs%0*Q!D85M+YI0oF>isXh_-5B2;m^U}I#;3to_Rh#^rArgBjX%ofg%6)MsN9@+;{ zODq#hiU!V{SjzkNQk(5_Z$=F5>ft_q`!X;ZyMtcQLiA7#+8K3cma#NmYsDqlj)br^ zC=I<>V)NE&YqD)2nzj^onq~UWXJ z*CTBoW@HyirKgHuuKWCy+H17g#kn>!wly?yd05o*wO5wC4FUBf;Bib!U zZRWHs(I%uY`M_+Hz#~iswT;Rwdxn5HaOU^zYUswAH-1!s%~P||>_vVLeg;4`q$Un> z)p*Pqv6J@3?=gpjZof(LrPI-Ln4M`o3)araar?cKg!&WDJq z)>FX4oT?3Y0sjc)?6I!1?7)t}f`*Odqne5gd)#b(32FleO3ckFFfJ3Kqwi?CClYZZ zqya97SKC*QP4dvk;vkkbto_Q`(;*I*w6PIvKb)U)MA#`GAvUh-UmqB}{Tz-yIAHWB z6Pn(E1Rxt6QJ(B1JZ*cfLtQ43hr=k|*5x4c%4lyNu{zWEi)bltpV>RErhqO7`lAnL z%GpoW*Uo8)@gcDq1olu7k{*%dp zu)ii9k@%|qYP{+4a+P7hreGhk4mB+Ll7mGWRa43$oUfbddg!i69;f?kKv zM{(cB6i;jOgOh0oGPK-QFBUQJl6(E*t0GtiU^B?uxcYAm%W&-}P-J?%OdO50p$u6a z;V#@!6#$SO8>$ z&;!}-+gRW-C)8ql#!{Z+?O?I{SO`hCEF5ZU*1Ssi`lo8ZV2YnFjMM3S7G z6P4Q*ML)uga^-OmV@aCZtE7&iBBHQ0MN3iZBn(+Tu#JeIth5anaqOE^?FSnRA+XJy zr9}}MuZH_+u|Rg(lL=qchGJ6%{kYaYmkkmhuNY)F3~8!Fp+a;`yr7!$LR zHcl*#v-4M-IOF2xjNW9_L3Sz@akJR$&TR{PZ|QBIn4|l6PusGuk$BV7ndP)$m08%( zny~zbBE;;!(T8%sIo?eu^2%PrC)gRpGr_W(5rJosMUz?%j?!wc<9^!cgFXOuJNMGZt(qbu|41 z&6PjEI!E6-|3Nl5{U962ZHrQvc$|Ka4V@2U18LsVDNzSvU8CBE9p+K3-%Hvl;Lv^h zkb@pwmPq=SWCQyz_W1wh(EUR}9}YnaSbTMdy5E57vTTwF%s_2Elr84{6wu?XExMZb zCQ??U=X~{kPIWxM$-KJ!*0)1jG+iI-+aX#G4Tx)q=_drZhBX$zf(Q??Ai@F`#JjiK zYG>T5xF8GS#7_&N|APh5?a2ezfcS%JkTtdN;Ckb6uF@d$4m;&&877a!AN|2Ve?f{H zo{PQjYA0*IvjefOb#2EjIR5Qb%qHZYx}19cvCGMI3-Jh0v5cbop@VyIg=Sar3o;wU zoBE)NRS`xaq1eot;zIN$ZKj+$J+t8NuWjQ#cWv0KV~bg^%Z9OOz7suf{LI>Fw(hR! zhw6fJ1x8_*BtT)yw;)V-AP;moLI1hSX=@)4&{r7q$&J{NJK1l*9x6O>iey}IKDy^BkY6ynQK7BlCZ!+zxak_;p1Hbpke_zOJxBS3l^Ya zF$Sqvcmp+3maz%69^oQ5PT3CL92Z?aNPM|kpEbT3mI0oOYMarKpSt?dI>Fb!4cg8s zBmj`&lQ---?+DgY){~|@kVX$<0Tm1PV2Jwn9jlyvn(~jUwK%p5&j1xm4Cito=N;jN zvYOP&G)rerxe0R@_nd~I<}w5|RuP$GuYys0E331@Y9 z7{fai{_kZ@hseK3PqY6blkkiAwWqEPa19uM^ueA)mYVX^XPQeEH)l2ws?7gAFA+vV zXlx3ni!rx=+V|FjddW&>oq~3@V)w)Ki<8U9VF&BXx=jrTD$)?%QzQUuNPm0!`uL#1 zx$@)_pkhHCm5@$hG*qCvAL>S2)Gqz@fHjaS{Yk}gfNw_xQnB=O5gN#;?&7CqX83U6 zxCPvP7|sxnhc+(}BRhOimp+F5Q6NuPOebzCHuOz~o@pQ!tFtx?z3!?0LC!}M!21)M z+}QLzSTz|()YHUEbI?s`6JP$w>Bxv~&}^TCoxE zjLxl#pO^<@CtWIvAR%og1n2wal_PF%NVw6nnHKN03H@= zz{B#T21Ot3hlhpLlqz{o)%dLB@hsjMw*gr&&a0dA`3pUf#@CF_ima8}pGDo>xRAxV z@f7n*XTLUUScfeHqde1C`8qE0=?BmtfyRnP>(&)j&3k&kn|)y0az(3d;SfZBi&itk zFkHGmCHvXFx<%oxFMIN$<&n)n0v6KvMpOPA8?in(w3S)=eyngqLA?=1h))_37_4*! z3x)G&pL&ZftCY~fX1#5}v z403lrvEpvQqZK}lmw?2wSS~04`|`!lI7w{tZHOV_JmY4g;5ib%l9&`&eAg8CLgVM+ zG@3{172vI7!$@KppS-Cm^fgp5H4UCA;!sdA%j!vhi>AFQi@Vkk^3*1o<4`A_u9i)xO8QJ7yka^>5lX ziN1&K(igVtx5{u*5(m*V6~W!?vHCDRDG)!L(ah#Y<1>Y~)R3Io-R;1W9}Ltr)_K&J zN-t%QKVs=emp-Q*+|N@b^-eU>q*O(Ru7w&4Pfzj#s^G}5oqTwFQgT*La;HeGulH|OU<6Z>4=gh(uQix?5XBmnc~`60o`2tR zRLFHxfL}sdBK+Egsql&$S#TYiiGIc55q~tcH&sz6N$g8YZ7EMS9=)}*nUalM+tmF_)T zBM}ZWXNfN6H$WQ*es0yDv?gcZ`7}dzUmiE+;>$^UKl(;7|B-0k`FT3`Rsb~wrR`ZTY?hEWJ;P71ZLUD zwbZA=g*eXfaZ*byP>iGl6Da-P9Xe~28kLk%y!bJI6MLaLL^v50;0Y5utEO^tP*v^}INpMBU zNZqDb6H@8o;P=?IN;J zzSrsAnyhA!j%MhWVrzO+S$e9&GBA{sZ0TG>L^`n)yXHcF)`=tT0)~L$yq$V zAshdvnK;h<7aozb@67xv2cJ%zvrrtus%#f*g*$8Ypudgo(wVEx>o8&?_;4#^X(JLD z7dGAadIi=&M`02D>;~4JD606T*^I&}-+bYj&7*<=)shyPfv%??=-+5UDN#}2Vd&E@ zf60w3hqG18S7ke)#9evjD0SL-sa>Ee(qH7zRDKEZfVwL);E2vWOOqx&8- z<7hA4m~N~jljJq+`LDlENS%D}f4$VQ|Lz;|d_V+3Q9;<-o!*glo{K9~?@*&a`D&4( zA^YOFT18f|O9`{0%Z^Wt!rl{cvie0q%yhF*Rrn6J6W&<~7Bz!OnAGPBaOJo!k1CFv z$8?3L(-h&vtG~pmQN*Lgvtm}Zx00jK)dZbpS;ujE&f}>~iLddDbX`St`lDN_CeJ5( z2gIgb9EHImQ4spY>NrsQ`1$}Y#0mt8<=0<4;PB+Qs5rJTa~@Al?<^?5P$hwM3um$Tg8CTywhw!A_wW=abr)!L~^wnw8M(QK2v2;;}YFB*Dr z9{e;duZtogH);(@)(`3JVCR=*iAtP7EPp5?ADa7wFFghGa7QnVahYF~_Em*PvFoT< zIM&z-l~+Gggp7?yo*Zdbs$nIzh%Z^VM(oymHtpAdJ((C@9vmD*xFi^^tzR0_u6loW zVsG!6%X=E4bsu%NZgc@3F6!RSiJH~GLpwM0>h?_1pn*~=0{i1NIAVudM3gg`I6tj; zv@AZ%y)kdDh&u&&13}{K6cuFb(EPWM5S#5xG!)nEEni}XNCo!NO}WsVrY9JKhCR*M zz9%;JzMdyFzRVK@!I>f6dq?|WUN(yTm&S6aI~EP^usBzg#7@FA%G^auvM#wuR7l6} z_o{rI8dRkz-K8lB-*s6cI|fdcJ&8l3DSBD1(3T;&K>FWe$i(jBVek%Ot;O&F56GU~MpFOq3MTTkX)fy8yE-R@%0aprC!ccuq)A8Itm4 zhY7yDq*@C`&1BnOEMJ@#QEfntQM=?S!<$3BP9s_*+D_`dCM`difKi{qdsH(I;~3q* z97I2rm#lB$T}N<|8{bUT`*5r)LDHBCZBf}iOUr52c&W#q-S^hfa;BHMx&K=UdryucE`JKnCfeT-`JVQiA|20X811gRVHrBCF}Rxqby z2LrNJFP`AXMz}sZ#bkdJXBXjrp@c{OviI3@F4W3wF3gI%FY~Zh!Mey#btFnKMilzk zy5!q25O+gkEy~}1VY2XMy)aC=Ft)CDTjSop0f#kjSJpRfv z6YMK=A;_$hePnC)(E#VZa}}OtR2Dgw_iooBOFp}M@?hrsx72%cSEu{ws#KUj0%+3R z7iD)wOm7#Tc_wRdsI5by`Q5t*^S_3xlV3)CYv{Wt5ZGXh$lE}a?NAoccb{AqDzWHS zG$R7$j2byZBtoHtZXm>XZ@iY|XA<%hv#dE9eZd`RuZdet%Pv()+J-z>5Dcv3Ne1SVI41Vj=b%L43!kLG{DLavd6C*H%sNrX9+A^UH&YZP#6$4zFLxJJ+RMgh?J@!W04Ny^>il_IAm0VuAZT!lQhEF(K`)4mb-px``B#V{bKD(gIUkp%1 z6j0my`bC`7Y+sFEZl6ymV2f9g>LyFd=;|dm0zQx@%Y#pCm&tR421?#P3d@@nhL4MN zU_?Z8ZTV$6WuwkDNUjVu^E8lCXVSO_^fZ2O&*XX#HQN{!Q~DeW!R>0!Qz@87L@S-r zO2~~;I3X_W?&Tn)08fKVy2vaAOl3-!rdV)#baC6Bnj^vOQ5moJ+u%t>$YfSEt)F7V z(~jH*=@%@o5Ft=8-_|hFnzfoU`Ni2sz&VmINi&Ja_==fj4)KehjOj=Nre=jVNhQ5! zop0P}XhvYmrP=)GKM@{u^{5{;Q4zj$foFf5G_t z2j8$JZH@Stjb#&sXb;?ao*+ypi>%K;IvSkn*%A2tIlk1fD0`0;92qiZ=gHBW+mQIN z*{PriCn@{XCYJF+@t}CjqS9S?8a|!w_~&;zu?QwH1>xo@L#+-+pT#2%2ym7Wn43}E z3q}hJM92!#vY-V?0lp!?$A6tW8X|5r9Bu*N8@?Glx5xb0nv&G{O=1m<^mo1iw8U#q zJV|?g8f(E_knl202gTcE1pR7vXt{3UbX}UlrKGCoo!rDgvjB>l5L)}RDGH<&iw5>E z)N=Bi8vr=atej9qeU80v^#C}O%1Go;jCPG>_VXe3ya5lzMuM3& z63<>5rG$x8cqur@%Ii8(;cWyio^ST8@m&Zb@t9EV7vWAa(J&KH0q-+f2n*Xl;dJ^4 zHTfjAh-;HT@u2!KCh>-wMk7$=B+}Fl`eH+(>&chbM-w^v+M`~bW|}k zN+ON0E9%AVc%6NZB?aNnqAv?~C;8Im`;I*?TY2~2sC8A5OwR^#TLkaO427N3X6}@9 zf{T*qp~ZjLEvj@4PAS(N%IzI@RJ!LrD=w}b-?XC4*A_)CR9V83%(rHp7eSK)AH0Az z77kH|Uqb-*m|lLofdTnFsrv1>q@Yc_nx57+*#ySf2LF9aJG6b)C23r(HdttX5>5DpZavx*?X zfzlke2NAA>K{o>+9576{yHU|sDY@91AT+j=8J;It=?MqNB^qOkGE5ea%f@<3`LcMr zIEqZ))Nw0A7mf@{pBk zHvU%gs|?CO+fs_{Eamm6mPEVn=v2lRC^XG6H*w7TpQ-vO&y|nNk)j$wfP>@%;P6%0 zkfOV3FcRkl&1=uYaHz(7{Oa;Y`eR5ZQY=&Bv4D&v21eoJ3bh~mRyGejrG(9x){Pc-vskPN*W3FfgNZ4;h>Up6d6n4dMiR{Jh83>IH5C9H$X%7ZAggc*Gcs3I zY&d$834-5Y0r^rD{zw`qRU@~+Vgxg?Asmg!cC;Q?FuY|y`*U;d`6R4Q`dUzmJPU38 znp^9c?nKKia3B}UvrYt`xDY614Mu1Yktd%!WQ6?&l;53(j$0SWg3DxfY)7%=!rmr| z5U`*ri!46A;Oemw(79h-ygNH`y|;~>hqoo>IM3xF8{_tI6oz5?mKCTneL4aF4!941 zL$GpzDF8Tl?*f3sk@ExKz+U$NI1C7J0)PYO?|=iPM#dokIF!MORyl%Qn5bHt52EDgjps>G?1Xg&Ue@_4WZ|j$-IgVe%V1Mxhny+F2n%%bpPIxAf$#wW!-Kk{0 z+TqFbmRxy+mE1Ik}U$Rr#1Pxd_w?{!Uj z&iP`V{BuDbgOn$8_PmNSUE^u04mSma&KoBx)5Rbei(!@gUu7&CCmy?2!`% zr3b7z_AcrvMzey_gb!DGwD0a9NX8=mAY)0ClS$J|PSWYk|BH;pzGvZ1N|yj}^PMhb zUhuWCtR@bKaKMXU0}&3e0c-IOgoDcMGFecey?X{jm+mNCg7WwvM5?NNS7vNf@U^#AHL$>wwZ`Le zoDCTaVK#YW&iK)cE7wj(owAPAX7kmp4N>L{PI|wLO*Ux=0mtdZ4J}r4>_!TO*z$zZ zNK?t9{rNmeJEhDQ%(^nwje_;tHo>jxYBg(W^;04ikwEG@G*u z)?xZDbUXUPC%FYP>x|iH2FgQN+>`Z~faR!h4L?Afm4v?oSzI8`JxIgw=_y8BR((vv z76*MB5qBkDB00mc9uL0aClN>>^v&FFDiGe4aP@v((F_(VhJL>< z@C>SeKKr$I`4WDKdA~=#|0-iS7JE*ybnJg4m}8~f3hCOtK4n_t%N7AH{T-aPDn%Xq zPcjyq8kLbBG8RV4dI-l-4pNYeP|dnJmyGe2_tv2>V)+nJ*2cxM*o& zi83GEpXUuAV~M@K1r?J~5HD(7`=0xg%Y(`0euEIE&Q?*HAnwN`<3Zei{8nx13Lu%CpX1)Et{x8FWEr6x~>#3bGxu zX-v0+J3b&YC-#t;inW z;V7=AmT;@(kCkX(NN$3?8Z<^4;(f@>Jrn}R{U`*E`=WRbDg;i1ENp!!1ZJfJ3V|VQ zl!;0UMl#8w9*W6E`+7@6tuI-xib0tW#-0z&&(Meq5zq>y7erb7n=b>BSAoB?)*n)ZUlTp|80|t)kdQDU`W(gJ-yLx z{7J}5h(C(S0te_m=l@bGjE88^%YKx5b7Ja{%u!`@&Qhi`4N6rP z6-v#j5H-n;t4mI3(mH~WCG#(TL6%ToUW1S&(snS&*7IeMNbH5e@lz18G^T6_vf<;kKZqq0 z5V3?a*gJE!GuM?#C4P?!r73Ear1ve|cDd00SbH z=$B|Lh1Y9A#8T@IVu^MFiR%dPvB(cCI{BriSq+qdh@}I7SaOXuYq9Wl8Fc1+AeQRz z?_QW#f`}yGYge2`$4V!$6M$qrqpv_O-bxPGa)K zu{W-Xgku&qmGR-(V{78U9yf=kxidAfaS*`pT28uUq-$;{6Jdi9A8R!Vbfl88!A+*|W4_WGw1+;XBANymM z7PCNZKGfMWT_h|h{nf^zmz7*46`)6!`xTi&{B^Ajvn<4*fV1pXLrhtwiS|;<_4VAD zHlE}`okASzi~Af;-L;g(568`AuR-|}R0_kGw|{Yo5ly2_hOOj{l@R@dOH4o@)I?pg zMY(pK#FoZWSxbC=XY*OE9%kBIB++rZHJq9Ac||J=i6_KK;Tb=u~Pdqt{>i80j7~L$^S3| zh@Th;{b&57u|BtvC{y4eeo{z~7UXOh;l&W+^ETjHNO}o za=Qpx@7%eXM4Vi&QPZ=ZMPNUOF)*=&mAkNryobJvi;r<*8+>POHH~nD#Garp7D=5`#)+mL3Q@6 zk5i4x;U7P!#IXLR5`(bA+i)EDT_t9&%t`UzRbru0*u@3U9k8*7Oet@3BqIyYHeXZ( z(yYh0(&;*7>wis89$0~{Ui+;p z*4TO7vMC&rb>Zz|!s0|b2-i{7I@Pa zIw+Dg<;;}bpasizq(SjXwkC?oVenxTRfWf5Rbk2@?^o15dpf+YvyH}i9vG{owL0SW z&4k6wza!lkrx`gWiBd^PSd}(j60SoerWpq^eVr-RrQHH%uu%&?;B6ptj>D40O&Y5sZCh*Y}fHl2SPSgzub*Rs({| z6$8m5M)O)qqB6H0AKR^lge|x(XR$^+iRxE%`^|$%1++r{?!g4Pst@{X8vqaHkYx5K zD&VSSn>*+%fz)rApQz1_p5?RQOQU)m2sQ4R<9Hz!{|T_M1S5T2gdPi`(yUR&l_7XR18;!QX4(9V=VJ*T0T&1#00XFDN>?GifT$r zy<;tKW@@C$k;#ye&o|20PIe9FOOPUY_$^CxkcR25ScBwm(JMv;YsGb^O5Wgr*jJLhrzwi?YJ;~K^Z zHl@Bsfi|DA!CQ|<-&$ST`Q{DfI~9g@v{HAvw%oKElF5LlXP*#b6r~|;;yw`ILxKgw zNE4lhF&{g{5rbr63tXBS6~+k#nlA1Hhy=-w4X=`_SI7S%6QcxVVs36$VIY~9mD&B2 zpu!c}2(5=GV;&s!Gg0*SUXmRrOV1x!y!-$IV^~%&7MI1Gt&;B0yJY1ee`|(naH_Ht zh?XH|G&a*J2E-?!S|!M<2#4fQ_GUIxU{V4$QxV{wO*0O=wn3jG1M$gPP<)cos^w1X z{OU6z5T6uPMJL^*K1G-7A|r08cXtfrkmXy<4(eRkGtvXR=72aPOS_ zh%331^A-@F^m_%@Us3B(X?-m^r+QXfE!Q|TE=|5j%W+L0xHfEAViAv%ZMm5OuE`!` zkivIfxJa~O_ERRd_8=33FWLiSVm#76Z7hi<`SR@EsDO>-I&KTp3eD%Wv23y~0=>hr zLO;Xj_0eDXlgsi4B)*6YP%AXgpKL6%6$4+Tf48yt+HKja9=AxW?(Nu3m$xf}&YJ4E zmM^oTqy!V8lu`I%P^6hymN)2SR1@+U`g)71Bda8?xsE`=7tkCeLz8R4`68o0G7yp1D5zE64Y<(^x=W4L(O-ouskz{{a%_99&_u=R76k-%4ZTpMB$ zHrE9U)f0IG)?adWuFG!GFC{4B8H#?r2GSZ#{m>eip}Qfs%OrlAkV~#FbwQ5e_A`6k zR|EzuCyIqN9VpfUIOa6j|K4f#0_Ze@7!6_#Vr4FtIsHDq{{56XEOHSlqMIbzWyk_% z60nKA`h9-c`Kz(R05-9WcOfDC9>N!@Jr@mFlfF+ZQCPsx`z+`o8E%a;{P>SC37&|> z8zG}8V?TK!63ru3{u(hhXNaDq3b!;C~yPF3{Rc9iN$ zR{z;L$+?Vp3U-lo=4WgebKfRWYj#4zqif&W}Qfe-j({dtu*eF^=1dz;vc1 zf22g{pSDX0Cw``r>zDR=AO-<7_F(jM_41p!S_&SO5G6(msxqsOS5LOR2_#hZ%fU}0 zf_I}x{8HVE%)7g-n}ui;rqU6~;q&^FCT<%s@a-mH#8AWUwjr7tf)9uqnZHAC*f!kE4V+;~Vrb;34*)GGXam3mqn*s%b^SyagC-48{9w=n zGl5J*AiyX*^aga8q#n%8PE2(=z0OSXp;#%r$aQV+`tkZZeVRNO-Q%?=+Bo9 zwjhRLnQx9Rn*pO3Dqs}L)l&2V9uSGVoVem8+^JJTaIX5SkuWnJoj*Y@>uM}57RpMRb z+4OcNV21I;1vq+FAj1i&fo7QC>wFyY!?_<1h!z`#4-bf$2BY6eM?{iJ@ zUR@Hb5$y*07l06R>W>%1sb+1k*wUKyr$=@_Ul23-m_9bv0R%v|;W$y|JvitEkv_aP z$OMQRgFf7U{-oi46*uPm-_~D1+}LiL3+;ZPwIiZh=?w=91Dtcs7(q=MO+Kt z>`Xz_e+V02RqwU2v@ZrGkfBr%hxqSEJ-=i#I_5XN$~C-g`4qP}wkRxQ0XS7g? z9^RpyN81t-lATpEQcnHJ5ZLx0X7$rqoea6mjivvGv$tT&a%t z?(S}oPU%)cx{*{u2?;6b?r!+58=cRb@AFR9x2^RXxCUd~=eZyIaeBOZ20qi?(#rk3 zWT?xPZFtJN1SczmyK#xHyOf>f%zZ9fXf7LK75Xbn4$LdjXDYm2)`ViAVcgHIVR{Zl zp35L>eo&ecpQ%~8BH@_$C5>@spgDv{htU36Vi7N9&s@%Kz0uBEbl-^3nCq-fx=-6? zIK;4JPT191s^&ChPdg^m>Hx2rtx44t75a_8ZI{3d&KEQ7WdeXUW+nAB%pV~UX$=e? zrkDidJnTSOdS{Ln#xP(-LGhrN$Jg}voiMJ9-nJYvR5=;DHBQakD>J$N`9`0&$X`BX z#q#o+WL=}_ih4M5ytslcn7W2grXeYEq6#@fow#SL;Bu-m zoi^g8DT>K$U)Uq>tKXtB)Vf#L$64Lt zeth9C!=B7AYZk0~W8;K=`T1}|IfJ{e43n_K%ucnW(l4eX-TZ{fy3j_Y74lS2oZ8M1yJ31a?GgVmrm9Ku&#!TV>Z7l61*<$Dh(8g8u z6kRXa2ui#$%h`n~B_S-d1?>Z=a7*?ep`vfz99awuo4Zs)!fvRH4VVHF_3R8i@^y7* zjgMoT=^ax!diF4t1a)#MhaohD)%j5YS=uB;qS?c-b|Gm$wpoxBg-MxoU)YW_6x%6s3zqoK+cqKn9rGhu zE&{C~_CxH=l~H}`QVl)1!!mgOh)YIN1s0(tx27Uzw_p3-CoVD-6MV)Zh4`Wx zZuwNgCFYy3C6z5Y2CBJ;av_f8FelVd`$pN)A6O}ApH2$rw&VpuOSE*UFQHJzfx?n(0_?iVQfdEbV zD}Oph>br?cE32Glk9k-wMv4AdxX_eFNDW8QrTms(-ElfBWei>L*Sjh{;S=m=7QrLj zOwkZ>GDqX%#iKOVvx=Q>4NUTx<)pop4ayomMPKH<^r zk5Rb>+9Ijp?_+aCtSWEkf(OBK55)CtWact8Xk5!iQ*-%2>>(aPcZdu8z>xH{A8%}i` zsTD=he%Hm^c*ZgLIXxDM*RSj#$9E}Z=N_nK(A2g?^B=ofyKYbRQ&Wc3+GtN)Gpt~H z%O*)TV>iJjgRWW~9Efy(QF0X!5)=^j6U)sCac>6}7hMfn#Fc9O>?+Q6ryAk(hA6(w z(Sm6egFtR?pb$DOi6(G9B4uhtvG(D{u(=k@8s9nnE4=x@eNmPgq^_KSEEpo5csSHf5X)+heM+#u zce!1_EmTE4bZ%j!ZWTctZF!vYj%<|leCw#2t#s`$N0I^|#?N5*5GY34B<~X#$T}jC zkn8jc255;$?o)8?#g&!4W2yvt?aQMRHU@>OeJI-U2D!q*;k|sbsz@?`F{asl|L{0%m}@?< z=K&aF&~xnHA8Ha$F~(d^pc_wcN_EP!HfT)9-V2qf+A!47V{@~@$fzY4REpGbIfwEp zaWy*yHf`qvDaqsm9gdkY8OURBNUa~+R#wuu98GSM`FDn-_<3_(6Qiu$`|e8RIg*?o zWCX(TSy=YJ;wXp{$m`5gc^98IOhOs;;U|0?{{`WMDu$B{+qF>c!V_&-1yN%{%J00O> z4{WTh$(k_;a>(-t-5JGxX5DB~t&Xbck6P|FBd}kS<_d#O?Mk!ngdhB(RH;Z=tLZpz zOIZR3+e6<|vN;M&}f>fiBY5C=!YBEHk}S_vK4)$ChgtMc3yt zviqrNW3D!I@w#=w0?&A61^JIyaLY=6315IUu#|j^7Qozf-?^~50<8FA?P*GFM zMiX0^+W+RY?#j>lNjaqtRfWqsA=YIGUnm)Bg*#?cpU1r-ky+CCl-XSjy$t19LzU1* zwXc#!ABt0Y`a!wl*J z32l1clr{2q6WyZRD;wTU=5`t(7dfc}7Zc1wLNp)#uzd$byNif?QBo#-h^JYXY_a87 z=XAX|U*zF_V)vU~!r(K!PXfjdpb!1uk3jj81Ggt8=ueCv{@c&WBk}Kk*0TThv!=0_ znl5~P1Nd37Kz_S%WwlNErq&Oxu{znW{f#|hFh8yr3Ia{$vSm>2wKX>HaB8HhB)N7* z6_AZm9mDM^-5?M0j0RX~NAj|0tkL{;=_Neo6H4mTjRwAf_sLfdt`Up(f}4h@Qolz& zq_Mcj!^VuH4Dg2aIp`dw=#6d5>68-e-h3u5?;jkW2U|0Z-?gHLl2hA^3niF3wq0;) z5kJ>I?m({QKr$v1E$9#74+HK?8+oz&vSQ*$h_Lw^7}NM1C7z^VTCcCtB$>?2y`#C( z>-eejcXwY;J-BphqFN=2jmBgrLWfWNFFB69#SWUQrqX93zN}er8@0~>vo^wNWya_S z(tLi6S)aB>PUrynu$w&Xk9|e&;n+D=#T)(J;S!wdO+PF7<+7bV^3~RL2gz14J7;v& ze+$1ok6jsX5s$p^Y!S?C;(Gh_?OqNP!TiN$e=ir-&w<9iaT@MAWakP5f1U=7nKV^X z4rwf3o!TNLYhy;Tl#&-h?}7(#LPBS$(l%fq>_1!$#eC*Se@eR9NChC zaQAREOsmq^4)Tm#u2!;P{SB+fAi!}X7B;~xA!vJE#(P>E>EZ&uIWw*0QcQz)U> zuV~cq^E*jZ8tUzLCQ{IHfh~R#A9lo^g?&mFS7Lk*TUeGTHPBwj(eudafy+W4_H%~x z-9!9cBI*!i_TMFwYoz}^GfB0qlq%;e_`F~X?@w92#*36dYc zx@zu#ZIO4sS8C3J^t#^u-?9y;-?EK{Lg(?e7#-$oM{QA5bZ#dBPisWM(-0jNuXRS7-aWBoy`?&iAs18rJksLIo=&P7LGaT&}02k z`;z7pM6MGSzxC3}rhVm{zq_uhS7!j_&r@bC1{0`)|b451L4}qC$c*csoS}~t~wWe^gM0&2K=Hw&g^gKJ2 zU7f+&UXG7(3ru4@*^U@)LP(C+V|R1|tFlRtZ-8wOVD-TAmfi#|E@7o3Rc@|zofLOf&ldy}1Pd(LLFv|4u67li5kL#kGviuDpPID4tJq4-WtDkth6 zpX{6_AFb7~SxXFU?3}wjyZ}wwN}Mo!L9_$t)c*T<2K5CKKO=f3@8~XT>@00=V<}^6 z|Z_`$`oZVSfQ!Gd_DRd?rOm` z;(fv3SVa{>a=B+a0PD%xF6Ix>6$7@YV7=TOS`hBQ)>3Sb4v|G%mIE1OfKiNS()<1$ z((*#}g0zydI%n%MjY zVwO|}YGP|0)LpTtb;{tYL2Q`_Qr4}U?F7?nzuT6}PkBh%oEDF$IR*i<+MbW?9-_to zuq-0Y=hC+{#U)0hPTR}>LV zXr;0_9e|pz`W+DiuBhiMaf2^TM{=P6SClrhEDw$nD;uiZEATtZ=*4^Mr`o~1x0ez^1_l|Dpzr;Tu%^%-xWArm!43yb!V};76WgLE zzIPC=H}{ud+zNhA%(JFV%nzMO5!+7jh4oBm9srxynh3{%G1RMcKm1yf^h{`IK@2!I zmWVT59~vS+ydu3SNky~s7W);yIXRyFdEV}-PGgTUpA436LJEi->%bf`Qg%q55rJ+n zqj5zc%f}k*&pC^zU)z?WS|b(7o&kb!C`!k{B~q370rdKX8O7_y;JlCF*LdAKBpml`YFQ~5OkEX<0w|)=sAK&ZlyBy~d>I62^G8E# zUBPeK+`ch>^H`qg0`ByEFaBE(D7O-T*Q@^U@mGSiT2Fa(!x%S?PAC%6VgssdM=0{$ zKY6_jVc&b^OdH=Y-H(~&JUz1N`M%`qV*%fDs~=vvALkAAzs9~TFj3PE23>aFe-ka%{$2NYegUc>oO#Gz%NSmj$0fyGda|5AHf*pa`?q1^`aX6mQ z(x2||_kVtYxxaAv#rkR?l=WBZeaJj6FBD*l^5&vaYq-imb~3L5UqXXXNYxk4()y<@ z3g_knZL>#CGf_t%dr6Aw93U8P7W_IF|8`K=c?2z%!1HwC42nMIOJfNBjwXXR>#Y?s z?`_xdlJ(tv%=WSB2Zl!Lkl8o=Rm^pOV4U>}AQ&Hzl;BM>QLj~G7isN?F2sbTeAgsM zAt5Cg<9x?HBBb)hHo$JosH*vtsmB~`xe{o>4(JR#)_TPCG~WFa*CXf_OdkeXus1|f z56;$8K)WJ9Fs_LOd3#5i5<`z-fA+@Xs^9aO)pv0@ zJX@{8(kKkWgcD5@r*>j1h%7HN=Z%oa>Gxy)xQv$Njo*@1kzq&esqHo|`%xCy;B%ZS zi~Mf6O79*B6LhIYhYB1K0reQm>CHkPGovKP_8|_{_DSI{d%0*k@SS;vsBqt_Z8jMc zTqoVPkSoTlMuq$osMncEO1}h&^A3m{@A4U(V5=_@4lP2GauZAg4c*Uw6rfTw`R3(X z)89dg%sSqoGwq_OzIPyiCdj(E$82nTtrPbw+rB&u| z63Pi@Cxg*4QFF;9)RrW>^>0gS*Drs%4;jU^(0VeQumv?&k0 zu1`bO`9rTyX25NTYP7g~INrySAelP9=!{DlA^9Tb0~5njD$#KD9kc0doS{sFyjQcd z{8(0QJ$T^I#bEIWHFNo=mchVD^7C@dkdySWAMKu1a-YxD-uF&AyoO4;9wGLUlMIDQ zVh$h#bglb3Kfg`7JN{Je;F2vfVTjT=hL6K8c{9X1&7{se>zU;^ch&D8O!zKw%mc@_m&IzDRa+!!B~Eq{R|l-NsY91GL-D1;sUpo?xwH7u(sM&&eq%U1 zkHE(n;KDr*mQ6VO{?XFvq$`gVVL2sMN&XD_y63?j4`(%B-?i zpgz}6FK=uz<`q#EN4(EF{@I);_%38zVFS(NPP?;q&hbOs zJ5FDEVUKHg#52)R1uO+HG!`x*#GL6M9bHdzBucBCGj&a0vL`0twdfwng@UN#^nA5?Hg@T7tf1T-d~g zv{+`)w>7W=;=x-|RI5Cv9Cls$coypYz-?E^5#wd^DNf^6YT8fn?z}}C*jlV$!zhiw zeTaaq)D8BjK4YTeG?vf9gKpJcc3yS|d??MWYSr+egM$<2_t%f1H(#%k8Bm24=%3TS z|J(ZX^uzm!9rqIt5x~;=4^S&-q%yJEIlVR1L?o6z=A)G8}qi)4n3|7=}pL>N}*YKsoa zck>%iJX!V{D4t}cp${h!%hZU&rVvYz_X{^9rz(c%LV1E%?Xy}F0Gps zg0agX^X45&#VEh5sEtHS*wKaC@zybKmg>4}1hQnX37K+q)`IO{o>mm6IN!$JSjD3Y z{FZBm>+l})X?a5mQVwMq9G&I`VY^a;C?RM}maYcFRo>DM%LdKgt;pkNh3Q^YknV`m zKj0JGK7v|@SXgWzPz&^7|NAHZe|UTU-)#&2j>`OJc+>-w`ygn7i_bor-{+^p1S3~O z)QUiNlPH6{x3JHUQI)P1;GUYWQ1Ck|qi7IL^gAn)^Kmi0vwRs{=M6P1W1bpN?gM&z5C345pa7Dd zG$}BcB0$p9Pg&H?0QB~LKAb0n+H2S0YZ;~Go85Uad7s^u)2}4>W0UJ2s@CFvRkbes z&{`HafzGD{zUNCLd=bM3uy0W+=~rUdy*`{=!nM{J=E!7*k8ke+;Oz}|Fu$+?yuHOe zI)X`RkFK%icSxpgvywK9AzuG9-h7i8wgk^@f3o__+l%wn3myDmMv0UXRdmWERfr(d z8eM?p^;_re(355+6y98AGNK$I5527@D0vKcROu*Bc zU4LkQ+Nky?C{q=BJ-luQ1Z9jQvlWY&$ZmMf^?_=i_Z%{V7sSyVl^yyVkVrtakG&S+ zz?wBb6}Serw2&JTF{VlCXAwBTK%xDa@-$rnHV|p-EQaMOigif&19GSE)qW`yg*MUP z92mWEF85~mn^?Z<<>QxFAKx^YfPyk2thWZh%Ukkf$@%U)Wb;__LnZL?&Uk!z8~r0F z!wdvvq=2BzJ$tgcDyRPd1dW6u^SJt&Ey7D>=Tly2N*VA$DMIZ`c6uL|v_33&f<%Gs zKc3ddoab*(YXd0f8TUKqS;>(N%6SHka!ocf4lM{cEPVw-!SXFsv*mA>tF-jYH-dNj zA)ms+x8M_;`!FLxrf*HN!6?-8M&vC z6kJ&|`JPRv6^T;6>iqgA#*WdCOs#>JH-$il!~ynya-Mwrx8R`P((*=Ql=}b2mp2-q zFL>g&^b-$DTuIs3=UU+6THtY_=?aRofogjrt6OMFwx!^c@QuKIqR7yk zj#8BLmHAMZKlyD)B#%$+nGUhYN2cyl@me9CWQ=&GHE;zaZ@XLnp10BAyyTtej%6ed zMs7a$TULv@@Xf{DzYe=DN{@vC%BW$yhb=h^ad>mZUq>}kc@A&Tdt7m`xNqg&cr6cz zb-dJW*Ot@9xw=xE@4kr9#tCoLRbNnM$=L8Pg}b%5E=@Lx4-JF+j@CDVSG>ywS1)}+ zx;H?KbYU^BGwH&>x8E?Wc+&cF7@aLQ_Y8NFLH(^`vR|kZgxchp8Z41^uYS}AKSQ(JqK#%dvSpI`di*(W&EI)3v%8ZrN|Xp<({~~1L9F(K zsvB9QMh&5f3U2f$fn-IlbZo-gOXn5;mD!tA13LRvOA^w7PelPY9yHRp~`~v-AG*%<417Q+o`LO~6C2(t^^^;@t9r~R2MGq*l-OR(W}6YV zzy9HlekPW<3q{L{Spqj?L`jX{6$8`q0D$70t*jUrID`7aU1sR@WWV5~ba zx#DAs?-o*$iOU#~l*yfn;VR4@w$gTnl7=e6Sz5q`_|i>Bl7)h5PAa)8lqf5FEWJtD z#B?fh#m5)FHhEBa2BMglPL0^=;=_zz20OF_Cn>HPG2O-?#pdY?ULGgxrKJgIpRNg_ zF5A7+>uZVK&-$+Va(N-0C`3~Mn2Lu3a#&oKl#hWn3n0)YnZF!QrPz~(5e`gvrwJ~= zKEfwQIA|`u?CBcp=~mO5$uMAC9kIlO&Xu*lO$FJ%)J#>s*tEVGc9(<3G|OjUM?8%UZBTj4 zm$FHHw;3VW+Q_wh@=5 z(z-`}X>Re(Z!W9=ubZ<*1LR*3imvp|!QbuXQs(NNKBG=xI7_$~tD>vP zt6{14VZ#DCUJ)BfO>rbain*9DDhcynez%*?BwB&-Mfp}^5i0fWaLINPE)A0-F0r#R zr!{I=mmSWn;05GI`h*ax8wIzZ^u3%)viH$$s7LL$CZ3cfY}*m7u+751g9M#}_h*;; zX~2sU^tJ!{If#AwhI(Q-{{)}%U;3}!da_x2R`{PIIU;_wa6nfu6|8?eRp=`**n-3! zxZ4F+s{Z#eOH#_bZgr^je=N)4qvaP0sZDEaj6ag1mMvll|@L9(8&j)sPZu^G;YH~ zH<_WmpY6RC>ebdSxlr9dhXrRfU~)$_Y=prLQK=rT@tVTl8Od5H4SI+nhVe6i1Yq2U zS{B#Gg;+ZlRxc+b(=$w=N#w*yIA^2s^rBx&txOZ~Cg2SV`ss`L@vot{9~0{Jh?)tz zFa|z1WOy3ILIuJzWf%=W^j*DAkfhP(qfi}1h!UQ_&+aHcldovsq zcFG~n62JWCr< z$4qFAmWI#pL&F!5(FjgeGS3V<(>=SrX^Eq(RiNQqJx6?1D$gN#;u_R&J_9tI^Oc+Y zZa8mytt&!(M+3@uuZbv;M?ekdU#QGJ7X)Lrp^vMWBRn>o zFEPS7Pq@vS>11byw{4zl4eN8-39C{Xln+Ak)uvl5oOkR^9*f&r6o+Dm@D>J$-~$5B zJ~5FPKGf!RU)qLUSs#YFQqKPA7>|JhEkPk0{06F$CMRbj%NV8c9{QM6>^BPYyRdqN za)CYS72XIqiaPQ7=xxfyE&?)$)OVx1(}O6+Cvx}~H;xBGp7nF~gLAshBavC}!7x;v z9qN|`32;jj#5QK$zD>{?w+G{>{m`YsS|-dh25fBt!=9fa%n}hPvt%Z2sBp(xfvqgFgi^$u8OLd= z#|0^cPYG@n1l2X4Pr22{%ktU=C^&cfU2wkc9|T1U6r5)yMzLHf5a2dd={*k%qj?vc z76O)5tha?$JguGqp2{CQL1_LO?V2H+*Nm(vtR3pmIcCbQ@%~H4-QLQ<{X9hr)|{~2 z{u_7k0yST!zx`kEe-)eq{x49$`9J($?Qa187s=4FYy8mu(f>8~ zs{-Wzg1J;tvHtD<3N=)rQ&VeEvb=fZ>QLq}FC>BK5a{32zyIT^dU`ta1ZVNY)B3+V zd|F<80~*e)(3}7N8qUABy-};&%9s4Oh^6Eq()qEhvaxMwO}oaEbu$|ffN`&cM17a) zd^t40Q%bNp`n_3*Vu-QR!uWAIbjX^Lxv!EnGE0=s^bOXgF^k$d&;5HD6)ybufbRxz zR6JO`1t!|`NI=KA-=B{2c_HL6P;qA1;>GhjE2Z0?aJX&YtcZxv5EzdJFy`T}0!NrY z$N3bf<2*V>QcWgN6DQ+szY5p$?boD@ix8imjV>gCLVQybYx%@yv%$g^jdC1`N%|oV zzxT4)Qg|$ht@Tj%jzv?0H*ABK@bm~bcII!+msq0x{A9ecx3%%kJ$|?ysaN(l$Z4pv-t`OACd!_TqBkmY837bv+^46@-t=yi{cN$(T}_2KuG-J% zT4a`gedO=ZZ%#D_O3o){b)L0VmytJgwe?9l@CsW_C#4iF5g6 zqX7k#C15dL(Ffzu7o9omQen^XIg!D+p9eD5Liiq90RKhYDu%%)`w>U^fMw0~qLs#H zBGc|?rL69856k^QiWD?J$@vi7pkJy_B>jT6cbG|ClLB*rVs~sRP;x$t@J9hAQ-XA# z@X|-T_W+0HCh%TS|84zviP3UV*pAg_t&a8Y~Dp_(wOCE zo^4V=V<)}^Yf{RL)6Iv6@Y_B+?CbAxAg?(k8I0c|!{udMb9l~V{^bh`2?j2MOg;a>9V)c5;?z?^AELG#l9J!uSO$p{LgtN5UePLXG0+a@T;J?b)r z)Iz4hrQ9p=om0Cp*TW=ThmVU{_Gk&q2UK7^2N=@6 zqR3Y@=@y#CuS;;7D+DLC^-~GN1v`X#;zB2ey+0Oy;sQd|{OZ*9<12sNC5FPU| zh+adQm|uK2-aR-FjyP%Gqh0HL=><_7wb%6nltVxNmmIp||IVSG?NkCebVE=MT?NRY z7ygw)fA^R}H~e2Y^pp!Ahc5R!hd%xyfgjnSflRGhneTK6ltXtv3;Aab9rt$*{YPti zVRQ57?;N`E|5XkhONi5wR9*Xa(_p6gvU3eL8_1#GoBhe5e;?6{f$l@t#{j^h{70~e z=MVsk?tZma0>zmE|D!mQUch*&qwR*T#eBE??;JYyV~NlI&Y|nB{4L6U7$&cX6ig1cpkY;d$ijF2UvTWO@Y!J^kd%<0D*; zZUnTJ1)bY}KhaO$U`$Vd7Ehe$#~Vx#*=wp&9R^lb-b$(-&5pXZrxtu{e5A;mN-VQ5 zDR0$@a-X!+;&!3MwBKc$+dPMXd&{MkrN+|}uz(HGeTl?r_1CM&psh_QY_xQh%cFwj z?N;`OjH0oO`2!Xi+0K)&m|&Ym{|1W~86dja=vdtF$Q+B*gCQs0(j?d!xk{k+DvW5+ zvzTMyi6KFrK^H|e$s5d5rQ6EL+eZpSRlQeF1)lyW(5?M`1sFp%Btc6IDG1< zJ`0>B6Ts%DmqG=VMoU_3Q1q9J%TV{u$O^WXB7glv!sx$fYr5p$wi4aqfZkoDMHG~L zVH!Y~=ofU|WE>PUDxYI)%toVQO2+jrY{?D)Gf zvyopsFy1p$x;8X86w|FqFZo`X4Nq1WFsMRhZ8-Yqo_#o5J$3vJE5mn`vZ&O52d(CO2%&WDqnTaBUAi)W75eqJ^DQScp`&LG@l3(AB3-{fHZVmQo2DY zaDnBKMOZ4NXRVN6Pt%_j&m^i`!PQW&Mp0>Gm2i0in# zbpG~v@s{d;Qrbbc2mG?w0iTkISg2g;Ux#AZMycG9z}zlY3+! zZ)s>@ri{NF)}d)C@8MoyuajN79{%xLq!OCIBl`b*gZ)DefLiFp+6$tw;H*Y- zo+4vyBY+ER1}lMNIFp41;EFg}RmWjm>=7RXos73RhO`~73IVRDpY!h;I(=E6xTk99 zE;%F>{7~9(b-2^CDvY@C05q21O&d?yTMBjfZ*IO|iYT4okJZn&7=NR&4E~kncMkv> z`vm}t&_1XBL1W?mLSxf{d>8g3oJK=&=yOdSZ+tRItg-mDx?l=t56$5S-yq$rVOnACUB)|kKaBnEN~C_v*R7j`(gHPQOU{ z1JJ0vI{43W!|1Q&#_clsKbISNz;Z+SWJ%?2Ieqh3^20D_xsd|GivGFW_=g4TUzQtW zkIRk7$K^)5V_Kgnd{*Lxb1z6@iQff-JA^FM)Y&%Yu?gq&!kAy&##0ALzkMO2A!&{L zp>Go;v0OhOYO*u(4?B_DT07yW9Q!b%LyPm`7yI~{^l(GA!6tweNvH!7%P9GW+Wgpx zyeqKb%a~j8cK)8%xTI+}G2={<`Ir-FGzeZtqNxUkz-~4>SoDQTpf5_%&s{MGRM44dHu8S#WW=YIvB+6 z@)u>o`Xw{lbRFJ_4~L0dO*d_IJjNgqSW4`=)gJA+h4Z32{|1tA<(#xE!$!f#Pc&Z$ z{^Yq>y?HQsJBB)Iv)y{AZ=sfh$hn=TaV7ZeiiHZcv-NQ=UWW=c*Fp@5(j$$05B3xOWEdsMb!zOx|eh$*8z&SmFv5mW>qy0THDq7*DkN4Lj7)$PjN4n*%^mu>C(lov5 zrC6Z{?yvRXn%`jT3oZN0|L6PbiRJtg-}~Rc!0b^OKO?MSO*_oEVOQk}g+4RK%h|cs z#|D@D-J|-q3GA<5V6MaSS4(jM-Jl*-wUk3RQH~<_FLP`ux-%2{@dXNc;X2H4swbL^ zNRG#m$`=kl@i@dK=}-z3zI!;KgC{1vWo}f;?5b1<0gu}J>lc`Nym3EkG$fL8)7vWS znWH24|71HZsd32_;E_TbiW<+@}B(91r~m1=h8J=VO=5gE;GEnze+wH@x%+t z%kDw`r;nx7)(U3C2>bQ{d}Zw1<%N#B!h~LscA*bR7&hwW+P4JK{goACoTFp1AcS_DP<(WXCwgnNW;IXZvYhUfkR~~%@Jj_mw zmV}fb5A*D!hxzW>(#7%+8_0IJPLybBZ{;fj*$&hoB;-8SZg2vgXSh+YQ`X<^jtlM% zac*{Kn_DTHg9OspsZwQs0aute(cnC4g%Mx^OM4^7cYVB#fMq4vWCAf+ry&$;IMEwRTx14*rFy)ADQ(=Z|wJxDe)!F6m zW%Su2VkADGrZ}bPn~pee6+PK-z*a z`afP_?oX1jOi%3RpE%P0MaI%;wUhrv#?C`p|9IqW6c9Pksl$v=*}irF+Yb(n`c)7a zYr~uA$eE6Va7KDfCH(E{8OAk?Mg#MW=qbV)QtuD{o$nuGy0+q}3KW4dRlRfpyQ3bc zq!U~mgJ7KRL8_3}&fLV|RFsmM+d)A65RtW^>uzg!<4i>}3|1hgt8fQ+sFn~eSSBpK_7r@lx$v8L^gSaZ z0IF82q?9$W+MtA+;V)!z*dofSTJkx@DrX*U_s4&cf0tcp#-r zJck`)08MGuj9E!~o7JSFLa`|?w}#vvPi49@-dWU}I*&d}K8Q*B;k#--l_Rvk*n)GZ zx8^p5YvmtVlmsbQ{s!-XOho4%Yh67*u73o9xnlhbZ^Dz0WbGzhSjc=>K zThD+0SaU~jXz69M$DO}@7J>w}CMJazLaD5@nYhD~t?Klk>pFhx&);4+P@V3J1X1h%-xlo>?vc+19Cofykm|7DTs*Q?3sul)SVcZxQPdYB z`R2z3MiFBOhEMBSnJXKwVfuc*KwYRk94h>W@bbZoT(D|lfzNqp+B1O_%8u;p!rgY^x$4!4H1W~{l_ zBRV16L|XCgAvm#_^db3Fa^xYwJ$)!E~Od9C9gVj1&l@N@DvUSvUQ|Dx3h>0Vpg zKDoa}0u7Ev&p*q(0ahQN|4sisIBI%w^}+OiC)EC*tB=KX9%>_a_H<|PG4_vp=Zql= zR?{l9Qm+a3THigifA}fU%=+r3=O?SJOOU}(?O^FGMe+|o;Uvg7wFXhgLb3m0e#OQW z9UvE?0CE8XL@rzi^GB?-IY<6iZyCX3Z<)5Hk~&MufT*s3?)*?ie#tBzMD!YS97w@C z(X7e#Hy7(F^Z_rZ>TSE-&+cx*JA&IX!&~04?G6|==$L0Le-wssfWk2O-xP+j)St&c zVg6PaZcGP5ou_BO;|%PK19fGevaKY5P+EI%ZC>e%3mrIoG-=Nj6!z!j7B!jHZXqu= zCSW}3C|j>AnB==K?bWYX1Q1)4n~s(1x*0?3jSJw;H=j_TKh6(ltoJOL(l> zPLQuUAOWO@010pMuY;~K$edMGtDXGZ*?ncg?^y)Gf&HWk)L!`jSTWP?|H9?M-$|b( zH9~nm_Xcs^akKw4>Mw&~H*&2lYrw24U@%N6ZV+8_BtJM*Z*Xn+FN2{cU@*it{M%r- z@n|sQJihfIcji+oF_t%Ps)h@eKV5C6GwWUyiS0aCK@#~>Qg(`d#dsgBSF{v{9Kyxo zt%03fv%rcYBH8%*1s6&hISvWMLjH7dB*SPcYlC8tJQXWxk;Uz4mdMfoEWGeedUS5! zZ9bn*Vnm1O&l%s*VY2T-F}N%^Eu&?FhhuzL0a6%-@WNH8DD4PwCk15@kz46CC{i|# zpAZ|PKN>dOJ35^{w*7d1m|F&cv7it8-#_`}r?($Z;5tvtO86#yyv%v5uyJchl_V8klK6{Vr!WaE_F3L+8^3nYu^Us)<=vbp2KNf<5<^hM00 z7#5bv#{#eUy9EV_;9pu$_=_(31=pmBmah39_8<16n`I$?>Px>~ z5aPN+ku(ygEi(s649zlLXt0IeTQw7cB!-@}rXOTKVN7l6rUCm8$C6#&uUWfE+!}+x z{Akx*WvdxaU~qm^xrLtc&D%UcVrX0*0g6^%hs|065<^s39N-cJB!)A#=~`79wPHve{|vn(1LLf7#* zUmx45`jm%k-Vzj$tbUV>WV@s0s+`iG>D>b)t9j)?$!gA}Vy=6_OC?pwa<&`peNMjLn1-Bb?prOSLd_UMA6Y?nR6*36mNrNvt?yXJ*;ghq;3@ z#F)EoBmt!MYkn2}oYM4nBNw=yvc|A-RPn^5^aBeuX^~mFCPAhI&D7ukC2Qw2X+>wW z@w#(Ybs_tJZI+of!pGc4$D9hSecYjxg4?+j?DFMPx$ucNG;hd zPqWzeQ(G3Ce`w9uI0#IQhlH@>3CdSDEz!MDy_Eb~G_4IIKRNDDkBmPTCK^yPju>ia zx__1+(g_A05mrrz8l%ghssQXagv{NwW;ak!)R#L{7gzGJl@|Va!UVg#(Jl}Zc zj%pbvpTZA2+;UP`=QWGN#QUot8SBQG?SWoAgYE<@S2NqV>?Il`ge z3{yF96r_pb(R)_K0tZ|fP*avW(fC(3IHi@C@573AeAXd7|p z3*jLX_rx9x;%)M$K4&e?+>Tg!k=m3~!tkjG(0o2ylZhf@4Q{3I%-QC^Y9g4dacbDSsrMSDh6n7|6910ZuLi_st_nmt` z7$#(rNzNqC-us+quk~BHU}B+?0w!)PeD5Ajdj+8=b(Te4w$3Xrg34;WVehh$pYtFY zv=#c~TECf~Z@4>!oDl7TDr`6@f@p{6VMi@bteXyx+g2b$rikzuXDF8om{4VJ+u-}_Vfjh+PtsV%KPYSdu-2a9SljPNTzUD58Xpz*6M=O>I&d9{L46R) zkgOA~^D0p_!E7VkU}ZZPl4%Zm^<6lH@U6#ZAR(l_>S0@7Q2q70@A?wc$t7m6eaZR& zv2)hG00-ZhcAI1Py@dd_8AboY!IF5VH65)?H^`&>>QBR*gEOeYyq_6GRaxES(`zH) z0p`WgGj2mH)BK!2CS9CQJc@Z=BYu!iuyurZ^0s2h4>qwMr^>%#`Y5Xk;y9GhUzl_B z%=g-&ZY+LhqiUQPEc{`Ldlz@$vUDMom+);3)cKh%_n`EZ5j-7;skGX|gge933UAGp zyO!pLy_(nom8osbMvGl_O5Dv zKAugGqidKT)*4$O#@ZDrjGqeB&iD4gRy*6UvI+b(AF7Xk zX2hz_DfbA-oI_zBkI#M!XC<(L!%0J0df#{f^0BYBFX_CeoL7k%D9_G4Q4u?Yt7T2@ z{c+;a&-DVK+v>PO^)kH^gS;K{N{G&*rEIPN=Ai*>Rb2G(@5d!g_yahpQ`BW8-alzr z)wN4uPJf!GP7#t5odzAm#VQvheND=0<>Dps7@LX0k@F2jOcwT$RG7LMK<2#fLRyJ4 z!FI_hTCU7|8Dn5`iKh(aSQ(xzP9x!9i9JgDB81Wiw}KTL3oJRg`_sUM@p4@Mu4OFy zgzk)V$Q$Lw;(INsUIUK#2%bn=Z*Vnnn8;}BJ4ynHN{`~2jK)}9?Jp{V$G*Y!ueEbe zEA^cu(Iu9Tp^l{HE$`_hqT7e2=qQ9Fizh{xqmqb`NQ%iN5hiF}5vRUiY8*(rg-3c| zQI>Gg%g`9*(-4KKe8yVpw8SO8d=%Sx@~tJl!A_TFCjI7!AZ5geXp7!!l+7GpJSB&d z2+qjVFvf$aI1CiA4(SjV8>`sW4ZqLQnf&gdwuz{?+<|mvW`M8CFL`L1LY$ydK^`rw zlUduYS-cZvtbDXAOetJ>na`GL1bj;?9@-{Tsxt9vOP|~Hp5lG)U~*4Jgh4s-J(9Ok zgkgwcD~^eDYHT9+fZ*HKvgVd;Yc&ZM#^HOJX1 zZwqkqqUxH=oBL5f%l^*Vjt$U{3=^-GDg!u0kV zkT)ifKT>}9%s3YN8LZ!pziz|VJC&H2=m@cblT2H)w}0Dp5?3g-fD1B;s)$wsjzSXv z?R(1ZMq(WWZyr-eZX{vqpV`c*$GkTjE9V}2Py1Ow&uq)wVU6x-UimBJ>@X}0jD(1o z$PRCMjaU+9Ce>wOXx8@_+bsDKO9Qpn+F{nCWd)gtxTGW6^f5uQLnx(06@1>YSet@~ z5Ft|TY|g6jsb+=w0f!X!pmN>1s_cZE zvuyK9m1%O#{XE3QtXz4*R@@}%+*NBVIqpgArKY%(tggb}gFDvuQk(W<5WJ}k1A(?d zQMDqFq^QF7^5mz)WZ6{)$5!3LE_V78yz%S-544?Kh3!Sp&v=BrjY4SY2)t=b24~unraoG$pinn0ugzo^>~~i0Q|(JC6_Ld>OsLO_OR9`h?r!L8 zK{aEpwX}AmN7aaL@~tOc<0sB^a6Q}|9tXUDgfQrBP>y&f42{mYkPaD888?$K-6eBR z<@@VP-6m{jt00fvMk#jnpMHu|;&IC%OJqm!z1*)Yv>|^AUI)q7(;JnFPV#~))Df$8 z?mqmsc-gd?SyPbd<>K^uax!#ysNX5tynm$dMbk^*@A&D8OrF*13SeR{;CGn1NOdki^0z54okJa! zK(`jPxaUBihsc;VJgw#T*NdOQ+BRD`^b_$IB0WkIisA5?em>QrHj}5AXy~^xLD`h{ zU^KE^H8QGVThcuVKiMmd$REcRK&iNoob7FnmFB4P!Uz(U~>u7wI zLw!E)%@wqHowdw)aipg2JF|r#Adb8E2%ci1xJCb9h=BOFE%@JICOL8c5yjPEg6R#L zkX~AuE^XTA8U)yXSDe;-(~o6+tknsrU9Q0bwQD6H*^aUR+{@}~%S8uwzg;to3ROpX z3a-H5kDmJey<<8r$S&_TP^<@-VD2}g3Jv^T{9iM3e-2yFqJgkwG8EyHgNC7D_yR(_%Qrn~8MR&)Vmg(i_Tw;GQa&t6|74L}TD6Cb$Ro zV@Jl&mrF*6kU=qF;( zzwxT7hg#owRVpj`fw4jlNR?VZ3CBf^yja}Vxj!9^h9KdJ5k&ScK2b$4H%%}&bz7iI zpf~!b5AVZ!9s)Mz<;_c?=Hv!ZWuyC-19JTU7N)L=*_U>{7Q*tZUlyh=k53Q3cvZH_ z01MNo6!eM4dX0iFdcJy0dQ5IQ}}G@)kB!dn5<>q4KN|{XjDI{sx5}L z#O1s3cS0D6n2Ij}@mvu?YKJdM40Dc-h#!;EUz9M_>hxJM1>~B=o;i?0NWqDU`4hg# zqvs@g*xD6B<|&nmN-dZ?E)TRPzdsCaB+cvw+QmQZA5yvW3`o2Xzh@2yZH9`LA+wWP zrE0%9`M7nMpQ8wgrnzkw#^lSvC=UEkUWswX76DXt+M z8jWX{ya#I)6~A>T12`LkwbMUX8&YM~*xSep06I!x6+r+eg6fSE;YEl)Fw)a1YzE*& zSpDWi%w$19BP5t!@JXPF#3x{tqba2q63CE@P{agHS|v-i?Xd)#>DGAta|mSS@FlLjM(d~rp?TmEjl{+T1t@Rttd=$|?i+Z&~oq}+X5uiDq7(o?5S({vf# z)WJ6$N(VZ00Ep|QaXG@OLp{4JefF`c`)JnM^Tx`jy0?NK6aUb7``)X8>#g8J>-6o& zN^$@Ne-hpWf-imcTZh6A(4p)!07z2ePtv5&n6#x>UDH&ic zjX;H?9WBFYhQ=|H_GKleKJrHe*P*BHTIqTe$|xrLFF;!+aOPFfTg3}cb;n+YLIEnC zF$D{PS6HgB8EM=Z2fcTiZ7zqZPwF<(jCmMpap`Y?bUy)v(Y_x;CW@KbltQJbx5PzO zw>PfBr=!p^LaTKX24PQgpl{7^^g}=(opktaU5OYuGrOuSX?DlDkb@rM2nSeX%7txU zVTr7!j*=VCuY%8vYMbXu&g3sy-+XJES%O;Z{r~DvnEs%<`os14KLKq3NW>DxO60AX zS0oUnQ6a-t;k%W31W^ECwBf%4+PqZ&LP{TSd%{-MALQTcnx9rGZxb&8fVLTFvmOAT z?HfQynS3mG%f0(fGMqGUepE9yCeXhwQ2O0ngt9w&kwkZx9s+*WFU07?980=P!3K&d z9SY4yGjBf7vaqfGw>)r=Uxc>4xnHgD^loEoKqB4r9&4`U%{zvYgk-4*uEJ&HUXKe@ z21}O?NFYWv6ieu0C}WG0sb6tE`fqVQUr=IPZJFT$ zO=Nxrs@CV2a&HAahW6fSYfIrbDFm}A`Wd_H?hSU`jb7J*021lm8sTvQq0H?2a?%0GB30n+fo-Y$G2wwV`e`?`4-QATzl4+&ZKKD$%dVnpR9@6oiQ6vs z2{Y^{e*?756ut$jTDO5zwh8&M`C+~Vs?z^50+3hFbTNIfOrO*=eHFgl6U|GEX1Sf1 z(o;{aZo8~l!!ekumyEq#|sq5B^_0ijt>+e zyM=wd!#y_Ar5SUbwe^iNIo@@n<3U&AFNj@BPFmZ@)sJ(YkFS((klu=PSmT|=j&MmWm&Lk{LLN)f)Z?>@e}mknIyk(ppz z^BMD=Utn$9tG_URY_=_CY>-(B(v?*OJbvng7szwTP@Q%TtUg16|;;^H{#r7xKE;Q~p1z}@6SyE`3E|Jmk$g-TS z)QJ@;spRPj=5kR))JTTnJjPU#AW2A+9u_-2O>d>C$Sq%rQ0^39~CCmJ0f5 zz2jqQ8CTbb`SW&+fS*bSH@ZF3}kSoy_LaD znsD|M+Q}l8i&b=Xt;+0EBT9@wrpNb%5gq}B9;}j*H#GLg7lkyOh>x5&t+BXFR`iAP z;3mX}d=E5FQjo_av8&#WP+=>kkXVnXiQ+Q$)B19b6ekiS&W~_ zda>PmH8hp@Lijzj_Ku_PS~Pt>YDr#aNw2f&{kt_Flt<$4h;e*_WDqlxzXjc}h5-Dr>xm76SKHs4uQrJmzHKk^no z)pc~KSblr_+P=;S`oYyzo{?Nuwn(xpgU0&C_+gPUKdSM{B>LE{cKqXpDQ2i=s_CcY zHvg`Tw#vdlPXigv!sG~+p{j{tEcMKIPHw~-aW2Im@>u6QpIN^aGE`l1M>k)Y=kZ~P z(&^4FEN|#pdBD;#rCuL}Em2b?@A13`gcR7+2y$Tvm<-kJSOnn#|RYToEcZgVG zPII?%wZ}c8sUd1E1PlK3Ek2!4(;UQp-_BDSo<9SdO|Vp(JYcf)~- zo7Iq6xhK6yP6h)l*y%a!dEj`0OUA{v50%_rGRN677Zr# z1^(;7nJHy1B{tAy4RK_dpb{v^E~)i`;rFcv4pg8i4Cf;K@bYGIzq=+CIgMw#@ymo{Qy@00E8{G2K5#|AoDh9{`D0BKg=Je)Bl^SuL^*$UFAUZSyWd5 zfo={U)=oKP(W03a4YC6*wDmvT&w~U2FHsv&DK8R4;I7WOllC%g$l;av3;S)YyI|#ej zYY};iLr%=O54H}Ok=La6l%9wnMBjM#u_OZ;4`P=CzPE+Fc`8^E!-GRe$3X0J;HM9L z7@qq-1K36G_(7Z@zI!u;eEOJJk*C{M#Jm3F(Yzwd1e}16!m(;p;~Lm2ETDYO3Q{Aj z1U+fRE)Bgm`N=EPf%f~bAR6->0el)`{La^2G*=L%q7af>QY`j&_zm|i!_@LfG;`n6 zuAbZ+J#O!F~@#4#$v#)HaG$?)kDyqA6?(o$oSt()PZsAsQ-+wrfO?jD}Dq ztlY4khlQ59FwG8UrNlmrtn>g5&J_v%&i!c4yDRFV63Ol9LP0W~_%8a3D6;L(%bnbN z!P@IXwUqmwevmvMC}Pq*{i1!bm)kEb(wC~mH1$6``ckjYpd|I@oR5z4!&aY5uWMp+ zcq0&+F0ReJn0P6K$mtJw350A-n`kU{**mfATYc;HAIex6n-)kSNF}lRwux@rtcJ6_ z3vKYKa1iezO7xkcm(-$zI}IQE^h^qz;LJ zAV5xnAf29pBsBpLS=qHT?!C#fqx`EZcs6(Pn0lfwRYOa%M+Y2=zI-{JK@B>=X2%?M z#Ppd^{Zp89sP0D_#OQqS*twzbK+sR~&iUy+ymqe|n>OM%lXjoS-*`Sz=(-)Uvd|ZK`vknA=x-`Tr@1C!- zeHq@zpd+VZAJZG;8&WT9Nt&Dw)8WWR#|WF~zNoFybz>Hqx2AN zpU+jA$;twKkxfW&b3riQEyxh-5UX5Do~ZquC>y^cfCf^KG!{ds@I%DwrXN_Vs?Rh^S) zY!fNi?ZSkw9L`%FloM&W7et!S=Rx-y8M7JbDrRmg(Yt!DOcu0gW%VEAgiD9^uW*7% zvXG(+5K}(5qttr|LPieovk|X7;Q_Ko`eTD~Unp}09p0s>3c^sj@0VQTJjFL%lrofK zdHJW!mNt__pgNj~8l|CfTk1N4(txOi0MI^!V11jIA0fU@E^5~Kf~}kRBU-_wVp{>i zeoK@WO=Ry6HSU8ApuT1sDl6oF?VHH3b>G)%6I#68%gIr|K&|V+9W&Z24%%4IJ3Urk zijaG8wLQ{ed_($-R-AR%g3JH1S-vRMM=t!ZG6x`ic(2g63c!64Qs98Vn^0D5;MSd* zUL4s(A>qhsL`*M|siHlxu_54>Rv~6vCbrc~oDkOC4u|u|o7n%3^y&PVd!384<{_0h zS=s7KQl3WnGkCT2iw`N+FQkv`-;qA7>y$Ix(K47~Xob0etX~-y?9)2vNuDl!Y29^J zK-O=AcjJc;1?N3!)!CGuSiGRkAy8qFLYZAXBb*y7fKo2Vpm_hcQcfRBtyZ4}S6HlB z?2!Yhg_IMZl-oH9ZHv9EWQqkS<)~PQ^Q_t*U-Eus{mKHeennqq@Bk*2UCzdP{2S$P zH}hXvzwfO_9iE>eQppRV4iBm~08DM;qXO#R5Rw3;9L&|G8$c;H z@I@K*UzKv(RrzwY->L7x^R<<`jL$xf)l69NW9o|pka~{L0F-h)JmVuZO1#6q=4%9$ zpGD%der4_`s5y*m={Z7oiBTdm-P`;D#dS7yQO=|BvaW@ZntyK(sJ*oZBL6q-fvLZ04_q0H zTmK{Mx7D}rci68>6*?g77Xu8TYuejgyQ`D9wSD0j#x8UF*7SGSuW!cgYXHA#5lOZz zGmRE)CLtiO{w?hH7r0?}&lRrE%CCSwBA@?Dx&4;-mXHdjS87|KG!Y z=>cKCSO9=K!ruVyci&>)0PYk2W>ZcszHZy7QZ`!I6fzFem)@EP5z?owE*9GuxQIG^`Tv32@>Jn{E4tkMHP z&$1re3Sqwagtfqs;-&aQD`!1zCcw0VEo?=1V@eD~=g2)+uD)tnc}KdT+LP+;KyJ0) zg@0`tzXFHzHRS^sPNq}#Q0!DX2f6+q9 z?PLI(@mV6&lGm({76v8ci&3gU4&iucIk-VD!>p}^)tdPPObXNz+Nvf`h!c!{I3Qu7 z#12sgcKRH2M2x(4PkTl<7S7XTI&xopbeNzxL8L|C$CkSrbkBw`jt2l6j6RBba0lEs z%Dsk0uMAF}k#0*uBA93vXbh2*dKRkVp~?PpxLuSCuNaII{so?A9QvTYZb%l+r9yG9 z5Xqy09hn|ouZ&()>>Vc-D}+}Rl(I;_iXqJ?+zm(w!3=K6&u#{yahasn5vPn0 zNyD4bkDxGtu&M-`)3tcK)bD~sm%)!S*a;kTa%2}3JdhoOHH@Kxh&SNcXVn(3AW}4+ zl>5DO;nLj6y91W4w_^V5r3>$mB5$TY2+#gd=Kox}1feLl&?0O`=p}hzHR7Cwf!JXl z>U>Zs;+>Y13cFJawPs#B3kp~2PAn}U#_?&Syw5z&V%mxQgzQN^XKifc<|sPUeCM`e zm+`jh${rXo-NLMFoG1A0D!!B)CU<0WKZ^sy9(VlhDu$#@%MFvr1n#5n`SxRBwQaR+ zbC&&FuVhg@Qx=Qftf%mkUq~=&uTzX3m?BPa*aQl@jDb4{BZ3}L9IYiKc#ThjZG6{N zr#oQpriOQqVGIbTCu&9){HSy3SoRDowGR6x&RVomir+G0Ca9|-vqJDuB1h*9UL zEwN|l`l3x#;QkFVH;JNz)MjtAbiMP`cF@N$s+EZ(Hb<22mGR01{ed#nTs(eK{PGgX zcl#owak(kbRb;uWz--4;?G?oR8 zY-Lbrm>G>+fI|4T#(<~Gr${5a5?wLFy2&X(t7M=nB{85||$ z8byZZTwtDOiK`?Yc3Qx zGTVxFD05L+=#ievY3gJf;^p{a@yWxS8T<~WS+%`}nv3zGWN|nrs0SG_7!V)=uthgz z;e#=wkb^TUCjK~ob45ocuqH7q&f_8*Szn?dvilWhQa9(UEq)V>Kf`W!ya_rL;tIPN^P+=Aqz+&=p+hRMmmqWs0O=zVLE8zKK}MZg z&EgPCos*<%p%oQw+FfX|p;7nmI%Jat8Y8?4y$P>74Vqcc(AR5}U$XT3wPsBKclql9 z)~vT7`!<-Nfgpi^fJiH?Q~@{t`1xzVD)S$-3Fh^-#&3Tv?l3^WU_XF?fWQ<0FZI^) z`Nu2%x(^D53V1une|tMsQs6)ET$%r{3H%2s#HsFg`&AC)*Rw7mMh5{Si^QdvIURTy z=lC1=K~mXiwl<0g0R`JZfHCSL-sw-+=X2=}3q>MR$rKyLYy`#eP!51gD73HSg>*er zSzUZr`^c^A(Sm-H_)d=5H~4eckI*2yA;l_8Cz7FAG_I>JErSSsPo_^P5AsfOs^SX^ zMMqq|XLcpbqA0ozaW`}G#j~r`PR5hy#Z{acmQipJn)JtD>jfk13)3epSBrh2uQ8ji z;w^q|uNT{2^4l>!=JPa9_PQixUUuXkN;JSuq)|Rc8s51#T$xxlXvxanRm5D#Bky}DG9RJmvx@nY$kMbN&eB8 zbZh~>ykHTk^0M-pngh;>g&v}tX37q(Wn7hAWM3$0IvHU zOqHQBx|_?}hb@xgU=`R# z8osVfzc-suBpI10sD+bp2tzHsA&Nr_;bnDLV}MGu43wPPA~mIqrk#5(=`r0$1l`*v zh}6PNJ3{gnTBjDA!17g;_`-zqZdx569K?Dw8y_&-;Q{hNQ^@HNQD>cLdvB5IQEE|~ zgZZe_UZM_?M>Ky-OFt(4;qdr$ddEpl$iEWi)M>S`5cpFgl%H)xh;O+sG7yF4IS{FNbC|@dygU)w9*# z@F5y`n`T3w?Wd_VSp*FwG7>b66!I0g8tYi9cEm2(*B4FIc(9z=TiX|lfi|E4(1F#^ zo#%;WR${^|9!Ygcmse+^u$rSw zYpi7E#)TMD!Yn8qjP^2h6@+);FUK_DOK;yAAKecy$UWto4-0Z-7?WLiA={WxaAF5!dzx`w@>qmj+ft2U1~t6*CAKpu|Kg z`E>}9%HRZ~0^K{P2s&v$$4k9POAVQFU|4A6dUocmdjZc{G0bG^*08XQ$DYniFtb-9COm zL^ka;bKuB5$>aZgU3ktTo6$N|YD#2~(5H*6S>Y61oGT@`^f4Av1LKd8vOqXjE3?0b z^rP$Wh63tZU^hP4)Tnrshbqp z<)_HXtEKs6ZK*m=sqT)A@-V-ME9C;Fad5UJ%5lI{$+FvIRib z^&pP<5kJTlkw-!;MAvg5>n1HUz(;~ouLaHiTP<%-;$fbhP!8ZuSmg;{b0#o|`ILP^ zVVTgOG_~xAEjTFxcQBW4h3fsp8zS(L+Y1aB>+EDXUMPqUrgX|6zarys9lCyuM}=i> zD$0j$V~#pN0m0gV%m zq7Osp3S=e2_U6AGlcA&3$AL1XwVBG^9Ti{+D8yQh0Z?=^KIUURD66#b0}!~kqWzKK z0C<*b+J8HO9aI{*0l*QM?N$JeU{S)r(dr*ZAh?r;pCqxFQO8%J_9F6iaDz1^j8XLA zgjCULN;i*attA80SXu)WSQM!q$*!7I?mvopO(&B8fqn5z*0s5%QAPM0lPuIy+O`qO z;i7P^#*g`%q#`TJ)s7BE+KLuIi01Q5>%PH9E?oj7oe>-i?cRJyiiq6kt+>R=VjAs+ z1R_F*VT374UX$e>Li&Zc78zv_HnZj>ux`Q{+TSl=ADZ=pGJM=;{~6fai1#eN?=`Pk zcIaokhgxt^ZtXz}U{(t9{r4%NkAw+_?CqPS0^9&m0)xOOUUY|i8pU*}0b{l41bmF4 zl3NTTfH?v_vA$2_TrF4kTV8&;ui43m%WOTU;0T3^rgj;Wa9wf3kM;~yc3^60y_O=H zz^c=#f(qb$Z(rOxMW3@V<(Y+=Ve4n0LNLfQyS)Gy@sdP zGQKt!gQF`(DWNMe$nJXKnZv+_uhW}{Y86kkFV9=PURtBd zYf(kDv3tMY0QXoQ&oV(ZbvsVMHGB(ehxC-BC1j+(jj{P1e6FqQs}Rt*CJeA;X7jQa&7&}R#+ z7^iBAm~i84K3+Z_=-e(JH|7@(U-z1|=gXh#p(-099Z~OmJ8J2f$Wv1*cl={rm2wH3 zh9ml!q&`i?SHEWEOiq+l6wIOPep+s>;MI~8*Z2|hkbrL5ZImOoMq#-po)ZeJ&wE*V zTI%&E-Jn;f~7P6lQh!Vc%1E9MEa(&O!N zFBAE2BV%jY6-=>7>4#uTnXj^vk(tOIr_P^GvYlf-=COQ?%p@?oBQm)wcwAu86=$9< z*$2VpzBkmD78ic%g*-^JVw20{@Kx*TFiMdB_zmZ=-jU7k(3+nH$F}+zU$&+QUA4)P zdREo=TRC&q?k#;KN%MPtx=;4Sgwh)RW!!>j-#ln}QU;vhbRZ)PI#MPXpT|&r#;JUL z%*sAQJbenU8qHuy*;{GiLtCOBHS>T_pfV98c zuSg)iW_KBVc2)|cxC|2Ca-LO%;?0xxhbinhMyX%$0 z@n$e{<6(ZxO#gJb^b|2Ev#>C70rvc{IlKc4=}x6U|9QqOki^j|#)dX52uE7`I5@uL zhMKi&&aLbmu84}ej;=;#(QtlkBU|bl7U^MTS=pf*JbvcjY(1+u^HCgEf5sRzC-G>5 z4C`ul&ftL`ytv=im!9SOC+5KqQjBQo6X{KKHVM$4qQ~6oI>m^|uQQf`rH! z+TSmlTFHbQ6k1`!VUcWamyx$mq;?v}ccPC=%Xe(`?xZ~f3Mhg#N>kBan6CgYce(xQ z70mUuyerh->_}tD)g;tlV$gtRqLYj&`}-A9J;K##&}|PN1VtSJ1RDf70T-h`{)LCl zVRkoL0Ifgd<%>EDYaD{kU4Z+x^@QuadZ5-t1tQNeR@xfkfT>qdbj|Quy+rUq=Jy`; zgZODL*_b6t*Zv}#k#}r+nc|N9R&^=`M!r87#s&ziN@^`#m&%=$)s%IPRc-tzHFe3X zt1+Za68u0pJK^E_Hzlq)@n1<;I_l!c?s>2zQfnZkJ@f!kE7#+QdFAX_dicn7G#18|Ooil4GJQAOUh5eX*)A*yMY8b&kb*1_{@ zEvVbds}`MBjzX6+)Es*j*78b##}Xt*%#Zpchue7sP*+&dSb$TqTkWQj2~8Ya2TPr= zEK^BBS>}DFwTsvE%vDu%y;>kxDvBxYDz8kcZp-Xdq}*Lc`CMPCC#VNPih0Y_cGK@To(+Xs3%e8EnW za(C_63eIodn3l0P%>7o{(~(Mo)9MP!sC%Luc9M~9vK394azj%od^w%VimfA`3tEt@ ztyH>R`TX7&V2(Q_}!kLElb!uLw4YI>WVhi;m^eC$ zeM>ZIEk&`n*NUL2RrG~t2;X~6_8GcM+7a81u~51X~3(fcFg_R6(KFI&i_7}EqANjhU_cgESBiNlZI zn03pUp)aTlVhDYM9*DA}7Gc3kq(pnQ*7T8a_4`4oEgTr0>NDZk=*6AqTqWzoUZ{^H zq2RZs!rq@~>Mdv?-el(`a|d6gj?$--K|GbeG%AmxO?Hv z!-XAX=Fa8CkG&N+0`=(0HqQLRdRmVz_UBjDqV&i0gJxXM)l-5i(h~>t_oIxMPcDoY zxcmjWObdp}sQ3n{_nu5qMM<(Ej-j%JZ7U{`_d@SV`VgkV{HP=S9_{Y!Z>AWI3oq@? zyW5mO&WRiqJFFjId|7qZKMPKLeOzxop_uG%&$ihyl)2rOxlk~p z)-Q9OyTOaEmv*%)sBAXWPCx1QajwMH?lkq<;{~~4Ou2VgS3|zis@YLIv!MUbwUSh5 zwFw---xi$#V*8?~2W<6vQ3FJJX%_qSxnwT^bqa4I$V4=i_DUq$hww~j=9tOHhjJ=< z>H8ih^kxhHp0%P}=u?@uEFO-brt>UfQuqErWHJ702-in@*~KnQ*}lgvsl*OkqrMka z?E);X9vOiqVBlaiWyhd0HW}Q{lpFYvT6?T#du=uoGvY?e1_X4K;sFX0C<)YV0Uw?1 zic3kwXqk=^w;^Q7AIr^16{#WiTb&)s?KdRd<|_~iIX=PCfDZtwdV?f3BwB7O^yi1mSKJ;gn%kHZK@g3{ zFo>U@lDrk()9DL{#v%Gy3GGCP^hrI?fI|$cJaiSjGrITRQUh%P0?ld=oT)}2RSphc zR+m|4EiD-k!_{W&pyn3vyBY7k^<^|dBQa~!~L!J-6b6S!4Clh^tLAd z^=f_gN9z{z9~8fTSgjjWtO3bzsL$E@jwqJMqDky3Mj7=`D^O*$7POhgUCQWQOabvgOz?5~fl)Xm#buD>qTv`FfEvYg9 zJ9YAno%-9jg&)tz7@R1iwL2YfFu-CM*xO;k*bdkGdH5~bw0GphiK`7ZboYycGY^h_ z6#2pSQQOoIes4G{wgNM(_PU;eCXJEhLDpA;&y_u{T{LjbWr1(gRIhb9PM~FI%m|JbJQ9=ds? z)u;x(FZDE86z4U!OX05;S%HKTLl7aoh^LA^StOlz>HbO=H)#uP96yZfJe0Qz&x@7A zAV#Q+e1(jcu4*YA2-Ci2N z+=UD(K#7~2-x60HMIFD#hEm}k8{k5s&i)9o`KzzrL5?Th0@xFL8x4OwPQw1kh++PN zl=BaJg8%D$s{FM7l23(7%)WStZn7~lA)aqX`KmUlw{+Ia;*?b-8FLj?9A%NVv)Syv zhhBA#hq;Zz+>S)GXfek?gsdz#9h8gouyE4@)U(VT501--$1t#?_AL4&i|bRhlV z2?y#ugHhf=RS5jB0lCqqe9mM z(fdY3{4eoTCSlwDwY;}Ps64HZfmr2~7R|*6@GA}WGFzoLCo1T-h-4yK7W1AKX@wGX z?T{A3*&#Ud5g%84psNBX$XW1GknrB`Nb$qY%Jf^*GjU0^%i+|>NoAP(MlfgHGxbL~ zTjQQuJsWLX5KjvQ$AcY1q~zz-HN7J}XnV(zq(J>Y;;B+d{QhS=6@ZDAvbgdVPqq9X z@l<^jfOskjKs?njZeTG`?^pOE1j86Ms)T)G{>xN*wYLL zwCId!yPFN(*d)QO1!C3dimM^Lk){VnR3`7R_Z#BAqv`k*qfVpGDm)pxTJ|S%3Z15t zafVSCUJ_)ImnND0ur;Lpyq;pj>(gv2YCsH}P^zR_hew(}P1R!Wz72w^2mGXEUNyXpc9aq;!OsmteSy=!&vE>IhC@<}3usGr8G3BdVUT5fvo|^xS4Nm5+YZ1C-n?6BjFSV?NYj=OY-iB&7f&^vtQUpD z4-G5}M*nu}YDy82EZtpGZp9n~dwt>S#-cXhs2y-OhNw+p2FWJ|j=KnlcswjS)PB&tUKFMA%$H%9WD zLeF?~@=ee;dmhWEro6bIfMySAkof?mcSIey>8#T#4!gSCw-oAhxGypc`J}0)Y#6cy zlV*>2zcA7Il%EhU=#Js?F*X?BW}Mmm^&QyzL#Ue2PF`MUGG4g3B3k_yR4bK~SO|4f z5#<79{lO2-(EQEGkiJr*=9vd(qpuuHf6O#OHBL4&pJsc?6cnoow8#oL|8Z%94WJ`W zsn)DaG^j%=Vac$q2`q`*0Il-+!*{Hm${PEv+55S31rz4aJcS&B_%hf!7?gg8BSwIi z6Kr%Pr7_8qJDi^rmY+gb*Qtfy&)C=SU=E^aVTB}*Wsz5#*aw|ikg5=8$=q}2O zH%$AByJ9ok$jEfTwER8{s8P5Tr|&b;GN7h*q)oUrp%{&UnKcdj?*W=KSZE^HNI^tR zE+rzy*K}I`P45x4XdeZpk6OD?PQ~VaUE<^V+}_mhSDT#P`ON2J9U7M(>RBmuB@bCve9HWsEFLewh8OEn|6Q z7vlmfuy5t~*M-Xb=M&NYn_jd^@({w0fmK-5&0D^$W8$qDy$FZ-U`@g|Nn}jwZpFcSU{Rd3+3;+}LL7b?h zd_xH2FYj1&Hb;(y(5hOj`9&7xfHvd|RTnJ5 zuiuW%1YLV#^T&^qXxqDaOw8T>AW&d{z|fcVP3dA*J_YkWbYrxjx76tb*a*5HKae6p zyzKpFLnWSMqLbq0O_*904en9Pm~94`^o@& zmU!N9C{f)qEuXUZEI3`!It5`W(&%fDk5@$8HCx;G_$_^HZ+AuBZ&;@uxZ0kcxr~SB zzv}rv4BCBvw2^!R`@4CN*I?BrIwe>-PwNN7zWeE;!49Od9B%u?Wh=$&M=Z0W|FnUp zyb5$q*(^mZqsN~Avo3+|3an84DDKU49t1ouoIGYpRWKzMcKAAwpj#dw1XU8i)XLI9 zNuEk>@3CE9?_^#VN(G!`7JmkuWO@*StI4~;u*eWAN5q4VP!1*~C)yP#C-r0>AXt~F zhW@|O&NQsXtq4HxtEHqLdv@2nXMyL zv%OcQAO|q*9CpAYG*N@2noroNLKtF}y3AvudVN<>nVCZJ*2iP?(FuX%H9a!TkOM{v zM*i^-oe&smzx@9HuG4MUYOB>CbGJC~aOkU=6SwDk?H!_9AKar!JHzLLbjcIrx zm$@l_D57eZp7V^>4g93@Hl1mMQzku;>aCoj>0m5S94fCX@1S$aI_Y;8LpdE8e&l)w z-jMA{IvYn{xZKml4^50YIP@hO8)*Ui?>;ZO?YQMXK7Z&x|BdW z=e8|ptIs`+mz{X&w=)|o@3!9X@7H0k{^gC%f|nH~Yd`yGZ8-O7|Cpy2FNBPCsddeE z%)NiFN_FwbgFafh9?mnzWSz+x5xjHsMM16It-=Gxl=;`H$9(M@W*u-=v7m3n%>jG# z(?9y3bUAI|7O1Q#9GvfQZpD5RZ?}g1Pax7cMG3b(2Z3W)W~4RLz++5dd_eb=p?McToOo@i$253UQ^S-knYn;GwHf>uYL zKCcyS+qUc$42$&gNzU4w5NoIswKzThfo!->4Z7*j`>;|}Reh~SpM5s-AfWg6A&pSI0=7mfx~N!96E^Y1rf5 z`;;6FT5Ae6PIOs$b8_Q|nBv6rggLUQN8{a78j7#lmVSLx_VK^RYd)@hq$EP#VI1OQ z->tif)Q^obvP}I~{685VH03&={A>6qj37cor z<7_P2ME5G0CsB-7eE}F@ZHN+PWV{l+nZo0>qAXgX~oPv=J9BZ z%!W0t1z?0xF{Lu!w;B$AAZ6{Cvg9xTMi|*p!oE2t1dm6`VvsUzR?{yH2VkW4ISIXp zWI1Xj34=?Kv}Z^fo8}!2(Fkt`Qqump`R8n=)^KgHu@H@$)t&`J>HLYby~DIw3n3c8 zYn0k{XO+g_61UAq&dobrH=P(`I@AqH)WT<7?Kh--c*vn6_*A z5RH%}l=Y@)^LZSsO>`Kd5we7GL=xGUKW#hGhPO$dB8W!F5=w2UZ#FsVfc3Vd7@`rf zgpzh~^^vYLM8n%8uN0yYvV<}x+pndK(?c}eZ#~K&8X-$4X{YkTzimXc4s;5M?e3i? zAsQh|C~1YcE{Tk`-!Q)=oq=eCETOEond+5s*@#w+`EArWfJQopLFoVsopinNXqddb zcQ=WWG^v1Sge;-dcG$z9kdLMzDi)IU_V5}+-dPev9^o zXxv5$nM%7(!P;gmfoOy*q0|;SFNX(b$#_4A#w|-kL0x~s+WM}5Xxy^o?&0jd%pHMv zPJUYj(YR$v)!V=7nD3Ws&>M5?RIUjEXr_eKJIZ<+lrb}z8FL-S91yn-q7kx$lBU#s z=60w};-jF9Z4ixHmYAnT6m0-g*)a{G5we7` z-qgaK+Tc{S+5yoBSwczUFSiPWlS?iKqH)U-b8Y@!IF-A00W>piS>jsUHJll};=SqR zeGrXXmNa=ju7|b#u^*xlvV^kUw%t4aoN<63mdde*AR4zUF;`pQ%3LLN1WV0v-CvpR#PvP}gD$iSUvI0(JK_x`vmL(o#nrd(=f2aaz=G?L* zUePuK&XPwWh(>TX5RIF=YaZBTz&WYj4ABVgrcAEd0UZK(q#)A*(FpFQEXkHh z76reeoNVtcnOwIYLo|ZBDYaclTDf`>%E>d>=yhouK(pZHZi6}3-@XAY=(8t=Zb!9o7~j4biw|iP}Js9jwho7NT*>lDVpXnZl_oP=IKJETQai z227n5z&zrO&+=820h%Sj-IVoaAiNR-&+>crgJ=YIQ_}98e7lNyic1blt~+WFjho+8 z$1eQJ%)6&xtD#Za5RKqBN_Trj)W$GvvJjgG#0-UKgjPaXk_Q7-!x_KDVXHH%heI?% zk3-oezi5@@F^}mmbDUq$O4s*)xsd|&ITah38yONBC1rOivi-ZAF6{$cZ~sSu8|BahMu zu_>CFDd5G*sb+(8_{=9IY5f_o2{%shc*W}PR9<2JsWxGbu>4{9oD zSO;`@A&4WS3T4X_e@mWScR7ne93fRG3sd|-R5p(92jU2+La9#t#ac?$*ID*Y7n~M0&-E3TCB8Vf{oDwH~w=f%5h<1|# zbw{xIzu@Gg$QKs>7(A>)lQ0}d+BQeLzL>q$Mf@T!wz_99j$m`j%oM+ggpGT?7gR^E zIVDc~LI*Z(#~~2Mtv44xht0;FF9dM}oBvBw5kK=Jewvnz3qi+ffTqID=Hf>e*|;I4 oAdZ{O#ZP;&al^_W9BGFcu`s34T!+WofPTiKdvQEZp>YxKKb*2KH2?qr literal 0 HcmV?d00001 diff --git a/Scripts/Models (Under Development)/Nback/stim/Kane stimuli.py b/Scripts/Models (Under Development)/Nback/stim/Kane stimuli.py new file mode 100644 index 00000000000..ea1f28283dc --- /dev/null +++ b/Scripts/Models (Under Development)/Nback/stim/Kane stimuli.py @@ -0,0 +1,50 @@ +r""" +Stimuli from `Kane et al., 2007 `_ +Constructed from unique stimuli: B, F, H, K, M, Q, R, X + +From Kane et al: +Control targets are designated with 1\1\0 and there are 6 of them +Experimental targets are designated with 1\2\0 and there are 2 of them +Control foils are designated with 2\1\0 and there are 34 of them +All experimental foils are designated with 2\2\0 and there are 6 of them + +*** 48 trials per block +*** Each letter should appear as a target once (i.e., 8 targets; 6 control, 2 experimental) +*** The 2 experimental target letters should not be used as experimental target letters in lists b, c, or d (the 2 experimental target letters in this list are B and F) +*** Each letter should appear as an experimental foil once if possible (the 6 in this list are B, F, H, K,M, R, and X) +*** Each letter should appear in the list 6 times +*** There should not be any 3-back lures + +""" + +Nback2_stims_a = ['Q', 'F', 'B', 'R', 'X', 'X', 'X', 'M', 'M', 'K', 'B', 'B', 'M', 'Q', 'M', 'X', + 'H', 'B', 'H', 'X', 'K', 'Q', 'F', 'F', 'F', 'K', 'K', 'M', 'R', 'H', 'H', 'M', + 'B', 'R', 'B', 'F', 'Q', 'H', 'Q', 'R', 'F', 'R', 'H', 'K', 'X', 'K', 'R', 'Q'] +Nback2_stims_b = ['R', 'Q', 'H', 'K', 'F', 'F', 'R', 'B', 'B', 'B', 'F', 'M', 'K', 'H', 'X', 'B', + 'X', 'H', 'Q', 'H', 'F', 'K', 'Q', 'Q', 'Q', 'K', 'M', 'K', 'R', 'X', 'R', 'B', + 'M', 'H', 'M', 'R', 'R', 'F', 'X', 'F', 'B', 'H', 'K', 'M', 'M', 'Q', 'X', 'X'] +Nback2_stims_c = ['F', 'X', 'H', 'M', 'F', 'X', 'X', 'M', 'H', 'F', 'Q', 'R', 'Q', 'B', 'B', 'M', + 'X', 'M', 'F', 'H', 'F', 'K', 'M', 'H', 'H', 'H', 'B', 'Q', 'B', 'K', 'K', 'K', + 'R', 'B', 'R', 'X', 'Q', 'X', 'M', 'K', 'R', 'R', 'F', 'Q', 'Q', 'K', 'R', 'B'] +Nback2_stims_d = ['K', 'F', 'X', 'B', 'R', 'H', 'Q', 'Q', 'K', 'F', 'K', 'M', 'R', 'R', 'R', 'X', + 'B', 'X', 'Q', 'R', 'Q', 'K', 'K', 'X', 'R', 'B', 'H', 'B', 'F', 'F', 'H', 'B', + 'H', 'M', 'M', 'M', 'Q', 'F', 'X', 'F', 'B', 'H', 'H', 'M', 'K', 'Q', 'X', 'M'] +Nback2_stims_e = ['F', 'M', 'Q', 'H', 'B', 'R', 'B', 'F', 'M', 'F', 'X', 'R', 'R', 'F', 'X', 'B', + 'X', 'Q', 'K', 'K', 'H', 'Q', 'B', 'Q', 'K', 'X', 'K', 'Q', 'Q', 'R', 'M', 'R', + 'H', 'H', 'H', 'B', 'B', 'K', 'F', 'X', 'M', 'M', 'M', 'R', 'X', 'F', 'H', 'K'] +Nback2_stims_f = ['Q', 'H', 'K', 'M', 'Q', 'Q', 'F', 'K', 'F', 'X', 'X', 'M', 'R', 'M', 'H', 'B', + 'H', 'M', 'K', 'K', 'K', 'B', 'R', 'B', 'M', 'X', 'Q', 'X', 'R', 'B', 'H', 'H', + 'X', 'B', 'F', 'Q', 'H', 'Q', 'F', 'F', 'B', 'X', 'K', 'R', 'R', 'F', 'R', 'M'] +Nback2_stims_g = ['R', 'B', 'Q', 'F', 'X', 'X', 'X', 'K', 'B', 'K', 'X', 'H', 'R', 'H', 'F', 'M', + 'F', 'H', 'B', 'B', 'M', 'H', 'M', 'Q', 'F', 'F', 'K', 'M', 'Q', 'Q', 'Q', 'R', + 'X', 'K', 'K', 'R', 'H', 'R', 'M', 'M', 'X', 'B', 'Q', 'B', 'H', 'K', 'F', 'R'] +Nback2_stims_h = ['R', 'X', 'K', 'Q', 'R', 'M', 'M', 'H', 'B', 'H', 'F', 'F', 'F', 'X', 'K', 'Q', + 'R', 'R', 'K', 'B', 'K', 'R', 'H', 'R', 'Q', 'F', 'Q', 'K', 'H', 'H', 'F', 'X', + 'X', 'M', 'B', 'M', 'H', 'Q', 'B', 'B', 'B', 'M', 'F', 'X', 'K', 'X', 'Q', 'M'] + +Nback2_conds_a = ['2\1\0', '2\1\0', '2\1\0', '2\1\0', '2\1\0', '2\2\0', '1\2\0', '2\1\0', + '2\2\0', '2\1\0', '2\1\0', '2\2\0', '2\1\0', '2\1\0', '1\1\0', '2\1\0', + '2\1\0', '2\1\0', '1\1\0', '2\1\0', '2\1\0', '2\1\0', '2\1\0', '2\2\0', + '1\2\0', '2\1\0', '2\2\0', '2\1\0', '2\1\0', '2\1\0', '2\2\0', '2\1\0', + '2\1\0', '2\1\0', '1\1\0', '2\1\0', '2\1\0', '2\1\0', '1\1\0', '2\1\0', + '2\1\0', '1\1\0', '2\1\0', '2\1\0', '2\1\0', '1\1\0', '2\1\0', '2\1\0'] \ No newline at end of file diff --git a/Scripts/Models (Under Development)/N-back/N-back_WITH_OBJECTIVE_MECH.py b/Scripts/Models (Under Development)/Nback/stim/__init__.py similarity index 100% rename from Scripts/Models (Under Development)/N-back/N-back_WITH_OBJECTIVE_MECH.py rename to Scripts/Models (Under Development)/Nback/stim/__init__.py diff --git a/Scripts/Models (Under Development)/Nback/stim/ckm_2_back_a.doc b/Scripts/Models (Under Development)/Nback/stim/ckm_2_back_a.doc new file mode 100644 index 0000000000000000000000000000000000000000..a98c118ac7f3067c60b5fd4d64a25a6473be6ee1 GIT binary patch literal 22131 zcmeI4du$xV9mnU+kJtu7NP-Cok4S2ek@zC(R= zlbJiyn9dtLp04pFmjw!ilSkqv^4kna*Yid(C|T-P^mySJ#rzVV6#W+$r|@|4MDkoh z`WYqp*U;-<5zZFNeO~!SoiSnb{2M7(sXRjc4h3h^xc*|fQ(X=IgYCaZXfHs0#B;R5~9WU%3h)x`ru()#^V=3L9Wm4jr)_EL($0Ac-0W+j zeQ_rl%je0Qi>HT@{Z2HKiAA$cl+F|jIq6hi%$c8PjkUHo==V&15w`5}Mxn)N_jSS3 zePuj0Cf@4vD$Lm$Q|Y|Z8*_$o3>?+^jNw&QY+}wyq?}|T$9VU)IDIWnI_va1^BK-b zDxz3Ep2)dn-D$AaofcabI2BkS!=Y+)joekD_fkkEzY_Y zXG4p#v87n+B$c{KVqRkzIOb^1YbO?E3Kz$zXtii+x6`qxH_A9ChqAF;V`F=yJ#u;E z298XBH?%e~P!d`h=`9Ie5y_T>wnTQ7g_v`$7Nxb3>V-B&21{~!A)B@#vb`kDU!NDU zIsIDvtyY?Q3Xjx2+M{)m_=GhmZK87imig=RTgeOAb}BU5cfS@dWOMp^-0c?j1JOcaS1Na#96uSx%Q;bQ$PPh^7f_vdN@LTv7ybWj9p%Y@T4RWvx zeh7EN@8J)il|(g9yPdEcqA&n^;m4r$r}@}?U=EwN%s=4+c#D{ieSU=lhXv|aE^N%i z_jb6x+Du$8Wrz!9;xZ4u8pRdl3oDNF^R&)i3Ae+GFr9IpQ5^4m$n~sho`H8^7Gpab z7Ch4VYp693+9~{3%76^wzA)Sk-!93`I9@qre!FJdUx57Q}AUqDw!U1>{-h>aK zhHLe7m;;Mp1#Ez8U@Hv4EpR97gNY|-kn*KkXLNqM=fB3{F?b4ezpc>vqgn#r@oE8$ zf0bj}_QXSH??1TM*?(xooc(oE=E6C9?yqTvdHavo&IRqR%mOpa$)d9=z?a`*=)L+&9c?xKG&>k=20YiN$D3HuQ|(2PxpI;2h4Ib zHks9E96;v~ZONnYGW=QC>K-#DPTX4Bn<90r)AuR*$QLW`JEi1GxE)@E>6{TWU={3x z^>pw=+JP41&vv2@jg*V|tsG%=9sgXs!yJJAB2|ZEj zNmAE~1~{RZra1nyLqsWcCg)zmliVA2aQ#jh$E-3JnuytGcH>{oHN1};gIw=*EzgnC zXSNf*-n2t6WfZ<1*~%S8xyyB4R@8a5S0|x-F=kLBX!46tqMim zRwyk?d=3VPjZjwi!G5>=5HV?U1+D8t?m%YUcIPOqkmHteNS&z7Q8f1lZFa}t%eQl( zJM8UiVRCW{_XBqd^EBv+brAS$WL^W^$KC*4bp8u^5ZWo& zQk&g)fnc<`;akX)kXw-2Ir?qnRHSZI(~%jZ>i-9oo`qCi-=-aIj*2InF>G^@U-jmO zlF7u5Xv(n3VkDiov3|Ar(sXt(nl!12%Bucl#w)8fZA|oK)4B9O-nlxR?RS>7E}=if zn@7+2M7D7K|5@(5w&nSd>;Ks!OSD^1_ES40b<361ZC5gIfdv8!1QrM^5Lh6vKwyEu z0)Yhr3j`JjED%`Wi(24V>;Le}2VTCfb@tS|e#H8}<)fdolr`^z*63QR#zE`y3~0T7 zBWR8LUC=v%AHZpFH|Q(SeW3OGPe5zsM?rr{_ylOZulEMh{Z}Ym*SqU?UN&kitoHb);b_}wo~wCG~#i_Nqe(Ou? zXRZ6Sme)F8-|)4bp8-nOnqKSvnV{`UvtTxy0ZP{@`)qJv4$Osfpc$0@DV$gID=Y%{ zrb<7&JALiX%teC!H*JNr!wq<$`Q(LduWXyB!kw3*KanJqv@9r-nEg){V z7~jT*$SB(wlWc|jJnkfBudQFQ5*_XjXK3=`?4I7S`;EQS$&a&4IHY%9whfb)@N(#P z)8uN{$f;e<+Wx5>#OoP_0Y=38dmFnm*~>fLaX$6S;yQOL?Z5kLV#WT~lpxQ+Db^oihqgAqX4sqEYqlxnGr4KhuWOjDS(VnW3T#bYt=h$%=WmI0E+my) zv1{Uu;$;1|^WMsel~wnRRWPF?5>ut#!He>5Mx$Vo*9V=Lya&&LK^d-8dH6i<_-btt9Z Oq?_B=(Acn~vEhHK)KhW* literal 0 HcmV?d00001 diff --git a/Scripts/Models (Under Development)/Nback/stim/ckm_2_back_b.doc b/Scripts/Models (Under Development)/Nback/stim/ckm_2_back_b.doc new file mode 100644 index 0000000000000000000000000000000000000000..0325c032c36738fae58ecb75901072eae0f6c2bc GIT binary patch literal 22016 zcmeI4Yiv}<701tAKfnYN0@S#W$3V=()G^ow8noQ6Wl{1{${i|Ghi* zy}N!aM3D-av3`5zd1mI!?7cJh&ieeDcU(ApblTq}%RM9oaxGmV#jbb@{@FS_QDiRs zjJuXjrx`s9z-!!1WPyLZct#pjNukK4NvAo8A{bdX#GEVDLj-Mk36f8 zey1zHHCE&|2q!Y-ZdCS-5%EFK_aN!=krl}IThy;m@SSlY9SFaK^539>ET{Xg+n1F* zYWt~|PM6pEwAArVuEBgKP>z#9rw=Ey^<;->AGgzb+;RE%bvdn%;R(1b>3FR}r|WiT zsnf4lYJIw%PSnf!bjWIXXZO}K7Uy3xz7cxf>3q7Ju1|;Gf;|=>lJk#F$3JkJ6V~N* zSf}fFJ>Rv|{TojDsm)PSJD^54q%#(d4Bd>{A(~OQa6my* zB%*@uwv41Hwm8eU@C>o$xy5}(xC$JcrcJq0dyu)NX4S*<1{ zdt)e=6|eO<6()6!(O62g1yyen1BYro#&9ajG%=~dQ56X%Io?&Oy-LLrszc4iaE?+D zilsu~q+QmY2F>=gSXZg?utL<^A`XvZk#MJq$CAl#TLji&BkI|{tct4CiYm3HO0BC> z>#OWqw~S_bTS?GqECvp9H0iVx7R40Kj8n#Hf#^jO{cWzCPDtlus0OJ{Ij_z&{vKC3Z|&Ze zZSn7L<@DCaH2dpqx6E5Ex3_~iy>0Qc>Fu9a zr?-Dh8!YFo!3pVBdwc7xooRzwytRAl8>zQWJ9T|t3wrBwLd;p}+!xHJv4F9FvA_*l z0Q1>OxHNq}#vy$U_xa6WH#iF>W1OdC#`_@TO6+P*fy-bT z#(Fxa1P!1CYybfe15blJU_W>n90zXz?rh!%AAzf&1mpjO+xDKoRdp0f6utDadySwyro@;Ka8~@kB0zS*K_+6X|W`R1;1lEGbKsVS4o(0c?Bj6}F4bFl0 zz=z;7P=sssWH1BF2TQ?fuoZNHUa$-70|&wI6V#-9rPdjo+4lFJWAQ3D0eHX7)B1y2 zjPGW(0LQ;TNzLAnbx;5K`KtfI(i#0@$IJxx?mbjg0cQ2zterFM&d&l<*^2?*cH@SR4vEL&ABkGEQd0mz=#lENfNfkv?!E#zrL z$r5RmAWC&WdsMmvbImSmNpXd>s6rYlWO0QoS&rPZrL_V}k-#eV8yu%O4@y(x$LYiJ zAT*ZAa%k*?&R(=71&xoup8#FgQk(%xJehM$Q@W#q{Pf6Jfc7wBEGQJU0 zz%pqH#h50i-7FYj_0rhV8h1M@7jpxnKOUMs~u#9M|x6 zn^;@8I*LZj4*E@f!;m3m6OB5*E0IdwbzH*9A7yE8ZF}{NUinpRWx~ zd}ZHQ`PQ6Yzsn2e4kX+G$@IN|CApU6GNK=FZFdszs(&8fwUPW4a4qlw;D!4?fGffA zP{PY}IV5k)3n6)et`U-l8@>ZM0dfl@PmX>Uav~&e_md&xkgWe7sPt|~=Jg!f!2*Nv z$R>g#F!ZzDWN#!A-X4ew4sHyjBQD#|Ha{Fo^aLUj9j+|vUnEXhwrNecJrPUBI#X(M zEYYEsR4+h(5HF_+|INI%Uuxd``<647?f*;QcRVdf`>~yrT((eh=|X9ov4F9Fv4F9F zv4F9Fv4F9Fv4F9Fv4F9FvA~zLz)1Ul-`nTjK2SY<;&VU2{=e$;Ut%jOp8)RBxmOJV z?#ttV`~9Z?_xs-md`9piFcth5@GZ~*!2SEr0r$$k0Q@E4F~I#kpBqs3p8@yEdS7qv z-|@hfdtp8&pkz8FA1_lL8du`HX@WoVgSi}eH!7#D+~)-2kwT5!<+BGH!>LHn*6Y-b zDC3L;j0KDZj0KDZj0KDZj0KDZj0KDZj0KDZZej~?f2;S>c-?I8M*X_)`PP3IJCdp2!U=62Adcf~cAIQt%5}L(sGrv`UbFJ8pB3nuoLYH` zJB8m8@w*UVa^9{CPZUS%zn=G6=B+%>jHA^*k{a61TwmQ5{CfExdvRBBYUPQ{<7B?=@&auc)tzPUU0 z-Zg8lspx~;vA=ufdCr+LkDZx2yBB}Ic>MXFtawM{xd(+$T+ft?lAL%C{OfeMN{IX6 zXWaEnCd23&0AAPqA6eiZFT5s>i*i4jZpp9Nh(e%c;Sh7aT!@7tax8Mp+GK4~v5rT% zzgn!S7NWY{qi{apNS+bMWEM@u74qv0UsRr@>|SuBxqcm&PnR+x>(*z>&PFScw;B0X zg6e#|sKrMJhqC2wmezoe7kp~7Rw;&Z1RchZ`w+5smkIGU!h2BuF3^0Z?V;te-hU_-dS|?HILR`(|)91K(a13UwOAC|4t)irH!Pt!s&sSWy)|m zCi`M$VjvZ^Vtw*N!ivkrP-Cb;hEsjgipP-7JX@vF&)`sH=gHw%Ix!SWS>d_ZG|gbs zG{r_M(r+Y^H&&zR25M|+o?_6ROv+fUF0*l(smXMODmQN2D4ScQl}?1A3Th`KNh3O_ z;yc69xJq)FwwA)IvK5#Mq2jC^=P*_ z1fiG}PnfE#YVR)9{(CpbBCL>dYQ%Q1Et2S$!-i=lB1u?>N^GTmPC>ys*;Oa^*2#l) z@=%?Udc-zQVbd!nF0nxe6T#oy-Y!h2lA(eW9V8&}`*wF6e5Cr?Z@sr<2RU(D57zheGL`5YyCnR6K*B zNKTp^y0JWGji}2s7pL@@noZ~28yd`!H(QNtE?O6c8d1`=gi~v0ZJAa`o-JK-VQ8AC z(@IVWOq=HEw1iVS)24Ge>*uVyQ=T2tnmO(1lwgN+&e?cQ*WuR8{4Q750 zU=SDvz6HDli~-kxe*$-6wYCYU0~VKJvjD6HBya@i2VMq#1-t?L5x5EntYTLJ8-X4m z3`Bro;M>46z>B~u!0W)9z#o9W0+)gJfCboxECYhT0U!x{8Q}h8a$@q@r1(Hw5tEnU ze;0pzPN?TKKKYoiW8a=i>pke-L1s|2Ess zOSA2K8ng+!kskqX13t8;A7I%nFO|Ki^NR{L{aSP1C3p%wmJ0OtrjKRf~dlfVyv+&O~t!#KdX z;S+%K!*ZY&Xa+b>bOT2LJy)ED|7n1821zzX5H3XX-44;-b~W?7*Z~gJq5~ZI!D9qESm4kNe;OE2&tVZq+%D*uLTavS z7LP&BRnZJC3s5SFGSotf9&kLVa(hwp3UOeMPaFh}0(9r`;fN(K-fjfcJ1wr0xOPCC z+B5NUlXEBI;6*L%Q*WIQ%M#@-fDhBLvH^z(U}W(G=##)%;2Gd`U>vxd#mAt;GH@W) zWf27J295%Qz-Sg+tv?M=q7Q1!ADIl#uW2V!jEX3%YQSO!tUM|~FVP+b z(nruMV#?+wQlerI;a<@ML||uzdtv!jJT0_S0&z)j?p1w>tx%6XK|Yn>O(`klv4F?H zRf>m^KO$2sJwaZA0iOov3v;eC?>Qh4@3zFA7RM)5;zu zbhWiM&`;$}*&+;SeY4ua)o`xiFIq66&*Jppwwj-(>2ZGjGh&7mgu9Tg|tbP$MT0|S4&Ug zz*MHNx3RqDvlV`O;sy&;G=)0ve+fcY zfd&D6jp(NDGhw+wUSN%_ddi~z4xrl{jmC|nF)%6yrqU6Y?`PiqMtUfmtnV`{J1^_7 z(-=r4ti{&X`psIBHa^b( z?JvAtmg}eOsV^xP)udclle*5efNKHQ0F765t;ARRAyD#Tx+604@ULy$Z~!liur-$Ke$A(`hqdjq*!CyOYU8 z%1D^@zL&hX|JC4&4*DDKDpPXdEl5{EA=V`b{QKt#72rnAxyz>|dJJX$;Up^|d;Ci@%*WEcJ8rz_Mi;+%W$sQt@0h&#muxJF!i<~{!; zE_B|ETZrDP<&ze}gXc-A^MX8z+NW``H2)h7#qeRC6~gP`n9({ETaLe*MHj$zZGMk} zm%M5F-Drh=r4wWLZ3teaE$r4qJ}u6%TWtN+uViuX!Ubm3e17`0LcXqhvGVzATiX5} zby;`_=YZOT89xG;ZJ?3^XPq#@SYl?)L^E+ XUZ>Z$iVwH{^!RK&&rSPZu)x0nITSX! literal 0 HcmV?d00001 diff --git a/Scripts/Models (Under Development)/Nback/stim/ckm_2_back_d.doc b/Scripts/Models (Under Development)/Nback/stim/ckm_2_back_d.doc new file mode 100644 index 0000000000000000000000000000000000000000..1889d7a7d167592137f56e0fb7d0c4ecadedba3b GIT binary patch literal 20992 zcmeHPYiv}<6`o!1`T>|Y4si(~WI`Om!#K9F!K5^`7n>M}!4OA@(k;%u>|I#zHG8iE z?n|guB`Wm?QL478Qc4s^BN0_@8>LbI6sk&%Dpe(wQngWkPzk9*YAZrQB-)tm_szZI z-MeP(H4$kO?!b4?JkL3EX70T1tk3=7_VX|PblKlTk$Xf`h-x5VZ zKjW_Daydrd55ViWPml%vaq6r%C~5*|y82(S5rsg@!Xf5ljS!1PbboZewa!|nVqK4# zK!aGZM2LoNpTfzYD|tpBm%D8`u9RPA_@eSGVfTU~E%fWSVhUxZ-D;=oY;+NFwOm{r>QPy=bC8`U9On2+yWFT<})Mg$k+Te)$O3EE?0_bJ!x+`>Jvuq)q2^n z(8ozH=A+Z+OEo_&e*kjXpU#B-es;;evp%iAro%|zi*#LXzVdEP{;fvF${J~DMY1Ca z%aoC9LdFwjY9te}5^;GbWhG^6xHa4&Bbm6gk_n_U&qiqsGdNn?dw497O^qfpR%9+V zZ8O-kO|#L83>&HR_0?#*jv5=JPvN8g#9WzQJ6HCa|spdp;lLRB%wx5O3Vz#-P zWSdLZHH!COGEo$-`Pl+ZEiq$QGMbR%CRzw$UA4A_IVzemm67R`iH40f$yk##vNA4L zquu5Zgkn}QWva5Oz57-B?_DFyutLVI5!=DGNNQM)8K#+vrePf_v61?@1#N4ReNA#_ zlk9Jjdzu`na|l&T61K7%rG?ID+IqsG=&MdUIaZ5gCgg_o(MaroOpj+1W^HX-xGlUh zd;m}Ma~0Ya9xezy6+T=L3WcKup}pa3L8vdBEC}rhADj_F$D30gby4q}%Gn*<<+-`IZRWNT)AADBmbXJ%!pw5{7GqkT zi`!;)NONH*ZzXr#-O}9_WLjRrY&Ej=ER>hm(``Ywbho^G9d6FZ?{LLT*$Auw4gh1o zw}8{YBya`zH*g2mcI$v9U^`YQJAh%p0*(R4fnNe|0)GN71ELx$D_||q32Xzt27DX% z0q_d&8{m(?dEj5bhrlYVeAI81hCD2=d|+@r=5?2wqe!uW8fX2 z0&N-q>H!Hf16u(0g{J`BAHEL1?h~)VU)V1$Ak4nuYA09VKZQni1zg#@z-&e=Z5`XX zJ}gG@U)=M*YJZLmi-F?tVGwaqU<@c8D_%$3^~Q?@7$24bdfW)Xug8uw{9gi&0^bEV zmSB|m@ahNe*_OB^p25oFIFkrgzOJ6X;M4CDc)hpiar)zKi_#s`OS!QZyG*;~`@4Go z3*U><9P))c->1d`Y=8f9aqr~0hvek>&)qj!U9}Rp|JaFOBk;iFr>3S}MM_$r9>={4 zuoU=$xJ{i_GGiD1znk-eF2ec$E@6nQ7!?tbMp`}4(I_@Giq1x{xyLX18o^)TcJgOnHveDPI3JVY}M2p=P(cN`9_oCPW z4h^CQ9EQPT96DIwFaUoR7*WqLkwn}!=$S!kp=%dUL(XN<4lWB&DvdJKLW)6fJgjp2 zQS(J&_l^p&2iOPD@hlt5R2(P=)He;Tlel(3v^pE-yvebXaXeVW(gZcN(-?wIlA<+gzVH1XjV5LqxE!Z@LxHLEqsdlHun$dpb zQw!ddl0hB|cotlxcntZY_#R}--6cYz8~&X)O;X5_bV#)(H9G`t!)VoB)OrBEZq(R9 z{9fo61Ero>We*d&+FBdvQ}Tvv5rMS$thVqpoU1>1HvBmH;az3Flkw?yJl9om#HYrf zmoN_fdEsyIK8017LmK1g{PuT3d#D`a2-u}YM)7FFF&~osl@-B2&|eX-2UJx?J&>r} zAYNV$N+w>yU7&|R_u^IMRFgrD4)F1dV9@9HRaN?fdPLP?$>RtgkK?r-#qh%cP>2ny zl=Ov_0Y6D|-AoDiSyP-UtrYW)77KTxs^hrV%Dj)RHTT8NbxqAyh+#VmFX19(zZ z@U^#|ewm<#h=ltq@I0mwA3m+qQs1b%gVB!jA>Q7TGosPX;-qYP5U0tw>g_2teMXR$K#WAc z=TyoiOe^fYyPY z1m)7{RZz;m0HG^DLx8?Ua>MsoF#VDjSPOy68uTv&2F!_A(nuR46JlgK9dX5e=G|pv zM(if#Wew99w&asWHZD7xu@2}$ zymnpmTzcy`ZxZ|5E~32?7I2Jj4kX9PS0 z;28nW4Nd@`0bT*PSDpm8r+)+B9{3E<0K5tC4B#9<-YdYII_bSWc|4K9emZNWtO@=I zWP3WD$`~or-uIFh_rDrE_@}@1&T1tW-jZ|$6k=UMz(+q%SOjj=oV$ElVlZW;6G|7# z&}S#`3JEZ7Bft`mIAQvEJ~A}0y=UmrC${%YTcR7WyMQm_BSa^@-*@0&D?T6e;toI? z{Jpppu^#a)xCPJwUkJ}mu^GM&NZEwc&G>lHiyR@uZ$jCxf}7_&3wRdrEZ|wdvw&v- z&jOwWJPUXh@GRh2;L~RT?l-x=ACmh-}vD^oO^P9&*mPV z`*!Z>`3;}x{Oc?HOC{XDbAP@RxC>YYFr9y8U-Rz5gNQrA z1h`h*+2?Nu(zxJwBW^Kzua-|*4iBC?sm=@XBx;|#HRRLce0R&OKhMT={ev@VK0kdrAz#KtapDFWl^^ngojAK^0@+m+|vZeAjA{Kr(!gnFfi14J*hQ8|Kq%; chA1_d(BG`nn>)q(n}2q2ww~vu{XbaXzu&B8bN~PV literal 0 HcmV?d00001 diff --git a/Scripts/Models (Under Development)/Nback/stim/ckm_2_back_e.doc b/Scripts/Models (Under Development)/Nback/stim/ckm_2_back_e.doc new file mode 100644 index 0000000000000000000000000000000000000000..6862c069b568a92b98fdd77a68af6ea3db563194 GIT binary patch literal 32768 zcmeHQYiv}<6`oytjU8g*0C5R|CSmq04zG%Uy#)UOwJ1?K771wwErKi)uYJKkD;WFxGl$89Lp|N2gDe zYJOV&EXZYllD|3r_KTl?!{@Ds)<>6{sJyAkzabH~l8KnKLdoukm6D-kM0Q3}(e8N2 zige1|QL9JR25W;WWGLP#t)2+dnP;_3bTQahzHv`~BpL0C#I4YHZ0bg_sT*OV73oSu zV`HmPH-;K3>qi)@kHuspTbF#CMr)F(Q2ElOOJ#kdw35*fR6*@zIF{(>Rq-1`9X&D@ zu`I-)A{R$%A~n?#jBMN94Wo6~=B}1?E?t)>-aC3C zo_NjA7MRi!;|WWKBXS^x7J^t;t!-hBiYXb5%UCpph7DKCj%t}m%1*f$?KX}e6tjAw zDOFarcbjVe_9e0aE5zLzu^nuSM7v~vB9)4UW3UdDSWW%hf|{#kTeaL=E!(T*1Jz3E zcH2C;O|O_lY-LwV3!O1#>j{gZuR87ISS=LaD_1TLhdO#?Y#w>MpF8oA4 zSE2S`UsmXW;A2^#wqQ?Is5uzU3T+Q2vqDT$&(63KC_R&NXK+sz7dKBkgzh^APj@-D zW=zXSaMx&aus2Jd9nuoqwRM+sOJG_?f}5vXGp1#7x_R0mE!|yrcdIZh!^N$q9nxGF zsznWLo4K{NLpsf^wYyKa>)_VK4rwk7W#qZ*;O63%;I2bPm-6!RsTuqYS7QNVfvejB zm}OJ}OMqUWA9x-Z0{#p94_JnIT{W1IPCXB0l=$>Tb9T zseMuN(h{H?2mpR@C+GRiz_)-u0CUjJbDj3y2U=Hx_5l746rqiaf!TlrYJj!CeSq#8 z7X0iV`+*+-x{tgG|6c&zSI%Xt?+Un@)fI4MGXXv$S+2l?zM}w@eRL&wsAxA~%BqQ!;Xl@6xmY+dpu#xNYdfXXViF zJ+}{)&R784vHwt@3b=FV(p1}Mbule4TU>|LdY||rsIws;77NkzVcJiM{lBduA(Em` zghUKs<|!2$?wcq2+RBR83g4G~h+B);GSMa?$ki#LA}+dx6t#$*2i_%BzP2i{s!B9e ziN-3iro}Is79!2-){6B_r_+ZOw@T3hZe8Fv0J#>pZG%4vbgQ5Jq6cxU&@GNsuj5V> zUKp@96;L~pdN+%|?C9rjJ&t^Q13*5tUoq23V;ue~_Z;}l00$HiYLFuZyXMp2Mn{9Kq8YL_h|Q4MA$q}O7qr_29YZR-3tVf!i!#zp zMT0)^Bh-;N`{(1(wO#pHdB)NC?fpO)3lW2~R}-cJNVvChzYod&lA=Iyz+Y6%xNL_+ z<#xa*tV{}(+X}iHbUPGamx?3DD)8}(K)~ns%_#8)R2v~$izSai{&*1P>O(~msw1vk zsHD#_ho$?_El!{x_b3Gy9r?oV+W<=NiJS2AZQwbyI6F3b zOHqaH4hSNa#Z*zyDsQEk#kc}3aJ1z0883;K(7dx(xQqPC>3r_kiG|pDJdiNkDaOab z9xjrq4@o?b98$W`CoR=G8RA3Tsd{TvJ?tT*MGzy^!)>Wyrxp(vu>!H|!%SrgA8D4? zd{)D6Kh$<=pQA~>2RUDwQ``_^0b>DU0b>DU0b>DU0b>DUfeW?3d7RuWRudgJ2OomN z-~8d}kMHlPc>X)3V%ee>|HgOpSCDW$DAV@?e91lta2|9Dz`RL(0Py|26q&GXB<6y0 zRwqF@+pY)YlxhnoU(R=e^2DG8$`f>hpgi&LB>LDKLw$; zf;I#C7}*7%XTxhfd4aVMIIKlKb=Z*Tjz_KN;}N+f6c#C&$m0SYSP3w*p_qH`uGNe$Xl!aBiSctP#tx$=NFEm9HKflpB?=5osHd zV>wb*!Pg8~&B(V3xm$2vpjCVw+>A38FcvTtFcvTtFcvTtFcvTtFcvTtFc!FSEx>sv z=b4-%bMKyWcFtD0UtbDvKFv8O=jxn?^R6%MhjaeTc{}Iy+{5SGowqMBo%46j@#g_I z0`mbLtmZ9COy`XFHb4Tm0}FsVfGU9L=YU0ypJBqdsj&bL;wNx;Sz=XyUsCX!go(IH z5%6iv*oR8W*qiM28<18NXqcmJL^`zl14Wl0a!@7k?86Qmmk#52e#D8H7@39MtL2jx zQd-@n!Ojc!FlwK~0dDVC7k29SKj#a-kI#(OVf+$y4&Iwa7r``ZPkmu6v>KZ((&D`95_hzYpt$_oEj) z%9IJY*^tjMjAK@z@+m+|vZeBy91Gv+@L7m4xnNg`mAmokulv2G1v4)&<9PD(v7znE z_?m6OtIO|0>lfh{JZtS8yE#O8&ZD&?ItF;vn|dc`e~@R?5Tzbw`fGH0O`|xo=I4*) z>v<;aQm{a-{U44C=KSZUbN;igkn^AQ|5(A_@lEVB+2=q1^~ZH@dCq^HKo-7>J%a+I z`CP8{=R3ufu>bc9v;T)(KePXr?-{tN(=bL{iuZpt^5w(a0d|qQ=BWF>7UsYIqYhs~ z5WG#G*YVpQK3`msRoY~O02N3rR@N~AN2N4)NI4XwA zDI9^+T@>&aa$iUqH>R+}Ls+lr1$YNYJnx+y6icms>}WOvVtLyL*D~^518{lXHj=MK z7k+yQZAZNS!~v|t_>^DUmDInOKjE~Y*)>);x0uwbKVQJ`mN|U0=F9SxbF0bBqxP*P zesu(mlLAf%%&jIc#FV$8o*KCW}EAtpC zcoWumfB%J>d;0IcaA?iva{bMh?08-N5$#d*Bz_Wf`RG`G|Ai-q>Bgem(OiIZ?o7VN zOmOA+Ex^D3@-?U2h0<5?r1D|?wf)2vM5~bhw~&7ZvVRF{4kJ8_^2#6xskHvu2 zN6XOZUOqZ~qEz$K^4CEo`_tep=i0a;NB!}lG1Bj}KDyjQEO4GgnyOK)sPz2m`?EiBA^`Gdibhg#fX}bt-(oF0A|G4KL^uE8QP-+?NuU-!4XQuo7m)U=s-u)-O3+iv`o8h1QckiG6{dwm6 T)YRRdV9rnRPK7J?{M7#eRrf*K literal 0 HcmV?d00001 diff --git a/Scripts/Models (Under Development)/Nback/stim/ckm_2_back_f.doc b/Scripts/Models (Under Development)/Nback/stim/ckm_2_back_f.doc new file mode 100644 index 0000000000000000000000000000000000000000..7a2d4e60b8f02df6e7b99bb7e7991ec73f4cc15b GIT binary patch literal 31232 zcmeHPYiwM_6+XN6IyS_K3BiQ$8iEs?#F*GIjspSt6_Yo?1d+B|!mhoxm#lZq?uL+7 ztyV}CY0HmPl}dyV4I-gbpvo9D8GzTi^T-1KI`X>QFIDAex&^PY5k=6laELivB{EOK_l56EuS~Dhv7Ucb<+XCv zLXp}IpT^;UCwWp}e0=^yTp_>7@I&o+&kd61`WZb{7|Kkzk6m9eIv07416Keh-x|Nj zO9(G!*FFP!0_DL@drZxs3rWj;a+97T8u!k2*NJ5zt7XMVbgFGR_guFloZ z@H5nub8=0#hbfm&S?&T9%IA|GrhTSzrX375`#Xt>I(k(d$0S9{)QTTOu3oLJ3INi67h605mV_r9@U~ss4{evTsWOO(ZPlu*s(>#ey^8_2C$Uq_* zn_7+LDb!fqGQprD7E_U2U5as6^b zbQ-a#p~Rk8zY2|vL_$dwg4V7XRU+OOQT5TrNMnNnBgeKk!)Seuxf@ioN7v<=_l}`R zUcBMw2uvA?@kCmMBWh0yEd;TiT06pA6;mo2SFvac4I6GyeGMv+RQ;+R?KX`d6iW|9 zQ@X5f@6EdXdzY&ctPuBV#CC8j5*<(@iBu{Yj=?%qqLuo21@$zj4Gn5@gSw+Z^)_g! zJ00^BHoayNag=RUX>`VvqbDqizUsD&B5I{p{>E;oRGJi6GG?GTzY~7IXOFn;hYfD^mC*lNp57WcA;X2CtaZJul9hgTMt+wo-ZJWd<-l%W z1b7x01wIG<2du<t|R4Lk%K0GIrzV*#MuB99Lq%DdEG*D&N1aoUQg3i~+ghL>t0ptO&zD0+=!5G5DVW zUII9FyayZw{sWu<^2d@+j33*9dw|?=(0b#rpp?Nk~$LB_uI~nWs{^HZ79j4OQjq z#CNX`aqAFUB^xAyT>TQ2xD1NYdj*TYyQ0pwp-yhBlQnhHRwrw_{nEY+CGy=m>1aPb z{-owsE8XBW0DgNQHw|vv;ZFjC`u~UwA+86y#gUrtxDt&Q11tvz^p1pC%|^6d+2*gF znQ7p~mU*GSlg2j${wUHtjRJGP0WZcP2h42npFI%B`P&!bLG~#((NT$dQ6w zi)nC~tHCzugsd*v44Hkh8(ellyIs&Rq{F))qY=C)W4ygAFA^C;KOWKwZaVz^H^TrW_~cUj9|N97i?d_1x0KcBt^q%t4}Z?PGedRxm1d-M zC0gJ}#cOk3kQdOr3s!lHyy$j9Z|uxMY&~vBE^v$Sv2ccqr0zos45Yi%uQTpLd>tUw}4;QftvFyW4WeOi@mN$G_;ddTtCw17>q}YR8EX^%$ ziM4>WfVF_NfVF_NfVF_NfVIHsTHq8;?v?9_j+cWE!J|L>?eQ;fAF6rwSCz8zy66AK zcl4K$a1kie4*-10J_v9gbPT|}Nj?Gi{$7bpoLw&j<*ZJDa;Dt^$|=EdWI(Gy9|4^QdJvQ+#}0wkfIb6yG3Y2L$k)c^jw??2vndCjjM#r(hF%Rh7@E|3@ZASRuh z&vV|+`R5S8x%vpeHGus9*8mOxTsPoa0XJC>0h}j418`113UL1WGEfV=25=4FEr7gh z!JImoxjtoVPwnd)O2iU_`_x?sWxLIc{ zU@c%RU@c%RU@c%RU@c%RU@c%RU@h?NwE*XxoM&>5%)NWg**RO~etjjt`84OCyyJ)S zaL&oOAI|wV=k1)+a}S^CynTf?m2m#fIsPKxGGH;lbl$RZCBPZ+)qn!70hRzvKM5># z{R|Vz7Zv}d55Og zAuhuNxF&of!7nIcI5>SeZXSBCkxyC*53W1u&I@ukYM;ab?)Est|XXK?2f8**}1%8di#xO0Ik zFLz{gb)5qkuD@VjmBmpLY}#tI_G-dGIirg{c4iqqvHoSNmy8#?kpk zrK?WXznRgYS^O7%`#QXum!g1wi<^u6JmvQDe}F!PI`Qg;cK7Or4tZ|A;HhnwkLNM6 zL*4;p_)Ww!U(dy)DEckB;-Tt@bH#&SuSSlcZuW`?7~!nDBle01y^5E6TTHPrtw4j! zz0@bS;33s_>iFMbF8;2;$@mRUK*pKjLuOiwm?g|J7oi6jQ;&dB87%f?c)skQ?C(AJ zSjHz0W$%AqSo122n3X~I_iJ|Ve|Hpe?h?5|zA?!E46@(A5BgDrYcUX)ySgAa;Y|IFeiI3< z$HW(+(;KR)hNI)hk0`Tj`A5I2d5yIivkw!|PnpRnV7%$ZCLl z$7au%=cv~JZNN>yjlj)--RGFxGPAM&cMaO2#vT2?hW$TbSH_Q}?#m!0xvz2mZ`%D& zv+rkUu7ay(VgH+^Kg>ZBj<(}-G2QdeF_S}fq4OWyQ!aM?gL}SHo&TWhB}ls#;M>nq zU>U$O0(@y*4lvJ3&<5aopb_90%ICicu+R16=epDw4*KTCOS}Fw`HswDb2YpEgon>K zyZ%)4ic!1%WID>ZrFQ*k@@>V%#`beouNnem^?f=YQ<}S-W3mvqI^`{ig#FvLZD`0;B&Z0h~=W^p`M45Ym;lGuj5Zq zeyLbpDnx0kOX9G{5j@S2N?knZm&vXoJRmJA*}Py#?RMptPFK>yG8Ifl8Pf?Z5j5S>!DqT8dIy8#O`ciz-`BFbK8C4I8s&tvC;;;Bo9zR;Il1t^G z!qfSv@VQdOPvzf@)U-lOdzlU+ec8foPwR3aH?;4q4Yo-rb>Kn!n;_W|&YpMq^2%Cuj^OP(#SpQED?YOT>f`jvA400u|d+W%O1V zvAEG^RG{8w$q31kp>RT`mG#{v>wni8BL^)+of5GgOpS#5jiFc~5$=gVI~1ak@;NhV zt1{ZFjE*W}Yn8FBO6I!5R8MBrOC~{6*hV9X#+WeWghtU;t$MPw7Kn}*b=5tA-n~X- zI37$C7yJA^{|mRtIgkIb9MP2wz=B<{WjMw|Ndz%G%m@7VYLV~U0eNoZ84oW zFiiHc$#he)bofKIbWS-LHkHmP**5>6Egi$AVmf&`Wiee!Y9|+GI)+X0%%&w%GR4!W z31<#YE~ZPRqiaf^PE9!L!I^{UQe2$roF!t|REeCDnJ&e}nV(aG&N47;il;L_r<~6E zo04;3Mt;pzYXBO+OsNc51MCHcfFA*`0%O2ez<+^DF}Gg}Q~_HsL)``R15W@4fwzJ8 zf#bkQVEGcvs(~h;9q0jufL{Qw0B-@ufX{%XnCV^)lmm6ZgFpoMA#ecTI%MM1#8(sI z3vohBd^YjvS8(#*rxI|Q78&P4uhqh~%7P7BRteBz(mc3jbrC{`Ly^sXkZ1l`=4vU&1cVdObhBQxwwz48Al8FJsn`e({opYS8Is`8340tE zkbj3n2!8F5Gm22VsTcPlpOc~1$+p6 z2CPDiIt%rn1XHTic}P7oQYpT&VfhkbMD#)%F_bh0?TmdY~V=cSB=M_?v`&!|;oM^KRJ&sJ$B0CHWMCH*<+1P7-(oTn%v- z;`fMhE8p#+O|-(kb-{9KvvS%eYnGDjgS39s@FtYH6Sh{AI0^q9kgpe%a>k`T5|GuD zI)?UT%u5#miU` z`OerJHb*&fw#boC+*x|ig*x+)(# zQN(BfAGh#$Ty9rkf!kw_qUu$($zqgbC%W|@h93@q7*5M&?iUs0yJ<4!-a0wV32{e* zOjDRwnA87JYmLxmoVt_ zAxTX=St8e^41%GG&Gk85WmFpCYBL`V94REdIuPqy~3e0&BF7KD$1cF#vCr!%#9a1mbE*#{WP z5Vkg^S9}^_Hy>VR=%6V`hUq}IN4Awv6RiQQ0j&Y8feS$c-{RzEzBMVam)8Zyv8R4@ z^3>g-l2?DSM6A8(jlc2v`8xz$0?P1%03U$Q0Gy*72RKjp9N@DxACb6TE&=6??P^f2 zi@l(n?sS0i!PyPUwet`tPf&dyv>5aN=yK4Xf%4?wv!ErQhe0m|eGQcPA4jIELE8Xz zjA-8HnefOZFR&H@hjnPb2i+o8A3s}j zRmo3(j`e@lso!jbpC`?46bsMw0N3qYgN6XE)rSD?0UQLl2f#f7?i)N0Tn@YfaIHKB zlmLGKxCTB7@awVn0qy~e1LVCL%qf#v>yyVlQLLxqiEwg+9|3KSM8eTnIAN}Pxxc{m zFVm26v!Ar~vL!NKSToYq5Qt@I15STGVHvnl3a;|0iSBSR5|pwq54Cp!t1tliH3Cd= zyXB^x`y;zMH*eg1=RKP@PHLhRzT1Jv@ZzBf@9Z1!uNJQiHsQU$5B5zs6;Ta;FW&q& zz}AMpP55rN8nhAL>$bzb0kP^4(gb&{cpTic(HhVi&>GMh&>GMh&>GMh&>GMh&>GMh zIR7-j^(NPsTpRPwxQl8vuq+0OgjQZbF-d(*O_I#&EFEz%Bs4p5VFo zxtLPnaVhE8h6;+1;Lm>|%u-~Z|Ne`p5UC=FnoI2Z2!3Sqs3%f52E1I%%e zi7M!YLg#e);wOg&P74_F`+D;u#IXF7F#?|iVoZIE*e0srYRBI$X;U9A z?z&Kvf6kRt-~W;M{txWy$Xyz)jaachOp0Cxc$des_L?4rW0WU(Ce0{pqtoMXn}#D9 zcPnVm^;~A+6(Z0i)hySCdm9S<+Nm-x_9Ide28BH&jXBg1mLZYXYM`{ z`#mM*r_uUu65gAl?=}wzWoeV z)ux*xD#uG2x0=k1z{5V6GSQ28H1O)EPU{YoF8Ko)_)!PpD;6!JyW}IKOFSrPr`RI> zJ4O4Y497UE?@Vz)34-^fjE&#-676|%dxKjW+}3c(g*@*}QMYa|j=nR6S~xT2NX9!; zxQ%kU2Hgba{Jk?pHeXrBa`9MmZGCfHeZ9B3p|!cTy0&&hdv((~ueaLkZEf{>8@;~9 zy0*tvibAyP@9{)?Eb6Oi*a{xYa1j$}x$FTr=xy>_*PIhS`Fb%F7z%cSMKM^BF1J{9 z#$)?Oc252N&&rZ_5DH_y8AM0R<@bO1l7HcT|Hl;Q^nU**YH4>S$~VD}BHaG_KR?FN zhp*xPk(F+~bQ$=tsV@hm$%U37-cvYs@g?#e!CM*r{*MpoA0UG?vocM-wBT9gCtsR9 zW`CUgy`D^bl^%YLe0{cbD!rnN$7fqQpL93%Gn1*X%B84Emx+F4#aH|&$7^W4N*$ien>^r}A{=B_;DZ--m8yMMv`2;3^&T=QF)_}}IE52cwqP?Q$y z{J|MG|KVao;u@VM7cDLJ`Hy-EiE)!=G0uPRcl(Z8y~fm9D#dpr=roWAzwrRj-+un9 z+VY$2{FmalP?U9@?Yu$*!a1Nd0X$2l&XC;-_j=$qfZtWM0Ji|GKpT*a=R%d`Y~Z^X z=YPD=_bOHr%o#ZUBj+=mmr|dSX%2tR;Q60f&p)YiKZ@GL^De^qH>!Ro4|KwC7+~AC z_kZe={mkC~8J**1w*Jp^W!cVua+a0t{3mBWHv;9rO+W>(2B-wq0vwk(4%GlR1C+ZK zl(UgKfa5Cre)-sha>ob#OLJm;MEeV_B5srX{^@?U;6 zzC+|fAEHb?)@c&8MtmiRThVDIA=VH_-p9JSI%Hb{AgT!ek1X(XV zkq{VJ2pWWJ(j}xB37rr+L0B!UmT+Y*nkt6G%8-!eYoyquDwF+LqOMNw6R)W_U0{zS z^08hbQuh|pysuFMx;juC1WX3i*ZUIHfqWL^n>H%fqa`6OFs&2o>wSxksBcuB?gc&r zYeLRJ`Zm-Dl|Bo!9q4xD3()+gD9UXdXn8cH`%Ra@^esrd zf_k7LNv^LiPq95M$d{KRpWZ9iJ}rlC?|UiTzi-L=tMZ%DgO*3ziJouNShqeex1Yru zqa@JtE)@D}1{}IxeKy$ud^Ck746;2?oPQYTa%C_NzLu8e{{1W64lRiuFVuqqCT(Y$ z()!V9n$q=~qI5ZVO3z1{HdUYI)9usgrl@>7begs^oo|*+3D{IP zTgZvzu|(`}YBEO@MLC2xgxFIoUN|L+VnaTbai;hYn2gr-jE`XpIMHmLi1jToj-QEf z{3M1}B!bW3epwC2FR04!oK(RpUdNWddUKpFQ;MlTlttk*BwQVmjTA0+_ z)|8`+u9gznfl+KZJ}pn$ppb5n#}`qdY${F&BLvOL^jg|jy^2DL!=tzyAq;G&H5Fz} z@dZ>kWdY;%jfS9_D2gML)Rl~PKgswHu%y1j3Oreja2%u-;Y3g|e4&sN$_4A763(cf zTtn8>0Bg$2n)0#swWb2BCGC!o8mMXGOT@6H>N-;*n2tiJreIT;t@R^SZ#Nb%fpW45 zWraM6#4#iwL;zQq>5|llkoZO^pOA@-QbR*x8>M9H`G!O{%9EyGvPoom zg+w*V>lYH*D1~{FbtWk}AS9wuo-{>E!IT5!{yjaVTGLWw`zNaz<~3+0+gsTlVO~Q? zSu4r*2=f|B%C;^mQ<|dnl$9boM$!~rQg$3%BX^L(EQw`%F+uqmo zlkTshjn5JA^UML3fQf(@z;H`iGXbZ3g3Hipv%6PrW|`|HSVP-G6^b3>5z%r=BN%|8KP1-xl4%Fsh`7FX?Bc z)_;%@iHSmY^%GB5%1Z*C-luZXGN01AF;SSWCe3fCt4hnOYv3`YJn)N&mb5%FtkKW! z2f#5uQ$NlI_2WDbs3W*?TL8ZURAEfj066w$0DC|mz+eE*g($!zKr%pnKAZtwDd0W; z=SD5){VB*KK73+BL?$6qOp*AS5u3J-)VfOWiVput8ypKfpT>a3Yc~MW;{iMXy^fn!P?7W32KGDc*08!Hg)g!-BNSE4HKZ%3a&%X{eh4I;3T)ngcrK5z;EJm|+CaNh`*!Ohx#Q zv%pvtCeepSfwqz!QUBCC6?`oq>43H0fHQm``W7a^-1!!olF)OV9O(rE(VkX!~g>Oh)e(lum`vS`~l+u zF@PjM8lb6qDAxk!P$}5^A%KFie?`uJdp-OjHUjVtpaJ$Y0t5r51Mus`I4>oYXfI6_ z9zX~X0f0@sV5nZyF@WkwzbgVyjSL}tA|TO(MYxcLjbqZyoOCxQF6N|%2NX7kkC3!P zTAB+{a#MmYYA%f=X@XofNC_t#!XuG{A`T!+1MX*P=1Ojzhzda`M?QEEcj{%`FT{iB zKyReM5K?Rs0kYyiClN^UhqwR`DY?dwDBukMy?Ky}AsE_xRksr;zVl+RSJNSZP$Lv1 zu%WbUFDcS9SevwZ3zNF*bv2@-N=LMmxrBvm27#AxVnHTSuj>kdl23BgC9)_o7}^v{ z+Gh}W>4wmojDp_21lrISEzliAN^C-bMr;yaUq=sW`3+JgLjhv}{yxeo1~49Yw{K5o zr5nr`THB+y(NgZB4jX<~H&5Usn$Emu z+^fZr>27oH@3wgMk2lA?GF}`nj;pcK_9)4mGW(RBed@`)O!0>r9rL7g?Gf8fIlU7ftCHeTQS;0Tz=8FF#Z#tGP5p`9&W zlNVhC6Y0!-o^o*ZrNp6^j~R8GWVbP`zuo#oa>rsul$psblP9=G2QMY{xUi3M)$A+3(wHs&h999OZsORyPwRw zyzW-Xy}Pr!Unn!Uw%SK`q;-b6ZYuk{?YuqWY1W%fvTu5sp0MWLIob05qj;m!yB4-6 z^FAdUn%Lo5T1rm-AJp~yF4x*WxBg@7KJy8#3kD894D8d`*t?=C=NhF)SN!%`Zaey z)U|E5Wn0O!lR9nm+gtkG99t_m-{1U2g4@!(sz2JkwaZEHvdU0Nf04ZFpylG*`3tMd z3Q|jgJVIxUPc$ouy|rN3ppu_gY|uM4V(fiO@A15Ip8sfhAT`F4`A*|u5tp+sc=VW5 z2jAYy%8gFlHeR&Fuh@U-YiB`)t6J{-B~y+>hGu%Vh)gL|Q;%DiHtYtQ(I)iz+a$ju zi{?}wddWN%Q0Q@M+sbu&cE!eN=0+|^AC zF8j9;S!&_!OoZ?9Z%zmk-Y7IlJJJ1tc;SVJJq3@;bx&0ce;}^yl5=g&g6^5~E>(_i zmec0u^T?~(r_y&;4SHvMq+~+tX>(>pS!DU_GlbUZ-b~j6p=!TEDq_TxGsHk+V z#j}cdbBpUb;+};YTlcCwyRZFuZSQf#f%ymZhUv6+SYQ3PraE_RU(b_zYuogF_vqc7 zV|Ug2j_P+|Yjj5Py&${8wNEdeY`4De+Uc)eUTSqMeq>SI`(vFC|M_P^H|FKYP8Sx~ zt^4!s)6)xb`_$GJ3~VX(ojH*GF0IHTYjveY)hi9SmKl??N|l}oF7ylu9K{;= z%4%hTi}%q9_d^4Us=}s?I8m^m#mU15y|v<;^n3=s;CYxjR2vO3Xy)C|;Ii<~q%^In zX915(<9>cW%coE9LtD=_gS+@TZs^c?Rm6hb9G&SagCFU9oW$KZRXxsS+vO<>O!s@8 z(mH3A+)n>~`iq`xElh_mHmyist~*_MTlSz+g@NzbA%0e3k(bfHgGpW^^Y5=9i9SQh z3jKez`s2{qO}{QMNNpQ<>A=h$b{+P-bMJmMc2r86VsopozG>>rETgK}Al0&ayg3gi zi+{)(4N8G0D9~iV8whURUMr8y&yoX>*NNH47h%Ih{CMaIkLL=`qVL zM%wJ!c~gr$yjpj+rL&FTKwkg0!-~`s$JkZ$B5jBizHis)Q9>(r?Ym(SOW~7_{bCj(_WEF1f}! zhS!tTRNQYdg2$%3+83d#-r}5BW)-jc_U|@D+N~a$wZ4-S-skd^iyyLi8Ou(l?$b;v z-6CGxr*chbzW$ySoj$1jexYK6V?hyP(!T23$9)o1E*{wM>a9o5-!^1qSi295&b=1G zn{w(T=a$K_m4_g=fvMFIPtrA_OWg$`5!Wl4r$|+ z5;%Iq0Ozjf0u33~mom2hdDhqdiKpfG4z=Ajj2Jn4{JWA3q}To(9q&542)c1&WAo@< zA66|7=rhzqC#zgzVX4Q{u$aTmf{Ht5>hI_@*t6I!X3?gc=uN*@WON);^Y%>nQ!qKqWJJ`8Swa3-E=nEr_JLTLtviU7%o=Qrs zm*=cBR|}s`?}xMxzWCPGX9+ubu*dS_%eSd{Qo8NJz29{#@BY!5tsZonQ<&9$(49l-kNW1-4ZY*^f!&Xv z-T#nYe#^qSYQ8#0)w13%iSC_dn>Hy%W#Q5`y9zgWkGv9*vvl{ULnFt49kn+|uAVhngU`RRpSifxAM=WHt|}*2cGGq4Yd-X8mF^B}4cob_Ln}E2Kd(96 zBFyo&L)2Ddjh>H|$F3@j-uv_?w*bQ-^KbMD&l^AMCw}*;mXV8nEPOke2UyrTqzjsL z9<(&0I5n^)X@|@DMZuM3>o>WdDk%1lOa1>UP$;7|)Ni zYiBs-<4||ujTBXjtfF(}9ae80ta4`b{{6?xK6PD*VFO-8Q}3lvtWo&FTdKd0tN{79 zwT9YVz@ClX-Q+9Ezx^~+G=O*H&t4}QnO|O#{HsWVq)T6tBzZBBmmxn18f2`7FEx#| z`O1{=_fds6B0Q#}O2$C%SHOoIa3lJ{ix4%5Ef8|}JXbRZTYEE#%?sm)b9j-iWdb^B^yRFNkJ|ASsBn3m32^gWzbcouj?Ivt2Zc!!x5oBOP4LxFQELN)!*V@QDx$ zjdVnwBk~{?J`rN9urTmRphSIy=0Lub^Qnvz&2gg3bfr06X-+qq(~agh(;R0rDvHbD zO@tmHF{2{*TrVkuj+xo@LF?hcS2)^=`$f`Yr{GWFhPq@!4dVEbh5mSnt`>}zwPB8^ED!0H)nY+ZX6POAoZxk))&~0_Sm=&?rN^1TvUJLyUqY6~XV; zBg$etOQ=Uw#1<01rudWl4QVuH26F)=uTtxc+}kut%ya{Nf5a569IfO8Lr4h z6VgFiN@{ZkZF%@YOUF@{fo;=36oa9rtg52QP*$01MJQk=(Q+gW0V5Dx@n~?vg^)c2 z=wzS)&@OI(@F2e%$YBswRV9X!x*9{3-iM`?hY} z$=2Or1A_xCVV%CNk~-?jnhb_2L#4?&{h^IGNj)}PX|3TT0SQ?gvijSIN#m5G>%^G6Z>f(16J9Z`>mwUMOc2zLx7X$_lc1g?Z^fGLjn zJ3%^{5($p8o8vf@mC-Bf3vxBQ{E@3EuG3S}Fo$?0j4K?NozWB|w+ATW_yu7wGrlk@ zn6Qh{z=nGQ-WNWFOz3Z29d1ZsaEEy;-S87g)*J;`3li5JAeViotbn*bu$4LM3c~+2tvalZ*FpfI;3lqYk z_*{Nu!sq$&WryIa4%arEFN&7b$390-n8@P67}2tE!eZPA;L8aU@P+&c5#<*V!3kqi z!})@6;JX3n;D(VAc2@Yyg2r*QJK6v#-X4+SjT5ONC@i3`fWiU_3n(n0uz3JWMK zps;|#0tyQ#EI?b}YyLmu+WBi4wkFJ_KZE~oU9$uI|K8wpqdyEpvYm&1c?QU&aX!wIf%!Re4P*_;152% zxv&q<=Ln^KF76#P_JeWLO6vdWNZN($CTUwx2>a3x@b~u)@QfRJ3JWMKps;|#0tyQ#ETFJ}!U75l{BKzR{Zjh-F@DEJKOQ|-{9cdW zt?8p`=)dFN`9wb*zsaNDkG?$m`S=YV{dxQy82<4(^y$&>N538G7z2f0U`T#H(grAc z$L#Obc_7MHxxZniL`Jtc>cQ3l_@9jS}@ME+5>ni<~_P;Li$8!Hp`#5(M_tVA}{O^h@ z!RRZ)20Hdy{@#tx;4hv{qk43=fs^}Alm|a74*&9_q1`l Q_$~IwrnRE<|9}Pl2@Z@}Z~y=R literal 0 HcmV?d00001 diff --git a/Scripts/Models (Under Development)/Nback/stim/ckm_3_back_a.doc b/Scripts/Models (Under Development)/Nback/stim/ckm_3_back_a.doc new file mode 100644 index 0000000000000000000000000000000000000000..3c8d650415d2039a9f85bd12ad094fb72e126c56 GIT binary patch literal 22016 zcmeHPdu&tJ8UO6Wc?<%9K}La=QwWqmNO1xow6u+rI3$n$9iVF~ z!Ahk~+f=qz>ZY-+Ok>^Ft+17KLu$41&pK%@G>J-;Hi>Q%o3#B=5UgT6vfuC6H@VkI zh|>w^#68K+=ly-(`Od@VyXRb=`NOPpM}IQ^eaUc-N{-wZE|fe|x&z^5M!Z;LIl@f4 zF+4oX{mfOLn3NdKL~TZ%h-JA>F&WcO_RQSL#&DM`3P> zES@J);&Euak#ET!6B!{hJYIZGx;(xAq{BD!r=v_BhVVjD z`M20|#49+o{|EnbpyNdSL)2}SeksZ&Tq2jiyC2$&BL02UpGyK6&S<|OXGZp<@{=xI zO}d>1J;TqWYs3xTsF&$8{}Z5_NhdQ-{fy@_`^%^|k#ERj(4#z?az@)_@G$Z-=@|J_ z^$mIk|Ha_T@wD?hzx~$n9j6_c?J)FY)SD{5TT{M17EQ)u5tR(Y`-91Z3dDn|H<$?b zM+3=Vui6(*hLqdy_ODifXs=3!g2-o?Y8C5abfCz)e=ryi4+Nvhz+_}9$B?NUA!7*A z7Yj$mh!IGr%5j9Ksv44K25cRyd*DBVrY;Sg}H_ zsZq&zIDlqBQ>r@>gKW#)4_1#T{SoD^uBcwOv`iQG26{p&5=Y&Pt4b9#bxn%4FBHs3H|V7pCJc_zSW~xo_o^ln7t6sGnLt_#~sFn;tS*)vvRHIhn<5?*o-NP2K z*xf0$g!|NBERhIzN1!4~Qmw{tt1DNH<*KP%Z7x@>|- z=^7JiX);z8$)T_9Wy;gi*@(_lwrufwI+Ul8w8*Hjxuer(WoIFv>bCk+ zowu>Q!>h}8dh6Rk(ITIv!X!gow>p>_9GECrUlK}JLGNJsNzaS4{sYk`)tb~ijSR(u ziB!Q6seRkpN1EB(rp0S%Mr+bKs`Dzy>g#AvX|S`XsM25MZ}j(Jk-);IW83}vO|cGt z+!Sl_hfJ}}{@tb+OX}rw9GZ{&_n1ni_+eQZDd|S;c-&L6KsH^n&fjgKVwEhFXV7f* z512|?+n35SN?O}6Ui*xaR=wPe151u-!rFE#zf_*Vfw56dSX*o5H@5Ag{H$%TN}b9x z_>CuvF$%2QS#lx{R#~jACBM;n);3rUh01z;E`LW33n zWxzVX3v>bjAO<`Gd=dCM@CxuVfNRS&xpvK@r@JEPaykT4yO#6uPV~ohvzF}$zu!#m z7Dh(7a4ooZvbk|vBpH>-4&6Qtu>bRwxDSUMi%y(*P@OpU$o(e@@)rY34!`Uw1(u$; zed5@h?t}(r%S=42cgPO-JguOs!S}+2VJH0QIR9&tn8am30un(U%gmGdP4lFy$&nj? zKhRW|n*fqPEp`P(Z$Rl~(o~9# zNMMus{04>LHi)P8;_y-U)1^cjK(P-rhagK56t^H82m1BrpoEat2zjH(&2&7S!Uh-T zRjE@-M)>Wo53La9Hz8L;reTAt&oB@iISE6ycE3P{e4KMq)&farSt4lTUdRyz|9W{y z{L(54^1u@cFFoL_~p7iK%lREJryg7^UK3tn~z#*qF1BngK9+FP{F-&Lp&eW+irYELf zEoUA+aLOkeOAb4*EIIRXT)8f1PA=2T4w>5dAS$O}0>ZYz-v_?~4dKGZW7f5x;}n<6 z;dJEZIbC{HK(fJ>G!F2i6BBy?O^xa0uxgp+J}ocTNjD*P4;H8}IB^MGjFsm?!RP16 z2WPOFhO~eOjy&ZLSWzDfhX^wng7L;kW z%T@!&^4`urDJSLN!P%>=b$)M*0&DV=V(4PH;o;foT18TcNl^tf0U>~KH()Ry`vq{b zDVHW|d$tB_4cHp6HDGJN)_|>nY&CELH+OTjRd21q?d7@WzJ2jZd#L!;uN2722VQ@V z8x(BSWfpwq9|pL+`5M4&_X&X8_BR1MTb45bcChj%fIF#w0JN<6u$|WQ91!8QNWsVB zd)WY=H|Uz+bF=G%KLh?w_`EsFt#UE^LHM79zaKvN{|!vwb>$a@%$p{q85Di7Wl%+NKn&ddE=P}f}#MXRRgT?Xgba?pKEk&X>RD+xV5=qqy-+NwgFGc8d-}Q z3u|#&+JN)YYFQ=g5nG4DpjF7LNBTO1YH&d0LCz}i;$*cEF*i!o=@K=%{xhIvJ6i*` z25b%38n88BYrxiktpQsDwgzks*cz}kFp&mmx6)3fy^LdYZL`zbUmyH8aWJg$w0;KZMuNsrzR1LZ6n2DZF3Lw^=53%fX)-=iJS<{rcC$ z((QMRY5COTsR4hZ?b-6bm(5A@ciNsI(zbjcQa33bfk_8xm;n4=gvJ`&C zoyq6(jIIUndhQ!!fxjPkRo1A=a*13yxj=ev?Y(tl&BhuX>-wlH zuaU)wUvsz33)5W5;{y5o%+a{X{3b(Ed;V4kQab&No+v!0ke}i|1^iH$=<#>`d|8)<3|5c0Q?_R%%8Fwb1DQD_4;k#gum5Ai{qciaxRd(2vH(`@* z;>~zB()4dK>67D7ye^%}WztENi)OaQa#siLWV zm5axa&OEDCdVs;fs?hC2u}oqxmdZt^Qd2)pP5r1E(~yC5A~~)Z(X6VUpb@K9jTU#c zqW-epXx}z9p(@>@`Uw6?>rt>2T|tMgL{-)J*}^5l|Zfy6WyR8x^klc>V6{Qe>rQ%55m!qC$yDtu$3tu9m}IP0Lkd zdrwzDgsrU>4mWn+ z5DFLa_o&9sX4Mup$o4>fkH#un;gMJ;XJJz;lsE(NM-V^}qXTDrPJI)6`ST^BUE z+UF`T#ZcG1j@g3)HwxyLtkOj=f-(IR?u&x`qp3U87506a9L~hDcE-_Ghc|VNcC)Qh zH?O@7y(!qJDWuTWaCet&!JewB`pBwCOJo3(1SUQm>WB~@$nB00 zbL#1H0-ZNSwmEXz^I?wR<$aonyUq*T@?E{cMWbUxh=t*W3{_$aJR+XTjm@qceWbb?PSie za&DdOv1F&29x*g_Rowk^*I=iyTwz6EuM7UiPptE6!E&$-41uS>bKsxg-=GPrm>WPJ zRwi-qBk&XOTksNi8@vbp0X_mtv4RbNEubIV1$Kf1;A!v|@K-Pw`-}OY19XECa5vZm z9t4NL5x{+jd?Kgil$@0J;5(`RPQ&*J!Y8$4>}hZgyUL~&6+`j-fJngyTfk=x#c34y z{798NeLluv)4rVhye9AfcmvGAb>0GQ0lUFTuoo-anb_OiIbGyg@D`YZ>pT}M0YPv* z=m48R0*ruN;1RGN90D(cH^IlwYxFN?;Pj*cS2iyoQlbqU2DR|6hOcY-d|@N89nd0VF90IS-c;QgZIH{P=1J{5yunELK7H}@#4fcV9;6?B{ zI0oJUAAmfl!nj%hmV-4Q1bRReeCjl@7x?lFGG=$q1sC#Kq`^JlA@C@85kiiUZ0et8y{BQs?N zmIVpOve9bSiZpzX_sLqk(X5b8NlQisB`Qhe;5+SkvhK!tvZd83k3tT%R+eW$4m5~$ zi-kN5C|N135<{teXiv#j!C2E}EvcxruB??cwX(KWu4+c^OQp3IOOfb}&NujWbFP)f zh7a@m#kzP22!iWD2iOb}U@N@nW1-^{nAp0~WyDcCL>E zO!|7Ai78Bf=gj>G*Q8u~VtT?eMDTpF(-=1$-h7^)(8YKcgP z+yQ?xo&tTyF^DGyp9Wc^^vO1ax5z5ci!uz;SBGa5bC1`#zMylHwBb3}EM2lmdZ4LM zhVaL_`=BQW$!890Whkm!!L$tGvtR(R5tQYVv|pDWMoe0|(Yij!?T{JW?kq|(q;x5T z)XCZ$h30;*&H5UA@pjHPlX$ZpijMO+H$S3-|Z{C>-4O{?(vd8g5t#?!xpA3Ye7gSbJ`dg-_7QeFFb z73Ds_?SXgPk0viCR^YPo*3mb z9Jg(uGZy5DT|fWueT?=Jn#an2XdAYm3H)As3@7irLF_sN##YToVxx$?+MO_ zHnx(xeD*q=$R%UCEp~emwXFC5=4fpii_s;|0 z0rSBE!1N0txno`g6u1~H21@|bKLwW-{0tM%eQ_4xp};hbKPd^}FjWo*r0#(AOLsN+ zIjLzK$52HjN0Z1a`V&9JaJUn*YV?1`S%$bBh4>(jDn@a@Fp1+=pT}K*(QE3bEJX+3 zF`1)|=0tAsY2i}zFwcDPS-fXV8%iz5|C>eU#}P}OQRcx++P)1}VE|Xe{=JPkS8V5< z=yMtBr^R{imfC;)8ynudaQ<=Uq(57G)}nsX_fqR$$chW~Tl&!7Al@N%VHDiTl+Wa5 zqkhg|oU=-;pB0#z>{@vSJBQyA@m@$Wxnx&^H;PmB-;8@BOIBWD#;NK*n;P2An`508 zJh%K7u0DVB&0ZJJ-F$u*olR>__YL!3ed+Uh9X&j+hA4eA(_d%O>({^SHnv;B-7D}0xepX99LpNP&xo+|(< zF`WO2X(F#7yog=<8t5kQ@uGYOC|x1668Y9m7dZ#n55W3=M0gP8KS>68PS;=SnU_4N z{^U#dfm2V-Pt$z9I;^QKmrt2q0lxWs^24;xST4W4bh+`U)<^TBK0)6|y|fISp3g_8 zPnBwZTK;UvWq&&Q%JVzs?tH_OUyt=je;Mhz+*IXVpZrZm(o7i%WrkAyQ8TSVsi^9W zrsMs|kQwb&`{QOz`Gfx8aurJUDl-;EI`gbnMjwL%Wi1B=qpA2nG--w=V^cedP3YfoUxqur`0l6r>*EDMsQj(M=dt1ZI*bS8Z^@BcsKzQ zQIpkbl(fcb)mpu|T6I>d?rKZ&ju8_%jXRosZnNYWDU{5$lckVTVq7DQDI@EKg_zSCX~y8PBX}Sy=V*B&;VRWuL9o(UIH!v{|4^Ds=5)_2z#nZq~fL{WC1I`2G zr6TizO~7`*1il131$-Mg2K)|q4-oD%0ds&lpamEN9s^zgUIk78?*iPDlFM@G!X-I} zEt^Xp;PXB{f5ivC7px!%LkPcri?_~{)SuQ^RnoNS(*o*s=nT>#!~cIvSzT)#p+0T9AslSaIpEscWi*e0! zd{96QupZzzu>^jL8ievTC{0zU>$030*U0A~R`Zp_7)Pz7-8Xaxd57|>(L*Wmva zz_H{t;4R>7fMdxOIBN^o@n+d;67PQ|Nx!+1y=5NpdOQ#Edb|U;@i=cY*mumV-m7v`G}DdnGPO=@%t_@R$ynC6%7~MYsX6ZbuQ9?bb^} z{n^a3vK|~Nq!}Fgz+(tHnBcG#{uIz}{SHbDajnoZiPU_@r_%WH5wCSRfpkv$0Sk`+K0)@OK^)ic<#HE3a$8JqjEtK_~?>L9~JJI5h0 z0lf>A_=qEMyR@SZH_H~;E_MB@_2@v2!m`M* zWS2C;dQGw!R=AqF+5KZ8NsW%A{m>w3^{4%?M^FONf_}O6e`I}E!)x!sx3&%MF1W`> zhR3~3ebFJ0HA8Tmp1a`}nBnX&rw_)_dF|6xm~Ig9dCr6N*or~I#)`upBzsGWe8oO* zQ8DAP9TF`Uz@@=KIgMea3v@qdH!fXH&`IQ21wLN!`8;0Fv=XmRAF|P6$>R`zdN5=S z;Ns>4AhKeSCH=ONVlUme(i?^cVvz-yhCZDBB({=H;?5yvDfo|5pL?F51TU&Df_0`G z!9?e=YFR8_o`nL#jwZ5qB^Xu7N@F5*-PKs&xsunX9gqWZ$Wy*NxBO1s3v#2U6hafj zWiq@0q~lO+-*9oFOo*~3&lIMRD17XlyhY<=*lYFHxHY+Oij5*hq2F?%;fIfX)K_9_S}PUj(K6GY~ot zw28RN^Q`{iqog)3uoeL?^5qV|*7U(h%t#ph2c@5JRyyMH{mk2Cqy|EXnqI@S^Rf<2 zMt?GH#`i^4M<^_5#d6fWR{E3lvp${iNXkeXeWu!Oq&<$A>vpa|V*uCcg8=scxJSS}0PYcR-{2_l zDc}WwYvtnr*YrOExCVY5;M>C|0PX>t1ju_nm{TXc)+di`Nvx++>9~22Z;`eq67i%F zPuuHWK4jqfSA$#N^!ssuRX@=mkIsWatVYdl@tTi*&;arpRc{cy^kLz}>>G=$w zYj>V4=RcKk{myi*!S4oU19Jc#!M_LKLYV0a^j=^Aun?#O?gJJ9Ouq~)ar_JuuKQ>$ zz@wuEj@v1mu;$wnJjp&4SAo|_wMOhiC8g|5_C5|sEB*Dgd#ry%Q^`{nU5dCvOn~#_ zY&XxCCve>FYTWJUy;?qLAv~;qA!O%;)93p0-*>Q6C;y#|wr2-)mYh^8zzYCVwI}w4FPyW?S&%<@ccVi?FfGTIY}5oWJwVq%|9nA^z2)wHMvl e!*gngvIZ0SYjk=|ot#_y^4^Jho||?vSm3`qwi1&7 literal 0 HcmV?d00001 diff --git a/Scripts/Models (Under Development)/Nback/stim/ckm_3_back_d.doc b/Scripts/Models (Under Development)/Nback/stim/ckm_3_back_d.doc new file mode 100644 index 0000000000000000000000000000000000000000..b53802bfb4180470acc5040851c734adafd52437 GIT binary patch literal 22131 zcmeHP3v3+48J;`)9NTaq31A?+CW#aBu#NAW1e-_XckINE#J(tLdm%dCoqd;_Z_VAB z#41z~P?ff-1Zt}W2`M2hZQ4>4`cg%u0zzu31Z~=iR#g#A)sWJvpc17)1c>{6vp4qc z`I)3C3Ei>3{pb1rGvCb4?CkFCkzdb0`ouE}|043-I#D3bjFyNZTf7JUWjZ`hh~@Az z?#$@uD5IAG@Vf32V&I=oz98-qC57m^1<$b)g+R~3A?A%zA?Av{oqapaRpu%c>o}AY zmWi9L6{4)cqwt1(usCvBfzi<`#$u=P>kQkK=k;8W((Y&UWZ|oF`6*81{L{(Hk>?C> zgDwAG(TX1{97spZcuek2nh6;Q2g21dl8#9;5l1@ntd&MTgMU3XhNpqris{W!Wd4)WMpV49?3{FHqu@Fg*$ko_(!@F#u&hVfuv^ji_Ez}zBw}qHfO*fNhX$vQ8IeWqxTZs0?%Q@@fwC>cxoa5!3 zb#b=AO2f3FYUZ@=Y|*yx9$U?rbKHW{y3?MOrY$fOfF6s_i8{x*w$gMtXKSqx@~T=p z>p58)v=-(Zx9()Z*#;|3mvh#0x_)5JabBEEOja{JvY2zcoYTT&)>*R)vvj&zT?Q_F z25_ZZ4y*tgfnC55@Oj|dz|+7f;C37vkEtu1|vZ!gpwF1(clckgehzIYp{|JPs)aDJ!(8h~zq^TaM-1kiKE*WrH#;C%54!1?0ufPJo5gn8grfb+#h;1j?A zzdpy!WY!v6<=^T!9`gAZ){O!R(C=M1mkBYa*!h;=+4KZ^eN(9X6S;96#L z7cekxg>%7;Q=J>XGL06CecSgOCm*e?gmoP*|42pU|UIH7BGQoTwQpeuNoRkM4O1CbW9{gS_}v&0?<(mn?>cG`enEPQJ15L zv%5P*L;Z=-C&Zo5P$rt8p&vSiQ3n$mHo>0(2Gns#BoNn%dZv+@@Ay;-7p}PCEmTh@ zdBwxkA93m_7gs>ef{eojs(N;TE2FU|U`ST(Z2Fq82tx z^m7sxQ>gcBEpE?Q+$!2JhMPsV*eZI_iUu)+zZCREq0xloxM7PKiYNw{mO*?L=tpcA zWqB6FRQX}V7@`NvL?Q2m%qY&XD9w;mr6f`(<7@gBMPM`b5pd`STO0>z@Ig+jc66Xc zeR$??jf^hQ2=t;QkVq>$)Xnnz6kD3KJ6TY_g7Z?*E73Sp3I6ANO?gA!rL=160ujiDBTUKR{c#!NZ zD)1Hhyak1fvpXaz7s9~fl$XUx+y%KCvIi#ym%lV}tbrb{@cBGm&zvH!Ppu({)@G?= zFAu$#5C(DTaC*f?cbT$&Sy7>vZc6J7W7C`vi!sr^Q~WTNts_{M6RLvqfBVUA613pO z0*co&aM*`s#G_hSC?1@L0%Ojj@~EPSD`%szJh|>%7X1H^Rg&_ zNt38nWjLu)R|5I0@>)+o1d+n!PNxsDB5;#0=+m9xRyp_RGT<`cGT<_B5i@WGZ|)Z2 zVj|yp5OMUg`%k>rotXFCM~lU(TYm6Iu6sX3!hA@kKLW7iK7iL2#{gbm{0qP}i6}%S zUYN~;H4>xsYFhzT;2Pi%z`gQefP4C%0NevV50n8f0lWur1fcGlp`3NndwuHIlE!{IlTDf< z{1sVyDwRwd$*i^SHfMp8mkq#mX+cQ_7oAA-1K7IRE537oTTy zkI#KO_w;;*&%HapzQlCy-?=~M9{f6BA#gpwuPia08{wM)2`mB@151E%faxCsx8(c` z6Rx{34DjVv11|(hykX5>Pw-9lvvFnkIjOE0$52rTN0W6Q2cnh!1B*APFCra1cu>g| zh}+8qxB%Yb)bTl+V%AvyaV+;igV)kFAwMxWTpl(TXRV4rgs=d9Vv6B;td-9)<4pFaQ$z0D`PI(g|Cir`-Y>uc&sOKp-JGKG-brgV zqQm^uo4O|G?B#oEh*Bpr{Z%@>s#ct=`~E%CjXXE)Vqm~uS~;By1ge8|!J0sApsr?h zoxhZiL(u(|&B-jjyoMJVjr8sK&R%A)G8Qo-cgS>PFn-tKb~8R$)nJ<{OYCAcPUyL%bxHuxc+|o-sax-!d)(e z5WwB!ch7!4`|PtHx4ZAYd;9-%>A@#on081cnOjAcI2ju+a%}!C*cWT}Bq5f9+V2 zvxImV?rCh=BcSiPg}}c!z>f@R0ph)m{69h7zw!ui5bonhe>rHf(dE~2CWmYPRFr&a z3gYF|{4~|+tXPBfq0=Q(rbELSs2|NInJ@KoDwAAaI$bKN`D=cZrw`C_={$6JG9Mj2 zT&nr${3jq!wx{xq&o#Vu*Q>7Na%g#Uy5Z(GI`OMQ!Du+tDWkq{n?D+nzOY}m_#=U~ zpfBogky``Nc3I*r@y?gNV2g~l`w`AKi)E;l&aS+gZQcHGpvxbO`i5dtI*3i_02?hx zYbek;NDN;@mYzb01&QPiDN5%x`IXQK%Wkr9$b+o2SuRZXW-DQlG_=nzVX*Q7%%R;#6!K&$KyMIwQw zPN;~IES7_FE0VQEa#fM6FOqAERMs2pI*Qlt;2egbC?%1OpHfY!vT(`_O|6(CE9%y* zE|N7H*3_(BSzpu8SWzcySFfy-bv2ERHEY+&bt_h{ud9{~s~cr~MPttx$o6?xys-DF%ik=*ix(%VD$6IE+o%n*=0}WXMU%WMW?Ix zcG-A3WpU=tmGAm3=?abY&7b{H1bmr%j%vnCh zOvvfv>C~qcrgNYxp?PNx&N4W;FlHh@XW5;SIZMl!iFD2!oKjn1lyh{xHZL5mnCBM& zbAS$@8+aCY9{3da45-9RbUAP@uod`Dt`Ls`F95FqZvy`WJ_Tlu7or%r3upy;fCqpb zz$3sDz;0k4Z~$;|jR8ytt_BK$r9dsP1^7DfEbvR<9pE6qb%{73J`=}8zc`A&6Qch! zSWn8jbWH9z$M-o#P0{kk7^EpWF14 zupUoXwe5J4Uy@TtJ1?j^&m}x=K!{t}AIG9T9AmO^@cYbWM$wVTg=4|hsrtrrgiLmGrKG6v`OMBr`|HKHDpdqoAx5rzL+ zq-q9b>af@j4iRv$c!f}2iE@V!qX!!ItFkQ;#fTAsUeif%Wn6*=QHi{.<3EIPnt zlUM*Wfd}22z_l2>m`6H(i_Tz)Qp+(q<%*SO^}56%G*u#c-*-9lRd!Y?{b=0Qb1rzG zlJ8+lDHnL?iD_j14#ao6bFw_y9(PtY{p^O!Xc|%3tfUAktpRi^=vpY2T_uR{Md0HW z9*@iI8kghtu$|H9Y-#W1$2v5FE>uuRb+84qRPGn&WV>ln=3as36F^Svl-Hu&ew_Q@ zR3Y}GaP3OK;L~4vPe54w*qG~uT!Jt9RhcDYOLm9%Mis?0<#jsn0l@;0*mn7YQk}hnd8ZqgO28;%b z28;%b28;%b28;$W*1$=e+{{)B0%sBzoCm-4#F1mG+b2EuNUoTB!!Q2Ax$GVUTnftY z9RMezj{#gS90u?>LHq~6Iz!|l66d>9K)L#npj=5W1m%jm4wRGF4WL+vizq0UB)y<~ zZu>Ck1klGod2(wm6G-S55i-s8nrP5S;1vHmYQ_Ix$`JV}1ru;}D^p6hn5 zKidJW)w==i0XztB4}g0G+&AD}!4<$xfa~OE0Iuo#0Iq-c0C;RK_5$1k*awhzKA2M` zz1C+QYa-j5+e4k9w(WBL$|`HE%eY+gGA=0}xRI7zo~!bMH6@(^L0E=LVC4G)5rfw}xBGkfG4R;OP3qdRKpsxg20{7Rz%^0Hr zqXDAmeNSmDm;Fn-qK}VK^oquaj!&*oJb(vo%@!Hz2jN_s)FxD|>N9LC*TE zx*2CS{TKjKf@Ac2gQ61$j8Df*KfQ3$02f`tUsy9_>zTW+Q)UoP9Ub@~d02*m+?*gGxR; zdCHK#E_9*=~|u%I~O#N>iG}H1#|xM?418xkjeSaPoFR4 zv!HS|8vFUr{TpZfF@FA&^8n7l^!d-{FfpM!ok>;yH5JJ zHVg}g^YQ+#r?`GHcYqC}+2X#h!S{a^r@#NBQ1DM;jn0>Vd8R<$BOu}4hE*G14Ypo` z=k5VtN0(~-8c;T_>ZAB$EWS`Y{8VxnB{WfouYn8m{*QzB*2nIp+GF9$hO?O*8H;gN z3j-T{EVB=@B*wS)HRCbhsstVacmin(yzd4!0b7B0fcwCrANT;%i+qftw*loq1K%S3@ig~2Q~n$KoocY*a7?m;F~?a1}OUh z;4p9m;5NtvU^-9;lmWFsBhUnN178Oo27UxQ1H25p2D}Zt4`4$>>?pv^X}eLl(XfN< z0Drq3Sr3@e$k*^3y*fP~b&JNP`!C!5;{%NkmR^Eq^ae~XY`ufy2=rdgMc8p@&*Wwy zYxEf>*5ezo1K@2z!L+yWkU3@KT|`_3rn^kT+(ne`cBQj@7g4&JI-NV4th5G?B^V3wP zvtlLm!L>-KEuBuUDbsO|lFTRBP5qq8d&nP_RO&wnhWkZjs8V{!iDX5{-3XDl$5)?zVE-Z%hqFer-qV;~$j|1~e}H`Mtr z&2O|Q>pIY+(Qs{dU zD+%Tdoc~c1Z%#d^PsKEYkInHoXW;zLkmsNDxgSmK;>{m}^KVrBdpyty!(o7J-`@YJ zOZGE+|7UcLo9X&LXJ6^ge{zz#O0umh@`~PBK3BWOczfsCQ&N#{aza0J?1HQ=p-|RmR>_wv`3>b8a0e@pR`^KZu2GH2d z{habU;!^CsyAkr7spIe(hC zk7g|HpBd}(|K|MBSf4)&#EUu(0>DrR6k*+Zi z+G?N=15F2}g}PHFF*Gh@nx;-dsHdHRp(#V!W@b5=h0 zN>>lR!1h_q|9<=YcK6%e@0_#youk*kH+lEdUqAoHCd=JyMw!FCH9z=vwGm&@^-cv9iXJK5S9PbZq4 z&B=6|QyMFcl{oRvW+&a2AYFOpIjI%}J50>w1#4sACBAAyb*i;9nNHr5aBAX>rrS}x zm#4Ke+kZA`D^gt@@%G~8RNBM`Kb?f~llPB0DZ z0-6J_0r8#&bG6Biwdz>eeP2^ssy)?upR;CZwbxhWRiCXaKB97kOV1tW)`vGIy_klm z4^`mU=g|bXsTCT1NuWNNZcn&vQ5|;8;580V+&rKXH~V4vwJsV~E?v;Dc+JuUeG;f7 zb_Mu6W#*bnQ)(7K7ZAGEl$q<9QKk6in1%S};;Y0zo4;$xu?WBNRPk4R?ETDNc<;tU zSF$6~nT`*&?8^EryE41%@~yyIQpxsy%QoKalnt(68lWu3 zUoH)h*`w0o@G1@PN_IP?zAnknpEYZiGiSb&?n=h#6@qda+fz*&-1vp@rZ%TNkxmoa z-Im(a-t5FTZcN0x9J(9#HJFp?Y)UvYlf{YRA_qpETDuW_)#PcqBB#tr1cu;hmv?-DC&c;-Cce1e^5ve8foPM=c6*)_aoMlDMsv@VZ zC|I>*#LV!X05NRTfDH|W*%a$xwR*}(V6F@ zS;*avokCm}{fX%dAl`YOGsjyH?VGw1-TC?XWwF_@wXu6yP+U~q(DGP^A990LvBp5+ zveiM0h1SHTslJ+N+6U#D1kFv z0(xdBhD{ZC7`z6i^3c8xG=RE8ul-AUN0d2Lm7G3}j>r0}Mz|RJj9zToKl^ zmUUjlNZ-IXV{|0^pc7yl8$KWhBP-cMhz*S%_s0OP*9k>g0f(O7O;8E~r;92l3@B;WY zcmwPP?}1-}UxD|*2jJIWFZdAb10S1@KMu&<^L`YJ`6wfVB!Y>-`~xKP&pOhWXnw@x zMFE>qT92Q^cpM5gIt&{!?@s_-*LMM{%Y!;x`nwiLXKw*kS4&6#5Ln$j9eru_v2^gm zK)UyNVD+ultMkuCXM=X|`(PV5|3a=8U)jgurRZp5vaZE<{L`Ww`HXqTh`we`?!hz!OtERSA>Pu2% z(%eJ)98*J!8aYp?HlN|-aI@R@X7X(|-Gu8ofmu$=W1PTjqEyP%QK|{5(p~00ICR6o z;@e;IW)nGJAh;M}KyHpbm>19cR` zOLg>CjOvM)4*DqUTKPz%j`qnfXK7T&b8-^u>ylxGcciv>+G7zWM0CukywQ2lQKNOt z7IY-Kt{$PNGu?D<1#~mCjs|E-b&_K)e4-{VFA|N69TUydDW{yRR{kFStVUcNbR^|Y z#MxK6wVyp^bW|>9?Ujf*Nllu(modKYkJ~bXelU;Q+y*}T<E7GQ3dYQPzGFtKmB^HFuT)!d`N~Pf$d{~&&VtP75rs5*GDwk$ zft;3wr!KnOIpnY)rBYg%lWBK$Dx^i-?9SGtyV&ETC5Um{#hrH5PAQ8^;B4NV!%9_( zmbA)SKJ)NzMcdDn5<- ze?Mibik%tu|GYO++>M+|dwV0E+?-x^H?8}7wtAG`v-9_J5UK(XgV(@RqSt{2uo=7w z9)QJu@D>$KV}tM(Pys4IEw~eCNButV2jB_t7vSsQCGcJF-{9w9KggpU=YUJWb>Jqj z4BQS{!6xt!cpQ8Q{54S9zXSdq{1p5eMCi*Ta1po$+yEAXHJ}OH4SolF0sJ}m3iw;_ zPvA}PWAGF37I+){FVKG1V}+FYDjhx%e%SW$5Bia9U;K%vH{^NZ#`H{+Ez88Nd%NE4 zTDNOFdu`h_ZP&A1%XSUhwQJX`U9WbXYHdD6y?w3+Uyt4`>2p2!dcN2fR7eQ(_#$zo zaeBzda23({=SWVuyer{YvxcLX4L~PXor7*rOKzHXIu}z-gKb!!n&)DJ_2_8M*^W0q z@7(T}TpNXVAAHi(@+ZHXrO^o}U^ywJ82A(LB9JoD(Y};XF}M{x2&9C*3Vr}4GQd(k zi@%M>r) zHlI#8N8wWUFkk6yZn(s}=+ly^Ub+tYmHvzyE-_Qj_4urJX{mY8PydM@|9>u(d`_P; z%Y*)z;^R&u{3-2kq ze?A-2+pD*>J(tUG)9Sb|%^~U%ddhxHmDZxNgh8_Pwx~>e*m;iR(+4k6SBdzbm2$tG*w3Bl25ZcC&3UZJ++PgzaJ#R6>BE0UB33fD5Vy;{Mbph*J?1iv zm}e!t5+nSb$QrZtiw>u_ZkRUdqaL{1=zSOFJ=s=wUh2@l=bLCmio>>H=n~8jWkPSt zijke6OqO!Hnba?dqv+f=Ni=>_+3e7L&GIE;^lj`YzB=?o#rp{nC7g$gDELRfK1PtHCUw%TEzd z+Lcg<>(rNP17RCK5G^O)P2grAd$SwCK>5z3xV(Nde`NDYOM#?zpg&F)G=XN204<;uw1FhhT-^XPGdqB; z2Pv=-+zq;b`gt@iJU>DOg=b7B>Rd0B?eNdnSjFM_QFvCxH1J#xZLQ$zvx6zPlRrXdR*lE_{ov4k^c1%r z9erwl|A8+#jOberXV&5l_LlEG^#A_Db>aITK^dKS{_y+nN4^JtCVu}t*Y_WXyApEE zA2$BN??;{j|3BjRe^q9%oB5;JP4TQzZC>N8eK&t>JRPh2T|U@O-Wb2?hJ9`LPNqwWEoPIJGJ zr#FtG_`)yoXg-`6mwQ*@u)olbrCyDmj;D1{EO&Z-iKoDnkDc_p?_G4bw>+AjQa&a< zac5qI>1kMZ$3x#Pc%{HRT2olG{;qcl|L0Ke>|K4a<#(hfHmutjA*Y|O=f>%qelh9A z#gT`0z4*{?W^AsR{hdd)6qb*EpttY~2J-5H!oo<)qpPEj;t|ujm2H27RNvWp=iwk{ z<%{n5fyn4R-xC?P=e5ixp-;D4`_HKbFS&!`e|U@M!eQBevNdb%Kcy}QbGd9jgYmZf zvD<(C%x6_T%JU3FCy=y+bnP$O{4b+$ULd^DFL$)`86>EQ|7ED$aA+a<6~DM7F8fcl|FcX0JMV0JY`Kxv zcVhCNO#4qO8Q1n^qSv3}VE-8z3oC2;G8`$nRTp?UKMDI!*(kXx2l8pQ|173`l55cZ zOLkX^n*@~z2eSVZ3gqH7-1eNR z`A!G>PdyKE(RAk}WLQo%b#kYH{b!%<(;W1AHTbXA#+hXQd8)3>lhJN}ru0Bhs;$vh zzhalTwSTc+Oqjet F{|_<25(oeQ literal 0 HcmV?d00001 diff --git a/Scripts/Models (Under Development)/Nback/stim/ckm_3_back_g.doc b/Scripts/Models (Under Development)/Nback/stim/ckm_3_back_g.doc new file mode 100644 index 0000000000000000000000000000000000000000..da4eb94ddb81bf707f5c91e7fd5bbf0b4e358d39 GIT binary patch literal 35328 zcmeHQeQea_9e?iV6^fQOktqT`V&%1z-W5tggzGEqXepGU$j&v__Szm?@5a&7hw=+`OpEFm^PPrC~P z0|T@!1EAIF3X;ITkDL(uMP3eyZqDm0L?KYJP_TJ_x)3u&>)zJA@s;tFhHb(}Ud}wR zXr2)B${hyYw^x<~pQd48VCJB0I=%96kD>X%GLqW$v`%H7YxyTRW$81~YvJbtu)ya3 z$1WjWg?S09_C?TU@NpviouE|N&;t0Yh7?aD?)B4zcn{|L5dL~HNKz{QDxFErBc`8x zshX^ODt?M4^HpX=Rk&nIcT|eO@=^Yi`BFZqRC0N#aOtSxulO-NPSjqNE)|DzPv)cC zM@tny75{w1%lah!ZH^iDoNy%PW1}S(6>hZgjgSAzNH`vigk;)aWJU;womm_yJyxcZk!I*R_+2pTSu|lpX zlksTKk6b}evNaTG+h^EU``fx?C=idsHr5sC4Yf;uPfx%fmB_B&DiIk8w*};~U{Ro` zP=b+JOt(Y+ZDz?8N{I#WqJM1 z#zI-WdwX?LT|@P@=JI-3(^yw8>#LiatDAPpowbd->Z|0o#%9@2-dtIm=(V}LX-jpp z@;pLCGo%$Ti>6e@(TZbI)gLr+t5#YP@P`k{HD+rF^+p4+{CtnE#8>OjKH+`R42w~=17H0j6E?&rknDam87~QlwCR#<729aBB<1=JVF0Pm<6#&bD z3Sb}513U`60K5#G0saTf#f+^2*aF-O8~`4|EcGz(8t@kIPvA>HT}KE>ZB zXU=|tzt5;K%(Me0S7Bw&&D4q&xs~6(L4Z0fI-PV1?16mPyJ*hg2W_3Akn22_JInhm ztBk*ka&587_Z`p&up;>q=$|UYbtv17z#YI=;BKH3=ml689s`~RUIJJr{s8<9Q1!xr zvi{nhjj!y6|E4u0c+&z<+@A0Q-y^fu#WZjY?n#&;qdkxF2{FU>|Y>cm+5C zybZ7)IVaAYv&m;-Y#uz;FxAp<7gO)g?+)obZz6J zQ!o^d)?J=UY^O;sk)RBm?LJ2>6jLL z2b8(c7Xs`Rge%vtMF*R3HQ zc_^hI>Xk%E1yM?DVQa^|JQ=@PYN_JCn6zgbHdD(POIZwfMW4gy9r$th7muT_8?Kz@ zIV(CIbST@|&v>yYhrccJ4#U6GnLEXm<8n^Pp`BeRW@cdpP~jyy=>QsLlbhqDN*jCmU}q%6VswX} zPkRKL3MbJWb{Prme`8Mv!5Ex|04_lHJbGdd4D5Lyn#JQXr5uJImA3+B8XdJ(W0Am- z+~ZRZii4U%m$_V zVStl?CjidZJ_a~v`y9YLPvpQ8=Nq#?Is04y%GGo+C|B9FpqxZBgL1{)1Ij7k1EBe! zkAThq{V6DK4n7Gw3-meA>p))wW&9r_(nX+^#2D{0xd*QjnY_SS2=F3boRf&{u|sWL zkx-=bkm#hH;SRfGJ^gNrM7#Z=qV`DK^vgU{Mmod6c<|nUtoOHym}EGn-6J}~)H6Q~ z!M1257U_t~yCTtcSyqI3kQer%=6AP}c=^%)`*ZoYU1rx)_Dm1y1&eZ`O|w>70$KuE z0$KuE0$KuE0$KuE0$KuE0$KuE0+c|y^*`69&UgRx?vq6~&HBmXSpOG({c;uTTuF8h zW8vuqxNhefvR56wSeSC}1RpkwB^a6Y3>E>r#M$Hd$usxku8ny{@Wd%dWbr zL1~o3b{lX%?g+SXbW(!b0;M>&t-@`C8qf;ZuY%n=m^{#}ho5z@U4>KOD%iX6S&GjZ z{8hqby?7AZw9*pL63`OR63`OR63`OR63`OR63`OR5|~^P;966yk$HB{H9HrnJg?_j zy}Aw0wL0H<;+mZ2*<9mu-Oe>V&+xf+r#oM&;rgBH^BaI0f%(8q0NodYav^*(Ac0$e z#lWoq-OmF{EIrMHR^ulDUKozxlBLA`e13a^H}OZK=Hct4Ds|jn$<1SJGWT&nYN+?t zd5*_?CmhFjJZQX(vkZ29bbxZ>&N;uK5W)q|v8WlSy()gvY2aj^GqIuUbRTP%;MJvfp!BC;W0|>5 z?z=gDC*4V_inR6esV`$Mx?v~pslm!<%+wbt_o6a!di}5WXR3MH?Q)Pny7M3Q3;O=& z_}~95$>#p2v;9$)hnE$`e*g2_lTXdYT~N-u=uSG?`=9B`pK}U*{-@9XPz|)2%o4!D z;Yxe|s|RbRDA!M;cn9q4CzH7l&Ag4_{jZ|T?|*pkWy}(w0%!$jHou^mZ|Lb(3v{J% z+YS4JNp6$B9Im7srhETq1KQV}Ry(`sw}7%_a3#rY5~P(=#%Wi_`!ARobG#VbA?n4b z_Ym+H@HFre@EX9ayqUm4U?s2~;A@wB^W`afk@vyS$9r3Yst>vpHe(C*vq84xnws;a zihXzrxtYlEm3wN`s~2);$8!O7CO|j!Le2o*3!>{T>{jjr`0h(M<2x*5ZoZ{M-A4${ z*L76Ms8X2$=@Y6{SRyr6iA=@rtN9WSU*oaEe1j*`*Z%d3I76=k=@)SlUt7{I;$&($ zV|fwhQu7<|PU&-q_s~HR%RYSq58pHydiEu>qjDS-l?9^u%|634fgi1ZT#BJ5n*=zI zS-#j0ejnf{#lG-GfTI*gDAgY>fSx0j>Jyo7J|6?b6-tAUIZbGyE>Aq9xs}MifmW<@uK8+{p`|F`g@93-@Ur5H{{+^TISiadC#4V zcNaAF?wh{7p>D^@;>`_#`puha>Pz>=x2&zmp)$c#eb(-N)QQicQaJT>dQ$4P|ybAmKtZ-wcuc9X=i(}SklnpI_ zeF|2_$qTP{Ci-j!X;Ds zL(&J!Nrg-1OZlWyrz}fFRk(Ci@mKtqo_-8PDqSiL<(|w(xsR4Aek%Trh?n)r`>(H# z@KlEBUyO27_3mM~H^EMY8*O~!Z~toQcz&w)AB{a^N*nc6>RUzzUza+a0p{x4@g zw*gClr2y}VE(cZsD*^UPZ1?PsRs&ofxB;fW1au9+zLo8N9Z&|a57+?EPpbCMv50-Z z)!6nS5^hSQ*tORC&wBrvFVyQ7+0FO!FZG^hwl9;XdJ$c}$Zr0I!pOg6Ftm@a_n-Ct zGe$Zn&UdITugCNC{hh%VV8E}a@R`o~7h%uHM)A=4O*fou{b#!N&A2-MH#eeY@C_h_#N zh%kJg_H*Co=lAKF|B!9skX}Cy%@__mpJ0CuNFU9VwDRN4yXIhjq9_#S=DnQrGev_Rjf}Vv#%&F-jGo^c1_bzj#xzdVteH0ax z%7Rjn(psN|Q_jKS%4r8iMrMu1PUP1ac37T2WrCDWKcmMB-=6C1)VdRbZYY z|FM9`j}czVp?wqb%h2OT`2&!2`AE*7ePHn%Z10{X@<)V^qWt|-kmYpybv?6^C#^sA z(mj@Gr`D%swq709QkTn?Ovk_(?H`>#TQB(;&t>b>DhX8 z`t?e!PurghyBtqX|K-GznLCd8vfELa;YF9bUVFDDf4z}3Q$|9W;Z$GLOsjAzs(Pa7 zcwaJXMtju0xEWKyP%u=f!pRit*XYU#Okr)u3A*B>JCSCt8rCXEvgFPFTWNA^U)P_=!&ORP;*nfV#Nxz zrdpY)co?k$QK~y(M0Q*8jp0a4C8DN@*mTSoO!TPmz(6#dQfO{C(<91AMxtt2ydqjr zuAs>7rjLXFh~0DLs!By2U9jBJ-4%;w#cO@G!L+uRG)&bURfB1CHq1ITWtWI2RU)3o z80juok#c3ERF7JQAux#{6f5n@hjsj;*)g?b%nZk=`-rtoOaAoAMQH-&aP%DHQ1r|EL; zy3~cb9p&saozq>KHaT#_XBf|*>hWPb8V+-d%4=+w#S^dgNzUO`|I**_cjnUR^OsKJ z?+m?G#QD?6!N8gGcTbm1)RL)nD+=pB`G7$BGjclR6vP9?hz~Gazz@dyrGooB_B;Fg zolGBp7X7+2)4wMmpTUmg0`UG+?Df#Ep8#rrTA&@+0dTi61aLfj6Zj6G$Hcqv{{i5* zxBy&qHsFFY@UIC2u54ZaL^`ROIN-?T!v+NZ67Wg!F9YJMxrZCy+tB|3%z3U2i-1Rg zCjhPyp91=TL4a$;3&6|3_ko`STrYkH{2BNsz;$C5FdyK$!8KzeuoZ{^Tsyu9d>!EW z@fz?G;BA2G2-lFya{01@Be(aGJ9xK*{KY;A_yJ61&d1x)A6J}YdI4U`9PSnd#;kBH zn4fEI{C8_HX4v!H5e%^Z3+Ktg;p300;ge4+8a^CY3@kZ#I8X*WJbXvg(yPQRWni|< z!fCBfJ_E_qYPuTy|6{}tIRnpsHj5!C>6fr1kTx4wTPEwvq`FKtH29^t3=a%;Hv0eGW(raZRXa5~128(h@aKEKxakZig4QGn^V54LXMcriJ%S!YGh}VR)y-GFhqsTP!oa_Ek5ieT`JZh6&E%u$Vx-^R>7m zV{x0bU_3NPyKIwAw4zo9@RxwT2sE0I>;tx#p|HgO(^7~}1HFh1p)5yKk5zsUF@|)2 znF!=w$dtud8l@RhRw;$l@%YN!qA+as+y)Md!4}6s5`2&oyB)1)QMWA3Fu7Ukz+Sy< z0t+|NHhX;JvQ(F`v=22%TH|RS_y|dtG-6zC`5&=g1bcCeYXzHe%mL2YJGW+Y&?lU? zN;oAUiNrxwt1$U@i3cB@poWeNzkl0eiL64}@)n z&r)<^iuPkLbHNi?z0|UPPho+dZbIv|=zM$O5m}7Y^}}fga13!ASDlzu!BcO4x|g5@ zzeT`R`1qKTmlCek&z0~2n|3}ck8WET8;zyib2q}kD}`@O-7otw3R9Kt@;vytS;yYv z#g`XMIxYbA#hMp}1V$k*7AY*5QTRAI8RFvKgU>WZ4Hr{hQlf}a7`I$ldC?DIvJ#b6 z87_0w6+=F|yw($xDx~lNq|^KDs!VWs$Th(;1wG+0;4$Da;4$Da@X=-9D&E{Iux_v1 zZSWy@@=GtDz1SWr`Q8iDWaT42{5|gseuRX3A(?&<;HCW~fOlUX0K5bH5WroV6d)7t z9!em2FF6mAyXi_u?y{R9c}vg*iOshRK=KCf8OUPD1CTQzUxehFgNGnXAioPa2l6N+ z?SBBH3n1%>8+@NBIQlA)tqZC}po@C(oJ6*yhaxc}Ve}12ALFca#AW-Lce9b|4<{;m z4AaibHq;w^$+#Kc8&$30Zb>VaW8JHyFG)Y!(-x1UjI_~fs%=K9M^#tguBHa@()5cZ z6gB)<`ZdM3%bb34&-zea%}CxhGweB!0gnNX0gnNX0gnNX0gnNX0gnNX0gnNX0WvVr z{-1kO|9kJgcc|i_l5czq`~UKbuQwnrkQMhlHl8&A_wC$+#sKcs2LPS{@Qi?G06Zh$ zxxr!JW5BBb_sYWn_w+vnxCed<;G_C?0G3$FoLRQNflwONx@T>4n z#9BOTsfKTjtVgH{z75E;9I=*G?!q!eK5Hf5JO=NF z^S$^4UfH}6Hxr{*+o#M&2hWDA!3&|ki;(>tjePVl&s_2QIA?Sn@-4^T&7uq7Womvi zgs)(e`xf*|DJkN}i+Rvef+<9xPlfnNlzYo1X1qaJ)b@tlL>vz^WX$?kXkU#Zhokh2G e@;x;~S(BOm3Y}h2E$24;U{8*L=Ox_{4EzU7fX{aT literal 0 HcmV?d00001 diff --git a/Scripts/Models (Under Development)/N-back/nback.results_nep_6250_lr_01.pnl.npy b/Scripts/Models (Under Development)/nback.results_nep_6250_lr_01.pnl.npy similarity index 69% rename from Scripts/Models (Under Development)/N-back/nback.results_nep_6250_lr_01.pnl.npy rename to Scripts/Models (Under Development)/nback.results_nep_6250_lr_01.pnl.npy index f0a8a235271ac5f17f547f5d66a552bf2bdb6042..35604e7c555edb474cce2bc12c145db8092fbd0f 100644 GIT binary patch delta 86 zcmZqRZQz~oZQ?(L$!2U`lN}hz-p1lES&h+QVxPcdHl~8fW{d`t^B6lO{__Be lOl(t_9LDGX#0w_(F&j*7V?HpsjJ06mKLMZ#24IR&3IJ;6AEy8S delta 56 zcmZqRZQz~oZSpq8f{ASlCYv!8Og_hCFj` (see `exponents ` for details of how exponents are applied). - operation : SUM or PRODUCT : default SUM + operation : SUM, PRODUCT or CROSS_ENTROPY : default SUM specifies whether the `function ` takes the elementwise (Hadamarad) - sum or product of the arrays in `variable `. + sum, product or cross entropy of the arrays in `variable `. scale : float or np.ndarray : default None specifies a value by which to multiply each element of the result of `function ` @@ -1078,8 +1079,8 @@ class LinearCombination( ` (if any are specified). operation : SUM or PRODUCT - determines whether the `function ` takes the elementwise (Hadamard) sum or - product of the arrays in `variable `. + determines whether the `function ` takes the elementwise (Hadamard) sum, + product, or cross entropy of the arrays in `variable `. scale : float or np.ndarray value is applied multiplicatively to each element of the array after applying the @@ -1176,7 +1177,8 @@ def __init__(self, # exponents: tc.optional(parameter_spec)=None, weights=None, exponents=None, - operation: tc.optional(tc.enum(SUM, PRODUCT)) = None, + operation: tc.optional(tc.enum(SUM, PRODUCT, CROSS_ENTROPY)) = None, + # operation: Union[SUM, PRODUCT, CROSS_ENTROPY] = None, scale=None, offset=None, params=None, @@ -1391,6 +1393,10 @@ def _function(self, combination = np.sum(variable, axis=0) elif operation == PRODUCT: combination = np.product(variable, axis=0) + elif operation == CROSS_ENTROPY: + v1 = variable[0] + v2 = variable[1] + combination = np.where(np.logical_and(v1 == 0, v2 == 0), 0.0, v1 * np.log(v2)) else: raise FunctionError("Unrecognized operator ({0}) for LinearCombination function". format(operation.self.Operation.SUM)) @@ -1429,6 +1435,16 @@ def _gen_llvm_combine(self, builder, index, ctx, vi, vo, params): elif operation == PRODUCT: val = ctx.float_ty(1.0) comb_op = "fmul" + elif operation == CROSS_ENTROPY: + raise FunctionError(f"LinearCombination Function does not (yet) support CROSS_ENTROPY operation.") + # FIX: THIS NEEDS TO BE REPLACED TO GENERATE A VECTOR WITH HADAMARD CROSS-ENTROPY OF vi AND vo + # ptr1 = builder.gep(vi, [index]) + # ptr2 = builder.gep(vo, [index]) + # val1 = builder.load(ptr1) + # val2 = builder.load(ptr2) + # log_f = ctx.get_builtin("log", [ctx.float_ty]) + # log = builder.call(log_f, [val2]) + # prod = builder.fmul(val1, log) else: assert False, "Unknown operation: {}".format(operation) diff --git a/psyneulink/core/components/functions/nonstateful/learningfunctions.py b/psyneulink/core/components/functions/nonstateful/learningfunctions.py index 49f15cf4f8d..d0c8bde085b 100644 --- a/psyneulink/core/components/functions/nonstateful/learningfunctions.py +++ b/psyneulink/core/components/functions/nonstateful/learningfunctions.py @@ -37,8 +37,8 @@ from psyneulink.core.globals.context import handle_external_context from psyneulink.core.globals.keywords import \ CONTRASTIVE_HEBBIAN_FUNCTION, TDLEARNING_FUNCTION, LEARNING_FUNCTION_TYPE, LEARNING_RATE, \ - KOHONEN_FUNCTION, GAUSSIAN, LINEAR, EXPONENTIAL, HEBBIAN_FUNCTION, RL_FUNCTION, BACKPROPAGATION_FUNCTION, MATRIX, \ - MSE, SSE + KOHONEN_FUNCTION, GAUSSIAN, LINEAR, EXPONENTIAL, HEBBIAN_FUNCTION, RL_FUNCTION, BACKPROPAGATION_FUNCTION, \ + MATRIX, Loss from psyneulink.core.globals.parameters import Parameter, check_user_specified from psyneulink.core.globals.preferences.basepreferenceset import is_pref_set from psyneulink.core.globals.utilities import is_numeric, scalar_distance, convert_to_np_array @@ -1695,6 +1695,7 @@ class BackPropagation(LearningFunction): default_variable=None, \ activation_derivative_fct=Logistic().derivative, \ learning_rate=None, \ + loss_function=None, \ params=None, \ name=None, \ prefs=None) @@ -1702,6 +1703,10 @@ class BackPropagation(LearningFunction): Calculate and return a matrix of weight changes and weighted error signal from arrays of inputs, outputs and error terms. + This implements the standard form of the `backpropagation learning algorithm + `_, using a form of loss determined by the `error_signal + ` of the `LearningMechanism` to which it is assigned. + `function ` calculates a matrix of weight changes using the `backpropagation `_ (`Generalized Delta Rule `_) learning algorithm, computed as: @@ -1798,6 +1803,10 @@ class BackPropagation(LearningFunction): supersedes any specification for the `Process` and/or `System` to which the function's `owner ` belongs (see `learning_rate ` for details). + loss_function : Loss : default None + specifies the operation to apply to the error signal (i.e., method of calculating the derivative of the errror + with respect to activation) before computing weight changes. + params : Dict[param keyword: param value] : default None a `parameter dictionary ` that specifies the parameters for the function. Values specified for parameters in the dictionary override any assigned to those parameters in @@ -1853,8 +1862,9 @@ class BackPropagation(LearningFunction): default_learning_rate : float the value used for the `learning_rate ` if it is not otherwise specified. - loss_function : string : default 'MSE' - the operation to apply to the error signal before computing weight changes. + loss_function : Loss or None + the operation to apply to the error signal (i.e., method of calculating the derivative of the errror + with respect to activation) before computing weight changes. owner : Component `Mechanism ` to which the Function belongs. @@ -1980,8 +1990,8 @@ def _validate_variable(self, variable, context=None): variable = super()._validate_variable(variable, context) if len(variable) != 3: - raise ComponentError("Variable for {} ({}) must have three items: {}, {}, and {})". - format(self.name, variable, ACTIVATION_INPUT, ACTIVATION_OUTPUT, ERROR_SIGNAL)) + raise ComponentError(f"Variable for '{self.name}' ({variable}) must have three items: " + f"{ACTIVATION_INPUT}, {ACTIVATION_OUTPUT}, and {ERROR_SIGNAL}).") return variable @@ -2005,13 +2015,6 @@ def _validate_params(self, request_set, target_set=None, context=None): or MappingProjection, its current value can be accessed at runtime (i.e., it can be used as a "pointer") """ - # # MODIFIED 3/22/17 OLD: - # # This allows callers to specify None as learning_rate (e.g., _instantiate_learning_components) - # if request_set[LEARNING_RATE] is None: - # request_set[LEARNING_RATE] = 1.0 - # # request_set[LEARNING_RATE] = request_set[LEARNING_RATE] or 1.0 - # # MODIFIED 3/22/17 END - super()._validate_params(request_set=request_set, target_set=target_set, context=context) if LEARNING_RATE in target_set and target_set[LEARNING_RATE] is not None: @@ -2025,26 +2028,25 @@ def _validate_params(self, request_set, target_set=None, context=None): from psyneulink.core.components.ports.parameterport import ParameterPort from psyneulink.core.components.projections.pathway.mappingprojection import MappingProjection if not isinstance(error_matrix, (list, np.ndarray, np.matrix, ParameterPort, MappingProjection)): - raise FunctionError("The {} arg for {} ({}) must be a list, 2d np.array, ParamaterState or " - "MappingProjection".format(ERROR_MATRIX, self.__class__.__name__, error_matrix)) + raise FunctionError(f"The '{ERROR_MATRIX}' arg for {self.__class__.__name__} ({error_matrix}) " + f"must be a list, 2d np.array, ParamaterPort or MappingProjection.") if isinstance(error_matrix, MappingProjection): try: error_matrix = error_matrix._parameter_ports[MATRIX].value param_type_string = "MappingProjection's ParameterPort" except KeyError: - raise FunctionError("The MappingProjection specified for the {} arg of {} ({}) must have a {} " - "paramaterState that has been assigned a 2d array or matrix". - format(ERROR_MATRIX, self.__class__.__name__, error_matrix.shape, MATRIX)) + raise FunctionError(f"The MappingProjection specified for the '{ERROR_MATRIX}' arg of " + f"of {self.__class__.__name__} ({error_matrix.shape}) must have a " + f"{MATRIX} ParamaterPort that has been assigned a 2d array or matrix.") elif isinstance(error_matrix, ParameterPort): try: error_matrix = error_matrix.value param_type_string = "ParameterPort" except KeyError: - raise FunctionError("The value of the {} parameterPort specified for the {} arg of {} ({}) " - "must be a 2d array or matrix". - format(MATRIX, ERROR_MATRIX, self.__class__.__name__, error_matrix.shape)) + raise FunctionError(f"The value of the {MATRIX} ParameterPort specified for the '{ERROR_MATRIX}' " + f"arg of {self.__class__.__name__} ({error_matrix.shape}).") else: param_type_string = "array or matrix" @@ -2056,24 +2058,24 @@ def _validate_params(self, request_set, target_set=None, context=None): error_signal_len = len(self.defaults.variable[LEARNING_ERROR_OUTPUT]) if error_matrix.ndim != 2: - raise FunctionError("The value of the {} specified for the {} arg of {} ({}) " - "must be a 2d array or matrix". - format(param_type_string, ERROR_MATRIX, self.name, error_matrix)) + raise FunctionError(f"The value of the {param_type_string} specified for the '{ERROR_MATRIX}' arg " + f"of '{self.name}' ({error_matrix}) must be a 2d array or matrix.") # The length of the sender outputPort.value (the error signal) must be the # same as the width (# columns) of the MappingProjection's weight matrix (# of receivers) # Validate that columns (number of receiver elements) of error_matrix equals length of error_signal if cols != error_signal_len: - raise FunctionError("The width (number of columns, {}) of the \'{}\' arg ({}) specified for {} " - "must match the length of the error signal ({}) it receives". - format(cols, MATRIX, error_matrix.shape, self.name, error_signal_len)) + raise FunctionError(f"The width (number of columns, {cols}) of the '{MATRIX}' arg " + f"({error_matrix.shape}) specified for '{self.name}' must match " + f"the length of the error signal ({error_signal_len}) it receives.") # Validate that rows (number of sender elements) of error_matrix equals length of activity_output, if rows != activity_output_len: - raise FunctionError("The height (number of rows, {}) of \'{}\' arg specified for {} must match the " - "length of the output {} of the activity vector being monitored ({})". - format(rows, MATRIX, self.name, activity_output_len)) + activation_input = self._get_current_parameter_value(ACTIVATION_INPUT, context) + raise FunctionError(f"The height (number of rows, {rows}) of '{MATRIX}' arg specified for " + f"'{self.name}' must match the length of the output {activity_output_len} " + f"of the activity vector being monitored ({activation_input}).") def _function(self, variable=None, @@ -2129,8 +2131,8 @@ def _function(self, owner_string = "" if self.owner: owner_string = " of " + self.owner.name - raise FunctionError("Call to {} function{} must include \'ERROR_MATRIX\' in params arg". - format(self.__class__.__name__, owner_string)) + raise FunctionError(f"Call to {self.__class__.__name__} function {owner_string} " + f"must include '{ERROR_MATRIX}' in params arg.") self.parameters.error_matrix._set(error_matrix, context) # self._check_args(variable=variable, context=context, params=params, context=context) @@ -2152,25 +2154,22 @@ def _function(self, # Derivative of error with respect to output activity (contribution of each output unit to the error above) loss_function = self.parameters.loss_function.get(context) - if loss_function == MSE: + if loss_function == Loss.MSE: num_output_units = self._get_current_parameter_value(ERROR_SIGNAL, context).shape[0] dE_dA = np.dot(error_matrix, self._get_current_parameter_value(ERROR_SIGNAL, context)) / num_output_units * 2 - elif loss_function == SSE: + elif loss_function == Loss.SSE: dE_dA = np.dot(error_matrix, self._get_current_parameter_value(ERROR_SIGNAL, context)) * 2 else: + # Use L0 (this applies to hidden layers) dE_dA = np.dot(error_matrix, self._get_current_parameter_value(ERROR_SIGNAL, context)) # Derivative of the output activity activation_output = self._get_current_parameter_value(ACTIVATION_OUTPUT, context) - # FIX: THIS ASSUMES DERIVATIVE CAN BE COMPUTED FROM output OF FUNCTION (AS IT CAN FOR THE Logistic) - # # MODIFIED 11/5/22 OLD: - # dA_dW = self.activation_derivative_fct(input=None, output=activation_output, context=context) - # MODIFIED 11/5/22 NEW: - dA_dW = self.activation_derivative_fct(input=activation_input, output=activation_output, context=context) - # MODIFIED 11/5/22 END + dA_dW = self.activation_derivative_fct(input=None, output=activation_output, context=context) # Chain rule to get the derivative of the error with respect to the weights dE_dW = dE_dA * dA_dW + # dE_dW = np.matmul(dE_dA,dA_dW) # Weight changes = delta rule (learning rate * activity * error) weight_change_matrix = learning_rate * activation_input * dE_dW diff --git a/psyneulink/core/components/functions/nonstateful/transferfunctions.py b/psyneulink/core/components/functions/nonstateful/transferfunctions.py index 4c09a109d6f..1841eb3f334 100644 --- a/psyneulink/core/components/functions/nonstateful/transferfunctions.py +++ b/psyneulink/core/components/functions/nonstateful/transferfunctions.py @@ -28,6 +28,11 @@ Functions that transform their variable but maintain its shape. +.. _TransferFunction_StandardAttributes: + +Standard Attributes +~~~~~~~~~~~~~~~~~~~ + All TransferFunctions have the following attributes: * **bounds**: specifies the lower and upper limits of the result; if there are none, the attribute is set to @@ -39,6 +44,21 @@ parameters and used by `ModulatoryProjections ` to modulate the output of the TransferFunction's function (see `Function_Modulatory_Params`). +.. _TransferFunction_Derivative: + +Derivatives +~~~~~~~~~~~ + +Most TransferFunctions have a derivative method. These take both an **input** and **output** argument. In general, +the **input** is used to compute the derivative of the function at that value. If that is not provided, some +Functions can compute the derivative using the function's output, either directly (such as `Logistic.derivative`) or by +inferring the input from the **output** and then computing the derivative for that value (such as `ReLU.derivative`) + + +TranferFunction Class References +-------------------------------- + + """ import numbers @@ -52,11 +72,11 @@ from psyneulink.core import llvm as pnlvm from psyneulink.core.components.component import parameter_keywords -from psyneulink.core.components.functions.nonstateful.combinationfunctions import LinearCombination from psyneulink.core.components.functions.function import ( DEFAULT_SEED, Function, Function_Base, FunctionError, _random_state_getter, _seed_setter, function_keywords, get_matrix, is_function_type, ) +from psyneulink.core.components.functions.nonstateful.combinationfunctions import LinearCombination from psyneulink.core.components.functions.nonstateful.selectionfunctions import OneHot from psyneulink.core.components.functions.stateful.integratorfunctions import SimpleIntegrator from psyneulink.core.components.shellclasses import Projection @@ -1052,21 +1072,11 @@ def derivative(self, input=None, output=None, context=None): Deriviative of logistic transform at output: number or array """ - # FIX: BackPropagation PASSES IN SAME INPUT AND OUTPUT - # if (output is not None and input is not None and self.prefs.paramValidationPref): - # if isinstance(input, numbers.Number): - # valid = output == self.function(input, context=context) - # else: - # valid = all(output[i] == self.function(input, context=context)[i] for i in range(len(input))) - # if not valid: - # raise FunctionError("Value of {} arg passed to {} ({}) " - # "does not match the value expected for specified {} ({})". - # format(repr('output'), self.__class__.__name__ + '.' + 'derivative', output, - # repr('input'), input)) - # + gain = self._get_current_parameter_value(GAIN, context) scale = self._get_current_parameter_value(SCALE, context) + # Favor use of output: compute it from input if it is not provided if output is None: output = self.function(input, context=context) @@ -1599,9 +1609,11 @@ def _gen_llvm_transfer(self, builder, index, ctx, vi, vo, params, state, *, tags @handle_external_context() def derivative(self, input=None, output=None, context=None): """ - derivative(input) + derivative(input or else output) - Derivative of `function ` at **input**. + Derivative of `function ` at **input** or **output**. If **input** is specified, that + is used to compute the derivative; if **input** is not specified, it is inferred from the **output** + and then used to compute the derivative. Arguments --------- @@ -1613,8 +1625,8 @@ def derivative(self, input=None, output=None, context=None): ------- derivative : number or array - """ + gain = self._get_current_parameter_value(GAIN, context) leak = self._get_current_parameter_value(LEAK, context) bias = self._get_current_parameter_value(BIAS, context) @@ -2503,7 +2515,7 @@ class SoftMax(TransferFunction): 0 for all others. per_item : boolean : default True - for 2d variables, determines whether the SoftMax function will be applied to the entire variable (per_item = + for 2d variables, determines whether the SoftMax function is applied to the entire variable (per_item = False), or applied to each item in the variable separately (per_item = True). bounds : None if `output ` == MAX_VAL, else (0,1) : default (0,1) @@ -2837,9 +2849,12 @@ def derivative(self, input=None, output=None, context=None): """ derivative(output) + .. technical note:: + If MAX_VAL is specified for the `output ` parameter, and there is a tie for the maximum + value, the element with the lower index is used to compute the derivative (see IMPLEMENTATION NOTE below). + Returns ------- - derivative of values returned by SoftMax : 1d or 2d array (depending on *OUTPUT_TYPE* of SoftMax) """ @@ -2854,7 +2869,7 @@ def derivative(self, input=None, output=None, context=None): output_type = self._get_current_parameter_value(OUTPUT_TYPE, context) if output_type == ALL: - # Return full Jacobian matrix of derivatives + # Return full Jacobian matrix of derivatives using Kronecker's delta method: derivative = np.empty([size, size]) for i, j in np.ndindex(size, size): if i == j: @@ -2862,12 +2877,18 @@ def derivative(self, input=None, output=None, context=None): else: d = 0 derivative[j, i] = sm[i] * (d - sm[j]) - elif output_type in {MAX_VAL, MAX_INDICATOR}: # Return 1d array of derivatives for max element (i.e., the one chosen by SoftMax) derivative = np.empty(size) - # Get the element of output returned as non-zero when output_type is not ALL - index_of_max = int(np.where(output == np.max(output))[-1][0]) + # Get the element of output returned as non-zero (max val) when output_type is not ALL + # IMPLEMENTATION NOTES: + # if there is a tie for max, this chooses the item in sm with the lowest index in sm: + index_of_max = int(np.where(sm == np.max(sm))[-1][0]) + # the following would randomly choose a value in case of a tie, + # but may cause problems with compilation: + # index_of_max = np.where(sm == np.max(sm))[0] + # if len(index_of_max)>1: + # index_of_max = int(np.random.choice(index_of_max)) max_val = sm[index_of_max] for i in range(size): if i == index_of_max: @@ -2875,7 +2896,6 @@ def derivative(self, input=None, output=None, context=None): else: d = 0 derivative[i] = sm[i] * (d - max_val) - else: raise FunctionError("Can't assign derivative for SoftMax function{} since OUTPUT_TYPE is PROB " "(and therefore the relevant element is ambiguous)".format(self.owner_name)) diff --git a/psyneulink/core/compositions/composition.py b/psyneulink/core/compositions/composition.py index 6c345800027..c61c155df38 100644 --- a/psyneulink/core/compositions/composition.py +++ b/psyneulink/core/compositions/composition.py @@ -74,10 +74,6 @@ Overview -------- - .. warning:: - As of PsyNeuLink 0.7.5, the API for using Compositions for Learning has been slightly changed! - Please see `this link ` for more details. - Composition is the base class for objects that combine PsyNeuLink `Components ` into an executable model. It defines a common set of attributes possessed, and methods used by all Composition objects. @@ -99,10 +95,10 @@ A Composition can be created by calling the constructor and specifying `Components ` to be added, using either arguments of the constructor and/or methods that allow Components to be added once it has been constructed. -.. hint:: - Although Components (Nodes and Projections) can be added individually to a Composition, it is often easier to use - `Pathways ` to construct a Composition, which in many cases can automaticially construct the - Projections needed without having to specify those explicitly. + .. hint:: + Although Components (Nodes and Projections) can be added individually to a Composition, it is often easier + to use `Pathways ` to construct a Composition, which in many cases can automaticially + construct the Projections needed without having to specify those explicitly. .. _Composition_Constructor: @@ -228,9 +224,9 @@ ` in the specified pathway; returns the `learning Pathway ` added to the Composition. -.. note:: - Only Mechanisms and Projections added to a Composition using the methods above belong to a Composition, even if - other Mechanism and/or Projections are constructed in the same Python script. + .. note:: + Only Mechanisms and Projections added to a Composition using the methods above belong to a Composition, even if + other Mechanism and/or Projections are constructed in the same Python script. A `Node ` can be removed from a Composition using the `remove_node ` method. @@ -824,21 +820,22 @@ .. .. _OBJECTIVE_MECHANISM: * *OBJECTIVE_MECHANISM* -- usually a `ComparatorMechanism`, used to `calculate an error signal - ` for the sequence by comparing the value received by the ComparatorMechanism's - *SAMPLE* `InputPort ` (from the `output ` of - the last Processing Mechanism in the `learning Pathway `) with the value received - in the *OBJECTIVE_MECHANISM*'s *TARGET* `InputPort ` (from the *TARGET_MECHANISM* - generated by the method -- see below); this is assigned the `NodeRole` `LEARNING` in the Composition. + ` (i.e., loss) for the sequence by comparing the value received by + the ComparatorMechanism's *SAMPLE* `InputPort ` (from the `output + ` of the last Processing Mechanism in the `learning Pathway + `) with the value received in the *OBJECTIVE_MECHANISM*'s *TARGET* + `InputPort ` (from the *TARGET_MECHANISM* generated by the method + -- see below); this is assigned the `NodeRole` `LEARNING` in the Composition. .. .. _LEARNING_MECHANISMS: * *LEARNING_MECHANISMS* -- a `LearningMechanism` for each MappingProjection in the sequence, each of which calculates the `learning_signal ` used to modify the `matrix - ` parameter for the coresponding MappingProjection, along with a `LearningSignal` and - `LearningProjection` that convey the `learning_signal ` to the + ` parameter for the coresponding MappingProjection, along with a `LearningSignal` + and `LearningProjection` that convey the `learning_signal ` to the MappingProjection's *MATRIX* `ParameterPort`; depending on learning method, additional MappingProjections may be created to and/or from the LearningMechanism -- see - `LearningMechanism_Learning_Configurations` for details); these are assigned the `NodeRole` `LEARNING` in the - Composition. + `LearningMechanism_Learning_Configurations` for details); these are assigned the `NodeRole` `LEARNING` in + the Composition. .. .. _LEARNING_FUNCTION: * *LEARNING_FUNCTION* -- the `LearningFunction` used by each of the `LEARNING_MECHANISMS` in the learning pathway. @@ -951,12 +948,12 @@ Animation of XOR Composition in example above when it is executed by calling its `learn ` method with the argument ``animate={'show_learning':True}``. -.. note:: - Since the `learning components ` are not executed until after the - processing components, the change to the weights of the MappingProjections in a learning pathway are not - made until after it has executed. Thus, as with `execution of a Projection `, those - changes will not be observed in the values of their `matrix ` parameters until after - they are next executed (see `Lazy Evaluation ` for an explanation of "lazy" updating). + .. note:: + Since the `learning components ` are not executed until after the + processing components, the change to the weights of the MappingProjections in a learning pathway are not + made until after it has executed. Thus, as with `execution of a Projection `, those + changes will not be observed in the values of their `matrix ` parameters until after + they are next executed (see `Lazy Evaluation ` for an explanation of "lazy" updating). .. _Composition_Learning_AutodiffComposition: @@ -967,32 +964,50 @@ Change reference to example below to point to Rumelhart Semantic Network Model Script once implemented COMMENT -`AutodiffCompositions ` provide the ability to execute a composition using `PyTorch -`_ (see `example ` in `BasicsAndPrimer`). The -AutodiffComposition constructor provides arguments for configuring the PyTorch implementation in various ways; the -Composition is then built using the same methods (e.g., `add_node`, `add_projection`, `add_linear_processing_pathway`, -etc.) as any other Composition. Note that there is no need to use any `learning methods ` -— AutodiffCompositions automatically creates backpropagation learning pathways ` between -all input - output `Node ` paths. It can be run just as a standard Composition would - using `learn -` for learning mode, and `run ` for test mode. - -The advantage of this approach is that it allows the Composition to be implemented in PsyNeuLink, while exploiting -the efficiency of execution in PyTorch (which can yield as much as three orders of magnitude improvement). However, -a disadvantage is that there are restrictions on the kinds of Compositions that be implemented in this way. -First, because it relies on PyTorch, it is best suited for use with `supervised -learning `, although it can be used for some forms of `unsupervised learning -` that are supported in PyTorch (e.g., `self-organized maps -`_). Second, all of the Components in the Composition are be subject to and must -be with compatible with learning. This means that it cannot be used with a Composition that contains any -`modulatory components ` or that are subject to modulation, whether by -ControlMechanisms within or outside the Composition; this includes a `controller ` -or any LearningMechanisms. An AutodiffComposition can be `nested in a Composition ` -that has such other Components. During learning, none of the internal Components of the AutodiffComposition (e.g., -intermediate layers of a neural network model) are accessible to the other Components of the outer Composition, -(e.g., as sources of information, or for modulation). However, when learning turned off, then the AutodiffComposition -functions like any other, and all of its internal Components accessible to other Components of the outer Composition. -Thus, as long as access to its internal Components is not needed during learning, an `AutodiffComposition` can be -trained, and then used to execute the trained Composition like any other. +`AutodiffCompositions ` provide the ability to execute backpropagation learning much more +efficiently than using a standard Composition. An AutodiffComposition is constructed in the same way, but there +is no need to specify any `learning components `>` or using any `learning methods +` -- in fact, they should *not* be specified (see `warning +`) -- an AutodiffComposition automatically creates backpropagation +`learning pathways ` from all input to all output `Nodes `. +While learning in an AutodiffComposition is restricted to the `BackPropagation` learning algorithm, its `loss +function can be specified (using the **loss_spec** parameter of its constructor), which implements different kinds of +`supervised learning ` (for example, `Loss.MSE` can be used for regression, +or `Loss.CROSS_ENTROPY` for classification). + +The advantage of using an AutodiffComposition is that it allows a model to be implemented in PsyNeuLink, and then +exploit the acceleration of optimized implementations of learning. This can be achieved by executing the `learn +` method in one of two modes (specified using its **execution_mode** argument): using direct +compilation (**execution_mode** = `ExecutionMode.LLVMRun`); or by automatically translating the model to `PyTorch +`_ for training (**execution_mode** = `ExecutionMode.PyTorch`). The advantage of these modes is +that they can provide up to three orders of magnitude speed-up in training a model. However, there are restrictions +on the kinds of Compositions that be implemented in this way. The features of the different ways to implement and +execute learning are outlined in the following table, and described in more detail in `AutodiffComposition`. + +.. _Composition_Compilation_Table: + +.. table:: + :align: left + + +-----------------+------------------------+------------------------------------------------+ + | | **Composition** | **AutodiffComposition** | + +-----------------+------------------------+-----------------------+------------------------+ + | | *Python* | *Direct Compilation* | *PyTorch* | + +=================+========================+=======================+========================+ + | execution_mode =| `ExecutionMode.Python` |`ExecutionMode.LLVMRun`|`ExecutionMode.PyTorch` | + +-----------------+------------------------+-----------------------+------------------------+ + | *learn()* | Python interpreted | LLVM compiled | PyTorch compiled | + | | | | | + | *run()* | Python interpreted | LLVM compiled | Python interpreted | + +-----------------+------------------------+-----------------------+------------------------+ + | *Speed:* | slow | fastest | fast | + +-----------------+------------------------+-----------------------+------------------------+ + | |* Backpropagation | * Backpropagation |* Backpropagation | + | |* Reinforcement learning| |* RNN, inclduing LSTM | + | *Supports:* |* Unspervised learning | |* Unsupervised learning | + | |* modulation, inspection| | | + +-----------------+------------------------+-----------------------+------------------------+ + .. _Composition_Learning_UDF: @@ -1045,15 +1060,15 @@ ` method for each `TRIAL `. The `execute ` method can also be called directly, but this is useful mostly for debugging. -.. hint:: - Once a Composition has been constructed, it can be called directly. If it is called with no arguments, and - has executed previously, the `result ` of the last `TRIAL ` - of execution is returned; otherwise None is returned. If it is called with arguments, then either `run - ` or `learn ` is called, based on the arguments provided: If the - Composition has any `learning_pathways `, and the relevant `TARGET_MECHANISM - `\\s are specified in the `inputs argument `, - then `learn ` is called; otherwise, `run ` is called. In either case, - the return value of the corresponding method is returned. + .. hint:: + Once a Composition has been constructed, it can be called directly. If it is called with no arguments, and + has executed previously, the `result ` of the last `TRIAL ` + of execution is returned; otherwise None is returned. If it is called with arguments, then either `run + ` or `learn ` is called, based on the arguments provided: If the + Composition has any `learning_pathways `, and the relevant `TARGET_MECHANISM + `\\s are specified in the `inputs argument `, + then `learn ` is called; otherwise, `run ` is called. In either case, + the return value of the corresponding method is returned. .. _Composition_Execution_Num_Trials: @@ -1137,17 +1152,18 @@ .. _Composition_Input_Internal_Only: -.. note:: - Most Mechanisms have only a single `InputPort`, and thus require only a single input to be specified for - them for each `TRIAL `. However some Mechanisms have more than one InputPort (for example, - a `ComparatorMechanism`), in which case inputs can be specified for some or all of them (see `below - `). Conversely, some Mechanisms have InputPorts that are designated - as `internal_only ` (for example, the `input_port ` for a - `RecurrentTransferMechanism`, if its `has_recurrent_input_port ` - attribute is True), in which case no input should be specified for those input_ports. Similar considerations - extend to the `external_input_ports_of_all_input_nodes ` of a - `nested Composition `, based on the Mechanisms (and/or additionally nested Compositions) that - comprise its set of `INPUT` `Nodes `. + .. note:: + Most Mechanisms have only a single `InputPort`, and thus require only a single input to be specified for + them for each `TRIAL `. However some Mechanisms have more than one InputPort (for example, + a `ComparatorMechanism`), in which case inputs can be specified for some or all of them (see `below + `). Conversely, some Mechanisms have InputPorts that + are designated as `internal_only ` (for example, the `input_port + ` for a `RecurrentTransferMechanism`, if its `has_recurrent_input_port + ` attribute is True), in which case no input should be + specified for those input_ports. Similar considerations extend to the `external_input_ports_of_all_input_nodes + ` of a `nested Composition `, + based on the Mechanisms (and/or additionally nested Compositions) thatcomprise its set of `INPUT` `Nodes + `. The factors above determine the format of each entry in an `inputs dictionary `, or the return value of the function or generator used for `programmatic specification ` of @@ -1333,10 +1349,10 @@ the function must return the input values for each `INPUT` `Node ` for a single `TRIAL `. -.. note:: - Default behavior when passing a function as input to a Composition is to execute for only one `TRIAL - `. Remember to set the num_trials argument of Composition.run if you intend to cycle through - multiple `TRIAL `\\s. + .. note:: + Default behavior when passing a function as input to a Composition is to execute for only one `TRIAL + `. Remember to set the num_trials argument of Composition.run if you intend to cycle through + multiple `TRIAL `\\s. Complete input specification: @@ -1370,10 +1386,10 @@ standard input specification. The only difference is that on each execution, the generator must yield the input values for each `INPUT` `Node ` for a single `TRIAL `. -.. note:: - Default behavior when passing a generator is to execute until the generator is exhausted. If the num_trials - argument of Composition.run is set, the Composition will execute EITHER until exhaustion, or until num_trials has - been reached - whichever comes first. + .. note:: + Default behavior when passing a generator is to execute until the generator is exhausted. If the num_trials + argument of Composition.run is set, the Composition will execute EITHER until exhaustion, or until num_trials + has been reached - whichever comes first. Complete input specification: @@ -1566,10 +1582,10 @@ def input_function(env, result): within another is added to the one in which it is nested, and all are treated as part of the same cycle. All Nodes within a cycle are assigned the `NodeRole` `CYCLE`. -.. note:: - A `RecurrentTransferMechanism` (and its subclaseses) are treated as single-Node cylces, formed by their - `AutoAssociativeProjection` (since the latter is subclass of MappingProjection and thus not designated as feedback - (see `below `). + .. note:: + A `RecurrentTransferMechanism` (and its subclaseses) are treated as single-Node cylces, formed by their + `AutoAssociativeProjection` (since the latter is subclass of MappingProjection and thus not designated as + feedback (see `below `). .. _Composition_Cycle_Synchronous_Execution: @@ -1587,13 +1603,13 @@ def input_function(env, result): FIGURE HERE COMMENT -.. note:: - Although all the Nodes in a cycle receive either the initial value or previous value of other Nodes in the cycle, - they receive the *current* value of any Nodes that project to them from *outisde* the cycle, and pass their current - value (i.e., the ones computed in the current execution of the cycle) to any Nodes to which they project outside of - the cycle. The former means that any Nodes within the cycle that receive such input are "a step ahead" of those - within the cycle and also, unless the use a `StatefulFunction`, others within the cycle will not see the effects of - that input within or across `TRIALS `. + .. note:: + Although all the Nodes in a cycle receive either the initial value or previous value of other Nodes in the cycle, + they receive the *current* value of any Nodes that project to them from *outisde* the cycle, and pass their + current value (i.e., the ones computed in the current execution of the cycle) to any Nodes to which they project + outside of the cycle. The former means that any Nodes within the cycle that receive such input are "a step + ahead" of those within the cycle and also, unless the use a `StatefulFunction`, others within the cycle will + not see the effects of that input within or across `TRIALS `. .. _Composition_Cycle_Initialization: @@ -1607,18 +1623,18 @@ def input_function(env, result): cycle in that run, whereas any Nodes not specified will retain the last `value ` they were assigned in the uprevious call to `run ` or `learn `. -Nodes in a cycle can also be initialized outside of a call to `run ` or `learn ` using -the `initialize ` method. +Nodes in a cycle can also be initialized outside of a call to `run ` or `learn ` +using the `initialize ` method. -.. note:: - If a `Mechanism` belonging to a cycle in a Composition is first executed on its own (i.e., using its own `execute - ` method), the value it is assigned will be used as its initial value when it is executed - within the Composition, unless an `execution_id ` is assigned to the **context** argument - of the Mechanism's `execute ` method when it is called. This is because the first time - a Mechanism is executed in a Composition, its initial value is copied from the `value ` - last assigned in the None context. As described aove, this can be overridden by specifying an initial value for - the Mechanism in the **initialize_cycle_values** argument of the call to the Composition's `run ` - or `learn ` methods. + .. note:: + If a `Mechanism` belonging to a cycle in a Composition is first executed on its own (i.e., using its own `execute + ` method), the value it is assigned will be used as its initial value when it is executed + within the Composition, unless an `execution_id ` is assigned to the **context** argument + of the Mechanism's `execute ` method when it is called. This is because the first time + a Mechanism is executed in a Composition, its initial value is copied from the `value ` + last assigned in the None context. As described aove, this can be overridden by specifying an initial value for + the Mechanism in the **initialize_cycle_values** argument of the call to the Composition's `run ` + or `learn ` methods. .. _Composition_Feedback: @@ -1880,41 +1896,60 @@ def input_function(env, result): specified and fails, an error is generated indicating the unsupported feature that failed. The compiled modes, in order of their power, are: -.. _Composition_Compilation_LLVM: +.. _Composition_Compilation_Modes: * *True* -- try to use the one that yields the greatesst improvement, progressively reverting to less powerful but more forgiving modes, in the order listed below, for each that fails; - * *LLVMRun* -- compile and run multiple `TRIAL `\\s; if successful, the compiled binary is - semantically equivalent to the execution of the `run ` method using the Python interpreter; + * `ExecutionMode.LLVMRun` - compile and run multiple `TRIAL `\\s; if successful, + the compiled binary is semantically equivalent to the execution of the `run ` method + using the Python interpreter; + + * `ExecutionMode.LLVMExec` -- compile and run each `TRIAL `, using the Python interpreter + to iterate over them; if successful, the compiled binary for each `TRIAL ` is semantically + equivalent the execution of the `execute ` method using the Python interpreter; + + * `ExecutionMode.LLVM` -- compile and run `Node ` of the `Composition` and their `Projections + `, using the Python interpreter to call the Composition's `scheduler `, + execute each Node and iterate over `TRIAL `\\s; note that, in this mode, scheduling + `Conditions ` that rely on Node `Parameters` is not supported; + + * `ExecutionMode.Python` (same as *False*; the default) -- use the Python interpreter to execute the `Composition`. + + * `ExecutionMode.PyTorch` -- used only for `AutodiffComposition`: executes `learn ` + using `PyTorch` and `run ` using Python interpreter (see `below + ` for additional details). + + .. warning:: + For clarity, `ExecutionMode.PyTorch` should only be used when executing an `AutodiffComposition`; + using it with a standard `Composition` is possible, but it will **not** have the expected effect of + executing its `learn ` method using PyTorch. - * *LLVMExec* -- compile and run each `TRIAL `, using the Python interpreter to iterate over them; - if successful, the compiled binary for each `TRIAL ` is semantically equivalent the execution - of the `execute ` method using the Python interpreter; + * `ExecutionMode.PTXrun` -- compile multiple `TRIAL `\\s for execution on GPU + (see `below ` for additional details). - * *LLVM* -- compile and run `Node ` of the `Composition` and their `Projections `, - using the Python interpreter to call the Composition's `scheduler `, execute each Node - and iterate over `TRIAL `\\s; note that, in this mode, scheduling `Conditions ` - that rely on Node `Parameters` is not supported; + * `ExecutionMode.PTXExec` -- compile individual `TRIAL `\\s for execution on GPU + (see `below ` for additional details). - * *Python* (same as *False*; the default) -- use the Python interpreter to execute the `Composition`. +.. _Composition_Compilation_PyTorch: + +*PyTorch support.* When using an `AutodiffComposition`, `ExecutionMode.PyTorch` can be used to execute its +`learn ` method using Pytorch; however, its `run ` method +will execute using the Python interpreter. See `Composition_Learning_AutodiffComposition` for additional details. .. _Composition_Compilation_PTX: *GPU support.* In addition to compilation for CPUs, support is being developed for `CUDA `_ capable `Invidia GPUs -`_. This can be invoked by specifying one -of the following modes in the **execution_mode** argument of a `Composition execution method -`: - - * *PTXExec|PTXRun* -- equivalent to the LLVM counterparts but run in a single thread of a CUDA capable GPU. - -This requires that a working `pycuda package `_ is -`installed `_, and that CUDA execution is explicitly enabled by setting -the ``PNL_LLVM_DEBUG`` environment variable to ``cuda``. At present compilation using these modes runs on a single -GPU thread, and therefore does not produce any performance benefits over running in compiled mode on a CPU; (see -`this `_ for progress extending support of parallization -in compiled modes). +`_. This can be invoked by +specifying either `ExecutionMode.PTXRun` or `ExecutionMode.PTXExec` oin the **execution_mode** argument +of a `Composition execution method `, which are equivalent to the LLVM +counterparts but run in a single thread of a CUDA capable GPU. This requires that a working `pycuda package +`_ is `installed `_, and that +CUDA execution is explicitly enabled by setting the ``PNL_LLVM_DEBUG`` environment variable to ``cuda``. At present +compilation using these modes runs on a single GPU thread, and therefore does not produce any performance benefits +over running in compiled mode on a CPU; (see `this `_ +for progress extending support of parallization in compiled modes). .. _Composition_Execution_Results_and_Reporting: @@ -2153,29 +2188,32 @@ def input_function(env, result): AS SOME InputPorts CAN HAVE FUNCTIONS THAT CHANGE THE SHAPE OF variable->value (e.g., Reduce) # Furthermore, Mechanisms can also have InputPorts with a `function ` that changes # the size of its input when generatings its `value `, in which case its `e -.. note:: - A `Node's ` `external_input_values` attribute is always a 2d list in which the index i - element is the variable of the i'th element of the Node's `external_input_ports` attribute. For Mechanisms, - the `external_input_values ` is often the same as its `variable - `. However, some Mechanisms may have InputPorts marked as `internal_only - ` which are excluded from its `external_input_ports ` - and therefore its `external_input_values `, and so should not receive an - input value. The same considerations extend to the `external_input_ports ` - and `external_input_values ` of a Composition, based on the Mechanisms and/or - `nested Compositions ` that comprise its `INPUT` Nodes. + .. note:: + A `Node's ` `external_input_values` attribute is always a 2d list in which the index i + element is the variable of the i'th element of the Node's `external_input_ports` attribute. For Mechanisms, + the `external_input_values ` is often the same as its `variable + `. However, some Mechanisms may have InputPorts marked as `internal_only + ` which are excluded from its `external_input_ports + ` and therefore its `external_input_values + `, and so should not receive an + input value. The same considerations extend to the `external_input_ports + ` and `external_input_values ` + of a Composition, based on the Mechanisms and/or `nested Compositions ` + that comprise its `INPUT` Nodes. MODIFIED 2/4/22 NEW: COMMENT -.. note:: - A `Node's ` `external_input_variables` attribute is always a 2d list in which the index i - element is the variable of the i'th element of the Node's `external_input_ports` attribute. For Mechanisms, - the `external_input_variables ` is often the same as its `variable - `. However, some Mechanisms may have InputPorts marked as `internal_only - ` which are excluded from its `external_input_ports ` - and therefore its `external_input_variables `, and so should not receive - an input value. The same considerations extend to the `external_input_ports_of_all_input_nodes - ` and `external_input_variables - ` of a Composition, based on the Mechanisms and/or `nested Compositions - ` that comprise its `INPUT` Nodes. + .. note:: + A `Node's ` `external_input_variables` attribute is always a 2d list in which the index i + element is the variable of the i'th element of the Node's `external_input_ports` attribute. For Mechanisms, + the `external_input_variables ` is often the same as its `variable + `. However, some Mechanisms may have InputPorts marked as `internal_only + ` which are excluded from its `external_input_ports + ` and therefore its `external_input_variables + `, and so should not receive + an input value. The same considerations extend to the `external_input_ports_of_all_input_nodes + ` and `external_input_variables + ` of a Composition, based on the Mechanisms and/or + `nested Compositions ` that comprise its `INPUT` Nodes. If num_trials is not in use, the number of inputs provided determines the number of `TRIAL `\\s in the run. For example, if five inputs are provided for each `INPUT` `Node `, and num_trials is not @@ -2520,9 +2558,9 @@ def input_function(env, result): as indicated by the results of S.run(), the original parameter values were used on trials 0 and 1, the runtime intercept was used on trials 2, 3, and 4, and the runtime slope was used on trial 3. -.. note:: - Runtime parameter values are subject to the same type, value, and shape requirements as the original parameter - value. + .. note:: + Runtime parameter values are subject to the same type, value, and shape requirements as the original parameter + value. .. _Composition_Examples_Execution_Context: @@ -2735,11 +2773,11 @@ def input_function(env, result): from psyneulink.core.components.component import Component, ComponentsMeta from psyneulink.core.components.functions.fitfunctions import make_likelihood_function from psyneulink.core.components.functions.function import is_function_type, RandomMatrix -from psyneulink.core.components.functions.nonstateful.combinationfunctions import LinearCombination, \ - PredictionErrorDeltaFunction +from psyneulink.core.components.functions.nonstateful.combinationfunctions import \ + LinearCombination, PredictionErrorDeltaFunction from psyneulink.core.components.functions.nonstateful.learningfunctions import \ LearningFunction, Reinforcement, BackPropagation, TDLearning -from psyneulink.core.components.functions.nonstateful.transferfunctions import Identity +from psyneulink.core.components.functions.nonstateful.transferfunctions import Identity, Logistic, SoftMax from psyneulink.core.components.mechanisms.mechanism import Mechanism_Base, MechanismError, MechanismList from psyneulink.core.components.mechanisms.modulatory.control.controlmechanism import ControlMechanism from psyneulink.core.components.mechanisms.modulatory.control.optimizationcontrolmechanism import AGENT_REP, \ @@ -2766,20 +2804,20 @@ def input_function(env, result): from psyneulink.core.components.shellclasses import Mechanism, Projection from psyneulink.core.compositions.report import Report, \ ReportOutput, ReportParams, ReportProgress, ReportSimulations, ReportDevices, \ - EXECUTE_REPORT, CONTROLLER_REPORT, RUN_REPORT, PROGRESS_REPORT + EXECUTE_REPORT, CONTROLLER_REPORT, RUN_REPORT, COMPILED_REPORT, PROGRESS_REPORT from psyneulink.core.compositions.showgraph import ShowGraph, INITIAL_FRAME, SHOW_CIM, EXECUTION_SET, SHOW_CONTROLLER from psyneulink.core.globals.context import Context, ContextFlags, handle_external_context from psyneulink.core.globals.keywords import \ - AFTER, ALL, ALLOW_PROBES, ANY, BEFORE, COMPONENT, COMPOSITION, CONTROL, CONTROL_SIGNAL, CONTROLLER, DEFAULT, \ - DICT, FEEDBACK, FULL, FUNCTION, HARD_CLAMP, IDENTITY_MATRIX, INPUT, INPUT_PORTS, INPUTS, INPUT_CIM_NAME, \ - LEARNED_PROJECTIONS, LEARNING_FUNCTION, LEARNING_MECHANISM, LEARNING_MECHANISMS, LEARNING_PATHWAY, \ - MATRIX, MATRIX_KEYWORD_VALUES, MAYBE, \ - MODEL_SPEC_ID_METADATA, \ + AFTER, ALL, ALLOW_PROBES, ANY, BEFORE, COMPONENT, COMPOSITION, CONTROL, CONTROL_SIGNAL, CONTROLLER, CROSS_ENTROPY, \ + DEFAULT, DICT, FEEDBACK, FULL, FUNCTION, HARD_CLAMP, IDENTITY_MATRIX, \ + INPUT, INPUT_PORTS, INPUTS, INPUT_CIM_NAME, \ + LEARNED_PROJECTIONS, LEARNING_FUNCTION, LEARNING_MECHANISM, LEARNING_MECHANISMS, LEARNING_PATHWAY, Loss, \ + MATRIX, MATRIX_KEYWORD_VALUES, MAYBE, MODEL_SPEC_ID_METADATA, \ MONITOR, MONITOR_FOR_CONTROL, NAME, NESTED, NO_CLAMP, NODE, OBJECTIVE_MECHANISM, ONLINE, OUTCOME, \ OUTPUT, OUTPUT_CIM_NAME, OUTPUT_MECHANISM, OUTPUT_PORTS, OWNER_VALUE, \ PARAMETER, PARAMETER_CIM_NAME, PORT, \ PROCESSING_PATHWAY, PROJECTION, PROJECTION_TYPE, PROJECTION_PARAMS, PULSE_CLAMP, RECEIVER, \ - SAMPLE, SENDER, SHADOW_INPUTS, SOFT_CLAMP, SSE, \ + SAMPLE, SENDER, SHADOW_INPUTS, SOFT_CLAMP, SUM, \ TARGET, TARGET_MECHANISM, TEXT, VARIABLE, WEIGHT, OWNER_MECH from psyneulink.core.globals.log import CompositionLog, LogCondition from psyneulink.core.globals.parameters import Parameter, ParametersBase, check_user_specified @@ -2793,7 +2831,8 @@ def input_function(env, result): from psyneulink.core.scheduling.time import Time, TimeScale from psyneulink.library.components.mechanisms.modulatory.learning.autoassociativelearningmechanism import \ AutoAssociativeLearningMechanism -from psyneulink.library.components.mechanisms.processing.objective.comparatormechanism import ComparatorMechanism, MSE +from psyneulink.library.components.mechanisms.processing.objective.comparatormechanism import \ + ComparatorMechanism, OUTCOME, MSE, SSE, L0, L1, CROSS_ENTROPY from psyneulink.library.components.mechanisms.processing.objective.predictionerrormechanism import \ PredictionErrorMechanism from psyneulink.library.components.mechanisms.processing.transfer.recurrenttransfermechanism import \ @@ -7183,7 +7222,7 @@ def add_linear_learning_pathway(self, specifies the type of `LearningFunction` to use for the `LearningMechanism` constructued for each `MappingProjection` in the **pathway**. - loss_function : MSE or SSE : default None + loss_function : Loss : default Loss.MSE specifies the loss function used if `BackPropagation` is specified as the **learning_function** (see `add_backpropagation_learning_pathway `). @@ -7249,7 +7288,7 @@ def add_linear_learning_pathway(self, # Preserve existing NodeRole.OUTPUT status for any non-learning-related nodes for node in self.get_nodes_by_role(NodeRole.OUTPUT): - if not any(node for node in [pathway for pathway in self.pathways + if not any(n for n in [pathway for pathway in self.pathways if PathwayRole.LEARNING in pathway.roles]): self._add_required_node_role(node, NodeRole.OUTPUT, context) @@ -7443,7 +7482,7 @@ def add_backpropagation_learning_pathway(self, pathway, learning_rate=0.05, error_function=None, - loss_function:tc.enum(MSE,SSE)=MSE, + loss_function:tc.enum(Loss)=Loss.MSE, learning_update:tc.optional(tc.any(bool, tc.enum(ONLINE, AFTER)))=AFTER, default_projection_matrix=None, name:str=None): @@ -7464,9 +7503,8 @@ def add_backpropagation_learning_pathway(self, specifies the function assigned to `ComparatorMechanism` used to compute the error from the target and the output (`value `) of the `TARGET` (last) Mechanism in the **pathway**). - loss_function : MSE or SSE : default MSE - specifies the loss function used in computing the error term; - MSE = mean squared error, and SSE = sum squared error. + loss_function : Loss : default Loss.MSE + specifies the loss function used in computing the error term; see `Loss` for values. learning_update : Optional[bool|ONLINE|AFTER] : default AFTER specifies when the `matrix ` parameters of the `learned_projections` are updated @@ -7723,7 +7761,7 @@ def _create_backpropagation_learning_pathway(self, pathway, learning_rate=0.05, error_function=None, - loss_function=MSE, + loss_function=Loss.MSE, learning_update=AFTER, default_projection_matrix=None, name=None, @@ -7733,7 +7771,7 @@ def _create_backpropagation_learning_pathway(self, if not error_function: error_function = LinearCombination() if not loss_function: - loss_function = MSE + loss_function = Loss.MSE # Add pathway to graph and get its full specification (includes all ProcessingMechanisms and MappingProjections) # Pass ContextFlags.INITIALIZING so that it can be passed on to _analyze_graph() and then @@ -7938,7 +7976,8 @@ def bfs(start): pathways = [p for n in self.get_nodes_by_role(NodeRole.INPUT) if NodeRole.TARGET not in self.get_roles_by_node(n) for p in bfs(n)] for pathway in pathways: - self.add_backpropagation_learning_pathway(pathway=pathway) + self.add_backpropagation_learning_pathway(pathway=pathway, + loss_function=self.loss_spec) def _create_terminal_backprop_learning_components(self, input_source, @@ -7962,17 +8001,47 @@ def _create_terminal_backprop_learning_components(self, # Otherwise, create new ones except KeyError: + # # MODIFIED 11/12/22 OLD: + # target_mechanism = ProcessingMechanism(name='Target', + # default_variable=output_source.output_ports[0].value) + # objective_mechanism = ComparatorMechanism(name='Comparator', + # target={NAME: TARGET, + # VARIABLE: target_mechanism.output_ports[0].value}, + # sample={NAME: SAMPLE, + # VARIABLE: output_source.output_ports[0].value, + # WEIGHT: -1}, + # function=error_function, + # output_ports=[OUTCOME, Loss.MSE], + # ) + # # MODIFIED 11/12/22 NEW: target_mechanism = ProcessingMechanism(name='Target', default_variable=output_source.output_ports[0].value) + # Base for object_mechanism output_ports: + sample={NAME: SAMPLE, + VARIABLE: output_source.output_ports[0].value} + target={NAME: TARGET, + VARIABLE: target_mechanism.output_ports[0].value} + if loss_function == Loss.CROSS_ENTROPY: + # error function: use LinearCombination to implement cross_entropy: (SoftMax(sample), SoftMax(target)) + sample.update({FUNCTION: SoftMax(output=ALL)}) + target.update({FUNCTION: SoftMax(output=ALL)}) + error_function = LinearCombination(operation=CROSS_ENTROPY) + output_ports = [OUTCOME, SUM] + else: + # error_function: use default for Comparator (LinearCombination) => target - sample + sample.update({WEIGHT: -1}) + if loss_function == Loss.L0: + output_ports = [OUTCOME, SUM] + elif loss_function == Loss.SSE: + output_ports = [OUTCOME, SSE] + else: + output_ports = [OUTCOME, MSE] objective_mechanism = ComparatorMechanism(name='Comparator', - target={NAME: TARGET, - VARIABLE: target_mechanism.output_ports[0].value}, - sample={NAME: SAMPLE, - VARIABLE: output_source.output_ports[0].value, - WEIGHT: -1}, + sample=sample, + target=target, function=error_function, - output_ports=[OUTCOME, MSE], - ) + output_ports=output_ports) + # MODIFIED 11/12/22 END learning_function = BackPropagation(default_variable=[input_source.output_ports[0].value, output_source.output_ports[0].value, @@ -8045,6 +8114,7 @@ def _create_non_terminal_backprop_learning_components(self, learning_function = BackPropagation(default_variable=[input_source.output_ports[0].value, output_source.output_ports[0].value, error_signal_template[0]], + loss_function=None, activation_derivative_fct=output_source.function.derivative, learning_rate=learning_rate) @@ -9299,9 +9369,9 @@ def _instantiate_input_dict(self, inputs): # Get number of trials of input specified for Port num_trials = len(port_input) if max_num_trials != 1 and num_trials not in {1, max_num_trials}: - raise CompositionError(f"Number of trials of input specified for {port.full_name} of {node.name} " - f"({num_trials}) is different from the number ({max_num_trials}) " - f"specified for one or more others.") + raise CompositionError(f"Number of trials of input specified for {port.full_name} of" + f"{INPUT_Node.name} ({num_trials}) is different from the" + f"number ({max_num_trials}) specified for one or more others.") max_num_trials = max(num_trials, max_num_trials) # Construct node_input_shape based on max_num_trials across all input_ports for mech @@ -9781,17 +9851,16 @@ def run( for the current and all future runs of the Composition. See `Scheduler_Execution` - execution_mode : enum.Enum[Auto|LLVM|LLVMexec|LLVMRun|Python|PTXExec|PTXRun] : default Python + execution_mode : bool or ExecutionMode : default ExecutionMode.Python specifies whether to run using the Python interpreter or a `compiled mode `. - False is the same as ``Python``; True tries LLVM compilation modes, in order of power, progressively + False uses the Python interpreter; True tries LLVM compilation modes, in order of power, progressively reverting to less powerful modes (in the order of the options listed), and to Python if no compilation - mode succeeds (see `Composition_Compilation` for explanation of modes). PTX modes are used for - CUDA compilation. + mode succeeds; see `ExecutionMode` for other options, and `Compilation Modes + ` for a more detailed explanation of their operation. default_absolute_time_unit : ``pint.Quantity`` : ``1ms`` - if not otherwise determined by any absolute **conditions**, - specifies the absolute duration of a `TIME_STEP`. See - `Scheduler.default_absolute_time_unit` + if not otherwise determined by any absolute **conditions**, specifies the absolute duration + of a `TIME_STEP`. See `Scheduler.default_absolute_time_unit` context : `execution_id ` : default `default_execution_id` context in which the `Composition` will be executed; set to self.default_execution_id ifunspecified. @@ -9841,6 +9910,12 @@ def run( execution_phase = context.execution_phase context.execution_phase = ContextFlags.PREPARING + # IMPLEMENTATION NOTE: Restore if ExecutionMode.PyTorch can be distinguished from ExecutionMode.Python + # from psyneulink.library.compositions.autodiffcomposition import AutodiffComposition + # if execution_mode is pnlvm.ExecutionMode.PyTorch and not isinstance(self, AutodiffComposition): + # warnings.warn(f"{pnlvm.ExecutionMode.PyTorch.name} is being used to execute {self.name} " + # f"but it is not an AutodiffComposition, therefore PyTorch will not be used.") + for node in self.nodes: num_execs = node.parameters.num_executions._get(context) if num_execs is None: @@ -9962,8 +10037,8 @@ def run( if not valid_reset_type: raise CompositionError( f"{reset_stateful_functions_when} is not a valid specification for reset_integrator_nodes_when " - f"of {self.name}. reset_integrator_nodes_when must be a Condition or a dict comprised of " - f" {Node: Condition} pairs.") + f"of {self.name}. reset_integrator_nodes_when must be a Condition or a dict comprised of " + + "{Node: Condition pairs.") self._reset_stateful_functions_when_cache = {} @@ -9988,33 +10063,60 @@ def run( # There's no mode to run simulations. # Simulations are run as part of the controller node wrapper. assert not is_simulation - try: - comp_ex_tags = frozenset({"learning"}) if self._is_learning(context) else frozenset() - _comp_ex = pnlvm.CompExecution.get(self, context, additional_tags=comp_ex_tags) - if execution_mode & pnlvm.ExecutionMode.LLVM: - results += _comp_ex.run(inputs, num_trials, num_inputs_sets) - elif execution_mode & pnlvm.ExecutionMode.PTX: - results += _comp_ex.cuda_run(inputs, num_trials, num_inputs_sets) - else: - assert False, "Unknown execution mode: {}".format(execution_mode) - # Update the parameter for results - self.parameters.results._set(results, context) + with Report(self, + report_output=report_output, + report_params=report_params, + report_progress=report_progress, + report_simulations=report_simulations, + report_to_devices=report_to_devices, + context=context) as report: + + report_num = report.start_report(self, num_trials, context) + + report(self, + [COMPILED_REPORT, PROGRESS_REPORT], + report_num=report_num, + scheduler=scheduler, + content='run_start', + context=context) + + try: + comp_ex_tags = frozenset({"learning"}) if self._is_learning(context) else frozenset() + _comp_ex = pnlvm.CompExecution.get(self, context, additional_tags=comp_ex_tags) + if execution_mode & pnlvm.ExecutionMode.LLVM: + results += _comp_ex.run(inputs, num_trials, num_inputs_sets) + elif execution_mode & pnlvm.ExecutionMode.PTX: + results += _comp_ex.cuda_run(inputs, num_trials, num_inputs_sets) + else: + assert False, "Unknown execution mode: {}".format(execution_mode) + + # Update the parameter for results + self.parameters.results._set(results, context) + + if self._is_learning(context): + # copies back matrix to pnl from param struct (after learning) + _comp_ex._copy_params_to_pnl(context=context) - if self._is_learning(context): - # copies back matrix to pnl from param struct (after learning) - _comp_ex._copy_params_to_pnl(context=context) + self._propagate_most_recent_context(context) + + report(self, + [COMPILED_REPORT, PROGRESS_REPORT], + report_num=report_num, + scheduler=scheduler, + content='run_end', + context=context, + node=self) - self._propagate_most_recent_context(context) - # KAM added the [-1] index after changing Composition run() - # behavior to return only last trial of run (11/7/18) - return results[-1] + # KAM added the [-1] index after changing Composition run() + # behavior to return only last trial of run (11/7/18) + return results[-1] - except Exception as e: - if not execution_mode & pnlvm.ExecutionMode._Fallback: - raise e from None + except Exception as e: + if not execution_mode & pnlvm.ExecutionMode._Fallback: + raise e from None - warnings.warn("Failed to run `{}': {}".format(self.name, str(e))) + warnings.warn("Failed to run `{}': {}".format(self.name, str(e))) # Reset gym forager environment for the current trial if self.env: @@ -10281,7 +10383,7 @@ def learn( `; see `Report_Simulations` for additional details. report_to_devices : list(ReportDevices) : default ReportDevices.CONSOLE - specifies where output and progress should be reported; see `Report_To_Devices` for additional + specifies where output and progress should be reported; see `Report_To_Device` for additional details and `ReportDevices` for options. Returns diff --git a/psyneulink/core/compositions/report.py b/psyneulink/core/compositions/report.py index c4d42de68c2..7f4476e8709 100644 --- a/psyneulink/core/compositions/report.py +++ b/psyneulink/core/compositions/report.py @@ -79,7 +79,6 @@ .. _Report_To_Device: - Devices ------- @@ -168,7 +167,7 @@ 'ReportLearning', 'CONSOLE', 'CONTROLLED', 'LOGGED', 'MODULATED', 'MONITORED', 'RECORD', 'DIVERT', 'PNL_VIEW', ] -# Used to specify self._run_mode +# Used to specify self._run_mode (specified as roots, that are conjugated in messages) DEFAULT = 'Execut' LEARNING = 'Train' SIMULATION = 'Simulat' @@ -182,6 +181,7 @@ CONTROLLER_REPORT = 'controller_report' LEARN_REPORT = 'learn_report' RUN_REPORT = 'run_report' +COMPILED_REPORT = 'compiled_report' PROGRESS_REPORT = 'progress_report' trial_sep_str = f'====================' @@ -925,7 +925,8 @@ def __call__(self, simulation_mode = context.runmode & ContextFlags.SIMULATION_MODE # Call report_output - if any(r in {EXECUTE_REPORT, MECHANISM_REPORT, CONTROLLER_REPORT, LEARN_REPORT, RUN_REPORT} for r in reports): + if any(r in {EXECUTE_REPORT, MECHANISM_REPORT, CONTROLLER_REPORT, + LEARN_REPORT, RUN_REPORT, COMPILED_REPORT} for r in reports): if content in {'run_start', 'execute_start'}: if simulation_mode: @@ -950,9 +951,7 @@ def __call__(self, if PROGRESS_REPORT in reports: # Just pass args relevant to report_progress() progress_args = {k:v for k,v in kwargs.items() if k in {'caller', 'report_num', 'content', 'context'}} - self.report_progress(caller, **progress_args) - - assert True + self.report_progress(caller, reports, **progress_args) def report_output(self, caller, @@ -1690,6 +1689,7 @@ def is_logged(node, name): def report_progress(self, caller, + reports, report_num:int, content:str, context:Context) -> None: @@ -1737,6 +1737,29 @@ def report_progress(self, # Update progress report if self._use_rich: + + # MODIFIED 11/12/22 NEW: + if content == 'run_start' and COMPILED_REPORT in reports and self._run_mode == LEARNING: + composition_type_name = list(self.output_reports.keys())[0].componentCategory + composition_name = list(self.output_reports.keys())[0].name + message = f"{composition_type_name} '{composition_name}' training " \ + f"{output_report.num_trials} trials using PyTorch..." + # Method 1: (direct print) + # self._rich_progress.console.print(message) + output_report.run_report = message + # Method 2: (ensure it is also recorded if that is enabled) + if self._record_reports: + with self._recording_console.capture() as capture: + self._recording_console.print(node_report) + self._recorded_reports += capture.get() + # Method 3: (shadow standard processing) + # self._rich_progress.update(output_report.rich_task_id, + # total=1, + # description=message, + # advance=1, + # refresh=True) + # MODIFIED 11/12/22 END + if content == 'run_end': # If it is the end of a run, and num_trials was not known (and so rich progress was "indeterminate"), # close out progress bar @@ -1776,12 +1799,18 @@ def report_progress(self, advance=1, refresh=True) - # FIX: NEED COMMENT ON WHY THIS IS NEEDED: - # WITHOUT THIS, WHEN RECORD_DEVICES IS ACTIVE, - # EITHER PROGRESS REPORT IS MISSING OR IT IS DUPLICATED ABOVE THE OUTPUT REPORT + # This is needed so that, when _record_reports is active, progress report is generated and not duplicated if self._report_output is ReportOutput.OFF or self._report_progress is ReportProgress.OFF: - self._print_and_record_reports(PROGRESS_REPORT) - assert True + # # MODIFIED 11/12/22 OLD: + # self._print_and_record_reports(PROGRESS_REPORT) + # # MODIFIED 11/12/22 NEW: FIX: MAY BE A PROBLEM IF RUN_REPORT IS ALSO EVER PASSED IN HERE + # self._print_and_record_reports(reports) + # # MODIFIED 11/12/22 NEWER + if COMPILED_REPORT in reports: + self._print_and_record_reports(COMPILED_REPORT, output_report) + else: + self._print_and_record_reports(PROGRESS_REPORT) + # MODIFIED 11/12/22 END def _print_and_record_reports(self, report_type:str, output_report:OutputReport=None) -> None: """ @@ -1800,14 +1829,18 @@ def _print_and_record_reports(self, report_type:str, output_report:OutputReport= OutputReport for caller[_run_mode] in self.output_reports to use for reporting. """ - # Print and record output report as they are created (progress reports are printed by _rich_progress.console) - if report_type in {EXECUTE_REPORT, RUN_REPORT}: + # Print and record output reports as they are created (progress reports are printed by _rich_progress.console) + # MODIFIED 11/12/22 OLD: + if report_type in {EXECUTE_REPORT, RUN_REPORT, COMPILED_REPORT}: + # # MODIFIED 11/12/22 NEW: + # if any(report in {EXECUTE_REPORT, RUN_REPORT, COMPILED_REPORT} for report in report_type): + # MODIFIED 11/12/22 END # Print output reports as they are created if self._rich_console or self._rich_divert: - if output_report.trial_report and report_type is EXECUTE_REPORT: + if output_report.trial_report and report_type == EXECUTE_REPORT: self._rich_progress.console.print(output_report.trial_report) self._rich_progress.console.print('') - elif output_report.run_report and report_type is RUN_REPORT: + elif output_report.run_report and report_type in {RUN_REPORT, COMPILED_REPORT}: self._rich_progress.console.print(output_report.run_report) self._rich_progress.console.print('') # Record output reports as they are created @@ -1818,14 +1851,13 @@ def _print_and_record_reports(self, report_type:str, output_report:OutputReport= with self._recording_console.capture() as capture: if report_type == EXECUTE_REPORT: self._recording_console.print(output_report.trial_report) - elif report_type == RUN_REPORT: + elif report_type in {RUN_REPORT or COMPILED_REPORT}: self._recording_console.print(output_report.run_report) self._recorded_reports += capture.get() # Record progress after execution of outer-most Composition if (self._report_output is not ReportOutput.OFF or (len(self._execution_stack)<=1 and not self._simulating)): - if report_type is PROGRESS_REPORT: # add progress report to any already recorded for output progress_reports = '\n'.join([t.description for t in self._rich_progress.tasks]) diff --git a/psyneulink/core/globals/keywords.py b/psyneulink/core/globals/keywords.py index 5bd996546bf..267e0eac33a 100644 --- a/psyneulink/core/globals/keywords.py +++ b/psyneulink/core/globals/keywords.py @@ -70,7 +70,7 @@ 'LEARNING_OBJECTIVE', 'LEARNING_MECHANISM', 'LEARNING_MECHANISMS', 'LEARNING_PATHWAY', 'LEARNING_PROJECTION', 'LEARNING_PROJECTION_PARAMS', 'LEARNING_RATE', 'LEARNING_SIGNAL', 'LEARNING_SIGNAL_SPECS', 'LEARNING_SIGNALS', 'LESS_THAN', 'LESS_THAN_OR_EQUAL', 'LINEAR', 'LINEAR_COMBINATION_FUNCTION', 'LINEAR_FUNCTION', - 'LINEAR_MATRIX_FUNCTION', 'LOG_ENTRIES', 'LOGISTIC_FUNCTION', 'LOW', 'LVOC_CONTROL_MECHANISM', 'L0', 'L1', + 'LINEAR_MATRIX_FUNCTION', 'LOG_ENTRIES', 'LOGISTIC_FUNCTION', 'Loss', 'LOW', 'LVOC_CONTROL_MECHANISM', 'MAPPING_PROJECTION', 'MAPPING_PROJECTION_PARAMS', 'MASKED_MAPPING_PROJECTION', 'MATRIX', 'MATRIX_KEYWORD_NAMES', 'MATRIX_KEYWORD_SET', 'MATRIX_KEYWORD_VALUES', 'MATRIX_KEYWORDS','MatrixKeywords', 'MAX_ABS_DIFF', 'MAX_ABS_INDICATOR', 'MAX_ONE_HOT', 'MAX_ABS_ONE_HOT', 'MAX_ABS_VAL', @@ -85,7 +85,7 @@ 'MODEL_SPEC_ID_RECEIVER_MECH', 'MODEL_SPEC_ID_RECEIVER_PORT', 'MODEL_SPEC_ID_PARAMETER_INITIAL_VALUE', 'MODEL_SPEC_ID_PARAMETER_SOURCE', 'MODEL_SPEC_ID_PARAMETER_VALUE', 'MODEL_SPEC_ID_TYPE', - 'MSE', 'MULTIPLICATIVE', 'MULTIPLICATIVE_PARAM', 'MUTUAL_ENTROPY', + 'MULTIPLICATIVE', 'MULTIPLICATIVE_PARAM', 'MUTUAL_ENTROPY', 'NAME', 'NESTED', 'NEWEST', 'NODE', 'NOISE', 'NORMAL_DIST_FUNCTION', 'NORMED_L0_SIMILARITY', 'NOT_EQUAL', 'NUM_EXECUTIONS_BEFORE_FINISHED', 'OBJECTIVE_FUNCTION_TYPE', 'OBJECTIVE_MECHANISM', 'OBJECTIVE_MECHANISM_OBJECT', 'OFF', 'OFFSET', 'OLDEST', 'ON', @@ -110,7 +110,7 @@ 'RELU_FUNCTION', 'REST', 'RESULT', 'RESULT', 'ROLES', 'RL_FUNCTION', 'RUN', 'SAMPLE', 'SAVE_ALL_VALUES_AND_POLICIES', 'SCALAR', 'SCALE', 'SCHEDULER', 'SELF', 'SENDER', 'SEPARATE', 'SEPARATOR_BAR', 'SHADOW_INPUT_NAME', 'SHADOW_INPUTS', 'SIMPLE', 'SIMPLE_INTEGRATOR_FUNCTION', 'SIMULATIONS', - 'SINGLETON', 'SIZE', 'SLOPE', 'SOFT_CLAMP', 'SOFTMAX_FUNCTION', 'SOURCE', 'SSE', 'STABILITY_FUNCTION', + 'SINGLETON', 'SIZE', 'SLOPE', 'SOFT_CLAMP', 'SOFTMAX_FUNCTION', 'SOURCE', 'STABILITY_FUNCTION', 'STANDARD_ARGS', 'STANDARD_DEVIATION', 'STANDARD_OUTPUT_PORTS', 'SUBTRACTION', 'SUM', 'TARGET', 'TARGET_MECHANISM', 'TARGET_LABELS_DICT', 'TERMINAL', 'TARGETS', 'TERMINATION_MEASURE', 'TERMINATION_THRESHOLD', 'TERMINATION_COMPARISION_OP', 'TERSE', 'TEXT', 'THRESHOLD', @@ -126,6 +126,7 @@ # ****************************************** KEYWORD CLASSES ********************************************************** # ********************************************************************************************************************** import operator +from enum import Enum, auto class MatrixKeywords: """ @@ -288,6 +289,58 @@ def _is_metric(metric): CONVERGENCE = 'CONVERGENCE' +class Loss(Enum): + """Loss function used for `learning `. + + Each keyword specifies a loss function used for learning in a `Composition` or `AutodiffComposition`, + and the comparable loss functions used by `PyTorch` when an AutodiffComposition is executed in + `ExecutionMode.PyTorch` mode. + COMMENT: + Get latex for remaining equations from https://blmoistawinde.github.io/ml_equations_latex/#cross-entropy + COMMENT + + Attributes + ---------- + + L0 + sum of errors: :math:`\\sum\\limits^{len}_i|target_i - output_i|` + + COMMENT: + L1 + mean + COMMENT + + SSE + sum of squared errors: :math:`\\sum\\limits^{len}_i(target_i - output_i)^2` + + MSE + mean of squared errors: :math:`\\frac{\\sum\\limits^{len}_i(target_i - output_i)^2}{len}` + + CROSS_ENTROPY + cross entropy: :math:`\\sum\\limits^{len}_ioutput_i\\log(target_i)` + + KL_DIV + `Kullback-Leibler (KL) divergence + `_: + :math:`\\sum\\limits^{len}_itarget_i\\log{(\\frac{target_i}{output_i})}` + + NLL + `Negative log likelihood loss `_: + :math:`-{\\log(target_i)}` + + POISSON_NLL + `Poisson negative log likelihood loss `_ + """ + L0 = auto() + L1 = auto() + SSE = auto() + MSE = auto() + CROSS_ENTROPY = auto() + KL_DIV = auto() + NLL = auto() + POISSON_NLL = auto() + + # ********************************************************************************************************************** # ****************************************** CONSTANTS ************************************************************* # ********************************************************************************************************************** @@ -657,9 +710,10 @@ def _is_metric(metric): #endregion -#region --------------------------------------- AUTODIFF COMPOSITION ---------------------------------------------- +#region ------------------------------------------ AUTODIFF COMPOSITION ---------------------------------------------- TRAINING_SET = 'training set' +LEARNING_RATE = "learning_rate" #endregion @@ -673,7 +727,6 @@ def _is_metric(metric): HARD_CLAMP = "hard_clamp" PULSE_CLAMP = "pulse_clamp" NO_CLAMP = "no_clamp" -LEARNING_RATE = "learning_rate" # CONTROL = 'CONTROL' PROCESS_DEFAULT_PROJECTION_FUNCTION = "Default Projection Function" PROCESS_EXECUTE = "ProcessExecute" @@ -980,8 +1033,6 @@ def _is_metric(metric): GAMMA = 'gamma' -MSE = 'MSE' -SSE = 'SSE' #endregion # model spec keywords diff --git a/psyneulink/core/globals/utilities.py b/psyneulink/core/globals/utilities.py index 61c987cb823..ef11156d5b6 100644 --- a/psyneulink/core/globals/utilities.py +++ b/psyneulink/core/globals/utilities.py @@ -1261,17 +1261,14 @@ def __copy__(self): def __getitem__(self, key): if key is None: - raise KeyError("None is not a legal key for {}".format(self.name)) + raise KeyError(f"None is not a legal key for '{self.name}'.") try: return self.data[key] except TypeError: key_num = self._get_key_for_item(key) if key_num is None: - # raise TypeError("\'{}\' is not a key in the {} being addressed". - # format(key, self.__class__.__name__)) - # raise KeyError("\'{}\' is not a key in {}". - raise TypeError("\'{}\' is not a key in {}". - format(key, self.name)) + # raise TypeError(f"'{key}' is not a key in {self.name}.") + raise TypeError(f"'{key}' is not in {self.name}.") return self.data[key_num] def __setitem__(self, key, value): diff --git a/psyneulink/core/llvm/__init__.py b/psyneulink/core/llvm/__init__.py index a62f8c875e3..3f1c5e21067 100644 --- a/psyneulink/core/llvm/__init__.py +++ b/psyneulink/core/llvm/__init__.py @@ -29,6 +29,49 @@ __all__ = ['LLVMBuilderContext', 'ExecutionMode'] class ExecutionMode(enum.Flag): + """Specify execution a `Composition` in interpreted or one of ithe compiled modes. + These are used to specify the **execution_mode** argument of a Composition's `execute `, + `run `, and `learn ` methods. See `Compiled Modes + ` under `Compilation ` for additional details concerning + use of each mode by a Composition. + + Attributes + ---------- + + Python + Execute using the Python interpreter; this is the default mode. + + LLVM + compile and run Composition `Nodes ` and `Projections ` individually. + + LLVMExec + compile and run each `TRIAL ` individually. + + LLVMRun + compile and run multiple `TRIAL `\\s. + + Auto + progressively attempt LLVMRun, LLVMexec. LLVM and then Python. + + PyTorch + execute the `AutodiffComposition` `learn ` method using PyTorch, and its + `run ` method using the Python interpreter. + + .. warning:: + For clarity, this mode should only be used when executing an `AutodiffComposition`; using it + with a standard `Composition` is possible, but it will **not** have the expected effect of executing + its `learn ` method using PyTorch. + + PTX + compile and run Composition `Nodes ` and `Projections ` using CUDA for GPU. + + PTXExec + compile and run each `TRIAL ` using CUDA for GPU. + + PTXRun + compile and run multiple `TRIAL `\\s using CUDA for GPU. + """ + Python = 0 LLVM = enum.auto() PTX = enum.auto() @@ -41,6 +84,7 @@ class ExecutionMode(enum.Flag): LLVMExec = LLVM | _Exec PTXRun = PTX | _Run PTXExec = PTX | _Exec + PyTorch = Python _binary_generation = 0 diff --git a/psyneulink/library/components/mechanisms/processing/objective/comparatormechanism.py b/psyneulink/library/components/mechanisms/processing/objective/comparatormechanism.py index 3a8e169b380..600a421e2ad 100644 --- a/psyneulink/library/components/mechanisms/processing/objective/comparatormechanism.py +++ b/psyneulink/library/components/mechanisms/processing/objective/comparatormechanism.py @@ -152,7 +152,8 @@ from psyneulink.core.components.ports.outputport import OutputPort from psyneulink.core.components.ports.port import _parse_port_spec from psyneulink.core.globals.keywords import \ - COMPARATOR_MECHANISM, FUNCTION, INPUT_PORTS, NAME, OUTCOME, SAMPLE, TARGET, VARIABLE, PREFERENCE_SET_NAME, MSE, SSE + COMPARATOR_MECHANISM, FUNCTION, INPUT_PORTS, NAME, OUTCOME, SAMPLE, TARGET, \ + VARIABLE, PREFERENCE_SET_NAME, Loss, SUM from psyneulink.core.globals.parameters import Parameter, check_user_specified from psyneulink.core.globals.preferences.basepreferenceset import is_pref_set, REPORT_OUTPUT_PREF from psyneulink.core.globals.preferences.preferenceset import PreferenceEntry, PreferenceLevel @@ -160,10 +161,13 @@ is_numeric, is_value_spec, iscompatible, kwCompatibilityLength, kwCompatibilityNumeric, recursive_update from psyneulink.core.globals.utilities import safe_len -__all__ = [ - 'ComparatorMechanism', 'ComparatorMechanismError' -] +__all__ = ['ComparatorMechanism', 'ComparatorMechanismError', 'MSE', 'SSE', 'SSE', 'L0', 'L1', 'CROSS_ENTROPY'] +MSE = Loss.MSE.name +SSE = Loss.SSE.name +L0 = Loss.L0.name +L1 = Loss.L1.name +CROSS_ENTROPY = Loss.CROSS_ENTROPY.name class ComparatorMechanismError(Exception): def __init__(self, error_value): @@ -245,13 +249,19 @@ class ComparatorMechanism(ObjectiveMechanism): .. _COMPARATOR_MECHANISM_SSE + *SUM* + the sum of the terms in in the array returned by the Mechanism's function. + *SSE* - the value of the sum squared error of the Mechanism's function + the sum of squares of the terms in the array returned by the Mechanism's function. .. _COMPARATOR_MECHANISM_MSE *MSE* - the value of the mean squared error of the Mechanism's function + the mean of the squares of the terms returned by the Mechanism's function. + + .. _COMPARATOR_MECHANISM_MSE + """ componentType = COMPARATOR_MECHANISM @@ -316,12 +326,15 @@ class Parameters(ObjectiveMechanism.Parameters): # ComparatorMechanism parameter and control signal assignments): standard_output_ports = ObjectiveMechanism.standard_output_ports.copy() - standard_output_ports.extend([{NAME: SSE, + standard_output_ports.extend([{NAME: SUM, + FUNCTION: lambda x: np.sum(x)}, + {NAME: SSE, FUNCTION: lambda x: np.sum(x * x)}, {NAME: MSE, - FUNCTION: lambda x: np.sum(x * x) / safe_len(x)}]) + FUNCTION: lambda x: np.sum(x * x) / safe_len(x)}] + ) standard_output_port_names = ObjectiveMechanism.standard_output_port_names.copy() - standard_output_port_names.extend([SSE, MSE]) + standard_output_port_names.extend([SUM, Loss.SSE.name, Loss.MSE.name]) @check_user_specified @tc.typecheck diff --git a/psyneulink/library/compositions/autodiffcomposition.py b/psyneulink/library/compositions/autodiffcomposition.py index 9c3518e2237..0a182c7bf84 100644 --- a/psyneulink/library/compositions/autodiffcomposition.py +++ b/psyneulink/library/compositions/autodiffcomposition.py @@ -16,8 +16,11 @@ * `AutodiffComposition_Overview` * `AutodiffComposition_Creation` * `AutodiffComposition_Execution` + - `AutodiffComposition_LLVM` + - `AutodiffComposition_PyTorch` + - `AutodiffComposition_Nested_Modulation` - `AutodiffComposition_Logging` - - `AutodiffComposition_Nested_Execution` + * `AutodiffComposition_Examples` * `AutodiffComposition_Class_Reference` @@ -26,35 +29,64 @@ Overview -------- -.. warning:: As of PsyNeuLink 0.7.5, the API for using AutodiffCompositions has been slightly changed! - Please see `this link ` for more details! +AutodiffComposition is a subclass of `Composition` for constructing and training feedforward neural network +either, using either direct compilation (to LLVM) or automatic conversion to `PyTorch `_, +both of which considerably accelerate training (by as much as three orders of magnitude) compared to the +`standard implementation of learning ` in a Composition. Although an +AutodiffComposition is constructed and executed in much the same way as a standard Composition, it largely restricted +to feedforward neural networks using `supervised learning `, and in particular the +the `backpropagation learning algorithm `_. although it can be used for +some forms of `unsupervised learning ` that are supported in PyTorch (e.g., +`self-organized maps `_). -AutodiffComposition is a subclass of `Composition` used to train feedforward neural network models through integration -with `PyTorch `_, a machine learning library that executes considerably more quickly -than using the `standard implementation of learning ` in a Composition, using its -`learning methods `. An AutodiffComposition is configured and run similarly to a standard -Composition, with some exceptions that are described below. -COMMENT: -FIX: UPDATE runtimes WITH COMPILED VERSION -COMMENT .. _AutodiffComposition_Creation: Creating an AutodiffComposition ------------------------------- -An AutodiffComposition can be created by calling its constructor, and then adding `Components ` using the -standard `Composition methods ` for doing so. The constructor also includes an number of -parameters that are specific to the AutodiffComposition. See `AutodiffComposition_Class_Reference` for a list of -these parameters. +An AutodiffComposition can be created by calling its constructor, and then adding `Components ` using +the standard `Composition methods ` for doing so (e.g., `add_node `, +`add_projection `, `add_linear_processing_pathway +`, etc.). The constructor also includes a number of parameters that are +specific to the AutodiffComposition (see `AutodiffComposition_Class_Reference` for a list of these parameters, +and `examples ` below). Note that all of the Components in an AutodiffComposition +must be able to be subject to `learning `, but cannot include any `learning components +` themselves. Specifically, it cannot include any `ModulatoryMechanisms +`, `LearningProjections `, or the ObjectiveMechanism ` +used to compute the loss for learning. + + .. _Autodiff_Learning_Components_Warning: + .. warning:: + When an AutodiffComposition is constructed, it creates all of the learning Components + that are needed, and thus **cannot include** any that are prespecified. + +COMMENT: +FIX: IS THIS STILL TRUE? SEEMS TO CONTRADICT STATEMENT BELOW: +This means that it cannot be used with a Composition that contains any `modulatory components +` or ones that are subject to modulation, whether by ModulatoryMechanisms within or +outside the Composition; +?MAYBE THE FOLLOWING IS BETTER: +COMMENT +This means that an AutodiffComposition also cannot itself include a `controller ` or any +`ControlMechanisms `. However, it can include Mechanisms that are subject to modulatory control +(see `Figure `, and `modulation `) by ControlMechanisms +*outside* the Composition, including the controller of a Composition within which the AutodiffComposition is nested. +That is, an AutodiffComposition can be `nested in a Composition ` that has such other Components +(see `AutodiffComposition_Nested_Modulation` below). + +A few other restrictions apply to the construction and modification of AutodiffCompositions: -.. warning:: Mechanisms or Projections should not be added to or deleted from an AutodiffComposition after it has - been run for the first time. Unlike an ordinary Composition, AutodiffComposition does not support this - functionality. + .. hint:: AutodiffComposition does not (currently) support the *automatic* construction of separate bias parameters. + Thus, when comparing a model constructed using an AutodiffComposition to a corresponding model in PyTorch, the + `bias ` parameter of PyTorch modules should be set + to `False`. Trainable biases *can* be specified explicitly in an AutodiffComposition by including a + TransferMechanism that projects to the relevant Mechanism (i.e., implementing that layer of the network to + receive the biases) using a `MappingProjection` with a `matrix ` parameter that + implements a diagnoal matrix with values corresponding to the initial value of the biases. -.. warning:: When comparing models built in PyTorch to those using AutodiffComposition, - the `bias ` parameter of PyTorch modules - should be set to `False`, as AutodiffComposition does not currently support trainable biases. + .. warning:: Mechanisms or Projections should not be added to or deleted from an AutodiffComposition after it + has been executed. Unlike an ordinary Composition, AutodiffComposition does not support this functionality. .. _AutodiffComposition_Execution: @@ -62,11 +94,149 @@ Execution --------- +COMMENT: +- Execute learn method using Execute_mode == Python (uses Python) or LLVMRun (direct compilation) using + +It can be run just as a standard Composition would - using `learn ` for learning mode, +and `run ` for test mode. +FIX: CHECK WITH SAMYAK THAT THIS IS CORRECT +COMMENT + An AutodiffComposition's `run `, `execute `, and `learn ` -methods are the same as for a `Composition`. +methods are the same as for a `Composition`. However, the **execution_mode** in the `learn ` +method has different effects than for a standard Composition, that determine whether it uses `LLVM compilation +` or `translation to PyTorch ` to execute learning. +This `table ` provides a summary and comparison of these different modes of execution, +that are described in greater detail below. + + +.. _AutodiffComposition_LLVM: + +*LLVM mode* +~~~~~~~~~~~ + +This is specified by setting **execution_mode** = `ExecutionMode.LLVMRun` in the `learn ` method +of an AutodiffCompositon. This provides the fastest performance, but is limited to `supervised learning +` using the `BackPropagation` algorithm. This can be run using standard forms of +loss, including mean squared error (MSE) and cross entropy, by specifying this in the **loss_spec** argument of +the constructor (see `AutodiffComposition ` for additional details, and +`Compilation Modes ` for more information about executing a Composition in compiled mode. + + .. note:: + Specifying `ExecutionMode.LLVMRUn` in either the `learn ` and `run ` + methods of an AutodiffComposition causes it to (attempt to) use compiled execution in both cases; this is + because LLVM compilation supports the use of modulation in PsyNeuLink models (as compared to `PyTorch mode + `; see `note ` below). -The following is an example showing how to create a -simple AutodiffComposition, specify its inputs and targets, and run it with learning enabled and disabled. + +COMMENT: +The advantage of using an AutodiffComposition is that it allows a model to be implemented in PsyNeuLink, and then +exploit the acceleration of optimized implementations of learning. This can be achieved by executing the `learn +` method in one of two modes (specified using its **execution_mode** argument): using direct +compilation (**execution_mode** = `ExecutionMode.LLVMRun`); or by automatically translating the model to `PyTorch +`_ for training (**execution_mode** = `ExecutionMode.PyTorch`). The advantage of these modes is +that they can provide up to three orders of magnitude speed-up in training a model. However, there are restrictions +on the kinds of Compositions that be implemented in this way. The features of the different ways to implement and +execute learning are outlined in the following table, and described in more detail in `AutodiffComposition`. + TABLE: + * AutodiffComposition: + * Execute_mode.Python: + - execution: + - executes `learn ` using PyTorch + - executes `run ` using Python + - advantage: - fast (but slightly slower than direct compilation) + - disadvantage :broader support (RNN including LSTM, convnet, ?transformer?) + * Execute_mode.LLVNRun: + - execution: executes `learn ` *and* `run ` in compiled mode + - advantage: fastest (direct compilation of PNL code) + - disadvantage: but (currently) more limited; not suppored: + * RNN (including LSTM) + * convnet (though "in the wings") + * transformer + * ?external memory + * Composition: + - execution: executes `learn ` *and* `run ` in Python mode + - disadvantage: learning is extremely slow + - advantage: + - broadest support (including RL, TDLearning, Hebbian, Kohonen / SOM) + - can be used to implement effects of modulation and control during learning + - useful for examination of individual operations (e.g., for teaching purposes) +COMMENT + +.. _AutodiffComposition_PyTorch: + +*PyTorch mode* +~~~~~~~~~~~~~~ + +This is specified by setting **execution_mode = `ExecutionMode.PyTorch` in the `learn ` method of +an AutodiffCompositon (see `example ` in `BasicsAndPrimer`). This automatically +translates the AutodiffComposition to a `PyTorch `_ model and uses that for execution. This is +almost as fast as `LLVM compilation <_AutodiffComposition_LLVM>`, but provides greater flexiblity. Although it too is +best suited for use with `supervised learning `, it can also be used for some forms +of `unsupervised learning ` that are supported in PyTorch (e.g., `self-organized +maps `_). + + .. _AutodiffComposition_PyTorch_Note: + + .. note:: + While specifying `ExecutionMode.PyTorch` in the `learn ` method of an AutodiffComposition + causes it to use PyTorch for training, specifying this in the `run ` method causes it to be + executing using the *Python* interpreter (and not PyTorch); this is so that any modulation can take effect + during execution (see `AutodiffComposition_Nested_Modulation` below), which is not supported by PyTorch. + +.. technical_note:: + `ExecutionMode.PyTorch` is a synonym for `ExecutionMode.Python`, that is provided for clarity of the user interface: + the default for an AutodiffComposition (i.e., if **execution_mode** is not specified, or it is set to + `ExecutionMode.Python`) is to use PyTorch translation in `learn ` but the Python interpreter + for `run `. The use of `ExecutionMode.PyTorch` is simply to make it clear that, during learning, + it will use PyTorch. This contrasts with the use of `ExecutionMode.LLVMrun`, in which case both the `learn + ` and `run ` methods use LLVM compilation. + + +.. _AutodiffComposition_Nested_Modulation: + +*Nested Execution and Modulation* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Like any other `Composition`, an AutodiffComposition may be `nested ` inside another +(see `example ` below). However, learning, none of the internal +Components of the AutodiffComposition (e.g., intermediate layers of a neural network model) are accessible to the +other Components of the outer Composition, (e.g., as sources of information, or for `modulation +`). However, when +COMMENT: +learning turned off, +COMMENT +it is executed using its `run ` method, then the AutodiffComposition functions like any other, +and all of its internal Components are accessible to other Components of the outer Composition. Thus, as long as access +to its internal Components is not needed during learning, an `AutodiffComposition` can be trained, and then used to +execute the trained Composition like any other. + + +.. _AutodiffComposition_Logging: + +*Logging* +~~~~~~~~~ + +Logging in AutodiffCompositions follows the same procedure as `logging in a Composition `. +However, since an AutodiffComposition internally converts all of its Mechanisms either to LLVM +or to an equivalent PyTorch model, then its inner components are not actually executed. This means that there is +limited support for logging parameters of components inside an AutodiffComposition; Currently, the only supported +parameters are: + +1) the `matrix` parameter of Projections + +2) the `value` parameter of its inner components + + +.. _AutodiffComposition_Examples: + +Examples +-------- + +.. _AutodiffComposition_Creation_Example: + +The following is an example showing how to create a simple AutodiffComposition, specify its inputs and targets, +and run it with learning enabled and disabled: >>> import psyneulink as pnl >>> # Set up PsyNeuLink Components @@ -89,26 +259,8 @@ >>> # Run Composition in test mode >>> my_autodiff.run(inputs = input_dict['inputs']) -.. _AutodiffComposition_Logging: - -Logging -~~~~~~~ - -Logging in AutodiffCompositions follows the same procedure as `logging in a Composition `. -However, since an AutodiffComposition internally converts all of its mechanisms to an equivalent PyTorch model, -then its inner components are not actually executed. This means that there is limited support for -logging parameters of components inside an AutodiffComposition; Currently, the only supported parameters are: - -1) the `matrix` parameter of Projections - -2) the `value` parameter of its inner components -.. _AutodiffComposition_Nested_Execution: - -Nested Execution -~~~~~~~~~~~~~~~~ - -Like any other `Composition`, an AutodiffComposition may be `nested inside another `. +.. _AutodiffComposition_Nested_Example: The following shows how the AutodiffComposition created in the previous example can be nested and run inside another Composition:: @@ -157,7 +309,7 @@ import ReportOutput, ReportParams, ReportProgress, ReportSimulations, ReportDevices, \ LEARN_REPORT, EXECUTE_REPORT, PROGRESS_REPORT from psyneulink.core.globals.context import Context, ContextFlags, handle_external_context -from psyneulink.core.globals.keywords import AUTODIFF_COMPOSITION, SOFT_CLAMP +from psyneulink.core.globals.keywords import AUTODIFF_COMPOSITION, SOFT_CLAMP, Loss from psyneulink.core.scheduling.scheduler import Scheduler from psyneulink.core.globals.parameters import Parameter, check_user_specified from psyneulink.core.scheduling.time import TimeScale @@ -184,8 +336,8 @@ def __str__(self): class AutodiffComposition(Composition): """ - Subclass of `Composition` that trains models using `PyTorch `_. - See `Composition ` for additional arguments and attributes. + Subclass of `Composition` that trains models using either LLVM compilation or `PyTorch `_; + see and `Composition ` for additional arguments and attributes. Arguments --------- @@ -203,10 +355,11 @@ class AutodiffComposition(Composition): weight_decay : float : default 0 specifies the L2 penalty (which discourages large weights) used by the optimizer. - loss_spec : str or PyTorch loss function : default 'mse' - specifies the loss function for training. The current string options are 'mse' (the default), 'crossentropy', - 'l1', 'nll', 'poissonnll', and 'kldiv'. Any PyTorch loss function can work here, such as ones from - https://pytorch.org/docs/stable/nn.html#loss-functions + loss_spec : Loss or PyTorch loss function : default Loss.MSE + specifies the loss function for training; see `Loss` for arguments. + + Attributes + ---------- losses : list of floats tracks the average for each weight update (i.e. each minibatch) @@ -238,7 +391,7 @@ def __init__(self, learning_rate=None, optimizer_type='sgd', weight_decay=0, - loss_spec='mse', + loss_spec=Loss.MSE, disable_learning=False, refresh_losses=False, disable_cuda=True, @@ -325,13 +478,13 @@ def _make_optimizer(self, optimizer_type, learning_rate, weight_decay, context): return optim.Adam(params, lr=learning_rate, weight_decay=weight_decay) def _get_loss(self, loss_spec): - if not isinstance(self.loss_spec, str): + if not isinstance(self.loss_spec, (str, Loss)): return self.loss_spec - elif loss_spec == 'mse': + elif loss_spec == Loss.MSE: return nn.MSELoss(reduction='mean') - elif loss_spec == 'sse': + elif loss_spec == Loss.SSE: return nn.MSELoss(reduction='sum') - elif loss_spec == 'crossentropy': + elif loss_spec == Loss.CROSS_ENTROPY: # Cross entropy loss is used for multiclass categorization and needs inputs in shape # ((# minibatch_size, C), targets) where C is a 1-d vector of probabilities for each potential category # and where target is a 1d vector of type long specifying the index to the target category. This @@ -342,20 +495,20 @@ def _get_loss(self, loss_spec): x.unsqueeze(0), y.type(torch.LongTensor) ) - elif loss_spec == 'l1': + elif loss_spec == Loss.L1: return nn.L1Loss(reduction='sum') - elif loss_spec == 'nll': + elif loss_spec == Loss.NLL: return nn.NLLLoss(reduction='sum') - elif loss_spec == 'poissonnll': + elif loss_spec == Loss.POISSON_NLL: return nn.PoissonNLLLoss(reduction='sum') - elif loss_spec == 'kldiv': + elif loss_spec == Loss.KL_DIV: return nn.KLDivLoss(reduction='sum') else: - raise AutodiffCompositionError("Loss type {} not recognized. Loss argument must be a string or function. " - "Currently, the recognized loss types are Mean Squared Error, Cross Entropy," - " L1 loss, Negative Log Likelihood loss, Poisson Negative Log Likelihood, " - "and KL Divergence. These are specified as 'mse', 'crossentropy', 'l1', " - "'nll', 'poissonnll', and 'kldiv' respectively.".format(loss_spec)) + raise AutodiffCompositionError(f"Loss type {loss_spec} not recognized. Loss argument must be a " + f"Loss enum or function. Currently, the recognized loss types are: " + f"L1 (Mean), SSE (sum squared error), CROSS_ENTROPY, NLL (negative log " + f"likelihood), POISSONNLL (Poisson negative log likelihood, " + f"and KL_DIV (KL divergence.") # performs learning/training on all input-target pairs it recieves for given number of epochs def autodiff_training(self, inputs, targets, context=None, scheduler=None): @@ -612,10 +765,9 @@ def save(self, path:PosixPath=None, directory:str=None, filename:str=None, conte else: path = Path(os.getcwd()) if filename: - # path = Path(path / filename) - path = Path(os.path.join(path / filename)) + path = Path(os.path.join(path, filename)) else: - path = Path(os.path.join(path / f'{self.name}_matrix_wts.pnl')) + path = Path(os.path.join(path, f'{self.name}_matrix_wts.pnl')) except IsADirectoryError: raise AutodiffCompositionError(f"'{path}' (for saving weight matrices of ({self.name}) " f"is not a legal path.") @@ -661,10 +813,8 @@ def load(self, path:PosixPath=None, directory:str=None, filename:str=None, conte else: path = Path(os.getcwd()) if filename: - # path = Path(path / filename) - path = Path(os.path.join(path / filename)) + path = Path(os.path.join(path, filename)) else: - # path = Path(path / f'{self.name}_matrix_wts.pnl') path = Path(os.path.join(path , f'{self.name}_matrix_wts.pnl')) except IsADirectoryError: raise AutodiffCompositionError(f"'{path}' (for saving weight matrices of ({self.name}) " diff --git a/psyneulink/library/compositions/compiledloss.py b/psyneulink/library/compositions/compiledloss.py index b82fd64cd49..53d0027ad40 100644 --- a/psyneulink/library/compositions/compiledloss.py +++ b/psyneulink/library/compositions/compiledloss.py @@ -1,7 +1,7 @@ from psyneulink.core import llvm as pnlvm from psyneulink.library.compositions.pytorchllvmhelper import * -__all__ = ['MSELoss'] +__all__ = ['MSELoss', "CROSS_ENTROPYLoss"] class Loss(): @@ -33,8 +33,8 @@ def _gen_loss_function(self, ctx): # args: # 1) pointer to network output - # 2) pointer to target - # 3) dimensionality + # 2) dimensionality + # 3) pointer to target args = [ctx.float_ty.as_pointer(), ctx.int32_ty, ctx.float_ty.as_pointer()] builder = ctx.create_llvm_function(args, self, name, return_type=ctx.float_ty) value, dim, target = builder.function.args @@ -79,3 +79,67 @@ def _gen_inject_loss_differential(self, ctx, builder, value, target, output=None tmp = gen_inject_vec_sub(ctx, builder, value, target) gen_inject_vec_add(ctx, builder, output, tmp, output) return output + + +class CROSS_ENTROPYLoss(Loss): + """Implements compiled CROSS_ENTROPY Loss""" + def __init__(self, reduction='cross_entropy'): + if reduction not in ['cross_entropy']: + raise Exception("Unsupported compiled reduction type " + reduction) + + super().__init__() + self.reduction = reduction + + def _gen_loss_function(self, ctx): + name = "LEARNING_CROSS_ENTROPY_CALL" + + # args: + # 1) pointer to network output + # 2) dimensionality + # 3) pointer to target + args = [ctx.float_ty.as_pointer(), ctx.int32_ty, ctx.float_ty.as_pointer()] + builder = ctx.create_llvm_function(args, self, name, return_type=ctx.float_ty) + value, dim, target = builder.function.args + + sum = builder.alloca(ctx.float_ty) + builder.store(ctx.float_ty(-0.0), sum) + + with pnlvm.helpers.for_loop_zero_inc(builder, dim, "cross_entropy_sum_loop") as (b1, index): + value_ptr = b1.gep(value,[index]) + target_ptr = b1.gep(target,[index]) + target_val = b1.load(target_ptr) + log_f = ctx.get_builtin("log", [ctx.float_ty]) + log = b1.call(log_f, [target_val]) + diff = b1.fmul(b1.load(value_ptr), log) + b1.store(b1.fadd(b1.load(sum),diff),sum) + + builder.ret(builder.load(sum)) + + return builder.function + + # inserts the computation for dC/da + def _gen_inject_loss_differential(self, ctx, builder, value, target, output=None, sum_loss=False): + + # FIX: FROM MSE_LOSS -- HERE JUST AS FILLER TO GET PAST THIS METHOD DURING DEBUGGING; + # NEEDS TO BE PROPERLY IMPLEMENTED + dim = len(value.type.pointee) + assert len(target.type.pointee) == dim + if output is None: + output = builder.alloca(pnlvm.ir.types.ArrayType(ctx.float_ty, dim)) + # zero output vector + builder.store(output.type.pointee(None), output) + assert len(output.type.pointee) == dim + + if sum_loss is False: + # we take mean + gen_inject_vec_sub(ctx, builder, value, target, output) + # multiply each element i by 2/n to get dC/da_i + scalar_mult = builder.fdiv(ctx.float_ty(2), ctx.float_ty(dim)) + with pnlvm.helpers.for_loop_zero_inc(builder, ctx.int32_ty(dim), "mse_mean_mult_loop") as (b1, index): + element_ptr = b1.gep(output, [ctx.int32_ty(0), index]) + b1.store(b1.fmul(b1.load(element_ptr),scalar_mult),element_ptr) + else: + # in this case, we add the loss + tmp = gen_inject_vec_sub(ctx, builder, value, target) + gen_inject_vec_add(ctx, builder, output, tmp, output) + return output diff --git a/psyneulink/library/compositions/pytorchmodelcreator.py b/psyneulink/library/compositions/pytorchmodelcreator.py index 916dfca438f..45dc323a792 100644 --- a/psyneulink/library/compositions/pytorchmodelcreator.py +++ b/psyneulink/library/compositions/pytorchmodelcreator.py @@ -5,9 +5,9 @@ from psyneulink.core.globals.context import Context, ContextFlags, handle_external_context from psyneulink.core import llvm as pnlvm from psyneulink.library.compositions.compiledoptimizer import AdamOptimizer, SGDOptimizer -from psyneulink.library.compositions.compiledloss import MSELoss +from psyneulink.library.compositions.compiledloss import MSELoss, CROSS_ENTROPYLoss from psyneulink.library.compositions.pytorchllvmhelper import * -from psyneulink.core.globals.keywords import TARGET_MECHANISM +from psyneulink.core.globals.keywords import TARGET_MECHANISM, Loss from psyneulink.core.globals.utilities import get_deepcopy_with_shared from .pytorchcomponents import * @@ -274,8 +274,10 @@ def _gen_llvm_training_function_body(self, ctx, builder, state, params, data): optimizer = self._get_compiled_optimizer() # setup loss loss_type = self._composition.loss_spec - if loss_type == 'mse': + if loss_type == Loss.MSE: loss = MSELoss() + elif loss_type == Loss.CROSS_ENTROPY: + loss = CROSS_ENTROPYLoss() else: raise Exception("LOSS TYPE", loss_type, "NOT SUPPORTED") diff --git a/tests/composition/test_autodiffcomposition.py b/tests/composition/test_autodiffcomposition.py index 6eba3a2b288..937096933d9 100644 --- a/tests/composition/test_autodiffcomposition.py +++ b/tests/composition/test_autodiffcomposition.py @@ -11,7 +11,7 @@ from psyneulink.core.components.functions.nonstateful.learningfunctions import BackPropagation from psyneulink.core.compositions.composition import Composition from psyneulink.core.globals import Context -from psyneulink.core.globals.keywords import TRAINING_SET +from psyneulink.core.globals.keywords import TRAINING_SET, Loss from psyneulink.core.components.mechanisms.processing.transfermechanism import TransferMechanism from psyneulink.core.components.projections.pathway.mappingprojection import MappingProjection from psyneulink.library.compositions.autodiffcomposition import AutodiffComposition @@ -252,7 +252,7 @@ def test_training_then_processing(self, autodiff_mode): # assert np.allclose(pt_weights_out_bp, pt_weights_out_ap) @pytest.mark.parametrize( - 'loss', ['l1', 'poissonnll'] + 'loss', [Loss.L1, Loss.POISSON_NLL] ) def test_various_loss_specs(self, loss, autodiff_mode): if autodiff_mode is not pnl.ExecutionMode.Python: @@ -1763,35 +1763,35 @@ def test_semantic_net_training_identicalness(self, eps, opt): default_variable=np.zeros(9), function=Logistic()) - # SET UP MECHANISMS FOR SYSTEM + # SET UP MECHANISMS FOR Composition - nouns_in_sys = TransferMechanism(name="nouns_input_sys", + nouns_in_comp = TransferMechanism(name="nouns_input_comp", default_variable=np.zeros(8)) - rels_in_sys = TransferMechanism(name="rels_input_sys", + rels_in_comp = TransferMechanism(name="rels_input_comp", default_variable=np.zeros(3)) - h1_sys = TransferMechanism(name="hidden_nouns_sys", + h1_comp = TransferMechanism(name="hidden_nouns_comp", default_variable=np.zeros(8), function=Logistic()) - h2_sys = TransferMechanism(name="hidden_mixed_sys", + h2_comp = TransferMechanism(name="hidden_mixed_comp", default_variable=np.zeros(15), function=Logistic()) - out_sig_I_sys = TransferMechanism(name="sig_outs_I_sys", + out_sig_I_comp = TransferMechanism(name="sig_outs_I_comp", default_variable=np.zeros(8), function=Logistic()) - out_sig_is_sys = TransferMechanism(name="sig_outs_is_sys", + out_sig_is_comp = TransferMechanism(name="sig_outs_is_comp", default_variable=np.zeros(12), function=Logistic()) - out_sig_has_sys = TransferMechanism(name="sig_outs_has_sys", + out_sig_has_comp = TransferMechanism(name="sig_outs_has_comp", default_variable=np.zeros(9), function=Logistic()) - out_sig_can_sys = TransferMechanism(name="sig_outs_can_sys", + out_sig_can_comp = TransferMechanism(name="sig_outs_can_comp", default_variable=np.zeros(9), function=Logistic()) @@ -1832,64 +1832,64 @@ def test_semantic_net_training_identicalness(self, eps, opt): sender=h2, receiver=out_sig_can) - # SET UP PROJECTIONS FOR SYSTEM - - map_nouns_h1_sys = MappingProjection(matrix=map_nouns_h1.matrix.base.copy(), - name="map_nouns_h1_sys", - sender=nouns_in_sys, - receiver=h1_sys) - - map_rels_h2_sys = MappingProjection(matrix=map_rels_h2.matrix.base.copy(), - name="map_relh2_sys", - sender=rels_in_sys, - receiver=h2_sys) - - map_h1_h2_sys = MappingProjection(matrix=map_h1_h2.matrix.base.copy(), - name="map_h1_h2_sys", - sender=h1_sys, - receiver=h2_sys) - - map_h2_I_sys = MappingProjection(matrix=map_h2_I.matrix.base.copy(), - name="map_h2_I_sys", - sender=h2_sys, - receiver=out_sig_I_sys) - - map_h2_is_sys = MappingProjection(matrix=map_h2_is.matrix.base.copy(), - name="map_h2_is_sys", - sender=h2_sys, - receiver=out_sig_is_sys) - - map_h2_has_sys = MappingProjection(matrix=map_h2_has.matrix.base.copy(), - name="map_h2_has_sys", - sender=h2_sys, - receiver=out_sig_has_sys) - - map_h2_can_sys = MappingProjection(matrix=map_h2_can.matrix.base.copy(), - name="map_h2_can_sys", - sender=h2_sys, - receiver=out_sig_can_sys) + # SET UP PROJECTIONS FOR COMPOSITION - # SET UP COMPOSITION FOR SEMANTIC NET - sem_net = AutodiffComposition(learning_rate=0.5, + map_nouns_h1_comp = MappingProjection(matrix=map_nouns_h1.matrix.base.copy(), + name="map_nouns_h1_comp", + sender=nouns_in_comp, + receiver=h1_comp) + + map_rels_h2_comp = MappingProjection(matrix=map_rels_h2.matrix.base.copy(), + name="map_relh2_comp", + sender=rels_in_comp, + receiver=h2_comp) + + map_h1_h2_comp = MappingProjection(matrix=map_h1_h2.matrix.base.copy(), + name="map_h1_h2_comp", + sender=h1_comp, + receiver=h2_comp) + + map_h2_I_comp = MappingProjection(matrix=map_h2_I.matrix.base.copy(), + name="map_h2_I_comp", + sender=h2_comp, + receiver=out_sig_I_comp) + + map_h2_is_comp = MappingProjection(matrix=map_h2_is.matrix.base.copy(), + name="map_h2_is_comp", + sender=h2_comp, + receiver=out_sig_is_comp) + + map_h2_has_comp = MappingProjection(matrix=map_h2_has.matrix.base.copy(), + name="map_h2_has_comp", + sender=h2_comp, + receiver=out_sig_has_comp) + + map_h2_can_comp = MappingProjection(matrix=map_h2_can.matrix.base.copy(), + name="map_h2_can_comp", + sender=h2_comp, + receiver=out_sig_can_comp) + + # SET UP AUTODIFFCOMPOSITION FOR SEMANTIC NET + sem_net_autodiff = AutodiffComposition(learning_rate=0.5, optimizer_type=opt, ) - sem_net.add_node(nouns_in) - sem_net.add_node(rels_in) - sem_net.add_node(h1) - sem_net.add_node(h2) - sem_net.add_node(out_sig_I) - sem_net.add_node(out_sig_is) - sem_net.add_node(out_sig_has) - sem_net.add_node(out_sig_can) - - sem_net.add_projection(sender=nouns_in, projection=map_nouns_h1, receiver=h1) - sem_net.add_projection(sender=rels_in, projection=map_rels_h2, receiver=h2) - sem_net.add_projection(sender=h1, projection=map_h1_h2, receiver=h2) - sem_net.add_projection(sender=h2, projection=map_h2_I, receiver=out_sig_I) - sem_net.add_projection(sender=h2, projection=map_h2_is, receiver=out_sig_is) - sem_net.add_projection(sender=h2, projection=map_h2_has, receiver=out_sig_has) - sem_net.add_projection(sender=h2, projection=map_h2_can, receiver=out_sig_can) + sem_net_autodiff.add_node(nouns_in) + sem_net_autodiff.add_node(rels_in) + sem_net_autodiff.add_node(h1) + sem_net_autodiff.add_node(h2) + sem_net_autodiff.add_node(out_sig_I) + sem_net_autodiff.add_node(out_sig_is) + sem_net_autodiff.add_node(out_sig_has) + sem_net_autodiff.add_node(out_sig_can) + + sem_net_autodiff.add_projection(sender=nouns_in, projection=map_nouns_h1, receiver=h1) + sem_net_autodiff.add_projection(sender=rels_in, projection=map_rels_h2, receiver=h2) + sem_net_autodiff.add_projection(sender=h1, projection=map_h1_h2, receiver=h2) + sem_net_autodiff.add_projection(sender=h2, projection=map_h2_I, receiver=out_sig_I) + sem_net_autodiff.add_projection(sender=h2, projection=map_h2_is, receiver=out_sig_is) + sem_net_autodiff.add_projection(sender=h2, projection=map_h2_has, receiver=out_sig_has) + sem_net_autodiff.add_projection(sender=h2, projection=map_h2_can, receiver=out_sig_can) # INPUTS & OUTPUTS FOR SEMANTIC NET: nouns = ['oak', 'pine', 'rose', 'daisy', 'canary', 'robin', 'salmon', 'sunfish'] @@ -1959,82 +1959,89 @@ def test_semantic_net_training_identicalness(self, eps, opt): targets_dict[out_sig_has].append(truth_has[i]) targets_dict[out_sig_can].append(truth_can[i]) - inputs_dict_sys = {} - inputs_dict_sys[nouns_in_sys] = inputs_dict[nouns_in] - inputs_dict_sys[rels_in_sys] = inputs_dict[rels_in] + inputs_dict_comp = {} + inputs_dict_comp[nouns_in_comp] = inputs_dict[nouns_in] + inputs_dict_comp[rels_in_comp] = inputs_dict[rels_in] - result = sem_net.run(inputs=inputs_dict) + sem_net_autodiff.run(inputs=inputs_dict) - # TRAIN COMPOSITION + # TRAIN AUTODIFFCOMPOSITION def g_f(): yield {"inputs": inputs_dict, "targets": targets_dict, "epochs": eps} g = g_f() - result = sem_net.learn(inputs=g_f) + sem_net_autodiff.learn(inputs=g_f) - # SET UP SYSTEM - sem_net_sys = Composition() + # SET UP COMPOSITION + sem_net_comp = Composition() - backprop_pathway = sem_net_sys.add_backpropagation_learning_pathway( + backprop_pathway = sem_net_comp.add_backpropagation_learning_pathway( pathway=[ - nouns_in_sys, - map_nouns_h1_sys, - h1_sys, - map_h1_h2_sys, - h2_sys, - map_h2_I_sys, - out_sig_I_sys + nouns_in_comp, + map_nouns_h1_comp, + h1_comp, + map_h1_h2_comp, + h2_comp, + map_h2_I_comp, + out_sig_I_comp ], learning_rate=0.5 ) - inputs_dict_sys[backprop_pathway.target] = targets_dict[out_sig_I] + inputs_dict_comp[backprop_pathway.target] = targets_dict[out_sig_I] - backprop_pathway = sem_net_sys.add_backpropagation_learning_pathway( + backprop_pathway = sem_net_comp.add_backpropagation_learning_pathway( pathway=[ - rels_in_sys, - map_rels_h2_sys, - h2_sys, - map_h2_is_sys, - out_sig_is_sys + rels_in_comp, + map_rels_h2_comp, + h2_comp, + map_h2_is_comp, + out_sig_is_comp ], learning_rate=0.5 ) - inputs_dict_sys[backprop_pathway.target] = targets_dict[out_sig_is] + inputs_dict_comp[backprop_pathway.target] = targets_dict[out_sig_is] - backprop_pathway = sem_net_sys.add_backpropagation_learning_pathway( + backprop_pathway = sem_net_comp.add_backpropagation_learning_pathway( pathway=[ - h2_sys, - map_h2_has_sys, - out_sig_has_sys + h2_comp, + map_h2_has_comp, + out_sig_has_comp ], learning_rate=0.5 ) - inputs_dict_sys[backprop_pathway.target] = targets_dict[out_sig_has] + inputs_dict_comp[backprop_pathway.target] = targets_dict[out_sig_has] - backprop_pathway = sem_net_sys.add_backpropagation_learning_pathway( + backprop_pathway = sem_net_comp.add_backpropagation_learning_pathway( pathway=[ - h2_sys, - map_h2_can_sys, - out_sig_can_sys + h2_comp, + map_h2_can_comp, + out_sig_can_comp ], learning_rate=0.5 ) - inputs_dict_sys[backprop_pathway.target] = targets_dict[out_sig_can] + inputs_dict_comp[backprop_pathway.target] = targets_dict[out_sig_can] - # TRAIN SYSTEM - results = sem_net_sys.learn(inputs=inputs_dict_sys, - num_trials=(len(inputs_dict_sys[nouns_in_sys]) * eps)) - - # CHECK THAT PARAMETERS FOR COMPOSITION, SYSTEM ARE SAME - - assert np.allclose(map_nouns_h1.parameters.matrix.get(sem_net), map_nouns_h1_sys.get_mod_matrix(sem_net_sys)) - assert np.allclose(map_rels_h2.parameters.matrix.get(sem_net), map_rels_h2_sys.get_mod_matrix(sem_net_sys)) - assert np.allclose(map_h1_h2.parameters.matrix.get(sem_net), map_h1_h2_sys.get_mod_matrix(sem_net_sys)) - assert np.allclose(map_h2_I.parameters.matrix.get(sem_net), map_h2_I_sys.get_mod_matrix(sem_net_sys)) - assert np.allclose(map_h2_is.parameters.matrix.get(sem_net), map_h2_is_sys.get_mod_matrix(sem_net_sys)) - assert np.allclose(map_h2_has.parameters.matrix.get(sem_net), map_h2_has_sys.get_mod_matrix(sem_net_sys)) - assert np.allclose(map_h2_can.parameters.matrix.get(sem_net), map_h2_can_sys.get_mod_matrix(sem_net_sys)) + # TRAIN COMPOSITION + sem_net_comp.learn(inputs=inputs_dict_comp, + num_trials=(len(inputs_dict_comp[nouns_in_comp]) * eps)) + + # CHECK THAT PARAMETERS FOR AUTODIFFCOMPOSITION, COMPOSITION ARE SAME + + assert np.allclose(map_nouns_h1.parameters.matrix.get(sem_net_autodiff), + map_nouns_h1_comp.get_mod_matrix(sem_net_comp)) + assert np.allclose(map_rels_h2.parameters.matrix.get(sem_net_autodiff), + map_rels_h2_comp.get_mod_matrix(sem_net_comp)) + assert np.allclose(map_h1_h2.parameters.matrix.get(sem_net_autodiff), + map_h1_h2_comp.get_mod_matrix(sem_net_comp)) + assert np.allclose(map_h2_I.parameters.matrix.get(sem_net_autodiff), + map_h2_I_comp.get_mod_matrix(sem_net_comp)) + assert np.allclose(map_h2_is.parameters.matrix.get(sem_net_autodiff), + map_h2_is_comp.get_mod_matrix(sem_net_comp)) + assert np.allclose(map_h2_has.parameters.matrix.get(sem_net_autodiff), + map_h2_has_comp.get_mod_matrix(sem_net_comp)) + assert np.allclose(map_h2_can.parameters.matrix.get(sem_net_autodiff), + map_h2_can_comp.get_mod_matrix(sem_net_comp)) def test_identicalness_of_input_types(self): # SET UP MECHANISMS FOR COMPOSITION @@ -2219,7 +2226,6 @@ def get_inputs_gen(): } g = get_inputs_gen() - result_gen = xor_gen.learn(inputs=g) # SET UP MECHANISMS FOR COMPOSITION @@ -3155,7 +3161,7 @@ def test_cross_entropy_loss(self): m1 = pnl.TransferMechanism() p = pnl.MappingProjection() m2 = pnl.TransferMechanism() - adc = pnl.AutodiffComposition(loss_spec='crossentropy') + adc = pnl.AutodiffComposition(loss_spec=Loss.CROSS_ENTROPY) adc.add_linear_processing_pathway([m1, p, m2]) adc._build_pytorch_representation() diff --git a/tests/composition/test_learning.py b/tests/composition/test_learning.py index cbba3e2d0c8..6659b86d36b 100644 --- a/tests/composition/test_learning.py +++ b/tests/composition/test_learning.py @@ -8,7 +8,9 @@ from psyneulink.core.compositions.composition import Composition, CompositionError, RunError from psyneulink.core.components.mechanisms.processing.transfermechanism import TransferMechanism from psyneulink.core.components.functions.nonstateful.learningfunctions import BackPropagation - +import psyneulink.core.llvm as pnlvm +from psyneulink.core.globals.keywords import Loss +from psyneulink.library.components.mechanisms.processing.objective.comparatormechanism import SSE, MSE, L0 class TestTargetSpecs: @@ -1708,7 +1710,7 @@ def test_stranded_nested_target_mech_error(self): f'as the target attribute of the relevant pathway in {inner_comp.name}.pathways. ' ) -class TestBackProp: +class TestBackPropLearning: def test_matrix_spec_and_learning_rate(self): T1 = pnl.TransferMechanism(size = 2, @@ -1773,7 +1775,121 @@ def test_back_prop(self): # else: # print(node.name, " EMPTY LOG!") - def test_multilayer(self): + expected_quantities = [ + ( + Loss.L0, + pnl.SUM, + # output_layer output values + [np.array([0.22686074, 0.25270212, 0.91542149])], + # objective_mechanism.output_port[] value + [np.array(-0.39498435)], + # Input Weights + [[ 0.09900247, 0.19839653, 0.29785764, 0.39739191, 0.49700232], + [ 0.59629092, 0.69403786, 0.79203411, 0.89030237, 0.98885379]], + # Middle Weights + [[ 0.09490249, 0.10488719, 0.12074013, 0.1428774 ], + [ 0.29677354, 0.30507726, 0.31949676, 0.3404652 ], + [ 0.49857336, 0.50526254, 0.51830509, 0.53815062], + [ 0.70029406, 0.70544225, 0.71717037, 0.73594383], + [ 0.90192903, 0.90561554, 0.91609668, 0.93385292]], + # Output Weights + [[-0.74447522, -0.71016859, 0.31575293], + [-0.50885177, -0.47444784, 0.56676582], + [-0.27333719, -0.23912033, 0.8178167 ], + [-0.03767547, -0.00389039, 1.06888608]], + # Results + [[np.array([0.8344837 , 0.87072018, 0.89997433])], + [np.array([0.77970193, 0.83263138, 0.90159627])], + [np.array([0.70218502, 0.7773823 , 0.90307765])], + [np.array([0.60279149, 0.69958079, 0.90453143])], + [np.array([0.4967927 , 0.60030321, 0.90610082])], + [np.array([0.4056202 , 0.49472391, 0.90786617])], + [np.array([0.33763025, 0.40397637, 0.90977675])], + [np.array([0.28892812, 0.33633532, 0.9117193 ])], + [np.array([0.25348771, 0.28791896, 0.9136125 ])], + [np.array([0.22686074, 0.25270212, 0.91542149])]] + ), + ( + Loss.SSE, + SSE, + # output_layer output values + [np.array([0.12306101, 0.12855051, 0.92795179])], + # objective_mechanism.output_port[] value + [np.array(0.03686019)], + # Input Weights + [[0.09944189, 0.19971589, 0.29997209, 0.40020673, 0.50041673], + [0.5979248, 0.69894361, 0.79989623, 0.90076867, 1.0015495]], + # Middle Weights + [[0.11871093, 0.12080358, 0.12913871, 0.14437706], + [0.32158068, 0.32166374, 0.32825218, 0.34203389], + [0.52434054, 0.52249285, 0.52740295, 0.53978486], + [0.72697833, 0.72328725, 0.72659469, 0.73763981], + [0.92948392, 0.92404372, 0.92583026, 0.93560663]], + # Output Weights + [[-0.93832915, -0.92583809, 0.36458405], + [-0.70446298, -0.69176289, 0.61576631], + [-0.47104248, -0.45856457, 0.86712447], + [-0.23778995, -0.22590794, 1.11863746]], + # Results + [[np.array([0.8344837, 0.87072018, 0.89997433])], + [np.array([0.71351724, 0.78641358, 0.90315634])], + [np.array([0.50994992, 0.62475304, 0.90595494])], + [np.array([0.32856147, 0.41172748, 0.90933295])], + [np.array([0.24083869, 0.2789737 , 0.91321678])], + [np.array([0.19538549, 0.21621273, 0.91684295])], + [np.array([0.16740723, 0.1806998 , 0.92008144])], + [np.array([0.14819045, 0.15753784, 0.92297786])], + [np.array([0.13402466, 0.14102997, 0.92558631])], + [np.array([0.12306101, 0.12855051, 0.92795179])]] + ), + ( + Loss.MSE, + MSE, + # output_layer output values + [np.array([0.34065762, 0.40283722, 0.90991679])], + # objective_mechanism.output_port[] value + np.array([0.09548014]), + # Input Weights + [[0.09878461, 0.19766035, 0.29665234, 0.39577252, 0.49502758], + [0.59548084, 0.69130054, 0.78755247, 0.88428106, 0.98151113]], + # Middle Weights + [[0.07706183, 0.09444972, 0.11723154, 0.14557542], + [0.27818676, 0.29420326, 0.3158414, 0.34327603], + [0.4792692, 0.49396883, 0.51450859, 0.54106987], + [0.68030443, 0.69374747, 0.71323898, 0.73896663], + [0.88128847, 0.89353987, 0.91203717, 0.93697403]], + # Output Weights + [[-0.59467351, -0.52912455, 0.29597305], + [-0.35770705, -0.29192171, 0.54683712], + [-0.12052892, -0.05468307, 0.79769116], + [ 0.11707288, 0.18282992, 1.04852107]], + # Results + [[np.array([0.8344837, 0.87072018, 0.89997433])], + [np.array([0.79924855, 0.84620706, 0.90106255])], + [np.array([0.75417448, 0.81457342, 0.90208226])], + [np.array([0.69827147, 0.77394099, 0.90306295])], + [np.array([0.63285507, 0.72284124, 0.90404476])], + [np.array([0.5625646 , 0.66140581, 0.90507175])], + [np.array([0.49415513, 0.59273088, 0.90617688])], + [np.array([0.4332465 , 0.52285839, 0.90736771])], + [np.array([0.38219876, 0.45825994, 0.90862524])], + [np.array([0.34065762, 0.40283722, 0.90991679])]] + ), + ] + # Indices into expected_quantities + @pytest.mark.parametrize("expected_quantities", expected_quantities, + # Rename L0 for test output as keyword actually = 'difference' + ids=['L0' if x[0] == Loss.L0 else x[0].name for x in expected_quantities]) + def test_multilayer_truth(self, expected_quantities): + + LOSS_FUNCTION = 0 + LOSS = 1 + OUTPUT_LAYER_VAL = 2 + OBJECTIVE_MECH_VAL = 3 + INPUT_WEIGHTS = 4 + MIDDLE_WEIGHTS = 5 + OUTPUT_WEIGHTS = 6 + RESULTS = 7 input_layer = pnl.TransferMechanism(name='input_layer', function=pnl.Logistic, @@ -1824,7 +1940,7 @@ def test_multilayer(self): p = [input_layer, input_weights, hidden_layer_1, middle_weights, hidden_layer_2, output_weights, output_layer] backprop_pathway = comp.add_backpropagation_learning_pathway( pathway=p, - loss_function='sse', + loss_function=expected_quantities[LOSS_FUNCTION], learning_rate=1. ) @@ -1838,42 +1954,15 @@ def test_multilayer(self): objective_output_layer = comp.nodes[5] - expected_output = [ - (output_layer.get_output_values(comp), [np.array([0.22686074, 0.25270212, 0.91542149])]), - # error here? why still MSE - (objective_output_layer.output_ports[pnl.MSE].parameters.value.get(comp), np.array(0.04082589331852094)), - (input_weights.get_mod_matrix(comp), np.array([ - [ 0.09900247, 0.19839653, 0.29785764, 0.39739191, 0.49700232], - [ 0.59629092, 0.69403786, 0.79203411, 0.89030237, 0.98885379], - ])), - (middle_weights.get_mod_matrix(comp), np.array([ - [ 0.09490249, 0.10488719, 0.12074013, 0.1428774 ], - [ 0.29677354, 0.30507726, 0.31949676, 0.3404652 ], - [ 0.49857336, 0.50526254, 0.51830509, 0.53815062], - [ 0.70029406, 0.70544225, 0.71717037, 0.73594383], - [ 0.90192903, 0.90561554, 0.91609668, 0.93385292], - ])), - (output_weights.get_mod_matrix(comp), np.array([ - [-0.74447522, -0.71016859, 0.31575293], - [-0.50885177, -0.47444784, 0.56676582], - [-0.27333719, -0.23912033, 0.8178167 ], - [-0.03767547, -0.00389039, 1.06888608], - ])), - (comp.parameters.results.get(comp), [ - [np.array([0.8344837 , 0.87072018, 0.89997433])], - [np.array([0.77970193, 0.83263138, 0.90159627])], - [np.array([0.70218502, 0.7773823 , 0.90307765])], - [np.array([0.60279149, 0.69958079, 0.90453143])], - [np.array([0.4967927 , 0.60030321, 0.90610082])], - [np.array([0.4056202 , 0.49472391, 0.90786617])], - [np.array([0.33763025, 0.40397637, 0.90977675])], - [np.array([0.28892812, 0.33633532, 0.9117193 ])], - [np.array([0.25348771, 0.28791896, 0.9136125 ])], - [np.array([0.22686074, 0.25270212, 0.91542149])] - ]), - ] - # Test nparray output of log for Middle_Weights + expected_output = [ + (output_layer.get_output_values(comp), expected_quantities[OUTPUT_LAYER_VAL]), + (objective_output_layer.output_ports[LOSS].parameters.value.get(comp), + expected_quantities[OBJECTIVE_MECH_VAL]), + (input_weights.get_mod_matrix(comp), expected_quantities[INPUT_WEIGHTS]), + (middle_weights.get_mod_matrix(comp), expected_quantities[MIDDLE_WEIGHTS]), + (output_weights.get_mod_matrix(comp), expected_quantities[OUTPUT_WEIGHTS]), + (comp.parameters.results.get(comp), expected_quantities[RESULTS])] for i in range(len(expected_output)): val, expected = expected_output[i] @@ -1882,13 +1971,10 @@ def test_multilayer(self): # which WILL FAIL unless you gather higher precision values to use as reference np.testing.assert_allclose(val, expected, atol=1e-08, err_msg='Failed on expected_output[{0}]'.format(i)) - @pytest.mark.parametrize('models', [ - # [pnl.SYSTEM,pnl.COMPOSITION], - # [pnl.SYSTEM,'AUTODIFF'], - [pnl.COMPOSITION,'AUTODIFF'] - ]) + models = ['PYTORCH','LLVM'] + @pytest.mark.parametrize('models', models, ids=[x for x in models]) @pytest.mark.pytorch - def test_xor_training_identicalness_standard_composition_vs_autodiff(self, models): + def test_xor_training_identicalness_standard_composition_vs_PyTorch_and_LLVM(self, models): """Test equality of results for running 3-layered xor network using System, Composition and Autodiff""" num_epochs=2 @@ -1910,89 +1996,135 @@ def test_xor_training_identicalness_standard_composition_vs_autodiff(self, model # SET UP MODELS -------------------------------------------------------------------------------- - # STANDARD Composition - if pnl.COMPOSITION in models: + # STANDARD Composition (used in all comparisons) + + input_comp = pnl.TransferMechanism(name='input_comp', + default_variable=np.zeros(2)) + + hidden_comp = pnl.TransferMechanism(name='hidden_comp', + default_variable=np.zeros(10), + function=pnl.Logistic()) + + output_comp = pnl.TransferMechanism(name='output_comp', + default_variable=np.zeros(1), + function=pnl.Logistic()) + + in_to_hidden_comp = pnl.MappingProjection(name='in_to_hidden_comp', + matrix=in_to_hidden_matrix.copy(), + sender=input_comp, + receiver=hidden_comp) + + hidden_to_out_comp = pnl.MappingProjection(name='hidden_to_out_comp', + matrix=hidden_to_out_matrix.copy(), + sender=hidden_comp, + receiver=output_comp) - input_comp = pnl.TransferMechanism(name='input_comp', + xor_comp = pnl.Composition() + + backprop_pathway = xor_comp.add_backpropagation_learning_pathway([input_comp, + in_to_hidden_comp, + hidden_comp, + hidden_to_out_comp, + output_comp], + learning_rate=10) + target_mech = backprop_pathway.target + inputs_dict = {"inputs": {input_comp:xor_inputs}, + "targets": {output_comp:xor_targets}, + "epochs": num_epochs} + result_comp = xor_comp.learn(inputs=inputs_dict) + + # AutodiffComposition using LLVM + if 'LLVM' in models: + + input_LLVM = pnl.TransferMechanism(name='input', default_variable=np.zeros(2)) - hidden_comp = pnl.TransferMechanism(name='hidden_comp', + hidden_LLVM = pnl.TransferMechanism(name='hidden', default_variable=np.zeros(10), function=pnl.Logistic()) - output_comp = pnl.TransferMechanism(name='output_comp', + output_LLVM = pnl.TransferMechanism(name='output', default_variable=np.zeros(1), function=pnl.Logistic()) - in_to_hidden_comp = pnl.MappingProjection(name='in_to_hidden_comp', + in_to_hidden_LLVM = pnl.MappingProjection(name='in_to_hidden', matrix=in_to_hidden_matrix.copy(), - sender=input_comp, - receiver=hidden_comp) + sender=input_LLVM, + receiver=hidden_LLVM) - hidden_to_out_comp = pnl.MappingProjection(name='hidden_to_out_comp', + hidden_to_out_LLVM = pnl.MappingProjection(name='hidden_to_out', matrix=hidden_to_out_matrix.copy(), - sender=hidden_comp, - receiver=output_comp) - - xor_comp = pnl.Composition() - - backprop_pathway = xor_comp.add_backpropagation_learning_pathway([input_comp, - in_to_hidden_comp, - hidden_comp, - hidden_to_out_comp, - output_comp], - learning_rate=10) - target_mech = backprop_pathway.target - inputs_dict = {"inputs": {input_comp:xor_inputs}, - "targets": {output_comp:xor_targets}, + sender=hidden_LLVM, + receiver=output_LLVM) + + xor_LLVM = pnl.AutodiffComposition(learning_rate=10, + optimizer_type='sgd') + + xor_LLVM.add_node(input_LLVM) + xor_LLVM.add_node(hidden_LLVM) + xor_LLVM.add_node(output_LLVM) + + xor_LLVM.add_projection(sender=input_LLVM, projection=in_to_hidden_LLVM, receiver=hidden_LLVM) + xor_LLVM.add_projection(sender=hidden_LLVM, projection=hidden_to_out_LLVM, receiver=output_LLVM) + xor_LLVM.infer_backpropagation_learning_pathways() + + inputs_dict = {"inputs": {input_LLVM:xor_inputs}, + "targets": {output_LLVM:xor_targets}, "epochs": num_epochs} - result_comp = xor_comp.learn(inputs=inputs_dict) + result_LLVM = xor_LLVM.learn(inputs=inputs_dict, execution_mode=pnlvm.ExecutionMode.LLVMRun) - # AutodiffComposition - if 'AUTODIFF' in models: + assert np.allclose(in_to_hidden_LLVM.parameters.matrix.get(xor_LLVM), + in_to_hidden_comp.get_mod_matrix(xor_comp)) + assert np.allclose(hidden_to_out_LLVM.parameters.matrix.get(xor_LLVM), + hidden_to_out_comp.get_mod_matrix(xor_comp)) + assert np.allclose(result_comp, result_LLVM) - input_autodiff = pnl.TransferMechanism(name='input', + elif 'PYTORCH' in models: + + input_PYTORCH = pnl.TransferMechanism(name='input', default_variable=np.zeros(2)) - hidden_autodiff = pnl.TransferMechanism(name='hidden', + hidden_PYTORCH = pnl.TransferMechanism(name='hidden', default_variable=np.zeros(10), function=pnl.Logistic()) - output_autodiff = pnl.TransferMechanism(name='output', + output_PYTORCH = pnl.TransferMechanism(name='output', default_variable=np.zeros(1), function=pnl.Logistic()) - in_to_hidden_autodiff = pnl.MappingProjection(name='in_to_hidden', + in_to_hidden_PYTORCH = pnl.MappingProjection(name='in_to_hidden', matrix=in_to_hidden_matrix.copy(), - sender=input_autodiff, - receiver=hidden_autodiff) + sender=input_PYTORCH, + receiver=hidden_PYTORCH) - hidden_to_out_autodiff = pnl.MappingProjection(name='hidden_to_out', + hidden_to_out_PYTORCH = pnl.MappingProjection(name='hidden_to_out', matrix=hidden_to_out_matrix.copy(), - sender=hidden_autodiff, - receiver=output_autodiff) + sender=hidden_PYTORCH, + receiver=output_PYTORCH) - xor_autodiff = pnl.AutodiffComposition(learning_rate=10, - optimizer_type='sgd') + xor_PYTORCH = pnl.AutodiffComposition(learning_rate=10, + optimizer_type='sgd') - xor_autodiff.add_node(input_autodiff) - xor_autodiff.add_node(hidden_autodiff) - xor_autodiff.add_node(output_autodiff) + xor_PYTORCH.add_node(input_PYTORCH) + xor_PYTORCH.add_node(hidden_PYTORCH) + xor_PYTORCH.add_node(output_PYTORCH) - xor_autodiff.add_projection(sender=input_autodiff, projection=in_to_hidden_autodiff, receiver=hidden_autodiff) - xor_autodiff.add_projection(sender=hidden_autodiff, projection=hidden_to_out_autodiff, receiver=output_autodiff) - xor_autodiff.infer_backpropagation_learning_pathways() + xor_PYTORCH.add_projection(sender=input_PYTORCH, projection=in_to_hidden_PYTORCH, receiver=hidden_PYTORCH) + xor_PYTORCH.add_projection(sender=hidden_PYTORCH, projection=hidden_to_out_PYTORCH, receiver=output_PYTORCH) + xor_PYTORCH.infer_backpropagation_learning_pathways() - inputs_dict = {"inputs": {input_autodiff:xor_inputs}, - "targets": {output_autodiff:xor_targets}, + inputs_dict = {"inputs": {input_PYTORCH:xor_inputs}, + "targets": {output_PYTORCH:xor_targets}, "epochs": num_epochs} - result_autodiff = xor_autodiff.learn(inputs=inputs_dict) + result_PYTORCH = xor_PYTORCH.learn(inputs=inputs_dict, + execution_mode=pnlvm.ExecutionMode.PyTorch) + + assert np.allclose(in_to_hidden_PYTORCH.parameters.matrix.get(xor_PYTORCH), + in_to_hidden_comp.get_mod_matrix(xor_comp)) + assert np.allclose(hidden_to_out_PYTORCH.parameters.matrix.get(xor_PYTORCH), + hidden_to_out_comp.get_mod_matrix(xor_comp)) + assert np.allclose(result_comp, result_PYTORCH) - # COMPARE WEIGHTS FOR PAIRS OF MODELS ---------------------------------------------------------- - if all(m in models for m in {pnl.COMPOSITION, 'AUTODIFF'}): - assert np.allclose(in_to_hidden_autodiff.parameters.matrix.get(xor_autodiff), in_to_hidden_comp.get_mod_matrix(xor_comp)) - assert np.allclose(hidden_to_out_autodiff.parameters.matrix.get(xor_autodiff), hidden_to_out_comp.get_mod_matrix(xor_comp)) - assert np.allclose(result_comp, result_autodiff) @pytest.mark.parametrize('configuration', [ 'Y UP', diff --git a/tests/composition/test_show_graph.py b/tests/composition/test_show_graph.py index 9d81254486b..a159f08eec9 100644 --- a/tests/composition/test_show_graph.py +++ b/tests/composition/test_show_graph.py @@ -72,7 +72,7 @@ def test_converging_pathways(self): class TestNested: def test_multiple_projections_to_node_of_nested_composition(self): - '''This is based on the N-back script''' + '''This is based on the Nback script''' stim = TransferMechanism(name='STIM', size=5) context = TransferMechanism(name='CONTEXT', size=5) diff --git a/tests/functions/test_transfer.py b/tests/functions/test_transfer.py index 15db649b2fb..9168e37c544 100644 --- a/tests/functions/test_transfer.py +++ b/tests/functions/test_transfer.py @@ -1,8 +1,9 @@ import numpy as np -import psyneulink.core.llvm as pnlvm +import pytest + import psyneulink.core.components.functions.nonstateful.transferfunctions as Functions import psyneulink.core.globals.keywords as kw -import pytest +import psyneulink.core.llvm as pnlvm SIZE=10 np.random.seed(0) @@ -98,6 +99,7 @@ def test_execute(func, variable, params, expected, benchmark, func_mode): tanh_derivative_helper = (RAND1 * (test_var + RAND2) + RAND3) tanh_derivative_helper = (1 - np.tanh(tanh_derivative_helper)**2) * RAND4 * RAND1 + derivative_test_data = [ (Functions.Linear, test_var, {kw.SLOPE:RAND1, kw.INTERCEPT:RAND2}, RAND1), (Functions.Exponential, test_var, {kw.SCALE:RAND1, kw.RATE:RAND2}, RAND1 * RAND2 * np.exp(RAND2 * test_var)), diff --git a/tests/log/test_log.py b/tests/log/test_log.py index 2229e2f3d30..ba103e5f582 100644 --- a/tests/log/test_log.py +++ b/tests/log/test_log.py @@ -1253,7 +1253,7 @@ def test_multilayer(self): p = [input_layer, input_weights, hidden_layer_1, middle_weights, hidden_layer_2, output_weights, output_layer] backprop_pathway = comp.add_backpropagation_learning_pathway( pathway=p, - loss_function='sse', + loss_function=pnl.Loss.L0, learning_rate=1. ) diff --git a/tests/log/test_rpc.py b/tests/log/test_rpc.py index 405706cf97c..c17b1cb292b 100644 --- a/tests/log/test_rpc.py +++ b/tests/log/test_rpc.py @@ -485,7 +485,8 @@ def test_multilayer(self): p = [input_layer, input_weights, hidden_layer_1, middle_weights, hidden_layer_2, output_weights, output_layer] backprop_pathway = comp.add_backpropagation_learning_pathway( pathway=p, - loss_function='sse', + # loss_function=pnl.Loss.SSE, + loss_function=pnl.Loss.L0, learning_rate=1. ) From 2deecb7e2895aceaf9ec7aedf05dd06243110594 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Sun, 20 Nov 2022 01:46:54 -0500 Subject: [PATCH 097/127] treewide: Drop mentions of ControlMechanismRegistry Not used in any place, always empty. Signed-off-by: Jan Vesely --- psyneulink/__init__.py | 1 - .../mechanisms/modulatory/control/controlmechanism.py | 4 +--- .../mechanisms/modulatory/control/agt/agtcontrolmechanism.py | 4 +--- .../mechanisms/modulatory/control/agt/lccontrolmechanism.py | 4 +--- 4 files changed, 3 insertions(+), 10 deletions(-) diff --git a/psyneulink/__init__.py b/psyneulink/__init__.py index 14911f370e7..7b7924ada43 100644 --- a/psyneulink/__init__.py +++ b/psyneulink/__init__.py @@ -84,7 +84,6 @@ def filter(self, record): primary_registries = [ CompositionRegistry, - ControlMechanismRegistry, DeferredInitRegistry, FunctionRegistry, GatingMechanismRegistry, diff --git a/psyneulink/core/components/mechanisms/modulatory/control/controlmechanism.py b/psyneulink/core/components/mechanisms/modulatory/control/controlmechanism.py index a100ed485a2..cecbe5bb451 100644 --- a/psyneulink/core/components/mechanisms/modulatory/control/controlmechanism.py +++ b/psyneulink/core/components/mechanisms/modulatory/control/controlmechanism.py @@ -612,7 +612,7 @@ from psyneulink.core.globals.utilities import ContentAddressableList, convert_to_list, convert_to_np_array, is_iterable __all__ = [ - 'CONTROL_ALLOCATION', 'GATING_ALLOCATION', 'ControlMechanism', 'ControlMechanismError', 'ControlMechanismRegistry', + 'CONTROL_ALLOCATION', 'GATING_ALLOCATION', 'ControlMechanism', 'ControlMechanismError', ] CONTROL_ALLOCATION = 'control_allocation' @@ -620,8 +620,6 @@ MonitoredOutputPortTuple = collections.namedtuple("MonitoredOutputPortTuple", "output_port weight exponent matrix") -ControlMechanismRegistry = {} - def _is_control_spec(spec): from psyneulink.core.components.projections.modulatory.controlprojection import ControlProjection if isinstance(spec, tuple): diff --git a/psyneulink/library/components/mechanisms/modulatory/control/agt/agtcontrolmechanism.py b/psyneulink/library/components/mechanisms/modulatory/control/agt/agtcontrolmechanism.py index 92c245d3275..dba0645d984 100644 --- a/psyneulink/library/components/mechanisms/modulatory/control/agt/agtcontrolmechanism.py +++ b/psyneulink/library/components/mechanisms/modulatory/control/agt/agtcontrolmechanism.py @@ -174,13 +174,11 @@ from psyneulink.core.globals.preferences.preferenceset import PreferenceLevel __all__ = [ - 'AGTControlMechanism', 'AGTControlMechanismError', 'ControlMechanismRegistry', 'MONITORED_OUTPUT_PORT_NAME_SUFFIX' + 'AGTControlMechanism', 'AGTControlMechanismError', 'MONITORED_OUTPUT_PORT_NAME_SUFFIX' ] MONITORED_OUTPUT_PORT_NAME_SUFFIX = '_Monitor' -ControlMechanismRegistry = {} - class AGTControlMechanismError(Exception): def __init__(self, error_value): self.error_value = error_value diff --git a/psyneulink/library/components/mechanisms/modulatory/control/agt/lccontrolmechanism.py b/psyneulink/library/components/mechanisms/modulatory/control/agt/lccontrolmechanism.py index f715396b12f..b709fd861f2 100644 --- a/psyneulink/library/components/mechanisms/modulatory/control/agt/lccontrolmechanism.py +++ b/psyneulink/library/components/mechanisms/modulatory/control/agt/lccontrolmechanism.py @@ -313,15 +313,13 @@ from psyneulink.core.globals.utilities import is_iterable, convert_to_list __all__ = [ - 'CONTROL_SIGNAL_NAME', 'ControlMechanismRegistry', 'LCControlMechanism', 'LCControlMechanismError', + 'CONTROL_SIGNAL_NAME', 'LCControlMechanism', 'LCControlMechanismError', 'MODULATED_MECHANISMS', ] MODULATED_MECHANISMS = 'modulated_mechanisms' CONTROL_SIGNAL_NAME = 'LCControlMechanism_ControlSignal' -ControlMechanismRegistry = {} - class LCControlMechanismError(Exception): def __init__(self, error_value): self.error_value = error_value From 10c749c75d25e09f9a2796c5c8e5001c2f3c29b6 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Sun, 20 Nov 2022 02:21:26 -0500 Subject: [PATCH 098/127] treewide: Drop mentions of GatingMechanismRegistry Not used in any place, always empty. Signed-off-by: Jan Vesely --- psyneulink/__init__.py | 1 - .../mechanisms/modulatory/control/gating/gatingmechanism.py | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/psyneulink/__init__.py b/psyneulink/__init__.py index 7b7924ada43..09c949c7170 100644 --- a/psyneulink/__init__.py +++ b/psyneulink/__init__.py @@ -86,7 +86,6 @@ def filter(self, record): CompositionRegistry, DeferredInitRegistry, FunctionRegistry, - GatingMechanismRegistry, MechanismRegistry, PathwayRegistry, PortRegistry, diff --git a/psyneulink/core/components/mechanisms/modulatory/control/gating/gatingmechanism.py b/psyneulink/core/components/mechanisms/modulatory/control/gating/gatingmechanism.py index 8aa950f2b4a..c41022d2ef5 100644 --- a/psyneulink/core/components/mechanisms/modulatory/control/gating/gatingmechanism.py +++ b/psyneulink/core/components/mechanisms/modulatory/control/gating/gatingmechanism.py @@ -196,11 +196,9 @@ from psyneulink.core.globals.utilities import ContentAddressableList, convert_to_list __all__ = [ - 'GatingMechanism', 'GatingMechanismError', 'GatingMechanismRegistry' + 'GatingMechanism', 'GatingMechanismError', ] -GatingMechanismRegistry = {} - def _is_gating_spec(spec): from psyneulink.core.components.projections.modulatory.gatingprojection import GatingProjection From 3c9662d8a3946acfcd70093553f694ff1c4792b5 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Sun, 20 Nov 2022 12:37:29 -0500 Subject: [PATCH 099/127] registry: Add configurable prefix to auto generated component names Names starting with "__pnl_" are reserved for PsyNeuLink internal use. Signed-off-by: Jan Vesely --- psyneulink/core/globals/registry.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/psyneulink/core/globals/registry.py b/psyneulink/core/globals/registry.py index 15c585d1c7f..71d2a34be7c 100644 --- a/psyneulink/core/globals/registry.py +++ b/psyneulink/core/globals/registry.py @@ -217,6 +217,8 @@ def register_category(entry, raise RegistryError("Requested entry {0} not of type {1}".format(entry, base_class)) +_register_auto_name_prefix = "" + def register_instance(entry, name, base_class, registry, sub_dict): renamed_instance_counts = registry[sub_dict].renamed_instance_counts @@ -224,11 +226,14 @@ def register_instance(entry, name, base_class, registry, sub_dict): # If entry (instance) name is None, set entry's name to sub_dict-n where n is the next available numeric suffix # (starting at 0) based on the number of unnamed/renamed sub_dict objects that have already been assigned names if name is None: - entry.name = '{0}-{1}'.format(sub_dict, renamed_instance_counts[sub_dict]) + entry.name = '{0}{1}-{2}'.format(_register_auto_name_prefix, sub_dict, renamed_instance_counts[sub_dict]) renamed = True else: entry.name = name + assert not entry.name.startswith("__pnl_") or entry.name.startswith(_register_auto_name_prefix), \ + "Using reserved name: {}".format(entry.name) + while entry.name in registry[sub_dict].instanceDict: # if the decided name (provided or determined) is already assigned to an object, get the non-suffixed name, # and append the proper new suffix according to the number of objects that have been assigned that name From 2795bf77388b08b5c39b923dadd41534b906d17b Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Sun, 20 Nov 2022 12:42:33 -0500 Subject: [PATCH 100/127] tests: Add custom name prefix to all components with default names generated during test collection This will help spot enumeration-time generated names in test results. These names should not be used, because they depend on test enumeration order. The current setup instantiates 419 components during enumeration. All their names are now prefixed with "__pnl_pytest_". Signed-off-by: Jan Vesely --- conftest.py | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/conftest.py b/conftest.py index 3d9c8ec1d91..8b840f77147 100644 --- a/conftest.py +++ b/conftest.py @@ -36,15 +36,6 @@ def pytest_addoption(parser): parser.addoption('--fp-precision', action='store', default='fp64', choices=['fp32', 'fp64'], help='Set default fp precision for the runtime compiler. Default: fp64') -def pytest_sessionstart(session): - precision = session.config.getvalue("--fp-precision") - if precision == 'fp64': - pnlvm.LLVMBuilderContext.default_float_ty = pnlvm.ir.DoubleType() - elif precision == 'fp32': - pnlvm.LLVMBuilderContext.default_float_ty = pnlvm.ir.FloatType() - else: - assert False, "Unsupported precision parameter: {}".format(precision) - def pytest_runtest_setup(item): # Check that all 'cuda' tests are also marked 'llvm' assert 'llvm' in item.keywords or 'cuda' not in item.keywords @@ -89,6 +80,30 @@ def pytest_generate_tests(metafunc): ] metafunc.parametrize("autodiff_mode", auto_modes) + +_old_register_prefix = None + +# Collection hooks +def pytest_sessionstart(session): + """Initialize session with the right floating point precision and component name prefix.""" + + precision = session.config.getvalue("--fp-precision") + if precision == 'fp64': + pnlvm.LLVMBuilderContext.default_float_ty = pnlvm.ir.DoubleType() + elif precision == 'fp32': + pnlvm.LLVMBuilderContext.default_float_ty = pnlvm.ir.FloatType() + else: + assert False, "Unsupported precision parameter: {}".format(precision) + + global _old_register_prefix + _old_register_prefix = psyneulink.core.globals.registry._register_auto_name_prefix + psyneulink.core.globals.registry._register_auto_name_prefix = "__pnl_pytest_" + +def pytest_collection_finish(session): + """Restore component prefix at the end of test collection.""" + psyneulink.core.globals.registry._register_auto_name_prefix = _old_register_prefix + +# Runtest hooks def pytest_runtest_call(item): # seed = int(item.config.getoption('--pnl-seed')) seed = 0 From 9b9c8c55f9fa2fa90ffef5a8f29767511c687f44 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Tue, 22 Nov 2022 16:30:31 -0500 Subject: [PATCH 101/127] github-actions: Check env var to tag selected OSes to run on self-hosted runners (#2546) Setting one of the following secrets: SELF_HOSTED_MACOS, SELF_HOSTED_LINUX, SELF_HOSTED_WINDOWS controls if jobs using that OS will run on self-hosted runners. If the value stored in any of them evaluates to true (based on GitHub actions expression truth values) the jobs for the respective OS will be tagged to use self-hosted runners. Without the secret, or if the value evaluates to false, the jobs will be tagged to use GitHub cloud runners. "runs-on" doesn't allow direct use of environment variables so we need to pass them through an extra job. Signed-off-by: Jan Vesely --- .github/workflows/pnl-ci.yml | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pnl-ci.yml b/.github/workflows/pnl-ci.yml index be57f480190..19775508b56 100644 --- a/.github/workflows/pnl-ci.yml +++ b/.github/workflows/pnl-ci.yml @@ -11,6 +11,11 @@ on: - 'v**' pull_request: +env: + SELF_HOSTED_MACOS: ${{ secrets.SELF_HOSTED_MACOS }} + SELF_HOSTED_LINUX: ${{ secrets.SELF_HOSTED_LINUX }} + SELF_HOSTED_WINDOWS: ${{ secrets.SELF_HOSTED_WINDOWS }} + # run only the latest instance of this workflow job for the current branch/PR # cancel older runs # fall back to run id if not available (run id is unique -> no cancellations) @@ -19,28 +24,47 @@ concurrency: cancel-in-progress: true jobs: + # A job to select self-hosted runner if requested by an env var + select-runner: + runs-on: ubuntu-latest + + outputs: + self_hosted_macos: ${{ steps.is_self_hosted.outputs.macos && 'macos' || '' }} + self_hosted_linux: ${{ steps.is_self_hosted.outputs.linux && 'linux' || '' }} + self_hosted_windows: ${{ steps.is_self_hosted.outputs.windows && 'windows' || '' }} + + steps: + - name: Add macos + id: is_self_hosted + run: | + echo "macos=$SELF_HOSTED_MACOS" | tee -a $GITHUB_OUTPUT + echo "linux=$SELF_HOSTED_LINUX" | tee -a $GITHUB_OUTPUT + echo "windows=$SELF_HOSTED_WINDOWS" | tee -a $GITHUB_OUTPUT + + # the main build job build: - runs-on: ${{ matrix.os }} + needs: select-runner + runs-on: ${{ (contains(needs.select-runner.outputs.*, matrix.os) && fromJSON(format('[ "self-hosted","{0}", "X64" ]', matrix.os))) || format('{0}-latest', matrix.os) }} strategy: fail-fast: false matrix: python-version: [3.7, 3.8, 3.9] python-architecture: ['x64'] extra-args: [''] - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu, macos, windows] include: # add 32-bit build on windows - python-version: 3.8 python-architecture: 'x86' - os: windows-latest + os: windows # code-coverage build on macos python 3.9 - python-version: 3.9 - os: macos-latest + os: macos extra-args: '--cov=psyneulink' exclude: # 3.7 is broken on macos-11, https://github.com/actions/virtual-environments/issues/4230 - python-version: 3.7 - os: macos-latest + os: macos steps: # increased fetch-depth and tag checkout needed to get correct From d36dd2dd5a91cf38e1da928c1afba2c002a33e4c Mon Sep 17 00:00:00 2001 From: jdcpni Date: Wed, 23 Nov 2022 18:02:39 -0500 Subject: [PATCH 102/127] =?UTF-8?q?=E2=80=A2=20utilities.py:=20(#2547)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - iscompatible: replace initial test with safe_equals() - safe_equals: instantiate Katherine's mods + special handling of defaultdicts --- psyneulink/core/globals/utilities.py | 47 ++++++++++++++-------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/psyneulink/core/globals/utilities.py b/psyneulink/core/globals/utilities.py index b56b10095f9..5d4eb532f27 100644 --- a/psyneulink/core/globals/utilities.py +++ b/psyneulink/core/globals/utilities.py @@ -438,21 +438,10 @@ def iscompatible(candidate, reference=None, **kargs): # If the two are equal, can settle it right here # IMPLEMENTATION NOTE: remove the duck typing when numpy supports a direct comparison of iterables try: - with warnings.catch_warnings(): - warnings.simplefilter(action='ignore', category=FutureWarning) - # np.array(...).size > 0 checks for empty list. Everything else create single element (dtype=obejct) array - if reference is not None and np.array(candidate, dtype=object).size > 0 and (candidate == reference): - return True - # if reference is not None: - # if (isinstance(reference, (bool, int, float)) - # and isinstance(candidate, (bool, int, float)) - # and candidate == reference): - # return True - # elif (isinstance(reference, (list, np.ndarray)) - # and isinstance(candidate, (list, np.ndarray)) and (candidate == reference).all()): - # return True - # elif is_iterable(reference) and is_iterable(candidate) and (candidate == reference): - # return True + if (reference is not None and np.array(candidate, dtype=object).size > 0 + and safe_equals(candidate, reference)): + return True + except ValueError: # raise UtilitiesError("Could not compare {0} and {1}".format(candidate, reference)) # IMPLEMENTATION NOTE: np.array generates the following error: @@ -1653,12 +1642,12 @@ def safe_len(arr, fallback=1): except TypeError: return fallback - def safe_equals(x, y): """ An == comparison that handles numpy's new behavior of returning an array of booleans instead of a single boolean for == """ + from collections import defaultdict with warnings.catch_warnings(): warnings.simplefilter('error') try: @@ -1670,14 +1659,26 @@ def safe_equals(x, y): except (ValueError, DeprecationWarning, FutureWarning): try: return np.array_equal(x, y) - except DeprecationWarning: + except (DeprecationWarning, FutureWarning): len_x = len(x) - return ( - len_x == len(y) - and all([ - safe_equals(x[i], y[i]) for i in range(len_x) - ]) - ) + try: + # IMPLEMENTATION NOTE: + # Handles case in which an element being compared is a defaultdict + # (makes copy to prevent indexing it from adding and entry to source) + if len_x != len(y): + return False + for i in range(len_x): + if isinstance(x[i],defaultdict) or isinstance(y[i],defaultdict): + copy_x = x[i].copy() + copy_y = y[i].copy() + if not safe_equals(copy_x, copy_y): + return False + else: + if not safe_equals(x[i],y[i]): + return False + return True + except KeyError: + return False @tc.typecheck From 26bf706a22c2df14e8e38fd7269eebe00b251999 Mon Sep 17 00:00:00 2001 From: jdcpni Date: Wed, 23 Nov 2022 23:33:30 -0500 Subject: [PATCH 103/127] =?UTF-8?q?=E2=80=A2=20composition.py=20(#2549)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - learn(): add error for use of ExecutionMode.LLVM or ExecutionMode.PyTorch • autodiffcomposition.py - learn(): add warning for use of ExecutionMode.Python • test_learning.py: - add test_execution_mode_pytorch_and_LLVM_errors • test_autodiffcomposition.py: - add test_execution_mode_python_warning --- conftest.py | 2 + psyneulink/core/compositions/composition.py | 6 + psyneulink/core/llvm/__init__.py | 2 +- .../compositions/autodiffcomposition.py | 16 +- tests/composition/test_autodiffcomposition.py | 2706 +++++++++-------- tests/composition/test_learning.py | 18 + 6 files changed, 1406 insertions(+), 1344 deletions(-) diff --git a/conftest.py b/conftest.py index 8b840f77147..393cb53e3c6 100644 --- a/conftest.py +++ b/conftest.py @@ -76,6 +76,7 @@ def pytest_generate_tests(metafunc): if "autodiff_mode" in metafunc.fixturenames: auto_modes = [ pnlvm.ExecutionMode.Python, + # pnlvm.ExecutionMode.PyTorch, pytest.param(pnlvm.ExecutionMode.LLVMRun, marks=pytest.mark.llvm) ] metafunc.parametrize("autodiff_mode", auto_modes) @@ -163,6 +164,7 @@ def llvm_current_fp_precision(): @pytest.helpers.register def get_comp_execution_modes(): return [pytest.param(pnlvm.ExecutionMode.Python), + # pytest.param(pnlvm.ExecutionMode.PyTorch, marks=pytest.mark.pytorch), pytest.param(pnlvm.ExecutionMode.LLVM, marks=pytest.mark.llvm), pytest.param(pnlvm.ExecutionMode.LLVMExec, marks=pytest.mark.llvm), pytest.param(pnlvm.ExecutionMode.LLVMRun, marks=pytest.mark.llvm), diff --git a/psyneulink/core/compositions/composition.py b/psyneulink/core/compositions/composition.py index c61c155df38..d65f8c4e1e6 100644 --- a/psyneulink/core/compositions/composition.py +++ b/psyneulink/core/compositions/composition.py @@ -10392,8 +10392,14 @@ def learn( the results of the final epoch of training : list """ from psyneulink.library.compositions import CompositionRunner + from psyneulink.library.compositions import AutodiffComposition runner = CompositionRunner(self) + if (execution_mode in {pnlvm.ExecutionMode.PyTorch, pnlvm.ExecutionMode.LLVM} + and not isinstance(self, AutodiffComposition)): + raise CompositionError(f"ExecutionMode.{execution_mode.name} cannot be used in the learn() method of " + f"'{self.name}' because it is not an {AutodiffComposition.componentCategory}") + context.add_flag(ContextFlags.LEARNING_MODE) # # MODIFIED 3/28/22 NEW: # context.source = ContextFlags.COMPOSITION diff --git a/psyneulink/core/llvm/__init__.py b/psyneulink/core/llvm/__init__.py index 3f1c5e21067..7c5c951d75f 100644 --- a/psyneulink/core/llvm/__init__.py +++ b/psyneulink/core/llvm/__init__.py @@ -73,6 +73,7 @@ class ExecutionMode(enum.Flag): """ Python = 0 + PyTorch = enum.auto() LLVM = enum.auto() PTX = enum.auto() _Run = enum.auto() @@ -84,7 +85,6 @@ class ExecutionMode(enum.Flag): LLVMExec = LLVM | _Exec PTXRun = PTX | _Run PTXExec = PTX | _Exec - PyTorch = Python _binary_generation = 0 diff --git a/psyneulink/library/compositions/autodiffcomposition.py b/psyneulink/library/compositions/autodiffcomposition.py index 0a182c7bf84..f7efc82f8e4 100644 --- a/psyneulink/library/compositions/autodiffcomposition.py +++ b/psyneulink/library/compositions/autodiffcomposition.py @@ -286,6 +286,7 @@ """ import logging import os +import warnings import numpy as np from pathlib import Path, PosixPath @@ -629,6 +630,19 @@ def learn(self, *args, **kwargs): if self._built_pathways is False: self.infer_backpropagation_learning_pathways() self._built_pathways = True + + if 'execution_mode' in kwargs: + execution_mode = kwargs['execution_mode'] + if execution_mode == pnlvm.ExecutionMode.Python: + warnings.warn(f"{self.name}.learn() called with ExecutionMode.Python; " + f"learning will be executed using PyTorch; " + f"should use ExecutionMode.PyTorch for clarity, " + f"or a standard Composition for Python execution.)") + # OK, now that the user has been advised to use ExecutionMode.PyTorch and warned *not* to ExecutionMdoe.Python, + # convert ExecutionMode.PyTorch specification to ExecutionMode.Python for internal use (nice, eh?) + if execution_mode == pnlvm.ExecutionMode.PyTorch: + kwargs['execution_mode'] = pnlvm.ExecutionMode.Python + return super().learn(*args, **kwargs) @handle_external_context() @@ -651,7 +665,7 @@ def execute(self, clamp_input=SOFT_CLAMP, targets=None, runtime_params=None, - execution_mode:pnlvm.ExecutionMode = pnlvm.ExecutionMode.Python, + execution_mode:pnlvm.ExecutionMode = pnlvm.ExecutionMode.PyTorch, skip_initialization=False, report_output:ReportOutput=ReportOutput.OFF, report_params:ReportOutput=ReportParams.OFF, diff --git a/tests/composition/test_autodiffcomposition.py b/tests/composition/test_autodiffcomposition.py index 937096933d9..6cf6f56b28f 100644 --- a/tests/composition/test_autodiffcomposition.py +++ b/tests/composition/test_autodiffcomposition.py @@ -26,91 +26,6 @@ # Unit tests for functions of AutodiffComposition class that are new (not in Composition) # or override functions in Composition -@pytest.mark.pytorch -def test_autodiff_forward(autodiff_mode): - # create xor model mechanisms and projections - xor_in = TransferMechanism(name='xor_in', - default_variable=np.zeros(2)) - - xor_hid = TransferMechanism(name='xor_hid', - default_variable=np.zeros(10), - function=Logistic()) - - xor_out = TransferMechanism(name='xor_out', - default_variable=np.zeros(1), - function=Logistic()) - - hid_map = MappingProjection(matrix=np.random.rand(2,10)) - out_map = MappingProjection(matrix=np.random.rand(10,1)) - - # put the mechanisms and projections together in an autodiff composition (AC) - xor = AutodiffComposition() - - xor.add_node(xor_in) - xor.add_node(xor_hid) - xor.add_node(xor_out) - - xor.add_projection(sender=xor_in, projection=hid_map, receiver=xor_hid) - xor.add_projection(sender=xor_hid, projection=out_map, receiver=xor_out) - - outputs = xor.run(inputs=[0,0], execution_mode=autodiff_mode) - assert np.allclose(outputs, [[0.9479085241082691]]) - -@pytest.mark.pytorch -def test_autodiff_saveload(tmp_path): - def create_xor(): - # create xor model mechanisms and projections - xor_in = TransferMechanism(name='xor_in', - default_variable=np.zeros(2)) - - xor_hid = TransferMechanism(name='xor_hid', - default_variable=np.zeros(10), - function=Logistic()) - - xor_out = TransferMechanism(name='xor_out', - default_variable=np.zeros(1), - function=Logistic()) - - hid_map = MappingProjection(matrix=np.random.rand(2,10), name='hid_map') - out_map = MappingProjection(matrix=np.random.rand(10,1), name='out_map') - - # put the mechanisms and projections together in an autodiff composition (AC) - xor = AutodiffComposition() - - xor.add_node(xor_in) - xor.add_node(xor_hid) - xor.add_node(xor_out) - - xor.add_projection(sender=xor_in, projection=hid_map, receiver=xor_hid) - xor.add_projection(sender=xor_hid, projection=out_map, receiver=xor_out) - return xor - - np.random.seed(0) - xor1 = create_xor() - xor1_outputs = xor1.run(inputs=[0,0]) - - # save - # path = xor1.save() - path = xor1.save(os.path.join(tmp_path, 'xor_1.pnl')) - - # del xor1 - pnl.clear_registry() - - # load - np.random.seed(1) - xor2 = create_xor() - xor2_outputs_pre = xor2.run(inputs=[0,0]) - # xor2.load(os.path.join(tmp_path, 'xor_1.pnl')) - xor2.load(path) - xor2_outputs_post = xor2.run(inputs=[0,0]) - - - # sanity check - make sure xor2 weights differ - assert not np.allclose(xor2_outputs_pre, xor2_outputs_post, atol=1e-9) - - # make sure loaded model is identical, and used during run - assert np.allclose(xor1_outputs, xor2_outputs_post, atol=1e-9) - @pytest.mark.pytorch @pytest.mark.acconstructor class TestACConstructor: @@ -147,49 +62,51 @@ def test_report_prefs(self): # comp = AutodiffComposition() # assert comp.patience == 10 + @pytest.mark.pytorch -@pytest.mark.acmisc -class TestMiscTrainingFunctionality: +def test_autodiff_forward(autodiff_mode): + # create xor model mechanisms and projections + xor_in = TransferMechanism(name='xor_in', + default_variable=np.zeros(2)) - # test whether pytorch parameters are initialized to be identical to the Autodiff Composition's - def test_weight_initialization(self): + xor_hid = TransferMechanism(name='xor_hid', + default_variable=np.zeros(10), + function=Logistic()) - # create xor model mechanisms and projections - xor_in = TransferMechanism(name='xor_in', - default_variable=np.zeros(2)) + xor_out = TransferMechanism(name='xor_out', + default_variable=np.zeros(1), + function=Logistic()) - xor_hid = TransferMechanism(name='xor_hid', - default_variable=np.zeros(10), - function=Logistic()) + hid_map = MappingProjection(matrix=np.random.rand(2,10)) + out_map = MappingProjection(matrix=np.random.rand(10,1)) - xor_out = TransferMechanism(name='xor_out', - default_variable=np.zeros(1), - function=Logistic()) + # put the mechanisms and projections together in an autodiff composition (AC) + xor = AutodiffComposition() - hid_map = MappingProjection(matrix=np.random.rand(2,10)) - out_map = MappingProjection(matrix=np.random.rand(10,1)) + xor.add_node(xor_in) + xor.add_node(xor_hid) + xor.add_node(xor_out) - # put the mechanisms and projections together in an autodiff composition (AC) - xor = AutodiffComposition() + xor.add_projection(sender=xor_in, projection=hid_map, receiver=xor_hid) + xor.add_projection(sender=xor_hid, projection=out_map, receiver=xor_out) - xor.add_node(xor_in) - xor.add_node(xor_hid) - xor.add_node(xor_out) + outputs = xor.run(inputs=[0,0], execution_mode=autodiff_mode) + assert np.allclose(outputs, [[0.9479085241082691]]) - xor.add_projection(sender=xor_in, projection=hid_map, receiver=xor_hid) - xor.add_projection(sender=xor_hid, projection=out_map, receiver=xor_out) - # mini version of xor.execute just to build up pytorch representation - xor._analyze_graph() - xor._build_pytorch_representation(context=xor.default_execution_id) - # check whether pytorch parameters are identical to projections - assert np.allclose(hid_map.parameters.matrix.get(None), - xor.parameters.pytorch_representation.get(xor).params[0].detach().numpy()) - assert np.allclose(out_map.parameters.matrix.get(None), - xor.parameters.pytorch_representation.get(xor).params[1].detach().numpy()) +@pytest.mark.pytorch +@pytest.mark.accorrectness +class TestTrainingCorrectness: - # test whether processing doesn't interfere with pytorch parameters after training - def test_training_then_processing(self, autodiff_mode): + # test whether xor model created as autodiff composition learns properly + @pytest.mark.benchmark(group="XOR") + @pytest.mark.parametrize( + 'eps, calls, opt, expected', [ + (100, 'single', 'adam', [[[0.09823965]], [[0.81092879]], [[0.78179557]], [[0.25593583]]]), + (50, 'multiple', 'adam', [[[0.31200036]], [[0.59406178]], [[0.60417587]], [[0.52347365]]]), + ] + ) + def test_xor_training_correctness(self, eps, calls, opt, autodiff_mode, benchmark, expected): xor_in = TransferMechanism(name='xor_in', default_variable=np.zeros(2)) @@ -201,10 +118,11 @@ def test_training_then_processing(self, autodiff_mode): default_variable=np.zeros(1), function=Logistic()) - hid_map = MappingProjection() - out_map = MappingProjection() + hid_map = MappingProjection(matrix=np.random.rand(2, 10)) + out_map = MappingProjection(matrix=np.random.rand(10, 1)) - xor = AutodiffComposition() + xor = AutodiffComposition(optimizer_type=opt, + learning_rate=0.1) xor.add_node(xor_in) xor.add_node(xor_hid) @@ -214,486 +132,200 @@ def test_training_then_processing(self, autodiff_mode): xor.add_projection(sender=xor_hid, projection=out_map, receiver=xor_out) xor_inputs = np.array( # the inputs we will provide to the model - [[0, 0], - [0, 1], - [1, 0], - [1, 1]]) + [[0, 0], [0, 1], [1, 0], [1, 1]]) xor_targets = np.array( # the outputs we wish to see from the model - [[0], - [1], - [1], - [0]]) + [[0], [1], [1], [0]]) - # train model for a few epochs - # results_before_proc = xor.run(inputs={xor_in:xor_inputs}, - # targets={xor_out:xor_targets}, - # epochs=10) - results_before_proc = xor.learn(inputs={"inputs": {xor_in:xor_inputs}, - "targets": {xor_out:xor_targets}, - "epochs": 10}, - execution_mode=autodiff_mode) + if calls == 'single': + results = benchmark(xor.learn, inputs={"inputs": {xor_in:xor_inputs}, + "targets": {xor_out:xor_targets}, + "epochs": eps}, execution_mode=autodiff_mode) - # get weight parameters from pytorch - pt_weights_hid_bp = xor.parameters.pytorch_representation.get(xor).params[0].detach().numpy().copy() - pt_weights_out_bp = xor.parameters.pytorch_representation.get(xor).params[1].detach().numpy().copy() + else: + input_dict = {"inputs": {xor_in: xor_inputs}, + "targets": {xor_out: xor_targets}, + "epochs": 1} + for i in range(eps - 1): + xor.learn(inputs=input_dict, execution_mode=autodiff_mode) + results = benchmark(xor.learn, inputs=input_dict, execution_mode=autodiff_mode) + + assert len(results) == len(expected) + for r, t in zip(results, expected): + assert np.allclose(r[0], t) - #KAM temporarily removed -- will reimplement when pytorch weights can be used in pure PNL execution - # do processing on a few inputs - # results_proc = xor.run(inputs={xor_in:xor_inputs}) - # results_proc = xor.run(inputs={"inputs": {xor_in:xor_inputs}}) - # - # # get weight parameters from pytorch - # pt_weights_hid_ap = xor.parameters.pytorch_representation.get(xor).params[0].detach().numpy().copy() - # pt_weights_out_ap = xor.parameters.pytorch_representation.get(xor).params[1].detach().numpy().copy() - # - # # check that weight parameters before and after processing are the same - # assert np.allclose(pt_weights_hid_bp, pt_weights_hid_ap) - # assert np.allclose(pt_weights_out_bp, pt_weights_out_ap) + # tests whether semantic network created as autodiff composition learns properly + @pytest.mark.benchmark(group="Semantic net") @pytest.mark.parametrize( - 'loss', [Loss.L1, Loss.POISSON_NLL] + 'eps, opt', [ + (50, 'adam'), + ] ) - def test_various_loss_specs(self, loss, autodiff_mode): - if autodiff_mode is not pnl.ExecutionMode.Python: - pytest.skip("Loss spec not yet implemented!") - - xor_in = TransferMechanism(name='xor_in', - default_variable=np.zeros(2)) + def test_semantic_net_training_correctness(self, eps, opt, autodiff_mode, benchmark): - xor_hid = TransferMechanism(name='xor_hid', - default_variable=np.zeros(10), - function=Logistic()) + # MECHANISMS FOR SEMANTIC NET: - xor_out = TransferMechanism(name='xor_out', - default_variable=np.zeros(1), - function=Logistic()) + nouns_in = TransferMechanism(name="nouns_input", + default_variable=np.zeros(8)) - hid_map = MappingProjection() - out_map = MappingProjection() + rels_in = TransferMechanism(name="rels_input", + default_variable=np.zeros(3)) - xor = AutodiffComposition(loss_spec=loss) + h1 = TransferMechanism(name="hidden_nouns", + default_variable=np.zeros(8), + function=Logistic()) - xor.add_node(xor_in) - xor.add_node(xor_hid) - xor.add_node(xor_out) + h2 = TransferMechanism(name="hidden_mixed", + default_variable=np.zeros(15), + function=Logistic()) - xor.add_projection(sender=xor_in, projection=hid_map, receiver=xor_hid) - xor.add_projection(sender=xor_hid, projection=out_map, receiver=xor_out) + out_sig_I = TransferMechanism(name="sig_outs_I", + default_variable=np.zeros(8), + function=Logistic()) - xor_inputs = np.array( # the inputs we will provide to the model - [[0, 0], - [0, 1], - [1, 0], - [1, 1]]) - - xor_targets = np.array( # the outputs we wish to see from the model - [[0], - [1], - [1], - [0]]) - - xor.learn(inputs = {"inputs": {xor_in:xor_inputs}, - "targets": {xor_out:xor_targets}, - "epochs": 10}, execution_mode=autodiff_mode) - - def test_pytorch_loss_spec(self, autodiff_mode): - if autodiff_mode is not pnl.ExecutionMode.Python: - pytest.skip("Loss spec not yet implemented!") - - import torch - ls = torch.nn.SoftMarginLoss(reduction='sum') - - xor_in = TransferMechanism(name='xor_in', - default_variable=np.zeros(2)) - - xor_hid = TransferMechanism(name='xor_hid', - default_variable=np.zeros(10), - function=Logistic()) - - xor_out = TransferMechanism(name='xor_out', - default_variable=np.zeros(1), - function=Logistic()) - - hid_map = MappingProjection() - out_map = MappingProjection() - - xor = AutodiffComposition(loss_spec=ls) - - xor.add_node(xor_in) - xor.add_node(xor_hid) - xor.add_node(xor_out) - - xor.add_projection(sender=xor_in, projection=hid_map, receiver=xor_hid) - xor.add_projection(sender=xor_hid, projection=out_map, receiver=xor_out) - xor_inputs = np.array( # the inputs we will provide to the model - [[0, 0], [0, 1], [1, 0], [1, 1]]) - - xor_targets = np.array( # the outputs we wish to see from the model - [[0], [1], [1], [0]]) - - xor.learn(inputs={"inputs": {xor_in:xor_inputs}, - "targets": {xor_out:xor_targets}, - "epochs": 10}, execution_mode=autodiff_mode) - xor.learn(inputs={"inputs": {xor_in: xor_inputs}, - "targets": {xor_out: xor_targets}, - "epochs": 10}, execution_mode=autodiff_mode) + out_sig_is = TransferMechanism(name="sig_outs_is", + default_variable=np.zeros(12), + function=Logistic()) + out_sig_has = TransferMechanism(name="sig_outs_has", + default_variable=np.zeros(9), + function=Logistic()) - @pytest.mark.benchmark(group="Optimizer specs") - @pytest.mark.parametrize( - 'learning_rate, weight_decay, optimizer_type, expected', [ - (10, 0, 'sgd', [[[0.9863038667851067]], [[0.9944287263151904]], [[0.9934801466163382]], [[0.9979153035411085]]]), - (1.5, 1, 'sgd', [[[0.33226742]], [[0.4492334]], [[0.75459534]], [[0.44477028]]]), - (1.5, 1, 'adam', [[[0.43109927]], [[0.33088828]], [[0.40094236]], [[0.57104689]]]), - ] - ) - def test_optimizer_specs(self, learning_rate, weight_decay, optimizer_type, expected, autodiff_mode, benchmark): - xor_in = TransferMechanism(name='xor_in', - default_variable=np.zeros(2)) + out_sig_can = TransferMechanism(name="sig_outs_can", + default_variable=np.zeros(9), + function=Logistic()) - xor_hid = TransferMechanism(name='xor_hid', - default_variable=np.zeros(10), - function=Logistic()) + # SET UP PROJECTIONS FOR SEMANTIC NET - xor_out = TransferMechanism(name='xor_out', - default_variable=np.zeros(1), - function=Logistic()) + map_nouns_h1 = MappingProjection(matrix=np.random.rand(8,8), + name="map_nouns_h1", + sender=nouns_in, + receiver=h1) - hid_map = MappingProjection() - out_map = MappingProjection() + map_rels_h2 = MappingProjection(matrix=np.random.rand(3,15), + name="map_relh2", + sender=rels_in, + receiver=h2) - xor = AutodiffComposition(learning_rate=learning_rate, - optimizer_type=optimizer_type, - weight_decay=weight_decay) + map_h1_h2 = MappingProjection(matrix=np.random.rand(8,15), + name="map_h1_h2", + sender=h1, + receiver=h2) - xor.add_node(xor_in) - xor.add_node(xor_hid) - xor.add_node(xor_out) + map_h2_I = MappingProjection(matrix=np.random.rand(15,8), + name="map_h2_I", + sender=h2, + receiver=out_sig_I) - xor.add_projection(sender=xor_in, projection=hid_map, receiver=xor_hid) - xor.add_projection(sender=xor_hid, projection=out_map, receiver=xor_out) + map_h2_is = MappingProjection(matrix=np.random.rand(15,12), + name="map_h2_is", + sender=h2, + receiver=out_sig_is) - xor_inputs = np.array( # the inputs we will provide to the model - [[0, 0], [0, 1], [1, 0], [1, 1]]) + map_h2_has = MappingProjection(matrix=np.random.rand(15,9), + name="map_h2_has", + sender=h2, + receiver=out_sig_has) - xor_targets = np.array( # the outputs we wish to see from the model - [[0], [1], [1], [0]]) + map_h2_can = MappingProjection(matrix=np.random.rand(15,9), + name="map_h2_can", + sender=h2, + receiver=out_sig_can) - # train model for a few epochs - # results_before_proc = xor.run(inputs={xor_in:xor_inputs}, - # targets={xor_out:xor_targets}, - # epochs=10) - results_before_proc = benchmark(xor.learn, inputs={"inputs": {xor_in:xor_inputs}, - "targets": {xor_out:xor_targets}, - "epochs": 10}, execution_mode=autodiff_mode) + # COMPOSITION FOR SEMANTIC NET + sem_net = AutodiffComposition(optimizer_type=opt, learning_rate=.001) - # fp32 results are different due to rounding - if pytest.helpers.llvm_current_fp_precision() == 'fp32' and \ - autodiff_mode != pnl.ExecutionMode.Python and \ - optimizer_type == 'sgd' and \ - learning_rate == 10: - expected = [[[0.9918830394744873]], [[0.9982172846794128]], [[0.9978305697441101]], [[0.9994590878486633]]] - # FIXME: LLVM version is broken with learning rate == 1.5 - if learning_rate != 1.5 or autodiff_mode == pnl.ExecutionMode.Python: - assert np.allclose(results_before_proc, expected) + sem_net.add_node(nouns_in) + sem_net.add_node(rels_in) + sem_net.add_node(h1) + sem_net.add_node(h2) + sem_net.add_node(out_sig_I) + sem_net.add_node(out_sig_is) + sem_net.add_node(out_sig_has) + sem_net.add_node(out_sig_can) + sem_net.add_projection(sender=nouns_in, projection=map_nouns_h1, receiver=h1) + sem_net.add_projection(sender=rels_in, projection=map_rels_h2, receiver=h2) + sem_net.add_projection(sender=h1, projection=map_h1_h2, receiver=h2) + sem_net.add_projection(sender=h2, projection=map_h2_I, receiver=out_sig_I) + sem_net.add_projection(sender=h2, projection=map_h2_is, receiver=out_sig_is) + sem_net.add_projection(sender=h2, projection=map_h2_has, receiver=out_sig_has) + sem_net.add_projection(sender=h2, projection=map_h2_can, receiver=out_sig_can) - # test whether pytorch parameters and projections are kept separate (at diff. places in memory) - def test_params_stay_separate(self, autodiff_mode): - if autodiff_mode is not pnl.ExecutionMode.Python: - pytest.skip("Compiled weights are always copied back!") + # INPUTS & OUTPUTS FOR SEMANTIC NET: - xor_in = TransferMechanism(name='xor_in', - default_variable=np.zeros(2)) + nouns = ['oak', 'pine', 'rose', 'daisy', 'canary', 'robin', 'salmon', 'sunfish'] + relations = ['is', 'has', 'can'] + is_list = ['living', 'living thing', 'plant', 'animal', 'tree', 'flower', 'bird', 'fish', 'big', 'green', 'red', + 'yellow'] + has_list = ['roots', 'leaves', 'bark', 'branches', 'skin', 'feathers', 'wings', 'gills', 'scales'] + can_list = ['grow', 'move', 'swim', 'fly', 'breathe', 'breathe underwater', 'breathe air', 'walk', 'photosynthesize'] - xor_hid = TransferMechanism(name='xor_hid', - default_variable=np.zeros(10), - function=Logistic()) + nouns_input = np.identity(len(nouns)) - xor_out = TransferMechanism(name='xor_out', - default_variable=np.zeros(1), - function=Logistic()) + rels_input = np.identity(len(relations)) - hid_m = np.random.rand(2,10) - out_m = np.random.rand(10,1) + truth_nouns = np.identity(len(nouns)) - hid_map = MappingProjection(name='hid_map', - matrix=hid_m.copy(), - sender=xor_in, - receiver=xor_hid) + truth_is = np.zeros((len(nouns), len(is_list))) - out_map = MappingProjection(name='out_map', - matrix=out_m.copy(), - sender=xor_hid, - receiver=xor_out) + truth_is[0, :] = [1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0] + truth_is[1, :] = [1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0] + truth_is[2, :] = [1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0] + truth_is[3, :] = [1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1] + truth_is[4, :] = [1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1] + truth_is[5, :] = [1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0] + truth_is[6, :] = [1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0] + truth_is[7, :] = [1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0] - xor = AutodiffComposition(learning_rate=10.0, - optimizer_type="sgd") + truth_has = np.zeros((len(nouns), len(has_list))) - xor.add_node(xor_in) - xor.add_node(xor_hid) - xor.add_node(xor_out) + truth_has[0, :] = [1, 1, 1, 1, 0, 0, 0, 0, 0] + truth_has[1, :] = [1, 1, 1, 1, 0, 0, 0, 0, 0] + truth_has[2, :] = [1, 1, 0, 0, 0, 0, 0, 0, 0] + truth_has[3, :] = [1, 1, 0, 0, 0, 0, 0, 0, 0] + truth_has[4, :] = [0, 0, 0, 0, 1, 1, 1, 0, 0] + truth_has[5, :] = [0, 0, 0, 0, 1, 1, 1, 0, 0] + truth_has[6, :] = [0, 0, 0, 0, 0, 0, 0, 1, 1] + truth_has[7, :] = [0, 0, 0, 0, 0, 0, 0, 1, 1] - xor.add_projection(sender=xor_in, projection=hid_map, receiver=xor_hid) - xor.add_projection(sender=xor_hid, projection=out_map, receiver=xor_out) + truth_can = np.zeros((len(nouns), len(can_list))) - xor_inputs = np.array( # the inputs we will provide to the model - [[0, 0], [0, 1], [1, 0], [1, 1]]) + truth_can[0, :] = [1, 0, 0, 0, 0, 0, 0, 0, 1] + truth_can[1, :] = [1, 0, 0, 0, 0, 0, 0, 0, 1] + truth_can[2, :] = [1, 0, 0, 0, 0, 0, 0, 0, 1] + truth_can[3, :] = [1, 0, 0, 0, 0, 0, 0, 0, 1] + truth_can[4, :] = [1, 1, 0, 1, 1, 0, 1, 1, 0] + truth_can[5, :] = [1, 1, 0, 1, 1, 0, 1, 1, 0] + truth_can[6, :] = [1, 1, 1, 0, 1, 1, 0, 0, 0] + truth_can[7, :] = [1, 1, 1, 0, 1, 1, 0, 0, 0] - xor_targets = np.array( # the outputs we wish to see from the model - [[0], [1], [1], [0]]) + # SETTING UP DICTIONARY OF INPUTS/OUTPUTS FOR SEMANTIC NET - # train the model for a few epochs - result = xor.learn(inputs={"inputs": {xor_in:xor_inputs}, - "targets": {xor_out:xor_targets}, - "epochs": 10}, execution_mode=autodiff_mode) + inputs_dict = {} + inputs_dict[nouns_in] = [] + inputs_dict[rels_in] = [] - # get weight parameters from pytorch - pt_weights_hid = xor.parameters.pytorch_representation.get(xor).params[0].detach().numpy().copy() - pt_weights_out = xor.parameters.pytorch_representation.get(xor).params[1].detach().numpy().copy() + targets_dict = {} + targets_dict[out_sig_I] = [] + targets_dict[out_sig_is] = [] + targets_dict[out_sig_has] = [] + targets_dict[out_sig_can] = [] - # assert that projections are still what they were initialized as - assert np.allclose(hid_map.parameters.matrix.get(None), hid_m) - assert np.allclose(out_map.parameters.matrix.get(None), out_m) + for i in range(len(nouns)): + for j in range(len(relations)): + inputs_dict[nouns_in].append(nouns_input[i]) + inputs_dict[rels_in].append(rels_input[j]) + targets_dict[out_sig_I].append(truth_nouns[i]) + targets_dict[out_sig_is].append(truth_is[i]) + targets_dict[out_sig_has].append(truth_has[i]) + targets_dict[out_sig_can].append(truth_can[i]) - # assert that projections didn't change during training with the pytorch - # parameters (they should now be different) - assert not np.allclose(pt_weights_hid, hid_map.parameters.matrix.get(None)) - assert not np.allclose(pt_weights_out, out_map.parameters.matrix.get(None)) - -@pytest.mark.pytorch -@pytest.mark.accorrectness -class TestTrainingCorrectness: - - # test whether xor model created as autodiff composition learns properly - @pytest.mark.benchmark(group="XOR") - @pytest.mark.parametrize( - 'eps, calls, opt, expected', [ - (100, 'single', 'adam', [[[0.09823965]], [[0.81092879]], [[0.78179557]], [[0.25593583]]]), - (50, 'multiple', 'adam', [[[0.31200036]], [[0.59406178]], [[0.60417587]], [[0.52347365]]]), - ] - ) - def test_xor_training_correctness(self, eps, calls, opt, autodiff_mode, benchmark, expected): - xor_in = TransferMechanism(name='xor_in', - default_variable=np.zeros(2)) - - xor_hid = TransferMechanism(name='xor_hid', - default_variable=np.zeros(10), - function=Logistic()) - - xor_out = TransferMechanism(name='xor_out', - default_variable=np.zeros(1), - function=Logistic()) - - hid_map = MappingProjection(matrix=np.random.rand(2, 10)) - out_map = MappingProjection(matrix=np.random.rand(10, 1)) - - xor = AutodiffComposition(optimizer_type=opt, - learning_rate=0.1) - - xor.add_node(xor_in) - xor.add_node(xor_hid) - xor.add_node(xor_out) - - xor.add_projection(sender=xor_in, projection=hid_map, receiver=xor_hid) - xor.add_projection(sender=xor_hid, projection=out_map, receiver=xor_out) - - xor_inputs = np.array( # the inputs we will provide to the model - [[0, 0], [0, 1], [1, 0], [1, 1]]) - - xor_targets = np.array( # the outputs we wish to see from the model - [[0], [1], [1], [0]]) - - if calls == 'single': - results = benchmark(xor.learn, inputs={"inputs": {xor_in:xor_inputs}, - "targets": {xor_out:xor_targets}, - "epochs": eps}, execution_mode=autodiff_mode) - - else: - input_dict = {"inputs": {xor_in: xor_inputs}, - "targets": {xor_out: xor_targets}, - "epochs": 1} - for i in range(eps - 1): - xor.learn(inputs=input_dict, execution_mode=autodiff_mode) - results = benchmark(xor.learn, inputs=input_dict, execution_mode=autodiff_mode) - - assert len(results) == len(expected) - for r, t in zip(results, expected): - assert np.allclose(r[0], t) - - - # tests whether semantic network created as autodiff composition learns properly - @pytest.mark.benchmark(group="Semantic net") - @pytest.mark.parametrize( - 'eps, opt', [ - (50, 'adam'), - ] - ) - def test_semantic_net_training_correctness(self, eps, opt, autodiff_mode, benchmark): - - # MECHANISMS FOR SEMANTIC NET: - - nouns_in = TransferMechanism(name="nouns_input", - default_variable=np.zeros(8)) - - rels_in = TransferMechanism(name="rels_input", - default_variable=np.zeros(3)) - - h1 = TransferMechanism(name="hidden_nouns", - default_variable=np.zeros(8), - function=Logistic()) - - h2 = TransferMechanism(name="hidden_mixed", - default_variable=np.zeros(15), - function=Logistic()) - - out_sig_I = TransferMechanism(name="sig_outs_I", - default_variable=np.zeros(8), - function=Logistic()) - - out_sig_is = TransferMechanism(name="sig_outs_is", - default_variable=np.zeros(12), - function=Logistic()) - - out_sig_has = TransferMechanism(name="sig_outs_has", - default_variable=np.zeros(9), - function=Logistic()) - - out_sig_can = TransferMechanism(name="sig_outs_can", - default_variable=np.zeros(9), - function=Logistic()) - - # SET UP PROJECTIONS FOR SEMANTIC NET - - map_nouns_h1 = MappingProjection(matrix=np.random.rand(8,8), - name="map_nouns_h1", - sender=nouns_in, - receiver=h1) - - map_rels_h2 = MappingProjection(matrix=np.random.rand(3,15), - name="map_relh2", - sender=rels_in, - receiver=h2) - - map_h1_h2 = MappingProjection(matrix=np.random.rand(8,15), - name="map_h1_h2", - sender=h1, - receiver=h2) - - map_h2_I = MappingProjection(matrix=np.random.rand(15,8), - name="map_h2_I", - sender=h2, - receiver=out_sig_I) - - map_h2_is = MappingProjection(matrix=np.random.rand(15,12), - name="map_h2_is", - sender=h2, - receiver=out_sig_is) - - map_h2_has = MappingProjection(matrix=np.random.rand(15,9), - name="map_h2_has", - sender=h2, - receiver=out_sig_has) - - map_h2_can = MappingProjection(matrix=np.random.rand(15,9), - name="map_h2_can", - sender=h2, - receiver=out_sig_can) - - # COMPOSITION FOR SEMANTIC NET - sem_net = AutodiffComposition(optimizer_type=opt, learning_rate=.001) - - sem_net.add_node(nouns_in) - sem_net.add_node(rels_in) - sem_net.add_node(h1) - sem_net.add_node(h2) - sem_net.add_node(out_sig_I) - sem_net.add_node(out_sig_is) - sem_net.add_node(out_sig_has) - sem_net.add_node(out_sig_can) - - sem_net.add_projection(sender=nouns_in, projection=map_nouns_h1, receiver=h1) - sem_net.add_projection(sender=rels_in, projection=map_rels_h2, receiver=h2) - sem_net.add_projection(sender=h1, projection=map_h1_h2, receiver=h2) - sem_net.add_projection(sender=h2, projection=map_h2_I, receiver=out_sig_I) - sem_net.add_projection(sender=h2, projection=map_h2_is, receiver=out_sig_is) - sem_net.add_projection(sender=h2, projection=map_h2_has, receiver=out_sig_has) - sem_net.add_projection(sender=h2, projection=map_h2_can, receiver=out_sig_can) - - # INPUTS & OUTPUTS FOR SEMANTIC NET: - - nouns = ['oak', 'pine', 'rose', 'daisy', 'canary', 'robin', 'salmon', 'sunfish'] - relations = ['is', 'has', 'can'] - is_list = ['living', 'living thing', 'plant', 'animal', 'tree', 'flower', 'bird', 'fish', 'big', 'green', 'red', - 'yellow'] - has_list = ['roots', 'leaves', 'bark', 'branches', 'skin', 'feathers', 'wings', 'gills', 'scales'] - can_list = ['grow', 'move', 'swim', 'fly', 'breathe', 'breathe underwater', 'breathe air', 'walk', 'photosynthesize'] - - nouns_input = np.identity(len(nouns)) - - rels_input = np.identity(len(relations)) - - truth_nouns = np.identity(len(nouns)) - - truth_is = np.zeros((len(nouns), len(is_list))) - - truth_is[0, :] = [1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0] - truth_is[1, :] = [1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0] - truth_is[2, :] = [1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0] - truth_is[3, :] = [1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1] - truth_is[4, :] = [1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1] - truth_is[5, :] = [1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0] - truth_is[6, :] = [1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0] - truth_is[7, :] = [1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0] - - truth_has = np.zeros((len(nouns), len(has_list))) - - truth_has[0, :] = [1, 1, 1, 1, 0, 0, 0, 0, 0] - truth_has[1, :] = [1, 1, 1, 1, 0, 0, 0, 0, 0] - truth_has[2, :] = [1, 1, 0, 0, 0, 0, 0, 0, 0] - truth_has[3, :] = [1, 1, 0, 0, 0, 0, 0, 0, 0] - truth_has[4, :] = [0, 0, 0, 0, 1, 1, 1, 0, 0] - truth_has[5, :] = [0, 0, 0, 0, 1, 1, 1, 0, 0] - truth_has[6, :] = [0, 0, 0, 0, 0, 0, 0, 1, 1] - truth_has[7, :] = [0, 0, 0, 0, 0, 0, 0, 1, 1] - - truth_can = np.zeros((len(nouns), len(can_list))) - - truth_can[0, :] = [1, 0, 0, 0, 0, 0, 0, 0, 1] - truth_can[1, :] = [1, 0, 0, 0, 0, 0, 0, 0, 1] - truth_can[2, :] = [1, 0, 0, 0, 0, 0, 0, 0, 1] - truth_can[3, :] = [1, 0, 0, 0, 0, 0, 0, 0, 1] - truth_can[4, :] = [1, 1, 0, 1, 1, 0, 1, 1, 0] - truth_can[5, :] = [1, 1, 0, 1, 1, 0, 1, 1, 0] - truth_can[6, :] = [1, 1, 1, 0, 1, 1, 0, 0, 0] - truth_can[7, :] = [1, 1, 1, 0, 1, 1, 0, 0, 0] - - # SETTING UP DICTIONARY OF INPUTS/OUTPUTS FOR SEMANTIC NET - - inputs_dict = {} - inputs_dict[nouns_in] = [] - inputs_dict[rels_in] = [] - - targets_dict = {} - targets_dict[out_sig_I] = [] - targets_dict[out_sig_is] = [] - targets_dict[out_sig_has] = [] - targets_dict[out_sig_can] = [] - - for i in range(len(nouns)): - for j in range(len(relations)): - inputs_dict[nouns_in].append(nouns_input[i]) - inputs_dict[rels_in].append(rels_input[j]) - targets_dict[out_sig_I].append(truth_nouns[i]) - targets_dict[out_sig_is].append(truth_is[i]) - targets_dict[out_sig_has].append(truth_has[i]) - targets_dict[out_sig_can].append(truth_can[i]) - - # TRAIN THE MODEL - results = benchmark(sem_net.learn, inputs={'inputs': inputs_dict, - 'targets': targets_dict, - 'epochs': eps}, execution_mode=autodiff_mode) + # TRAIN THE MODEL + results = benchmark(sem_net.learn, inputs={'inputs': inputs_dict, + 'targets': targets_dict, + 'epochs': eps}, execution_mode=autodiff_mode) # CHECK CORRECTNESS expected = [[[0.13455769, 0.12924714, 0.13288172, 0.1404659 , 0.14305814, @@ -1248,107 +880,589 @@ def test_pytorch_equivalence_with_autodiff_training_disabled_on_proj(self): 0.08293781, 0.08313539, 0.08310112, 0.08409653, 0.08289441, 0.08348761, 0.08323367, 0.35237628, 0.22024095, 0.08336799]) - assert np.allclose(output, comparator) + assert np.allclose(output, comparator) + + +@pytest.mark.pytorch +@pytest.mark.acidenticalness +class TestTrainingIdenticalness(): + + @pytest.mark.parametrize( + 'eps, opt', [ + # (1, 'sgd'), + (10, 'sgd'), + # (40, 'sgd') + ] + ) + def test_semantic_net_training_identicalness(self, eps, opt): + # SET UP MECHANISMS FOR SEMANTIC NET: + + nouns_in = TransferMechanism(name="nouns_input", + default_variable=np.zeros(8)) + + rels_in = TransferMechanism(name="rels_input", + default_variable=np.zeros(3)) + + h1 = TransferMechanism(name="hidden_nouns", + default_variable=np.zeros(8), + function=Logistic()) + + h2 = TransferMechanism(name="hidden_mixed", + default_variable=np.zeros(15), + function=Logistic()) + + out_sig_I = TransferMechanism(name="sig_outs_I", + default_variable=np.zeros(8), + function=Logistic()) + + out_sig_is = TransferMechanism(name="sig_outs_is", + default_variable=np.zeros(12), + function=Logistic()) + + out_sig_has = TransferMechanism(name="sig_outs_has", + default_variable=np.zeros(9), + function=Logistic()) + + out_sig_can = TransferMechanism(name="sig_outs_can", + default_variable=np.zeros(9), + function=Logistic()) + + # SET UP MECHANISMS FOR Composition + + nouns_in_comp = TransferMechanism(name="nouns_input_comp", + default_variable=np.zeros(8)) + + rels_in_comp = TransferMechanism(name="rels_input_comp", + default_variable=np.zeros(3)) + + h1_comp = TransferMechanism(name="hidden_nouns_comp", + default_variable=np.zeros(8), + function=Logistic()) + + h2_comp = TransferMechanism(name="hidden_mixed_comp", + default_variable=np.zeros(15), + function=Logistic()) + + out_sig_I_comp = TransferMechanism(name="sig_outs_I_comp", + default_variable=np.zeros(8), + function=Logistic()) + + out_sig_is_comp = TransferMechanism(name="sig_outs_is_comp", + default_variable=np.zeros(12), + function=Logistic()) + + out_sig_has_comp = TransferMechanism(name="sig_outs_has_comp", + default_variable=np.zeros(9), + function=Logistic()) + + out_sig_can_comp = TransferMechanism(name="sig_outs_can_comp", + default_variable=np.zeros(9), + function=Logistic()) + + # SET UP PROJECTIONS FOR SEMANTIC NET + + map_nouns_h1 = MappingProjection(matrix=np.random.rand(8,8), + name="map_nouns_h1", + sender=nouns_in, + receiver=h1) + + map_rels_h2 = MappingProjection(matrix=np.random.rand(3,15), + name="map_relh2", + sender=rels_in, + receiver=h2) + + map_h1_h2 = MappingProjection(matrix=np.random.rand(8,15), + name="map_h1_h2", + sender=h1, + receiver=h2) + + map_h2_I = MappingProjection(matrix=np.random.rand(15,8), + name="map_h2_I", + sender=h2, + receiver=out_sig_I) + + map_h2_is = MappingProjection(matrix=np.random.rand(15,12), + name="map_h2_is", + sender=h2, + receiver=out_sig_is) + + map_h2_has = MappingProjection(matrix=np.random.rand(15,9), + name="map_h2_has", + sender=h2, + receiver=out_sig_has) + + map_h2_can = MappingProjection(matrix=np.random.rand(15,9), + name="map_h2_can", + sender=h2, + receiver=out_sig_can) + + # SET UP PROJECTIONS FOR COMPOSITION + + map_nouns_h1_comp = MappingProjection(matrix=map_nouns_h1.matrix.base.copy(), + name="map_nouns_h1_comp", + sender=nouns_in_comp, + receiver=h1_comp) + + map_rels_h2_comp = MappingProjection(matrix=map_rels_h2.matrix.base.copy(), + name="map_relh2_comp", + sender=rels_in_comp, + receiver=h2_comp) + + map_h1_h2_comp = MappingProjection(matrix=map_h1_h2.matrix.base.copy(), + name="map_h1_h2_comp", + sender=h1_comp, + receiver=h2_comp) + + map_h2_I_comp = MappingProjection(matrix=map_h2_I.matrix.base.copy(), + name="map_h2_I_comp", + sender=h2_comp, + receiver=out_sig_I_comp) + + map_h2_is_comp = MappingProjection(matrix=map_h2_is.matrix.base.copy(), + name="map_h2_is_comp", + sender=h2_comp, + receiver=out_sig_is_comp) + + map_h2_has_comp = MappingProjection(matrix=map_h2_has.matrix.base.copy(), + name="map_h2_has_comp", + sender=h2_comp, + receiver=out_sig_has_comp) + + map_h2_can_comp = MappingProjection(matrix=map_h2_can.matrix.base.copy(), + name="map_h2_can_comp", + sender=h2_comp, + receiver=out_sig_can_comp) + + # SET UP AUTODIFFCOMPOSITION FOR SEMANTIC NET + sem_net_autodiff = AutodiffComposition(learning_rate=0.5, + optimizer_type=opt, + ) + + sem_net_autodiff.add_node(nouns_in) + sem_net_autodiff.add_node(rels_in) + sem_net_autodiff.add_node(h1) + sem_net_autodiff.add_node(h2) + sem_net_autodiff.add_node(out_sig_I) + sem_net_autodiff.add_node(out_sig_is) + sem_net_autodiff.add_node(out_sig_has) + sem_net_autodiff.add_node(out_sig_can) + + sem_net_autodiff.add_projection(sender=nouns_in, projection=map_nouns_h1, receiver=h1) + sem_net_autodiff.add_projection(sender=rels_in, projection=map_rels_h2, receiver=h2) + sem_net_autodiff.add_projection(sender=h1, projection=map_h1_h2, receiver=h2) + sem_net_autodiff.add_projection(sender=h2, projection=map_h2_I, receiver=out_sig_I) + sem_net_autodiff.add_projection(sender=h2, projection=map_h2_is, receiver=out_sig_is) + sem_net_autodiff.add_projection(sender=h2, projection=map_h2_has, receiver=out_sig_has) + sem_net_autodiff.add_projection(sender=h2, projection=map_h2_can, receiver=out_sig_can) + # INPUTS & OUTPUTS FOR SEMANTIC NET: + + nouns = ['oak', 'pine', 'rose', 'daisy', 'canary', 'robin', 'salmon', 'sunfish'] + relations = ['is', 'has', 'can'] + is_list = ['living', 'living thing', 'plant', 'animal', 'tree', 'flower', 'bird', 'fish', 'big', 'green', 'red', + 'yellow'] + has_list = ['roots', 'leaves', 'bark', 'branches', 'skin', 'feathers', 'wings', 'gills', 'scales'] + can_list = ['grow', 'move', 'swim', 'fly', 'breathe', 'breathe underwater', 'breathe air', 'walk', 'photosynthesize'] + + nouns_input = np.identity(len(nouns)) + + rels_input = np.identity(len(relations)) + + truth_nouns = np.identity(len(nouns)) + + truth_is = np.zeros((len(nouns), len(is_list))) + + truth_is[0, :] = [1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0] + truth_is[1, :] = [1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0] + truth_is[2, :] = [1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0] + truth_is[3, :] = [1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0] + truth_is[4, :] = [1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1] + truth_is[5, :] = [1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1] + truth_is[6, :] = [1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0] + truth_is[7, :] = [1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0] + + truth_has = np.zeros((len(nouns), len(has_list))) + + truth_has[0, :] = [1, 1, 1, 1, 0, 0, 0, 0, 0] + truth_has[1, :] = [1, 1, 1, 1, 0, 0, 0, 0, 0] + truth_has[2, :] = [1, 1, 0, 0, 0, 0, 0, 0, 0] + truth_has[3, :] = [1, 1, 0, 0, 0, 0, 0, 0, 0] + truth_has[4, :] = [0, 0, 0, 0, 1, 1, 1, 0, 0] + truth_has[5, :] = [0, 0, 0, 0, 1, 1, 1, 0, 0] + truth_has[6, :] = [0, 0, 0, 0, 0, 0, 0, 1, 1] + truth_has[7, :] = [0, 0, 0, 0, 0, 0, 0, 1, 1] + + truth_can = np.zeros((len(nouns), len(can_list))) + + truth_can[0, :] = [1, 0, 0, 0, 0, 0, 0, 0, 1] + truth_can[1, :] = [1, 0, 0, 0, 0, 0, 0, 0, 1] + truth_can[2, :] = [1, 0, 0, 0, 0, 0, 0, 0, 1] + truth_can[3, :] = [1, 0, 0, 0, 0, 0, 0, 0, 1] + truth_can[4, :] = [1, 1, 0, 1, 1, 0, 1, 1, 0] + truth_can[5, :] = [1, 1, 0, 1, 1, 0, 1, 1, 0] + truth_can[6, :] = [1, 1, 1, 0, 1, 1, 0, 0, 0] + truth_can[7, :] = [1, 1, 1, 0, 1, 1, 0, 0, 0] + + # SETTING UP DICTIONARY OF INPUTS/OUTPUTS FOR SEMANTIC NET + + inputs_dict = {} + inputs_dict[nouns_in] = [] + inputs_dict[rels_in] = [] + + targets_dict = {} + targets_dict[out_sig_I] = [] + targets_dict[out_sig_is] = [] + targets_dict[out_sig_has] = [] + targets_dict[out_sig_can] = [] + + for i in range(len(nouns)): + for j in range(len(relations)): + inputs_dict[nouns_in].append(nouns_input[i]) + inputs_dict[rels_in].append(rels_input[j]) + targets_dict[out_sig_I].append(truth_nouns[i]) + targets_dict[out_sig_is].append(truth_is[i]) + targets_dict[out_sig_has].append(truth_has[i]) + targets_dict[out_sig_can].append(truth_can[i]) + + inputs_dict_comp = {} + inputs_dict_comp[nouns_in_comp] = inputs_dict[nouns_in] + inputs_dict_comp[rels_in_comp] = inputs_dict[rels_in] + + sem_net_autodiff.run(inputs=inputs_dict) + + # TRAIN AUTODIFFCOMPOSITION + def g_f(): + yield {"inputs": inputs_dict, + "targets": targets_dict, + "epochs": eps} + g = g_f() + sem_net_autodiff.learn(inputs=g_f) + + # SET UP COMPOSITION + sem_net_comp = Composition() + + backprop_pathway = sem_net_comp.add_backpropagation_learning_pathway( + pathway=[ + nouns_in_comp, + map_nouns_h1_comp, + h1_comp, + map_h1_h2_comp, + h2_comp, + map_h2_I_comp, + out_sig_I_comp + ], + learning_rate=0.5 + ) + inputs_dict_comp[backprop_pathway.target] = targets_dict[out_sig_I] + + backprop_pathway = sem_net_comp.add_backpropagation_learning_pathway( + pathway=[ + rels_in_comp, + map_rels_h2_comp, + h2_comp, + map_h2_is_comp, + out_sig_is_comp + ], + learning_rate=0.5 + ) + inputs_dict_comp[backprop_pathway.target] = targets_dict[out_sig_is] + + backprop_pathway = sem_net_comp.add_backpropagation_learning_pathway( + pathway=[ + h2_comp, + map_h2_has_comp, + out_sig_has_comp + ], + learning_rate=0.5 + ) + inputs_dict_comp[backprop_pathway.target] = targets_dict[out_sig_has] + + backprop_pathway = sem_net_comp.add_backpropagation_learning_pathway( + pathway=[ + h2_comp, + map_h2_can_comp, + out_sig_can_comp + ], + learning_rate=0.5 + ) + inputs_dict_comp[backprop_pathway.target] = targets_dict[out_sig_can] + + # TRAIN COMPOSITION + sem_net_comp.learn(inputs=inputs_dict_comp, + num_trials=(len(inputs_dict_comp[nouns_in_comp]) * eps)) + + # CHECK THAT PARAMETERS FOR AUTODIFFCOMPOSITION, COMPOSITION ARE SAME + + assert np.allclose(map_nouns_h1.parameters.matrix.get(sem_net_autodiff), + map_nouns_h1_comp.get_mod_matrix(sem_net_comp)) + assert np.allclose(map_rels_h2.parameters.matrix.get(sem_net_autodiff), + map_rels_h2_comp.get_mod_matrix(sem_net_comp)) + assert np.allclose(map_h1_h2.parameters.matrix.get(sem_net_autodiff), + map_h1_h2_comp.get_mod_matrix(sem_net_comp)) + assert np.allclose(map_h2_I.parameters.matrix.get(sem_net_autodiff), + map_h2_I_comp.get_mod_matrix(sem_net_comp)) + assert np.allclose(map_h2_is.parameters.matrix.get(sem_net_autodiff), + map_h2_is_comp.get_mod_matrix(sem_net_comp)) + assert np.allclose(map_h2_has.parameters.matrix.get(sem_net_autodiff), + map_h2_has_comp.get_mod_matrix(sem_net_comp)) + assert np.allclose(map_h2_can.parameters.matrix.get(sem_net_autodiff), + map_h2_can_comp.get_mod_matrix(sem_net_comp)) + + def test_identicalness_of_input_types(self): + # SET UP MECHANISMS FOR COMPOSITION + from copy import copy + hid_map_mat = np.random.rand(2, 10) + out_map_mat = np.random.rand(10, 1) + xor_in_dict = TransferMechanism(name='xor_in', + default_variable=np.zeros(2)) + + xor_hid_dict = TransferMechanism(name='xor_hid', + default_variable=np.zeros(10), + function=Logistic()) + + xor_out_dict = TransferMechanism(name='xor_out', + default_variable=np.zeros(1), + function=Logistic()) + + # SET UP PROJECTIONS FOR COMPOSITION + + hid_map_dict = MappingProjection(name='hid_map', + matrix=copy(hid_map_mat), + sender=xor_in_dict, + receiver=xor_hid_dict) + + out_map_dict = MappingProjection(name='out_map', + matrix=copy(out_map_mat), + sender=xor_hid_dict, + receiver=xor_out_dict) + + # SET UP COMPOSITION + + xor_dict = AutodiffComposition() + + xor_dict.add_node(xor_in_dict) + xor_dict.add_node(xor_hid_dict) + xor_dict.add_node(xor_out_dict) + + xor_dict.add_projection(sender=xor_in_dict, projection=hid_map_dict, receiver=xor_hid_dict) + xor_dict.add_projection(sender=xor_hid_dict, projection=out_map_dict, receiver=xor_out_dict) + # SET UP INPUTS AND TARGETS + + xor_inputs_dict = np.array( # the inputs we will provide to the model + [[0, 0], + [0, 1], + [1, 0], + [1, 1]]) + + xor_targets_dict = np.array( # the outputs we wish to see from the model + [[0], + [1], + [1], + [0]]) + + input_dict = { + "inputs": { + xor_in_dict: xor_inputs_dict + }, + "targets": { + xor_out_dict: xor_targets_dict + } + } + + result_dict = xor_dict.learn(inputs=input_dict) + + # SET UP MECHANISMS FOR COMPOSITION + xor_in_func = TransferMechanism(name='xor_in', + default_variable=np.zeros(2)) + + xor_hid_func = TransferMechanism(name='xor_hid', + default_variable=np.zeros(10), + function=Logistic()) + + xor_out_func = TransferMechanism(name='xor_out', + default_variable=np.zeros(1), + function=Logistic()) + + # SET UP PROJECTIONS FOR COMPOSITION + + hid_map_func = MappingProjection(name='hid_map', + matrix=copy(hid_map_mat), + sender=xor_in_func, + receiver=xor_hid_func) + + out_map_func = MappingProjection(name='out_map', + matrix=copy(out_map_mat), + sender=xor_hid_func, + receiver=xor_out_func) + + # SET UP COMPOSITION + xor_func = AutodiffComposition() -@pytest.mark.pytorch -@pytest.mark.actime -class TestTrainingTime: + xor_func.add_node(xor_in_func) + xor_func.add_node(xor_hid_func) + xor_func.add_node(xor_out_func) - @pytest.mark.skip - @pytest.mark.parametrize( - 'eps, opt', [ - (1, 'sgd'), - (10, 'sgd'), - (100, 'sgd') - ] - ) - def test_and_training_time(self, eps, opt,autodiff_mode): + xor_func.add_projection(sender=xor_in_func, projection=hid_map_func, receiver=xor_hid_func) + xor_func.add_projection(sender=xor_hid_func, projection=out_map_func, receiver=xor_out_func) - # SET UP MECHANISMS FOR COMPOSITION + # SET UP INPUTS AND TARGETS - and_in = TransferMechanism(name='and_in', - default_variable=np.zeros(2)) + xor_inputs_func = np.array( # the inputs we will provide to the model + [[0, 0], + [0, 1], + [1, 0], + [1, 1]]) - and_out = TransferMechanism(name='and_out', - default_variable=np.zeros(1), - function=Logistic()) + xor_targets_func = np.array( # the outputs we wish to see from the model + [[0], + [1], + [1], + [0]]) - # SET UP MECHANISMS FOR SYSTEM + def get_inputs(idx): + return { + "inputs": { + xor_in_func: xor_inputs_func[idx] + }, + "targets": { + xor_out_func: xor_targets_func[idx] + } + } - and_in_sys = TransferMechanism(name='and_in_sys', + result_func = xor_func.learn(inputs=get_inputs) + + # SET UP MECHANISMS FOR COMPOSITION + xor_in_gen = TransferMechanism(name='xor_in', default_variable=np.zeros(2)) - and_out_sys = TransferMechanism(name='and_out_sys', + xor_hid_gen = TransferMechanism(name='xor_hid', + default_variable=np.zeros(10), + function=Logistic()) + + xor_out_gen = TransferMechanism(name='xor_out', default_variable=np.zeros(1), function=Logistic()) # SET UP PROJECTIONS FOR COMPOSITION - and_map = MappingProjection(name='and_map', - matrix=np.random.rand(2, 1), - sender=and_in, - receiver=and_out) - - # SET UP PROJECTIONS FOR SYSTEM + hid_map_gen = MappingProjection(name='hid_map', + matrix=copy(hid_map_mat), + sender=xor_in_gen, + receiver=xor_hid_gen) - and_map_sys = MappingProjection(name='and_map_sys', - matrix=and_map.matrix.base.copy(), - sender=and_in_sys, - receiver=and_out_sys) + out_map_gen = MappingProjection(name='out_map', + matrix=copy(out_map_mat), + sender=xor_hid_gen, + receiver=xor_out_gen) # SET UP COMPOSITION - and_net = AutodiffComposition() + xor_gen = AutodiffComposition() - and_net.add_node(and_in) - and_net.add_node(and_out) + xor_gen.add_node(xor_in_gen) + xor_gen.add_node(xor_hid_gen) + xor_gen.add_node(xor_out_gen) - and_net.add_projection(sender=and_in, projection=and_map, receiver=and_out) + xor_gen.add_projection(sender=xor_in_gen, projection=hid_map_gen, receiver=xor_hid_gen) + xor_gen.add_projection(sender=xor_hid_gen, projection=out_map_gen, receiver=xor_out_gen) + # SET UP INPUTS AND TARGETS + + xor_inputs_gen = np.array( # the inputs we will provide to the model + [[0, 0], + [0, 1], + [1, 0], + [1, 1]]) + + xor_targets_gen = np.array( # the outputs we wish to see from the model + [[0], + [1], + [1], + [0]]) + + def get_inputs_gen(): + yield { + "inputs": { + xor_in_gen: xor_inputs_gen + }, + "targets": { + xor_out_gen: xor_targets_gen + } + } + + g = get_inputs_gen() + result_gen = xor_gen.learn(inputs=g) + + # SET UP MECHANISMS FOR COMPOSITION + xor_in_gen_func = TransferMechanism(name='xor_in', + default_variable=np.zeros(2)) + + xor_hid_gen_func = TransferMechanism(name='xor_hid', + default_variable=np.zeros(10), + function=Logistic()) + + xor_out_gen_func = TransferMechanism(name='xor_out', + default_variable=np.zeros(1), + function=Logistic()) + + # SET UP PROJECTIONS FOR COMPOSITION + + hid_map_gen_func = MappingProjection(name='hid_map', + matrix=copy(hid_map_mat), + sender=xor_in_gen_func, + receiver=xor_hid_gen_func) + + out_map_gen_func = MappingProjection(name='out_map', + matrix=copy(out_map_mat), + sender=xor_hid_gen_func, + receiver=xor_out_gen_func) + + # SET UP COMPOSITION + + xor_gen_func = AutodiffComposition() + + xor_gen_func.add_node(xor_in_gen_func) + xor_gen_func.add_node(xor_hid_gen_func) + xor_gen_func.add_node(xor_out_gen_func) + xor_gen_func.add_projection(sender=xor_in_gen_func, projection=hid_map_gen_func, receiver=xor_hid_gen_func) + xor_gen_func.add_projection(sender=xor_hid_gen_func, projection=out_map_gen_func, receiver=xor_out_gen_func) # SET UP INPUTS AND TARGETS - and_inputs = np.zeros((4,2)) - and_inputs[0] = [0, 0] - and_inputs[1] = [0, 1] - and_inputs[2] = [1, 0] - and_inputs[3] = [1, 1] + xor_inputs_gen_func = np.array( # the inputs we will provide to the model + [[0, 0], + [0, 1], + [1, 0], + [1, 1]]) - and_targets = np.zeros((4,1)) - and_targets[0] = [0] - and_targets[1] = [1] - and_targets[2] = [1] - and_targets[3] = [0] + xor_targets_gen_func = np.array( # the outputs we wish to see from the model + [[0], + [1], + [1], + [0]]) - # TIME TRAINING FOR COMPOSITION + def get_inputs_gen_func(): + yield { + "inputs": { + xor_in_gen_func: xor_inputs_gen_func + }, + "targets": { + xor_out_gen_func: xor_targets_gen_func + } + } - start = timeit.default_timer() - result = and_net.run(inputs={and_in:and_inputs}, - targets={and_out:and_targets}, - epochs=eps, - learning_rate=0.1, - controller=opt, - execution_mode=autodiff_mode) - end = timeit.default_timer() - comp_time = end - start + result_gen_func = xor_gen_func.learn(inputs=get_inputs_gen_func) - msg = 'Training XOR model as AutodiffComposition for {0} epochs took {1} seconds'.format(eps, comp_time) - print(msg) - print("\n") - logger.info(msg) + assert result_dict == result_func == result_gen == result_gen_func - @pytest.mark.skip - @pytest.mark.parametrize( - 'eps, opt', [ - (1, 'sgd'), - (10, 'sgd'), - (100, 'sgd') - ] - ) - def test_xor_training_time(self, eps, opt, autodiff_mode): - # SET UP MECHANISMS FOR COMPOSITION +@pytest.mark.pytorch +@pytest.mark.acmisc +class TestMiscTrainingFunctionality: + + # test whether pytorch parameters are initialized to be identical to the Autodiff Composition's + def test_weight_initialization(self): + # create xor model mechanisms and projections xor_in = TransferMechanism(name='xor_in', default_variable=np.zeros(2)) @@ -1360,46 +1474,45 @@ def test_xor_training_time(self, eps, opt, autodiff_mode): default_variable=np.zeros(1), function=Logistic()) - # SET UP MECHANISMS FOR SYSTEM - - xor_in_sys = TransferMechanism(name='xor_in_sys', - default_variable=np.zeros(2)) - - xor_hid_sys = TransferMechanism(name='xor_hid_sys', - default_variable=np.zeros(10), - function=Logistic()) - - xor_out_sys = TransferMechanism(name='xor_out_sys', - default_variable=np.zeros(1), - function=Logistic()) + hid_map = MappingProjection(matrix=np.random.rand(2,10)) + out_map = MappingProjection(matrix=np.random.rand(10,1)) - # SET UP PROJECTIONS FOR COMPOSITION + # put the mechanisms and projections together in an autodiff composition (AC) + xor = AutodiffComposition() - hid_map = MappingProjection(name='hid_map', - matrix=np.random.rand(2,10), - sender=xor_in, - receiver=xor_hid) + xor.add_node(xor_in) + xor.add_node(xor_hid) + xor.add_node(xor_out) - out_map = MappingProjection(name='out_map', - matrix=np.random.rand(10,1), - sender=xor_hid, - receiver=xor_out) + xor.add_projection(sender=xor_in, projection=hid_map, receiver=xor_hid) + xor.add_projection(sender=xor_hid, projection=out_map, receiver=xor_out) - # SET UP PROJECTIONS FOR SYSTEM + # mini version of xor.execute just to build up pytorch representation + xor._analyze_graph() + xor._build_pytorch_representation(context=xor.default_execution_id) + # check whether pytorch parameters are identical to projections + assert np.allclose(hid_map.parameters.matrix.get(None), + xor.parameters.pytorch_representation.get(xor).params[0].detach().numpy()) + assert np.allclose(out_map.parameters.matrix.get(None), + xor.parameters.pytorch_representation.get(xor).params[1].detach().numpy()) - hid_map_sys = MappingProjection(name='hid_map_sys', - matrix=hid_map.matrix.base.copy(), - sender=xor_in_sys, - receiver=xor_hid_sys) + # test whether processing doesn't interfere with pytorch parameters after training + def test_training_then_processing(self, autodiff_mode): + xor_in = TransferMechanism(name='xor_in', + default_variable=np.zeros(2)) - out_map_sys = MappingProjection(name='out_map_sys', - matrix=out_map.matrix.base.copy(), - sender=xor_hid_sys, - receiver=xor_out_sys) + xor_hid = TransferMechanism(name='xor_hid', + default_variable=np.zeros(10), + function=Logistic()) - # SET UP COMPOSITION + xor_out = TransferMechanism(name='xor_out', + default_variable=np.zeros(1), + function=Logistic()) - xor = AutodiffComposition(execution_mode=autodiff_mode) + hid_map = MappingProjection() + out_map = MappingProjection() + + xor = AutodiffComposition() xor.add_node(xor_in) xor.add_node(xor_hid) @@ -1408,8 +1521,6 @@ def test_xor_training_time(self, eps, opt, autodiff_mode): xor.add_projection(sender=xor_in, projection=hid_map, receiver=xor_hid) xor.add_projection(sender=xor_hid, projection=out_map, receiver=xor_out) - # SET UP INPUTS AND TARGETS - xor_inputs = np.array( # the inputs we will provide to the model [[0, 0], [0, 1], @@ -1422,873 +1533,782 @@ def test_xor_training_time(self, eps, opt, autodiff_mode): [1], [0]]) - # TIME TRAINING FOR COMPOSITION - - start = timeit.default_timer() - result = xor.run(inputs={xor_in:xor_inputs}, - targets={xor_out:xor_targets}, - epochs=eps, - learning_rate=0.1, - controller=opt, - execution_mode=autodiff_mode) - end = timeit.default_timer() - comp_time = end - start - - # SET UP SYSTEM - - # xor_process = Process(pathway=[xor_in_sys, - # hid_map_sys, - # xor_hid_sys, - # out_map_sys, - # xor_out_sys], - # learning=pnl.LEARNING) + # train model for a few epochs + # results_before_proc = xor.run(inputs={xor_in:xor_inputs}, + # targets={xor_out:xor_targets}, + # epochs=10) + results_before_proc = xor.learn(inputs={"inputs": {xor_in:xor_inputs}, + "targets": {xor_out:xor_targets}, + "epochs": 10}, + execution_mode=autodiff_mode) - xor_process = Composition(pathways=([xor_in_sys, - hid_map_sys, - xor_hid_sys, - out_map_sys, - xor_out_sys], BackPropagation)) + # get weight parameters from pytorch + pt_weights_hid_bp = xor.parameters.pytorch_representation.get(xor).params[0].detach().numpy().copy() + pt_weights_out_bp = xor.parameters.pytorch_representation.get(xor).params[1].detach().numpy().copy() - msg = 'Training XOR model as AutodiffComposition for {eps} epochs took {comp_time} seconds.' - print(msg) - print("\n") - logger.info(msg) + #KAM temporarily removed -- will reimplement when pytorch weights can be used in pure PNL execution + # do processing on a few inputs + # results_proc = xor.run(inputs={xor_in:xor_inputs}) + # results_proc = xor.run(inputs={"inputs": {xor_in:xor_inputs}}) + # + # # get weight parameters from pytorch + # pt_weights_hid_ap = xor.parameters.pytorch_representation.get(xor).params[0].detach().numpy().copy() + # pt_weights_out_ap = xor.parameters.pytorch_representation.get(xor).params[1].detach().numpy().copy() + # + # # check that weight parameters before and after processing are the same + # assert np.allclose(pt_weights_hid_bp, pt_weights_hid_ap) + # assert np.allclose(pt_weights_out_bp, pt_weights_out_ap) - @pytest.mark.skip @pytest.mark.parametrize( - 'eps, opt', [ - (1, 'sgd'), - (10, 'sgd'), - (100, 'sgd') - ] + 'loss', [Loss.L1, Loss.POISSON_NLL] ) - def test_semantic_net_training_time(self, eps, opt): - - # SET UP MECHANISMS FOR COMPOSITION: - - nouns_in = TransferMechanism(name="nouns_input", - default_variable=np.zeros(8)) - - rels_in = TransferMechanism(name="rels_input", - default_variable=np.zeros(3)) - - h1 = TransferMechanism(name="hidden_nouns", - default_variable=np.zeros(8), - function=Logistic()) - - h2 = TransferMechanism(name="hidden_mixed", - default_variable=np.zeros(15), - function=Logistic()) - - out_sig_I = TransferMechanism(name="sig_outs_I", - default_variable=np.zeros(8), - function=Logistic()) - - out_sig_is = TransferMechanism(name="sig_outs_is", - default_variable=np.zeros(12), - function=Logistic()) + def test_various_loss_specs(self, loss, autodiff_mode): + if autodiff_mode is not pnl.ExecutionMode.Python: + pytest.skip("Loss spec not yet implemented!") - out_sig_has = TransferMechanism(name="sig_outs_has", - default_variable=np.zeros(9), - function=Logistic()) + xor_in = TransferMechanism(name='xor_in', + default_variable=np.zeros(2)) - out_sig_can = TransferMechanism(name="sig_outs_can", - default_variable=np.zeros(9), - function=Logistic()) + xor_hid = TransferMechanism(name='xor_hid', + default_variable=np.zeros(10), + function=Logistic()) - # SET UP MECHANISMS FOR SYSTEM + xor_out = TransferMechanism(name='xor_out', + default_variable=np.zeros(1), + function=Logistic()) - nouns_in_sys = TransferMechanism(name="nouns_input_sys", - default_variable=np.zeros(8)) + hid_map = MappingProjection() + out_map = MappingProjection() - rels_in_sys = TransferMechanism(name="rels_input_sys", - default_variable=np.zeros(3)) + xor = AutodiffComposition(loss_spec=loss) - h1_sys = TransferMechanism(name="hidden_nouns_sys", - default_variable=np.zeros(8), - function=Logistic()) + xor.add_node(xor_in) + xor.add_node(xor_hid) + xor.add_node(xor_out) - h2_sys = TransferMechanism(name="hidden_mixed_sys", - default_variable=np.zeros(15), - function=Logistic()) + xor.add_projection(sender=xor_in, projection=hid_map, receiver=xor_hid) + xor.add_projection(sender=xor_hid, projection=out_map, receiver=xor_out) - out_sig_I_sys = TransferMechanism(name="sig_outs_I_sys", - default_variable=np.zeros(8), - function=Logistic()) + xor_inputs = np.array( # the inputs we will provide to the model + [[0, 0], + [0, 1], + [1, 0], + [1, 1]]) - out_sig_is_sys = TransferMechanism(name="sig_outs_is_sys", - default_variable=np.zeros(12), - function=Logistic()) + xor_targets = np.array( # the outputs we wish to see from the model + [[0], + [1], + [1], + [0]]) - out_sig_has_sys = TransferMechanism(name="sig_outs_has_sys", - default_variable=np.zeros(9), - function=Logistic()) + xor.learn(inputs = {"inputs": {xor_in:xor_inputs}, + "targets": {xor_out:xor_targets}, + "epochs": 10}, execution_mode=autodiff_mode) - out_sig_can_sys = TransferMechanism(name="sig_outs_can_sys", - default_variable=np.zeros(9), - function=Logistic()) + def test_pytorch_loss_spec(self, autodiff_mode): + if autodiff_mode is not pnl.ExecutionMode.Python: + pytest.skip("Loss spec not yet implemented!") - # SET UP PROJECTIONS FOR COMPOSITION + import torch + ls = torch.nn.SoftMarginLoss(reduction='sum') - map_nouns_h1 = MappingProjection(matrix=np.random.rand(8,8), - name="map_nouns_h1", - sender=nouns_in, - receiver=h1) + xor_in = TransferMechanism(name='xor_in', + default_variable=np.zeros(2)) - map_rels_h2 = MappingProjection(matrix=np.random.rand(3,15), - name="map_rel_h2", - sender=rels_in, - receiver=h2) + xor_hid = TransferMechanism(name='xor_hid', + default_variable=np.zeros(10), + function=Logistic()) - map_h1_h2 = MappingProjection(matrix=np.random.rand(8,15), - name="map_h1_h2", - sender=h1, - receiver=h2) + xor_out = TransferMechanism(name='xor_out', + default_variable=np.zeros(1), + function=Logistic()) - map_h2_I = MappingProjection(matrix=np.random.rand(15,8), - name="map_h2_I", - sender=h2, - receiver=out_sig_I) + hid_map = MappingProjection() + out_map = MappingProjection() - map_h2_is = MappingProjection(matrix=np.random.rand(15,12), - name="map_h2_is", - sender=h2, - receiver=out_sig_is) + xor = AutodiffComposition(loss_spec=ls) - map_h2_has = MappingProjection(matrix=np.random.rand(15,9), - name="map_h2_has", - sender=h2, - receiver=out_sig_has) + xor.add_node(xor_in) + xor.add_node(xor_hid) + xor.add_node(xor_out) - map_h2_can = MappingProjection(matrix=np.random.rand(15,9), - name="map_h2_can", - sender=h2, - receiver=out_sig_can) + xor.add_projection(sender=xor_in, projection=hid_map, receiver=xor_hid) + xor.add_projection(sender=xor_hid, projection=out_map, receiver=xor_out) + xor_inputs = np.array( # the inputs we will provide to the model + [[0, 0], [0, 1], [1, 0], [1, 1]]) - # SET UP PROJECTIONS FOR SYSTEM + xor_targets = np.array( # the outputs we wish to see from the model + [[0], [1], [1], [0]]) - map_nouns_h1_sys = MappingProjection(matrix=map_nouns_h1.matrix.base.copy(), - name="map_nouns_h1_sys", - sender=nouns_in_sys, - receiver=h1_sys) + xor.learn(inputs={"inputs": {xor_in:xor_inputs}, + "targets": {xor_out:xor_targets}, + "epochs": 10}, execution_mode=autodiff_mode) + xor.learn(inputs={"inputs": {xor_in: xor_inputs}, + "targets": {xor_out: xor_targets}, + "epochs": 10}, execution_mode=autodiff_mode) - map_rels_h2_sys = MappingProjection(matrix=map_rels_h2.matrix.base.copy(), - name="map_relh2_sys", - sender=rels_in_sys, - receiver=h2_sys) - map_h1_h2_sys = MappingProjection(matrix=map_h1_h2.matrix.base.copy(), - name="map_h1_h2_sys", - sender=h1_sys, - receiver=h2_sys) + @pytest.mark.benchmark(group="Optimizer specs") + @pytest.mark.parametrize( + 'learning_rate, weight_decay, optimizer_type, expected', [ + (10, 0, 'sgd', [[[0.9863038667851067]], [[0.9944287263151904]], [[0.9934801466163382]], [[0.9979153035411085]]]), + (1.5, 1, 'sgd', [[[0.33226742]], [[0.4492334]], [[0.75459534]], [[0.44477028]]]), + (1.5, 1, 'adam', [[[0.43109927]], [[0.33088828]], [[0.40094236]], [[0.57104689]]]), + ] + ) + def test_optimizer_specs(self, learning_rate, weight_decay, optimizer_type, expected, autodiff_mode, benchmark): + xor_in = TransferMechanism(name='xor_in', + default_variable=np.zeros(2)) - map_h2_I_sys = MappingProjection(matrix=map_h2_I.matrix.base.copy(), - name="map_h2_I_sys", - sender=h2_sys, - receiver=out_sig_I_sys) + xor_hid = TransferMechanism(name='xor_hid', + default_variable=np.zeros(10), + function=Logistic()) - map_h2_is_sys = MappingProjection(matrix=map_h2_is.matrix.base.copy(), - name="map_h2_is_sys", - sender=h2_sys, - receiver=out_sig_is_sys) + xor_out = TransferMechanism(name='xor_out', + default_variable=np.zeros(1), + function=Logistic()) - map_h2_has_sys = MappingProjection(matrix=map_h2_has.matrix.base.copy(), - name="map_h2_has_sys", - sender=h2_sys, - receiver=out_sig_has_sys) + hid_map = MappingProjection() + out_map = MappingProjection() - map_h2_can_sys = MappingProjection(matrix=map_h2_can.matrix.base.copy(), - name="map_h2_can_sys", - sender=h2_sys, - receiver=out_sig_can_sys) + xor = AutodiffComposition(learning_rate=learning_rate, + optimizer_type=optimizer_type, + weight_decay=weight_decay) - # COMPOSITION FOR SEMANTIC NET + xor.add_node(xor_in) + xor.add_node(xor_hid) + xor.add_node(xor_out) - sem_net = AutodiffComposition() + xor.add_projection(sender=xor_in, projection=hid_map, receiver=xor_hid) + xor.add_projection(sender=xor_hid, projection=out_map, receiver=xor_out) - sem_net.add_node(nouns_in) - sem_net.add_node(rels_in) - sem_net.add_node(h1) - sem_net.add_node(h2) - sem_net.add_node(out_sig_I) - sem_net.add_node(out_sig_is) - sem_net.add_node(out_sig_has) - sem_net.add_node(out_sig_can) + xor_inputs = np.array( # the inputs we will provide to the model + [[0, 0], [0, 1], [1, 0], [1, 1]]) - sem_net.add_projection(sender=nouns_in, projection=map_nouns_h1, receiver=h1) - sem_net.add_projection(sender=rels_in, projection=map_rels_h2, receiver=h2) - sem_net.add_projection(sender=h1, projection=map_h1_h2, receiver=h2) - sem_net.add_projection(sender=h2, projection=map_h2_I, receiver=out_sig_I) - sem_net.add_projection(sender=h2, projection=map_h2_is, receiver=out_sig_is) - sem_net.add_projection(sender=h2, projection=map_h2_has, receiver=out_sig_has) - sem_net.add_projection(sender=h2, projection=map_h2_can, receiver=out_sig_can) + xor_targets = np.array( # the outputs we wish to see from the model + [[0], [1], [1], [0]]) - # INPUTS & OUTPUTS FOR SEMANTIC NET: + # train model for a few epochs + # results_before_proc = xor.run(inputs={xor_in:xor_inputs}, + # targets={xor_out:xor_targets}, + # epochs=10) + results_before_proc = benchmark(xor.learn, inputs={"inputs": {xor_in:xor_inputs}, + "targets": {xor_out:xor_targets}, + "epochs": 10}, execution_mode=autodiff_mode) - nouns = ['oak', 'pine', 'rose', 'daisy', 'canary', 'robin', 'salmon', 'sunfish'] - relations = ['is', 'has', 'can'] - is_list = ['living', 'living thing', 'plant', 'animal', 'tree', 'flower', 'bird', 'fish', 'big', 'green', 'red', - 'yellow'] - has_list = ['roots', 'leaves', 'bark', 'branches', 'skin', 'feathers', 'wings', 'gills', 'scales'] - can_list = ['grow', 'move', 'swim', 'fly', 'breathe', 'breathe underwater', 'breathe air', 'walk', 'photosynthesize'] + # fp32 results are different due to rounding + if pytest.helpers.llvm_current_fp_precision() == 'fp32' and \ + autodiff_mode != pnl.ExecutionMode.Python and \ + optimizer_type == 'sgd' and \ + learning_rate == 10: + expected = [[[0.9918830394744873]], [[0.9982172846794128]], [[0.9978305697441101]], [[0.9994590878486633]]] + # FIXME: LLVM version is broken with learning rate == 1.5 + if learning_rate != 1.5 or autodiff_mode == pnl.ExecutionMode.Python: + assert np.allclose(results_before_proc, expected) - nouns_input = np.identity(len(nouns)) - rels_input = np.identity(len(relations)) + # test whether pytorch parameters and projections are kept separate (at diff. places in memory) + def test_params_stay_separate(self, autodiff_mode): + if autodiff_mode is not pnl.ExecutionMode.Python: + pytest.skip("Compiled weights are always copied back!") - truth_nouns = np.identity(len(nouns)) + xor_in = TransferMechanism(name='xor_in', + default_variable=np.zeros(2)) - truth_is = np.zeros((len(nouns), len(is_list))) + xor_hid = TransferMechanism(name='xor_hid', + default_variable=np.zeros(10), + function=Logistic()) - truth_is[0, :] = [1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0] - truth_is[1, :] = [1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0] - truth_is[2, :] = [1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0] - truth_is[3, :] = [1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0] - truth_is[4, :] = [1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1] - truth_is[5, :] = [1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1] - truth_is[6, :] = [1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0] - truth_is[7, :] = [1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0] + xor_out = TransferMechanism(name='xor_out', + default_variable=np.zeros(1), + function=Logistic()) - truth_has = np.zeros((len(nouns), len(has_list))) + hid_m = np.random.rand(2,10) + out_m = np.random.rand(10,1) - truth_has[0, :] = [1, 1, 1, 1, 0, 0, 0, 0, 0] - truth_has[1, :] = [1, 1, 1, 1, 0, 0, 0, 0, 0] - truth_has[2, :] = [1, 1, 0, 0, 0, 0, 0, 0, 0] - truth_has[3, :] = [1, 1, 0, 0, 0, 0, 0, 0, 0] - truth_has[4, :] = [0, 0, 0, 0, 1, 1, 1, 0, 0] - truth_has[5, :] = [0, 0, 0, 0, 1, 1, 1, 0, 0] - truth_has[6, :] = [0, 0, 0, 0, 0, 0, 0, 1, 1] - truth_has[7, :] = [0, 0, 0, 0, 0, 0, 0, 1, 1] + hid_map = MappingProjection(name='hid_map', + matrix=hid_m.copy(), + sender=xor_in, + receiver=xor_hid) - truth_can = np.zeros((len(nouns), len(can_list))) + out_map = MappingProjection(name='out_map', + matrix=out_m.copy(), + sender=xor_hid, + receiver=xor_out) - truth_can[0, :] = [1, 0, 0, 0, 0, 0, 0, 0, 1] - truth_can[1, :] = [1, 0, 0, 0, 0, 0, 0, 0, 1] - truth_can[2, :] = [1, 0, 0, 0, 0, 0, 0, 0, 1] - truth_can[3, :] = [1, 0, 0, 0, 0, 0, 0, 0, 1] - truth_can[4, :] = [1, 1, 0, 1, 1, 0, 1, 1, 0] - truth_can[5, :] = [1, 1, 0, 1, 1, 0, 1, 1, 0] - truth_can[6, :] = [1, 1, 1, 0, 1, 1, 0, 0, 0] - truth_can[7, :] = [1, 1, 1, 0, 1, 1, 0, 0, 0] + xor = AutodiffComposition(learning_rate=10.0, + optimizer_type="sgd") - # SETTING UP DICTIONARIES OF INPUTS/OUTPUTS FOR SEMANTIC NET + xor.add_node(xor_in) + xor.add_node(xor_hid) + xor.add_node(xor_out) - inputs_dict = {} - inputs_dict[nouns_in] = [] - inputs_dict[rels_in] = [] + xor.add_projection(sender=xor_in, projection=hid_map, receiver=xor_hid) + xor.add_projection(sender=xor_hid, projection=out_map, receiver=xor_out) - targets_dict = {} - targets_dict[out_sig_I] = [] - targets_dict[out_sig_is] = [] - targets_dict[out_sig_has] = [] - targets_dict[out_sig_can] = [] + xor_inputs = np.array( # the inputs we will provide to the model + [[0, 0], [0, 1], [1, 0], [1, 1]]) - for i in range(len(nouns)): - for j in range(len(relations)): - inputs_dict[nouns_in].append(nouns_input[i]) - inputs_dict[rels_in].append(rels_input[j]) - targets_dict[out_sig_I].append(truth_nouns[i]) - targets_dict[out_sig_is].append(truth_is[i]) - targets_dict[out_sig_has].append(truth_has[i]) - targets_dict[out_sig_can].append(truth_can[i]) + xor_targets = np.array( # the outputs we wish to see from the model + [[0], [1], [1], [0]]) - inputs_dict_sys = {} - inputs_dict_sys[nouns_in_sys] = inputs_dict[nouns_in] - inputs_dict_sys[rels_in_sys] = inputs_dict[rels_in] + # train the model for a few epochs + result = xor.learn(inputs={"inputs": {xor_in:xor_inputs}, + "targets": {xor_out:xor_targets}, + "epochs": 10}, execution_mode=autodiff_mode) - targets_dict_sys = {} - targets_dict_sys[out_sig_I_sys] = targets_dict[out_sig_I] - targets_dict_sys[out_sig_is_sys] = targets_dict[out_sig_is] - targets_dict_sys[out_sig_has_sys] = targets_dict[out_sig_has] - targets_dict_sys[out_sig_can_sys] = targets_dict[out_sig_can] + # get weight parameters from pytorch + pt_weights_hid = xor.parameters.pytorch_representation.get(xor).params[0].detach().numpy().copy() + pt_weights_out = xor.parameters.pytorch_representation.get(xor).params[1].detach().numpy().copy() - # TIME TRAINING FOR COMPOSITION + # assert that projections are still what they were initialized as + assert np.allclose(hid_map.parameters.matrix.get(None), hid_m) + assert np.allclose(out_map.parameters.matrix.get(None), out_m) - start = timeit.default_timer() - result = sem_net.run(inputs=inputs_dict, - targets=targets_dict, - epochs=eps, - learning_rate=0.1, - controller=opt) - end = timeit.default_timer() - comp_time = end - start + # assert that projections didn't change during training with the pytorch + # parameters (they should now be different) + assert not np.allclose(pt_weights_hid, hid_map.parameters.matrix.get(None)) + assert not np.allclose(pt_weights_out, out_map.parameters.matrix.get(None)) - msg = 'Training Semantic net as AutodiffComposition for {0} epochs took {1} seconds'.format(eps, comp_time) - print(msg) - print("\n") - logger.info(msg) + def test_execution_mode_python_warning(self): + A = TransferMechanism(name="learning-process-mech-A") + B = TransferMechanism(name="learning-process-mech-B") + adc = AutodiffComposition(name='AUTODIFFCOMP') + pway = adc.add_backpropagation_learning_pathway(pathway=[A,B]) + # Call learn with default_variable specified for target (for comparison with missing target) + with pytest.warns(UserWarning) as warning: + adc.learn(inputs={A: 1.0, + pway.target: 0.0}, + execution_mode=pnl.ExecutionMode.Python, + num_trials=2) + assert repr(warning[1].message.args[0]) == '\'AUTODIFFCOMP.learn() called with ExecutionMode.Python; ' \ + 'learning will be executed using PyTorch; should use ' \ + 'ExecutionMode.PyTorch for clarity, or a standard Composition ' \ + 'for Python execution.)\'' @pytest.mark.pytorch -@pytest.mark.acidenticalness -class TestTrainingIdenticalness(): +@pytest.mark.actime +class TestTrainingTime: + @pytest.mark.skip @pytest.mark.parametrize( 'eps, opt', [ - # (1, 'sgd'), + (1, 'sgd'), (10, 'sgd'), - # (40, 'sgd') + (100, 'sgd') ] ) - def test_semantic_net_training_identicalness(self, eps, opt): - # SET UP MECHANISMS FOR SEMANTIC NET: - - nouns_in = TransferMechanism(name="nouns_input", - default_variable=np.zeros(8)) + def test_and_training_time(self, eps, opt,autodiff_mode): - rels_in = TransferMechanism(name="rels_input", - default_variable=np.zeros(3)) + # SET UP MECHANISMS FOR COMPOSITION - h1 = TransferMechanism(name="hidden_nouns", - default_variable=np.zeros(8), - function=Logistic()) + and_in = TransferMechanism(name='and_in', + default_variable=np.zeros(2)) - h2 = TransferMechanism(name="hidden_mixed", - default_variable=np.zeros(15), - function=Logistic()) + and_out = TransferMechanism(name='and_out', + default_variable=np.zeros(1), + function=Logistic()) - out_sig_I = TransferMechanism(name="sig_outs_I", - default_variable=np.zeros(8), - function=Logistic()) + # SET UP MECHANISMS FOR SYSTEM - out_sig_is = TransferMechanism(name="sig_outs_is", - default_variable=np.zeros(12), - function=Logistic()) + and_in_sys = TransferMechanism(name='and_in_sys', + default_variable=np.zeros(2)) - out_sig_has = TransferMechanism(name="sig_outs_has", - default_variable=np.zeros(9), + and_out_sys = TransferMechanism(name='and_out_sys', + default_variable=np.zeros(1), function=Logistic()) - out_sig_can = TransferMechanism(name="sig_outs_can", - default_variable=np.zeros(9), - function=Logistic()) + # SET UP PROJECTIONS FOR COMPOSITION - # SET UP MECHANISMS FOR Composition + and_map = MappingProjection(name='and_map', + matrix=np.random.rand(2, 1), + sender=and_in, + receiver=and_out) - nouns_in_comp = TransferMechanism(name="nouns_input_comp", - default_variable=np.zeros(8)) + # SET UP PROJECTIONS FOR SYSTEM - rels_in_comp = TransferMechanism(name="rels_input_comp", - default_variable=np.zeros(3)) + and_map_sys = MappingProjection(name='and_map_sys', + matrix=and_map.matrix.base.copy(), + sender=and_in_sys, + receiver=and_out_sys) - h1_comp = TransferMechanism(name="hidden_nouns_comp", - default_variable=np.zeros(8), - function=Logistic()) + # SET UP COMPOSITION - h2_comp = TransferMechanism(name="hidden_mixed_comp", - default_variable=np.zeros(15), - function=Logistic()) + and_net = AutodiffComposition() + + and_net.add_node(and_in) + and_net.add_node(and_out) + + and_net.add_projection(sender=and_in, projection=and_map, receiver=and_out) - out_sig_I_comp = TransferMechanism(name="sig_outs_I_comp", - default_variable=np.zeros(8), - function=Logistic()) + # SET UP INPUTS AND TARGETS - out_sig_is_comp = TransferMechanism(name="sig_outs_is_comp", - default_variable=np.zeros(12), - function=Logistic()) + and_inputs = np.zeros((4,2)) + and_inputs[0] = [0, 0] + and_inputs[1] = [0, 1] + and_inputs[2] = [1, 0] + and_inputs[3] = [1, 1] - out_sig_has_comp = TransferMechanism(name="sig_outs_has_comp", - default_variable=np.zeros(9), - function=Logistic()) + and_targets = np.zeros((4,1)) + and_targets[0] = [0] + and_targets[1] = [1] + and_targets[2] = [1] + and_targets[3] = [0] - out_sig_can_comp = TransferMechanism(name="sig_outs_can_comp", - default_variable=np.zeros(9), - function=Logistic()) + # TIME TRAINING FOR COMPOSITION - # SET UP PROJECTIONS FOR SEMANTIC NET + start = timeit.default_timer() + result = and_net.run(inputs={and_in:and_inputs}, + targets={and_out:and_targets}, + epochs=eps, + learning_rate=0.1, + controller=opt, + execution_mode=autodiff_mode) + end = timeit.default_timer() + comp_time = end - start - map_nouns_h1 = MappingProjection(matrix=np.random.rand(8,8), - name="map_nouns_h1", - sender=nouns_in, - receiver=h1) + msg = 'Training XOR model as AutodiffComposition for {0} epochs took {1} seconds'.format(eps, comp_time) + print(msg) + print("\n") + logger.info(msg) - map_rels_h2 = MappingProjection(matrix=np.random.rand(3,15), - name="map_relh2", - sender=rels_in, - receiver=h2) + @pytest.mark.skip + @pytest.mark.parametrize( + 'eps, opt', [ + (1, 'sgd'), + (10, 'sgd'), + (100, 'sgd') + ] + ) + def test_xor_training_time(self, eps, opt, autodiff_mode): - map_h1_h2 = MappingProjection(matrix=np.random.rand(8,15), - name="map_h1_h2", - sender=h1, - receiver=h2) + # SET UP MECHANISMS FOR COMPOSITION - map_h2_I = MappingProjection(matrix=np.random.rand(15,8), - name="map_h2_I", - sender=h2, - receiver=out_sig_I) + xor_in = TransferMechanism(name='xor_in', + default_variable=np.zeros(2)) - map_h2_is = MappingProjection(matrix=np.random.rand(15,12), - name="map_h2_is", - sender=h2, - receiver=out_sig_is) + xor_hid = TransferMechanism(name='xor_hid', + default_variable=np.zeros(10), + function=Logistic()) - map_h2_has = MappingProjection(matrix=np.random.rand(15,9), - name="map_h2_has", - sender=h2, - receiver=out_sig_has) + xor_out = TransferMechanism(name='xor_out', + default_variable=np.zeros(1), + function=Logistic()) - map_h2_can = MappingProjection(matrix=np.random.rand(15,9), - name="map_h2_can", - sender=h2, - receiver=out_sig_can) + # SET UP MECHANISMS FOR SYSTEM - # SET UP PROJECTIONS FOR COMPOSITION + xor_in_sys = TransferMechanism(name='xor_in_sys', + default_variable=np.zeros(2)) - map_nouns_h1_comp = MappingProjection(matrix=map_nouns_h1.matrix.base.copy(), - name="map_nouns_h1_comp", - sender=nouns_in_comp, - receiver=h1_comp) + xor_hid_sys = TransferMechanism(name='xor_hid_sys', + default_variable=np.zeros(10), + function=Logistic()) - map_rels_h2_comp = MappingProjection(matrix=map_rels_h2.matrix.base.copy(), - name="map_relh2_comp", - sender=rels_in_comp, - receiver=h2_comp) + xor_out_sys = TransferMechanism(name='xor_out_sys', + default_variable=np.zeros(1), + function=Logistic()) - map_h1_h2_comp = MappingProjection(matrix=map_h1_h2.matrix.base.copy(), - name="map_h1_h2_comp", - sender=h1_comp, - receiver=h2_comp) + # SET UP PROJECTIONS FOR COMPOSITION - map_h2_I_comp = MappingProjection(matrix=map_h2_I.matrix.base.copy(), - name="map_h2_I_comp", - sender=h2_comp, - receiver=out_sig_I_comp) + hid_map = MappingProjection(name='hid_map', + matrix=np.random.rand(2,10), + sender=xor_in, + receiver=xor_hid) - map_h2_is_comp = MappingProjection(matrix=map_h2_is.matrix.base.copy(), - name="map_h2_is_comp", - sender=h2_comp, - receiver=out_sig_is_comp) + out_map = MappingProjection(name='out_map', + matrix=np.random.rand(10,1), + sender=xor_hid, + receiver=xor_out) - map_h2_has_comp = MappingProjection(matrix=map_h2_has.matrix.base.copy(), - name="map_h2_has_comp", - sender=h2_comp, - receiver=out_sig_has_comp) + # SET UP PROJECTIONS FOR SYSTEM - map_h2_can_comp = MappingProjection(matrix=map_h2_can.matrix.base.copy(), - name="map_h2_can_comp", - sender=h2_comp, - receiver=out_sig_can_comp) + hid_map_sys = MappingProjection(name='hid_map_sys', + matrix=hid_map.matrix.base.copy(), + sender=xor_in_sys, + receiver=xor_hid_sys) - # SET UP AUTODIFFCOMPOSITION FOR SEMANTIC NET - sem_net_autodiff = AutodiffComposition(learning_rate=0.5, - optimizer_type=opt, - ) + out_map_sys = MappingProjection(name='out_map_sys', + matrix=out_map.matrix.base.copy(), + sender=xor_hid_sys, + receiver=xor_out_sys) - sem_net_autodiff.add_node(nouns_in) - sem_net_autodiff.add_node(rels_in) - sem_net_autodiff.add_node(h1) - sem_net_autodiff.add_node(h2) - sem_net_autodiff.add_node(out_sig_I) - sem_net_autodiff.add_node(out_sig_is) - sem_net_autodiff.add_node(out_sig_has) - sem_net_autodiff.add_node(out_sig_can) + # SET UP COMPOSITION - sem_net_autodiff.add_projection(sender=nouns_in, projection=map_nouns_h1, receiver=h1) - sem_net_autodiff.add_projection(sender=rels_in, projection=map_rels_h2, receiver=h2) - sem_net_autodiff.add_projection(sender=h1, projection=map_h1_h2, receiver=h2) - sem_net_autodiff.add_projection(sender=h2, projection=map_h2_I, receiver=out_sig_I) - sem_net_autodiff.add_projection(sender=h2, projection=map_h2_is, receiver=out_sig_is) - sem_net_autodiff.add_projection(sender=h2, projection=map_h2_has, receiver=out_sig_has) - sem_net_autodiff.add_projection(sender=h2, projection=map_h2_can, receiver=out_sig_can) - # INPUTS & OUTPUTS FOR SEMANTIC NET: + xor = AutodiffComposition(execution_mode=autodiff_mode) - nouns = ['oak', 'pine', 'rose', 'daisy', 'canary', 'robin', 'salmon', 'sunfish'] - relations = ['is', 'has', 'can'] - is_list = ['living', 'living thing', 'plant', 'animal', 'tree', 'flower', 'bird', 'fish', 'big', 'green', 'red', - 'yellow'] - has_list = ['roots', 'leaves', 'bark', 'branches', 'skin', 'feathers', 'wings', 'gills', 'scales'] - can_list = ['grow', 'move', 'swim', 'fly', 'breathe', 'breathe underwater', 'breathe air', 'walk', 'photosynthesize'] + xor.add_node(xor_in) + xor.add_node(xor_hid) + xor.add_node(xor_out) - nouns_input = np.identity(len(nouns)) + xor.add_projection(sender=xor_in, projection=hid_map, receiver=xor_hid) + xor.add_projection(sender=xor_hid, projection=out_map, receiver=xor_out) - rels_input = np.identity(len(relations)) + # SET UP INPUTS AND TARGETS - truth_nouns = np.identity(len(nouns)) + xor_inputs = np.array( # the inputs we will provide to the model + [[0, 0], + [0, 1], + [1, 0], + [1, 1]]) - truth_is = np.zeros((len(nouns), len(is_list))) + xor_targets = np.array( # the outputs we wish to see from the model + [[0], + [1], + [1], + [0]]) - truth_is[0, :] = [1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0] - truth_is[1, :] = [1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0] - truth_is[2, :] = [1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0] - truth_is[3, :] = [1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0] - truth_is[4, :] = [1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1] - truth_is[5, :] = [1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1] - truth_is[6, :] = [1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0] - truth_is[7, :] = [1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0] + # TIME TRAINING FOR COMPOSITION - truth_has = np.zeros((len(nouns), len(has_list))) + start = timeit.default_timer() + result = xor.run(inputs={xor_in:xor_inputs}, + targets={xor_out:xor_targets}, + epochs=eps, + learning_rate=0.1, + controller=opt, + execution_mode=autodiff_mode) + end = timeit.default_timer() + comp_time = end - start - truth_has[0, :] = [1, 1, 1, 1, 0, 0, 0, 0, 0] - truth_has[1, :] = [1, 1, 1, 1, 0, 0, 0, 0, 0] - truth_has[2, :] = [1, 1, 0, 0, 0, 0, 0, 0, 0] - truth_has[3, :] = [1, 1, 0, 0, 0, 0, 0, 0, 0] - truth_has[4, :] = [0, 0, 0, 0, 1, 1, 1, 0, 0] - truth_has[5, :] = [0, 0, 0, 0, 1, 1, 1, 0, 0] - truth_has[6, :] = [0, 0, 0, 0, 0, 0, 0, 1, 1] - truth_has[7, :] = [0, 0, 0, 0, 0, 0, 0, 1, 1] + # SET UP SYSTEM - truth_can = np.zeros((len(nouns), len(can_list))) + # xor_process = Process(pathway=[xor_in_sys, + # hid_map_sys, + # xor_hid_sys, + # out_map_sys, + # xor_out_sys], + # learning=pnl.LEARNING) - truth_can[0, :] = [1, 0, 0, 0, 0, 0, 0, 0, 1] - truth_can[1, :] = [1, 0, 0, 0, 0, 0, 0, 0, 1] - truth_can[2, :] = [1, 0, 0, 0, 0, 0, 0, 0, 1] - truth_can[3, :] = [1, 0, 0, 0, 0, 0, 0, 0, 1] - truth_can[4, :] = [1, 1, 0, 1, 1, 0, 1, 1, 0] - truth_can[5, :] = [1, 1, 0, 1, 1, 0, 1, 1, 0] - truth_can[6, :] = [1, 1, 1, 0, 1, 1, 0, 0, 0] - truth_can[7, :] = [1, 1, 1, 0, 1, 1, 0, 0, 0] + xor_process = Composition(pathways=([xor_in_sys, + hid_map_sys, + xor_hid_sys, + out_map_sys, + xor_out_sys], BackPropagation)) - # SETTING UP DICTIONARY OF INPUTS/OUTPUTS FOR SEMANTIC NET + msg = 'Training XOR model as AutodiffComposition for {eps} epochs took {comp_time} seconds.' + print(msg) + print("\n") + logger.info(msg) - inputs_dict = {} - inputs_dict[nouns_in] = [] - inputs_dict[rels_in] = [] + @pytest.mark.skip + @pytest.mark.parametrize( + 'eps, opt', [ + (1, 'sgd'), + (10, 'sgd'), + (100, 'sgd') + ] + ) + def test_semantic_net_training_time(self, eps, opt): + + # SET UP MECHANISMS FOR COMPOSITION: - targets_dict = {} - targets_dict[out_sig_I] = [] - targets_dict[out_sig_is] = [] - targets_dict[out_sig_has] = [] - targets_dict[out_sig_can] = [] + nouns_in = TransferMechanism(name="nouns_input", + default_variable=np.zeros(8)) - for i in range(len(nouns)): - for j in range(len(relations)): - inputs_dict[nouns_in].append(nouns_input[i]) - inputs_dict[rels_in].append(rels_input[j]) - targets_dict[out_sig_I].append(truth_nouns[i]) - targets_dict[out_sig_is].append(truth_is[i]) - targets_dict[out_sig_has].append(truth_has[i]) - targets_dict[out_sig_can].append(truth_can[i]) + rels_in = TransferMechanism(name="rels_input", + default_variable=np.zeros(3)) - inputs_dict_comp = {} - inputs_dict_comp[nouns_in_comp] = inputs_dict[nouns_in] - inputs_dict_comp[rels_in_comp] = inputs_dict[rels_in] + h1 = TransferMechanism(name="hidden_nouns", + default_variable=np.zeros(8), + function=Logistic()) - sem_net_autodiff.run(inputs=inputs_dict) + h2 = TransferMechanism(name="hidden_mixed", + default_variable=np.zeros(15), + function=Logistic()) - # TRAIN AUTODIFFCOMPOSITION - def g_f(): - yield {"inputs": inputs_dict, - "targets": targets_dict, - "epochs": eps} - g = g_f() - sem_net_autodiff.learn(inputs=g_f) + out_sig_I = TransferMechanism(name="sig_outs_I", + default_variable=np.zeros(8), + function=Logistic()) - # SET UP COMPOSITION - sem_net_comp = Composition() + out_sig_is = TransferMechanism(name="sig_outs_is", + default_variable=np.zeros(12), + function=Logistic()) - backprop_pathway = sem_net_comp.add_backpropagation_learning_pathway( - pathway=[ - nouns_in_comp, - map_nouns_h1_comp, - h1_comp, - map_h1_h2_comp, - h2_comp, - map_h2_I_comp, - out_sig_I_comp - ], - learning_rate=0.5 - ) - inputs_dict_comp[backprop_pathway.target] = targets_dict[out_sig_I] + out_sig_has = TransferMechanism(name="sig_outs_has", + default_variable=np.zeros(9), + function=Logistic()) - backprop_pathway = sem_net_comp.add_backpropagation_learning_pathway( - pathway=[ - rels_in_comp, - map_rels_h2_comp, - h2_comp, - map_h2_is_comp, - out_sig_is_comp - ], - learning_rate=0.5 - ) - inputs_dict_comp[backprop_pathway.target] = targets_dict[out_sig_is] + out_sig_can = TransferMechanism(name="sig_outs_can", + default_variable=np.zeros(9), + function=Logistic()) - backprop_pathway = sem_net_comp.add_backpropagation_learning_pathway( - pathway=[ - h2_comp, - map_h2_has_comp, - out_sig_has_comp - ], - learning_rate=0.5 - ) - inputs_dict_comp[backprop_pathway.target] = targets_dict[out_sig_has] + # SET UP MECHANISMS FOR SYSTEM - backprop_pathway = sem_net_comp.add_backpropagation_learning_pathway( - pathway=[ - h2_comp, - map_h2_can_comp, - out_sig_can_comp - ], - learning_rate=0.5 - ) - inputs_dict_comp[backprop_pathway.target] = targets_dict[out_sig_can] + nouns_in_sys = TransferMechanism(name="nouns_input_sys", + default_variable=np.zeros(8)) - # TRAIN COMPOSITION - sem_net_comp.learn(inputs=inputs_dict_comp, - num_trials=(len(inputs_dict_comp[nouns_in_comp]) * eps)) + rels_in_sys = TransferMechanism(name="rels_input_sys", + default_variable=np.zeros(3)) - # CHECK THAT PARAMETERS FOR AUTODIFFCOMPOSITION, COMPOSITION ARE SAME + h1_sys = TransferMechanism(name="hidden_nouns_sys", + default_variable=np.zeros(8), + function=Logistic()) - assert np.allclose(map_nouns_h1.parameters.matrix.get(sem_net_autodiff), - map_nouns_h1_comp.get_mod_matrix(sem_net_comp)) - assert np.allclose(map_rels_h2.parameters.matrix.get(sem_net_autodiff), - map_rels_h2_comp.get_mod_matrix(sem_net_comp)) - assert np.allclose(map_h1_h2.parameters.matrix.get(sem_net_autodiff), - map_h1_h2_comp.get_mod_matrix(sem_net_comp)) - assert np.allclose(map_h2_I.parameters.matrix.get(sem_net_autodiff), - map_h2_I_comp.get_mod_matrix(sem_net_comp)) - assert np.allclose(map_h2_is.parameters.matrix.get(sem_net_autodiff), - map_h2_is_comp.get_mod_matrix(sem_net_comp)) - assert np.allclose(map_h2_has.parameters.matrix.get(sem_net_autodiff), - map_h2_has_comp.get_mod_matrix(sem_net_comp)) - assert np.allclose(map_h2_can.parameters.matrix.get(sem_net_autodiff), - map_h2_can_comp.get_mod_matrix(sem_net_comp)) + h2_sys = TransferMechanism(name="hidden_mixed_sys", + default_variable=np.zeros(15), + function=Logistic()) - def test_identicalness_of_input_types(self): - # SET UP MECHANISMS FOR COMPOSITION - from copy import copy - hid_map_mat = np.random.rand(2, 10) - out_map_mat = np.random.rand(10, 1) - xor_in_dict = TransferMechanism(name='xor_in', - default_variable=np.zeros(2)) + out_sig_I_sys = TransferMechanism(name="sig_outs_I_sys", + default_variable=np.zeros(8), + function=Logistic()) - xor_hid_dict = TransferMechanism(name='xor_hid', - default_variable=np.zeros(10), - function=Logistic()) + out_sig_is_sys = TransferMechanism(name="sig_outs_is_sys", + default_variable=np.zeros(12), + function=Logistic()) - xor_out_dict = TransferMechanism(name='xor_out', - default_variable=np.zeros(1), - function=Logistic()) + out_sig_has_sys = TransferMechanism(name="sig_outs_has_sys", + default_variable=np.zeros(9), + function=Logistic()) + + out_sig_can_sys = TransferMechanism(name="sig_outs_can_sys", + default_variable=np.zeros(9), + function=Logistic()) # SET UP PROJECTIONS FOR COMPOSITION - hid_map_dict = MappingProjection(name='hid_map', - matrix=copy(hid_map_mat), - sender=xor_in_dict, - receiver=xor_hid_dict) + map_nouns_h1 = MappingProjection(matrix=np.random.rand(8,8), + name="map_nouns_h1", + sender=nouns_in, + receiver=h1) - out_map_dict = MappingProjection(name='out_map', - matrix=copy(out_map_mat), - sender=xor_hid_dict, - receiver=xor_out_dict) + map_rels_h2 = MappingProjection(matrix=np.random.rand(3,15), + name="map_rel_h2", + sender=rels_in, + receiver=h2) - # SET UP COMPOSITION + map_h1_h2 = MappingProjection(matrix=np.random.rand(8,15), + name="map_h1_h2", + sender=h1, + receiver=h2) - xor_dict = AutodiffComposition() + map_h2_I = MappingProjection(matrix=np.random.rand(15,8), + name="map_h2_I", + sender=h2, + receiver=out_sig_I) - xor_dict.add_node(xor_in_dict) - xor_dict.add_node(xor_hid_dict) - xor_dict.add_node(xor_out_dict) + map_h2_is = MappingProjection(matrix=np.random.rand(15,12), + name="map_h2_is", + sender=h2, + receiver=out_sig_is) - xor_dict.add_projection(sender=xor_in_dict, projection=hid_map_dict, receiver=xor_hid_dict) - xor_dict.add_projection(sender=xor_hid_dict, projection=out_map_dict, receiver=xor_out_dict) - # SET UP INPUTS AND TARGETS + map_h2_has = MappingProjection(matrix=np.random.rand(15,9), + name="map_h2_has", + sender=h2, + receiver=out_sig_has) - xor_inputs_dict = np.array( # the inputs we will provide to the model - [[0, 0], - [0, 1], - [1, 0], - [1, 1]]) + map_h2_can = MappingProjection(matrix=np.random.rand(15,9), + name="map_h2_can", + sender=h2, + receiver=out_sig_can) - xor_targets_dict = np.array( # the outputs we wish to see from the model - [[0], - [1], - [1], - [0]]) + # SET UP PROJECTIONS FOR SYSTEM - input_dict = { - "inputs": { - xor_in_dict: xor_inputs_dict - }, - "targets": { - xor_out_dict: xor_targets_dict - } - } + map_nouns_h1_sys = MappingProjection(matrix=map_nouns_h1.matrix.base.copy(), + name="map_nouns_h1_sys", + sender=nouns_in_sys, + receiver=h1_sys) - result_dict = xor_dict.learn(inputs=input_dict) + map_rels_h2_sys = MappingProjection(matrix=map_rels_h2.matrix.base.copy(), + name="map_relh2_sys", + sender=rels_in_sys, + receiver=h2_sys) - # SET UP MECHANISMS FOR COMPOSITION - xor_in_func = TransferMechanism(name='xor_in', - default_variable=np.zeros(2)) + map_h1_h2_sys = MappingProjection(matrix=map_h1_h2.matrix.base.copy(), + name="map_h1_h2_sys", + sender=h1_sys, + receiver=h2_sys) - xor_hid_func = TransferMechanism(name='xor_hid', - default_variable=np.zeros(10), - function=Logistic()) + map_h2_I_sys = MappingProjection(matrix=map_h2_I.matrix.base.copy(), + name="map_h2_I_sys", + sender=h2_sys, + receiver=out_sig_I_sys) - xor_out_func = TransferMechanism(name='xor_out', - default_variable=np.zeros(1), - function=Logistic()) + map_h2_is_sys = MappingProjection(matrix=map_h2_is.matrix.base.copy(), + name="map_h2_is_sys", + sender=h2_sys, + receiver=out_sig_is_sys) - # SET UP PROJECTIONS FOR COMPOSITION + map_h2_has_sys = MappingProjection(matrix=map_h2_has.matrix.base.copy(), + name="map_h2_has_sys", + sender=h2_sys, + receiver=out_sig_has_sys) - hid_map_func = MappingProjection(name='hid_map', - matrix=copy(hid_map_mat), - sender=xor_in_func, - receiver=xor_hid_func) + map_h2_can_sys = MappingProjection(matrix=map_h2_can.matrix.base.copy(), + name="map_h2_can_sys", + sender=h2_sys, + receiver=out_sig_can_sys) - out_map_func = MappingProjection(name='out_map', - matrix=copy(out_map_mat), - sender=xor_hid_func, - receiver=xor_out_func) + # COMPOSITION FOR SEMANTIC NET - # SET UP COMPOSITION + sem_net = AutodiffComposition() - xor_func = AutodiffComposition() + sem_net.add_node(nouns_in) + sem_net.add_node(rels_in) + sem_net.add_node(h1) + sem_net.add_node(h2) + sem_net.add_node(out_sig_I) + sem_net.add_node(out_sig_is) + sem_net.add_node(out_sig_has) + sem_net.add_node(out_sig_can) - xor_func.add_node(xor_in_func) - xor_func.add_node(xor_hid_func) - xor_func.add_node(xor_out_func) + sem_net.add_projection(sender=nouns_in, projection=map_nouns_h1, receiver=h1) + sem_net.add_projection(sender=rels_in, projection=map_rels_h2, receiver=h2) + sem_net.add_projection(sender=h1, projection=map_h1_h2, receiver=h2) + sem_net.add_projection(sender=h2, projection=map_h2_I, receiver=out_sig_I) + sem_net.add_projection(sender=h2, projection=map_h2_is, receiver=out_sig_is) + sem_net.add_projection(sender=h2, projection=map_h2_has, receiver=out_sig_has) + sem_net.add_projection(sender=h2, projection=map_h2_can, receiver=out_sig_can) - xor_func.add_projection(sender=xor_in_func, projection=hid_map_func, receiver=xor_hid_func) - xor_func.add_projection(sender=xor_hid_func, projection=out_map_func, receiver=xor_out_func) + # INPUTS & OUTPUTS FOR SEMANTIC NET: - # SET UP INPUTS AND TARGETS + nouns = ['oak', 'pine', 'rose', 'daisy', 'canary', 'robin', 'salmon', 'sunfish'] + relations = ['is', 'has', 'can'] + is_list = ['living', 'living thing', 'plant', 'animal', 'tree', 'flower', 'bird', 'fish', 'big', 'green', 'red', + 'yellow'] + has_list = ['roots', 'leaves', 'bark', 'branches', 'skin', 'feathers', 'wings', 'gills', 'scales'] + can_list = ['grow', 'move', 'swim', 'fly', 'breathe', 'breathe underwater', 'breathe air', 'walk', 'photosynthesize'] - xor_inputs_func = np.array( # the inputs we will provide to the model - [[0, 0], - [0, 1], - [1, 0], - [1, 1]]) + nouns_input = np.identity(len(nouns)) - xor_targets_func = np.array( # the outputs we wish to see from the model - [[0], - [1], - [1], - [0]]) + rels_input = np.identity(len(relations)) - def get_inputs(idx): - return { - "inputs": { - xor_in_func: xor_inputs_func[idx] - }, - "targets": { - xor_out_func: xor_targets_func[idx] - } - } + truth_nouns = np.identity(len(nouns)) - result_func = xor_func.learn(inputs=get_inputs) + truth_is = np.zeros((len(nouns), len(is_list))) - # SET UP MECHANISMS FOR COMPOSITION - xor_in_gen = TransferMechanism(name='xor_in', - default_variable=np.zeros(2)) + truth_is[0, :] = [1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0] + truth_is[1, :] = [1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0] + truth_is[2, :] = [1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0] + truth_is[3, :] = [1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0] + truth_is[4, :] = [1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1] + truth_is[5, :] = [1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1] + truth_is[6, :] = [1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0] + truth_is[7, :] = [1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0] - xor_hid_gen = TransferMechanism(name='xor_hid', - default_variable=np.zeros(10), - function=Logistic()) + truth_has = np.zeros((len(nouns), len(has_list))) - xor_out_gen = TransferMechanism(name='xor_out', - default_variable=np.zeros(1), - function=Logistic()) + truth_has[0, :] = [1, 1, 1, 1, 0, 0, 0, 0, 0] + truth_has[1, :] = [1, 1, 1, 1, 0, 0, 0, 0, 0] + truth_has[2, :] = [1, 1, 0, 0, 0, 0, 0, 0, 0] + truth_has[3, :] = [1, 1, 0, 0, 0, 0, 0, 0, 0] + truth_has[4, :] = [0, 0, 0, 0, 1, 1, 1, 0, 0] + truth_has[5, :] = [0, 0, 0, 0, 1, 1, 1, 0, 0] + truth_has[6, :] = [0, 0, 0, 0, 0, 0, 0, 1, 1] + truth_has[7, :] = [0, 0, 0, 0, 0, 0, 0, 1, 1] - # SET UP PROJECTIONS FOR COMPOSITION + truth_can = np.zeros((len(nouns), len(can_list))) - hid_map_gen = MappingProjection(name='hid_map', - matrix=copy(hid_map_mat), - sender=xor_in_gen, - receiver=xor_hid_gen) + truth_can[0, :] = [1, 0, 0, 0, 0, 0, 0, 0, 1] + truth_can[1, :] = [1, 0, 0, 0, 0, 0, 0, 0, 1] + truth_can[2, :] = [1, 0, 0, 0, 0, 0, 0, 0, 1] + truth_can[3, :] = [1, 0, 0, 0, 0, 0, 0, 0, 1] + truth_can[4, :] = [1, 1, 0, 1, 1, 0, 1, 1, 0] + truth_can[5, :] = [1, 1, 0, 1, 1, 0, 1, 1, 0] + truth_can[6, :] = [1, 1, 1, 0, 1, 1, 0, 0, 0] + truth_can[7, :] = [1, 1, 1, 0, 1, 1, 0, 0, 0] - out_map_gen = MappingProjection(name='out_map', - matrix=copy(out_map_mat), - sender=xor_hid_gen, - receiver=xor_out_gen) + # SETTING UP DICTIONARIES OF INPUTS/OUTPUTS FOR SEMANTIC NET - # SET UP COMPOSITION + inputs_dict = {} + inputs_dict[nouns_in] = [] + inputs_dict[rels_in] = [] - xor_gen = AutodiffComposition() + targets_dict = {} + targets_dict[out_sig_I] = [] + targets_dict[out_sig_is] = [] + targets_dict[out_sig_has] = [] + targets_dict[out_sig_can] = [] - xor_gen.add_node(xor_in_gen) - xor_gen.add_node(xor_hid_gen) - xor_gen.add_node(xor_out_gen) + for i in range(len(nouns)): + for j in range(len(relations)): + inputs_dict[nouns_in].append(nouns_input[i]) + inputs_dict[rels_in].append(rels_input[j]) + targets_dict[out_sig_I].append(truth_nouns[i]) + targets_dict[out_sig_is].append(truth_is[i]) + targets_dict[out_sig_has].append(truth_has[i]) + targets_dict[out_sig_can].append(truth_can[i]) - xor_gen.add_projection(sender=xor_in_gen, projection=hid_map_gen, receiver=xor_hid_gen) - xor_gen.add_projection(sender=xor_hid_gen, projection=out_map_gen, receiver=xor_out_gen) - # SET UP INPUTS AND TARGETS + inputs_dict_sys = {} + inputs_dict_sys[nouns_in_sys] = inputs_dict[nouns_in] + inputs_dict_sys[rels_in_sys] = inputs_dict[rels_in] - xor_inputs_gen = np.array( # the inputs we will provide to the model - [[0, 0], - [0, 1], - [1, 0], - [1, 1]]) + targets_dict_sys = {} + targets_dict_sys[out_sig_I_sys] = targets_dict[out_sig_I] + targets_dict_sys[out_sig_is_sys] = targets_dict[out_sig_is] + targets_dict_sys[out_sig_has_sys] = targets_dict[out_sig_has] + targets_dict_sys[out_sig_can_sys] = targets_dict[out_sig_can] - xor_targets_gen = np.array( # the outputs we wish to see from the model - [[0], - [1], - [1], - [0]]) + # TIME TRAINING FOR COMPOSITION - def get_inputs_gen(): - yield { - "inputs": { - xor_in_gen: xor_inputs_gen - }, - "targets": { - xor_out_gen: xor_targets_gen - } - } + start = timeit.default_timer() + result = sem_net.run(inputs=inputs_dict, + targets=targets_dict, + epochs=eps, + learning_rate=0.1, + controller=opt) + end = timeit.default_timer() + comp_time = end - start - g = get_inputs_gen() - result_gen = xor_gen.learn(inputs=g) + msg = 'Training Semantic net as AutodiffComposition for {0} epochs took {1} seconds'.format(eps, comp_time) + print(msg) + print("\n") + logger.info(msg) - # SET UP MECHANISMS FOR COMPOSITION - xor_in_gen_func = TransferMechanism(name='xor_in', - default_variable=np.zeros(2)) - xor_hid_gen_func = TransferMechanism(name='xor_hid', - default_variable=np.zeros(10), - function=Logistic()) +@pytest.mark.pytorch +def test_autodiff_saveload(tmp_path): + def create_xor(): + # create xor model mechanisms and projections + xor_in = TransferMechanism(name='xor_in', + default_variable=np.zeros(2)) - xor_out_gen_func = TransferMechanism(name='xor_out', - default_variable=np.zeros(1), - function=Logistic()) + xor_hid = TransferMechanism(name='xor_hid', + default_variable=np.zeros(10), + function=Logistic()) - # SET UP PROJECTIONS FOR COMPOSITION + xor_out = TransferMechanism(name='xor_out', + default_variable=np.zeros(1), + function=Logistic()) - hid_map_gen_func = MappingProjection(name='hid_map', - matrix=copy(hid_map_mat), - sender=xor_in_gen_func, - receiver=xor_hid_gen_func) + hid_map = MappingProjection(matrix=np.random.rand(2,10), name='hid_map') + out_map = MappingProjection(matrix=np.random.rand(10,1), name='out_map') - out_map_gen_func = MappingProjection(name='out_map', - matrix=copy(out_map_mat), - sender=xor_hid_gen_func, - receiver=xor_out_gen_func) + # put the mechanisms and projections together in an autodiff composition (AC) + xor = AutodiffComposition() - # SET UP COMPOSITION + xor.add_node(xor_in) + xor.add_node(xor_hid) + xor.add_node(xor_out) - xor_gen_func = AutodiffComposition() + xor.add_projection(sender=xor_in, projection=hid_map, receiver=xor_hid) + xor.add_projection(sender=xor_hid, projection=out_map, receiver=xor_out) + return xor - xor_gen_func.add_node(xor_in_gen_func) - xor_gen_func.add_node(xor_hid_gen_func) - xor_gen_func.add_node(xor_out_gen_func) + np.random.seed(0) + xor1 = create_xor() + xor1_outputs = xor1.run(inputs=[0,0]) - xor_gen_func.add_projection(sender=xor_in_gen_func, projection=hid_map_gen_func, receiver=xor_hid_gen_func) - xor_gen_func.add_projection(sender=xor_hid_gen_func, projection=out_map_gen_func, receiver=xor_out_gen_func) - # SET UP INPUTS AND TARGETS + # save + # path = xor1.save() + path = xor1.save(os.path.join(tmp_path, 'xor_1.pnl')) - xor_inputs_gen_func = np.array( # the inputs we will provide to the model - [[0, 0], - [0, 1], - [1, 0], - [1, 1]]) + # del xor1 + pnl.clear_registry() - xor_targets_gen_func = np.array( # the outputs we wish to see from the model - [[0], - [1], - [1], - [0]]) + # load + np.random.seed(1) + xor2 = create_xor() + xor2_outputs_pre = xor2.run(inputs=[0,0]) + # xor2.load(os.path.join(tmp_path, 'xor_1.pnl')) + xor2.load(path) + xor2_outputs_post = xor2.run(inputs=[0,0]) - def get_inputs_gen_func(): - yield { - "inputs": { - xor_in_gen_func: xor_inputs_gen_func - }, - "targets": { - xor_out_gen_func: xor_targets_gen_func - } - } - result_gen_func = xor_gen_func.learn(inputs=get_inputs_gen_func) + # sanity check - make sure xor2 weights differ + assert not np.allclose(xor2_outputs_pre, xor2_outputs_post, atol=1e-9) - assert result_dict == result_func == result_gen == result_gen_func + # make sure loaded model is identical, and used during run + assert np.allclose(xor1_outputs, xor2_outputs_post, atol=1e-9) @pytest.mark.pytorch @@ -2431,6 +2451,7 @@ def test_autodiff_loss_tracking(self): xor.clear_losses(context=xor) assert len(xor.losses) == 0 + @pytest.mark.pytorch @pytest.mark.acnested class TestNested: @@ -2905,6 +2926,7 @@ def test_semantic_net_nested(self, eps, opt, autodiff_mode): parentComposition.run(inputs=no_training_input) + @pytest.mark.pytorch class TestBatching: def test_call_before_minibatch(self): diff --git a/tests/composition/test_learning.py b/tests/composition/test_learning.py index 6659b86d36b..bfc001bab4b 100644 --- a/tests/composition/test_learning.py +++ b/tests/composition/test_learning.py @@ -408,6 +408,22 @@ def test_indepedence_of_learning_pathways_using_same_mechs_in_different_comps(se num_trials=2) assert np.allclose(comp2.results, comp1.results) + @pytest.mark.parametrize('execution_mode', + [pnlvm.ExecutionMode.LLVM, pnlvm.ExecutionMode.PyTorch]) + def test_execution_mode_pytorch_and_LLVM_errors(self, execution_mode): + A = TransferMechanism(name="learning-process-mech-A") + B = TransferMechanism(name="learning-process-mech-B") + comp = Composition() + pway = comp.add_backpropagation_learning_pathway(pathway=[A,B]) + # Call learn with default_variable specified for target (for comparison with missing target) + with pytest.raises(CompositionError) as error: + comp.learn(inputs={A: 1.0, + pway.target: 0.0}, + execution_mode=execution_mode, + num_trials=2) + assert error.value.error_value == f"ExecutionMode.{execution_mode.name} cannot be used in the learn() " \ + f"method of \'Composition-0\' because it is not an AutodiffComposition" + class TestNoLearning: @@ -1710,6 +1726,7 @@ def test_stranded_nested_target_mech_error(self): f'as the target attribute of the relevant pathway in {inner_comp.name}.pathways. ' ) + class TestBackPropLearning: def test_matrix_spec_and_learning_rate(self): @@ -2079,6 +2096,7 @@ def test_xor_training_identicalness_standard_composition_vs_PyTorch_and_LLVM(sel hidden_to_out_comp.get_mod_matrix(xor_comp)) assert np.allclose(result_comp, result_LLVM) + # AutodiffComposition using PyTorch elif 'PYTORCH' in models: input_PYTORCH = pnl.TransferMechanism(name='input', From 5d4ead378867f097939c02a290bbe2498114dfed Mon Sep 17 00:00:00 2001 From: jdcpni Date: Fri, 25 Nov 2022 10:43:42 -0500 Subject: [PATCH 104/127] Refactor/execution mode compiled (#2550) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * • composition.py - learn(): add error for use of ExecutionMode.LLVM or ExecutionMode.PyTorch • autodiffcomposition.py - learn(): add warning for use of ExecutionMode.Python • test_learning.py: - add test_execution_mode_pytorch_and_LLVM_errors • test_autodiffcomposition.py: - add test_execution_mode_python_warning * [skip ci] * [skip ci] * • llvm/__init__.py - add ExecutionMode.COMPILED (~ Python | PyTorch). • compositon.py, autodiffcomposition.py, compositionrunner.py: - modifed to sue ExecutionMode.COMPILED for condionals • test_autodiffcomposition.py: - test_execution_mode_python_error() replaces test_execution_mode_python_warning() * - * • Modification per Jan's suggestions * • test_autodiffcomposition.py - test_execution_mode_python_error(): fix bug --- conftest.py | 4 ++-- psyneulink/core/compositions/composition.py | 23 +++++++++++-------- psyneulink/core/llvm/__init__.py | 1 + .../compositions/autodiffcomposition.py | 7 +++--- .../library/compositions/compositionrunner.py | 5 ++-- tests/composition/test_autodiffcomposition.py | 14 +++++------ 6 files changed, 28 insertions(+), 26 deletions(-) diff --git a/conftest.py b/conftest.py index 393cb53e3c6..cae7d36ce73 100644 --- a/conftest.py +++ b/conftest.py @@ -75,8 +75,8 @@ def pytest_generate_tests(metafunc): if "autodiff_mode" in metafunc.fixturenames: auto_modes = [ - pnlvm.ExecutionMode.Python, - # pnlvm.ExecutionMode.PyTorch, + # pnlvm.ExecutionMode.Python, + pnlvm.ExecutionMode.PyTorch, pytest.param(pnlvm.ExecutionMode.LLVMRun, marks=pytest.mark.llvm) ] metafunc.parametrize("autodiff_mode", auto_modes) diff --git a/psyneulink/core/compositions/composition.py b/psyneulink/core/compositions/composition.py index d65f8c4e1e6..a83be7b2531 100644 --- a/psyneulink/core/compositions/composition.py +++ b/psyneulink/core/compositions/composition.py @@ -10395,7 +10395,7 @@ def learn( from psyneulink.library.compositions import AutodiffComposition runner = CompositionRunner(self) - if (execution_mode in {pnlvm.ExecutionMode.PyTorch, pnlvm.ExecutionMode.LLVM} + if ((execution_mode is not pnlvm.ExecutionMode.Python) and not isinstance(self, AutodiffComposition)): raise CompositionError(f"ExecutionMode.{execution_mode.name} cannot be used in the learn() method of " f"'{self.name}' because it is not an {AutodiffComposition.componentCategory}") @@ -10477,11 +10477,14 @@ def _execute_controller(self, context=context, node=self.controller) - if self.controller and not execution_mode: + if self.controller and not execution_mode & pnlvm.ExecutionMode.COMPILED: context.execution_phase = ContextFlags.PROCESSING self.controller.execute(context=context) - if execution_mode: + else: + assert (execution_mode == pnlvm.ExecutionMode.LLVM + or execution_mode & pnlvm.ExecutionMode._Fallback),\ + f"PROGRAM ERROR: Unrecognized compiled execution_mode: '{execution_mode}'." _comp_ex.execute_node(self.controller, context=context) context.remove_flag(ContextFlags.PROCESSING) @@ -10705,7 +10708,7 @@ def execute( # Run compiled execution (if compiled execution was requested # NOTE: This should be as high up as possible, # but still after the context has been initialized - if execution_mode: + if execution_mode & pnlvm.ExecutionMode.COMPILED: is_simulation = (context is not None and ContextFlags.SIMULATION_MODE in context.runmode) # Try running in Exec mode first @@ -10819,7 +10822,7 @@ def execute( inputs = self._validate_execution_inputs(inputs) build_CIM_input = self._build_variable_for_input_CIM(inputs) - if execution_mode: + if execution_mode & pnlvm.ExecutionMode.COMPILED: _comp_ex.execute_node(self.input_CIM, inputs, context) # FIXME: parameter_CIM should be executed here as well, # but node execution of nested compositions with @@ -11024,7 +11027,7 @@ def execute( # This ensures that the order in which nodes execute does not affect the results of this timestep frozen_values = {} new_values = {} - if execution_mode: + if execution_mode & pnlvm.ExecutionMode.COMPILED: _comp_ex.freeze_values() # PURGE LEARNING IF NOT ENABLED ---------------------------------------------------------------- @@ -11106,7 +11109,7 @@ def execute( context.replace_flag(ContextFlags.PROCESSING, ContextFlags.LEARNING) # Execute Mechanism - if execution_mode: + if execution_mode & pnlvm.ExecutionMode.COMPILED: _comp_ex.execute_node(node, context=context) else: if node is not self.controller: @@ -11129,7 +11132,7 @@ def execute( elif isinstance(node, Composition): - if execution_mode: + if execution_mode & pnlvm.ExecutionMode.COMPILED: # Invoking nested composition passes data via Python # structures. Make sure all sources get their latest values srcs = (proj.sender.owner for proj in node.input_CIM.afferents) @@ -11168,7 +11171,7 @@ def execute( execution_mode=nested_execution_mode) # Get output info from nested execution - if execution_mode: + if execution_mode & pnlvm.ExecutionMode.COMPILED: # Update result in binary data structure _comp_ex.insert_node_output(node, ret) @@ -11310,7 +11313,7 @@ def execute( context=context) # Extract result here - if execution_mode: + if execution_mode & pnlvm.ExecutionMode.COMPILED: _comp_ex.freeze_values() _comp_ex.execute_node(self.output_CIM, context=context) report(self, diff --git a/psyneulink/core/llvm/__init__.py b/psyneulink/core/llvm/__init__.py index 7c5c951d75f..651225c2a1f 100644 --- a/psyneulink/core/llvm/__init__.py +++ b/psyneulink/core/llvm/__init__.py @@ -85,6 +85,7 @@ class ExecutionMode(enum.Flag): LLVMExec = LLVM | _Exec PTXRun = PTX | _Run PTXExec = PTX | _Exec + COMPILED = ~ (Python | PyTorch) _binary_generation = 0 diff --git a/psyneulink/library/compositions/autodiffcomposition.py b/psyneulink/library/compositions/autodiffcomposition.py index f7efc82f8e4..ee5f37ecb31 100644 --- a/psyneulink/library/compositions/autodiffcomposition.py +++ b/psyneulink/library/compositions/autodiffcomposition.py @@ -634,10 +634,9 @@ def learn(self, *args, **kwargs): if 'execution_mode' in kwargs: execution_mode = kwargs['execution_mode'] if execution_mode == pnlvm.ExecutionMode.Python: - warnings.warn(f"{self.name}.learn() called with ExecutionMode.Python; " - f"learning will be executed using PyTorch; " - f"should use ExecutionMode.PyTorch for clarity, " - f"or a standard Composition for Python execution.)") + raise AutodiffCompositionError(f"{self.name} is an AutodiffComposition so its learn() " + f"cannot be called with execution_mode = ExecutionMode.Python; " + f"use ExecutionMode.PyTorch or ExecutionMode.LLVMRun.") # OK, now that the user has been advised to use ExecutionMode.PyTorch and warned *not* to ExecutionMdoe.Python, # convert ExecutionMode.PyTorch specification to ExecutionMode.Python for internal use (nice, eh?) if execution_mode == pnlvm.ExecutionMode.PyTorch: diff --git a/psyneulink/library/compositions/compositionrunner.py b/psyneulink/library/compositions/compositionrunner.py index 8e7a757a353..601bb6b6484 100644 --- a/psyneulink/library/compositions/compositionrunner.py +++ b/psyneulink/library/compositions/compositionrunner.py @@ -146,7 +146,8 @@ def run_learning(self, --------- Outputs from the final execution """ - if not execution_mode: + + if not (execution_mode & pnlvm.ExecutionMode.COMPILED): self._is_llvm_mode = False else: self._is_llvm_mode = True @@ -195,7 +196,7 @@ def run_learning(self, raise Exception("The minibatch size cannot be greater than the number of trials.") early_stopper = None - if patience is not None and not execution_mode: + if patience is not None and not self._is_llvm_mode: early_stopper = EarlyStopping(min_delta=min_delta, patience=patience) if callable(stim_input) and not isgeneratorfunction(stim_input): diff --git a/tests/composition/test_autodiffcomposition.py b/tests/composition/test_autodiffcomposition.py index 6cf6f56b28f..a6bbdd6ca26 100644 --- a/tests/composition/test_autodiffcomposition.py +++ b/tests/composition/test_autodiffcomposition.py @@ -14,7 +14,7 @@ from psyneulink.core.globals.keywords import TRAINING_SET, Loss from psyneulink.core.components.mechanisms.processing.transfermechanism import TransferMechanism from psyneulink.core.components.projections.pathway.mappingprojection import MappingProjection -from psyneulink.library.compositions.autodiffcomposition import AutodiffComposition +from psyneulink.library.compositions.autodiffcomposition import AutodiffComposition, AutodiffCompositionError from psyneulink.core.compositions.report import ReportOutput logger = logging.getLogger(__name__) @@ -1770,22 +1770,20 @@ def test_params_stay_separate(self, autodiff_mode): assert not np.allclose(pt_weights_hid, hid_map.parameters.matrix.get(None)) assert not np.allclose(pt_weights_out, out_map.parameters.matrix.get(None)) - def test_execution_mode_python_warning(self): + def test_execution_mode_python_error(self): A = TransferMechanism(name="learning-process-mech-A") B = TransferMechanism(name="learning-process-mech-B") adc = AutodiffComposition(name='AUTODIFFCOMP') pway = adc.add_backpropagation_learning_pathway(pathway=[A,B]) # Call learn with default_variable specified for target (for comparison with missing target) - with pytest.warns(UserWarning) as warning: + with pytest.raises(AutodiffCompositionError) as error: adc.learn(inputs={A: 1.0, pway.target: 0.0}, execution_mode=pnl.ExecutionMode.Python, num_trials=2) - assert repr(warning[1].message.args[0]) == '\'AUTODIFFCOMP.learn() called with ExecutionMode.Python; ' \ - 'learning will be executed using PyTorch; should use ' \ - 'ExecutionMode.PyTorch for clarity, or a standard Composition ' \ - 'for Python execution.)\'' - + assert error.value.error_value == 'AUTODIFFCOMP is an AutodiffComposition so its learn() ' \ + 'cannot be called with execution_mode = ExecutionMode.Python; ' \ + 'use ExecutionMode.PyTorch or ExecutionMode.LLVMRun.' @pytest.mark.pytorch @pytest.mark.actime From e6e8f844d5c8123a62138adab9582ad20e0e876d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 26 Nov 2022 03:31:41 -0500 Subject: [PATCH 105/127] requirements: update pandas requirement from <1.5.2 to <1.5.3 (#2548) Updates the requirements on [pandas](https://github.com/pandas-dev/pandas) to permit the latest version. - [Release notes](https://github.com/pandas-dev/pandas/releases) - [Changelog](https://github.com/pandas-dev/pandas/blob/main/RELEASE.md) - [Commits](https://github.com/pandas-dev/pandas/compare/0.3.0...v1.5.2) --- updated-dependencies: - dependency-name: pandas dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8900f425577..c881f2d8b9b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,5 +17,5 @@ torch>=1.8.0, <1.13.0; (platform_machine == 'AMD64' or platform_machine == 'x86_ typecheck-decorator<=1.2 leabra-psyneulink<=0.3.2 rich>=10.1, <10.13 -pandas<1.5.2 +pandas<1.5.3 fastkde==1.0.19 From b1a546f7847fbd21428b1a2ff1a20859ea9199cc Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Sat, 3 Dec 2022 00:40:42 -0500 Subject: [PATCH 106/127] github-actions/install-pnl: Only check version of a new package if it's in a requirements file Applies to packages that were remove from requirements file in the previous step. Signed-off-by: Jan Vesely --- .github/actions/install-pnl/action.yml | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/actions/install-pnl/action.yml b/.github/actions/install-pnl/action.yml index 06ed2b4ef9b..98787c8710b 100644 --- a/.github/actions/install-pnl/action.yml +++ b/.github/actions/install-pnl/action.yml @@ -60,14 +60,18 @@ runs: id: new_package run: | export NEW_PACKAGE=$(echo '${{ github.head_ref }}' | cut -f 4 -d/ | sed 's/-gt.*//' | sed 's/-lt.*//') - echo "new_package=$NEW_PACKAGE" >> $GITHUB_OUTPUT - # save a list of all installed packages (including pip, wheel; it's never empty) - pip freeze --all > orig - pip install "$(echo $NEW_PACKAGE | sed 's/[-_]/./g' | xargs grep *requirements.txt -h -e | head -n1)" - pip show "$NEW_PACKAGE" | grep 'Version' | tee new_version.deps - # uninstall new packages but skip those from previous steps (pywinpty, terminado on windows x86) - # the 'orig' list is not empty (includes at least pip, wheel) - pip uninstall -y $(pip freeze -r orig | sed '1,/## /d') + if grep "$NEW_PACKAGE" *requirements.txt; then + echo "new_package=$NEW_PACKAGE" >> $GITHUB_OUTPUT + # save a list of all installed packages (including pip, wheel; it's never empty) + pip freeze --all > orig + pip install "$(echo $NEW_PACKAGE | sed 's/[-_]/./g' | xargs grep *requirements.txt -h -e | head -n1)" + pip show "$NEW_PACKAGE" | grep 'Version' | tee new_version.deps + # uninstall new packages but skip those from previous steps (pywinpty, terminado on windows x86) + # the 'orig' list is not empty (includes at least pip, wheel) + pip uninstall -y $(pip freeze -r orig | sed '1,/## /d') + else + echo "new_package=''" >> $GITHUB_OUTPUT + fi - name: Python dependencies shell: bash @@ -88,7 +92,7 @@ runs: done - name: Check updated package - if: ${{ startsWith(github.head_ref, 'dependabot/pip') && matrix.pnl-version != 'base' }} + if: ${{ startsWith(github.head_ref, 'dependabot/pip') && matrix.pnl-version != 'base' && steps.new_package.outputs.new_package != '' }} shell: bash run: | if [ $(pip list | grep -o ${{ steps.new_package.outputs.new_package }} | wc -l) != "0" ] ; then From 42c70685f03b34c888c3e0a260974ec415eb9108 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Tue, 22 Nov 2022 23:26:38 -0500 Subject: [PATCH 107/127] requirements: update torch requirement from >=1.8.0,<1.13.0 to >=1.8.0,<1.14.0 Signed-off-by: Jan Vesely --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c881f2d8b9b..ad37fb4cd73 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ numpy<1.21.7, >=1.17.0 pillow<9.4.0 pint<0.21.0 toposort<1.8 -torch>=1.8.0, <1.13.0; (platform_machine == 'AMD64' or platform_machine == 'x86_64') and platform_python_implementation == 'CPython' and implementation_name == 'cpython' +torch>=1.8.0, <1.14.0; (platform_machine == 'AMD64' or platform_machine == 'x86_64') and platform_python_implementation == 'CPython' and implementation_name == 'cpython' typecheck-decorator<=1.2 leabra-psyneulink<=0.3.2 rich>=10.1, <10.13 From a694919f79e5cb36990e06c6ecf906313aa76bd3 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Sat, 3 Dec 2022 16:20:52 -0500 Subject: [PATCH 108/127] llvm/builtins: Implement binomial distribution for N==1 (#2552) Signed-off-by: Jan Vesely --- psyneulink/core/llvm/builtins.py | 50 ++++++++++++++- tests/llvm/test_builtins_mt_random.py | 62 ++++++++++++++++++ tests/llvm/test_builtins_philox_random.py | 78 ++++++++++++++++++++++- 3 files changed, 187 insertions(+), 3 deletions(-) diff --git a/psyneulink/core/llvm/builtins.py b/psyneulink/core/llvm/builtins.py index 977e941bdbe..59b2dae5a9b 100644 --- a/psyneulink/core/llvm/builtins.py +++ b/psyneulink/core/llvm/builtins.py @@ -868,6 +868,7 @@ def setup_mersenne_twister(ctx): gen_int = _setup_mt_rand_integer(ctx, state_ty) gen_float = _setup_mt_rand_float(ctx, state_ty, gen_int) _setup_mt_rand_normal(ctx, state_ty, gen_float) + _setup_rand_binomial(ctx, state_ty, gen_float, prefix="mt") _PHILOX_DEFAULT_ROUNDS = 10 @@ -1877,8 +1878,9 @@ def _setup_philox_rand_normal(ctx, state_ty, gen_float, gen_int, wi_data, ki_dat fptype = gen_float.args[1].type.pointee itype = gen_int.args[1].type.pointee if fptype != ctx.float_ty: - # We don't have numeric halpers available for the desired type + # We don't have numeric helpers available for the desired type return + builder = _setup_builtin_func_builder(ctx, "philox_rand_normal", (state_ty.as_pointer(), fptype.as_pointer())) state, out = builder.function.args @@ -1988,6 +1990,50 @@ def _setup_philox_rand_normal(ctx, state_ty, gen_float, gen_int, wi_data, ki_dat builder.branch(loop_block) +def _setup_rand_binomial(ctx, state_ty, gen_float, prefix): + fptype = gen_float.args[1].type.pointee + if fptype != ctx.float_ty: + # We don't have numeric helpers available for the desired type + return + + args = [state_ty.as_pointer(), # state + ctx.int32_ty.as_pointer(), # N - total number of draws + fptype.as_pointer(), # p - prob of success + ctx.int32_ty.as_pointer()] # output + + builder = _setup_builtin_func_builder(ctx, prefix + "_rand_binomial", args) + state, n_ptr, p_ptr, out_ptr = builder.function.args + + n = builder.load(n_ptr) + p = builder.load(p_ptr) + q = builder.fsub(p.type(1), p) + + success = out_ptr.type.pointee(1) + failure = out_ptr.type.pointee(0) + + # N > 1 (!=1) is not supported + is_large_n = builder.icmp_unsigned("!=", n, n.type(1)) + with builder.if_then(is_large_n): + builder.store(out_ptr.type.pointee(0), out_ptr) + builder.ret_void() + + uniform_draw_ptr = builder.alloca(fptype, name="tmp_fp") + builder.call(gen_float, [state, uniform_draw_ptr]) + draw = builder.load(uniform_draw_ptr) + + # If 'p' is large enough, success == draw < p + is_less_than_p = builder.fcmp_ordered("<", draw, p) + large_p_result = builder.select(is_less_than_p, success, failure) + + + # The draw check is reverted for small p + is_less_than_q = builder.fcmp_ordered("<", draw, q) + small_p_result = builder.select(is_less_than_q, failure, success) + + is_small_p = builder.fcmp_ordered("<=", p, p.type(0.5)) + result = builder.select(is_small_p, small_p_result, large_p_result) + builder.store(result, out_ptr) + builder.ret_void() def get_philox_state_struct(ctx): int64_ty = ir.IntType(64) @@ -2010,7 +2056,9 @@ def setup_philox(ctx): gen_int64 = _setup_philox_rand_int64(ctx, state_ty) gen_double = _setup_philox_rand_double(ctx, state_ty, gen_int64) _setup_philox_rand_normal(ctx, state_ty, gen_double, gen_int64, _wi_double_data, _ki_i64_data, _fi_double_data) + _setup_rand_binomial(ctx, state_ty, gen_double, prefix="philox") gen_int32 = _setup_philox_rand_int32(ctx, state_ty, gen_int64) gen_float = _setup_philox_rand_float(ctx, state_ty, gen_int32) _setup_philox_rand_normal(ctx, state_ty, gen_float, gen_int32, _wi_float_data, _ki_i32_data, _fi_float_data) + _setup_rand_binomial(ctx, state_ty, gen_float, prefix="philox") diff --git a/tests/llvm/test_builtins_mt_random.py b/tests/llvm/test_builtins_mt_random.py index 19dbeb7b818..09840843447 100644 --- a/tests/llvm/test_builtins_mt_random.py +++ b/tests/llvm/test_builtins_mt_random.py @@ -44,6 +44,8 @@ def f(): def f(): gen_fun.cuda_call(gpu_state, gpu_out) return out[0] + else: + assert False, "Unknown mode: {}".format(mode) res = [f(), f()] assert np.allclose(res, [3626764237, 1654615998]) @@ -88,6 +90,8 @@ def f(): def f(): gen_fun.cuda_call(gpu_state, gpu_out) return out[0] + else: + assert False, "Unknown mode: {}".format(mode) res = [f(), f()] assert np.allclose(res, [0.8444218515250481, 0.7579544029403025]) @@ -127,7 +131,65 @@ def f(): def f(): gen_fun.cuda_call(gpu_state, gpu_out) return out[0] + else: + assert False, "Unknown mode: {}".format(mode) res = [f(), f()] assert np.allclose(res, [0.4644982638709743, 0.6202001216069017]) benchmark(f) + +@pytest.mark.benchmark(group="Marsenne Twister Binomial distribution") +@pytest.mark.parametrize('mode', ['numpy', + pytest.param('LLVM', marks=pytest.mark.llvm), + pytest.helpers.cuda_param('PTX')]) +@pytest.mark.parametrize('n', [1]) +@pytest.mark.parametrize('p, exp', [ + (0, [0]), + (0.1, [0x20d00c]), + (0.33, [0xc224f70d]), + (0.5, [0xca76f71d]), + (0.66, [0x3ddb08f2]), + (0.95, [0xffffbffb]), + (1, [0xffffffff]), + ]) +# Python uses different algorithm so skip it in this test +def test_random_binomial(benchmark, mode, n, p, exp): + if mode == 'numpy': + # numpy promotes elements to int64 + state = np.random.RandomState([SEED]) + def f(): + return state.binomial(n, p) + elif mode == 'LLVM': + init_fun = pnlvm.LLVMBinaryFunction.get('__pnl_builtin_mt_rand_init') + state = init_fun.byref_arg_types[0]() + init_fun(state, SEED) + + gen_fun = pnlvm.LLVMBinaryFunction.get('__pnl_builtin_mt_rand_binomial') + c_n = gen_fun.byref_arg_types[1](n) + c_p = gen_fun.byref_arg_types[2](p) + c_out = gen_fun.byref_arg_types[-1]() + def f(): + gen_fun(state, c_n, c_p, c_out) + return c_out.value + elif mode == 'PTX': + init_fun = pnlvm.LLVMBinaryFunction.get('__pnl_builtin_mt_rand_init') + state_size = ctypes.sizeof(init_fun.byref_arg_types[0]) + gpu_state = pnlvm.jit_engine.pycuda.driver.mem_alloc(state_size) + init_fun.cuda_call(gpu_state, np.int32(SEED)) + + gen_fun = pnlvm.LLVMBinaryFunction.get('__pnl_builtin_mt_rand_binomial') + gpu_n = pnlvm.jit_engine.pycuda.driver.In(np.array([n], dtype=np.dtype(gen_fun.byref_arg_types[1]))) + gpu_p = pnlvm.jit_engine.pycuda.driver.In(np.array([p], dtype=np.dtype(gen_fun.byref_arg_types[2]))) + out = np.array([0.0], dtype=np.dtype(gen_fun.byref_arg_types[3])) + gpu_out = pnlvm.jit_engine.pycuda.driver.Out(out) + + def f(): + gen_fun.cuda_call(gpu_state, gpu_n, gpu_p, gpu_out) + return out[0] + else: + assert False, "Unknown mode: {}".format(mode) + + res = [f() for _ in range(32)] + res = int(''.join(str(x) for x in res), 2) + assert res == exp[n - 1] + benchmark(f) diff --git a/tests/llvm/test_builtins_philox_random.py b/tests/llvm/test_builtins_philox_random.py index 479e91379e7..0398fb9eda0 100644 --- a/tests/llvm/test_builtins_philox_random.py +++ b/tests/llvm/test_builtins_philox_random.py @@ -47,6 +47,8 @@ def f(): def f(): gen_fun.cuda_call(gpu_state, gpu_out) return out[0] + else: + assert False, "Unknown mode: {}".format(mode) # Get >4 samples to force regeneration of Philox buffer res = [f(), f(), f(), f(), f(), f()] @@ -89,6 +91,8 @@ def f(): def f(): gen_fun.cuda_call(gpu_state, gpu_out) return out[0] + else: + assert False, "Unknown mode: {}".format(mode) # Get >4 samples to force regeneration of Philox buffer res = [f(), f(), f(), f(), f(), f()] @@ -129,6 +133,8 @@ def f(): def f(): gen_fun.cuda_call(gpu_state, gpu_out) return out[0] + else: + assert False, "Unknown mode: {}".format(mode) res = [f(), f()] assert np.allclose(res, [0.014067035665647709, 0.2577672456246177]) @@ -168,6 +174,8 @@ def f(): def f(): gen_fun.cuda_call(gpu_state, gpu_out) return out[0] + else: + assert False, "Unknown mode: {}".format(mode) res = [f(), f()] assert np.allclose(res, [0.13562285900115967, 0.014066934585571289]) @@ -179,7 +187,7 @@ def f(): pytest.param('LLVM', marks=pytest.mark.llvm), pytest.helpers.cuda_param('PTX')]) @pytest.mark.parametrize('fp_type', [pnlvm.ir.DoubleType(), pnlvm.ir.FloatType()], - ids=lambda x: str(x)) + ids=str) def test_random_normal(benchmark, mode, fp_type): if mode != 'numpy': # Instantiate builder context with the desired type @@ -208,11 +216,13 @@ def f(): init_fun.cuda_call(gpu_state, np.int64(SEED)) gen_fun = pnlvm.LLVMBinaryFunction.get('__pnl_builtin_philox_rand_normal') - out = np.asfarray([0.0], dtype=dtype) + out = np.array([0.0], dtype=np.dtype(gen_fun.byref_arg_types[1])) gpu_out = pnlvm.jit_engine.pycuda.driver.Out(out) def f(): gen_fun.cuda_call(gpu_state, gpu_out) return out[0] + else: + assert False, "Unknown mode: {}".format(mode) res = [f() for i in range(191000)] if fp_type is pnlvm.ir.DoubleType(): @@ -250,3 +260,67 @@ def f(): 2.000257730484009, -1.129721999168396]) assert not any(np.isnan(res)), list(np.isnan(res)).index(True) benchmark(f) + +@pytest.mark.benchmark(group="Philox Binomial distribution") +@pytest.mark.parametrize('mode', ['numpy', + pytest.param('LLVM', marks=pytest.mark.llvm), + pytest.helpers.cuda_param('PTX')]) +@pytest.mark.parametrize('fp_type', [pnlvm.ir.DoubleType(), pnlvm.ir.FloatType()], + ids=str) +@pytest.mark.parametrize('n', [1]) +@pytest.mark.parametrize('p, exp_64, exp_32', [ + (0, [0], [0]), + (0.1, [0xa0c0100], [0x20440250]), + (0.33, [0xa2c8186], [0x20440650]), + (0.5, [0xa2c81c6], [0x226c8650]), + (0.66, [0xf5d37e79], [0xdfbbf9af]), + (0.95, [0xf7f3ffff], [0xffbffdaf]), + (1, [0xffffffff], [0xffffffff]), + ]) +def test_random_binomial(benchmark, mode, fp_type, n, p, exp_64, exp_32): + if mode != 'numpy': + # Instantiate builder context with the desired type + pnlvm.LLVMBuilderContext(fp_type) + + # numpy always uses fp64 uniform sampling + exp = exp_64 if fp_type is pnlvm.ir.DoubleType() or mode == 'numpy' else exp_32 + if mode == 'numpy': + state = np.random.Philox([SEED]) + prng = np.random.Generator(state) + def f(): + return prng.binomial(n, p) + elif mode == 'LLVM': + init_fun = pnlvm.LLVMBinaryFunction.get('__pnl_builtin_philox_rand_init') + c_state = init_fun.byref_arg_types[0]() + init_fun(c_state, SEED) + + gen_fun = pnlvm.LLVMBinaryFunction.get('__pnl_builtin_philox_rand_binomial') + c_n = gen_fun.byref_arg_types[1](n) + c_p = gen_fun.byref_arg_types[2](p) + c_out = gen_fun.byref_arg_types[-1]() + def f(): + gen_fun(c_state, c_n, c_p, c_out) + return c_out.value + elif mode == 'PTX': + init_fun = pnlvm.LLVMBinaryFunction.get('__pnl_builtin_philox_rand_init') + state_size = ctypes.sizeof(init_fun.byref_arg_types[0]) + gpu_state = pnlvm.jit_engine.pycuda.driver.mem_alloc(state_size) + init_fun.cuda_call(gpu_state, np.int64(SEED)) + + gen_fun = pnlvm.LLVMBinaryFunction.get('__pnl_builtin_philox_rand_binomial') + gpu_n = pnlvm.jit_engine.pycuda.driver.In(np.array([n], dtype=np.dtype(gen_fun.byref_arg_types[1]))) + gpu_p = pnlvm.jit_engine.pycuda.driver.In(np.array([p], dtype=np.dtype(gen_fun.byref_arg_types[2]))) + out = np.array([0.0], dtype=np.dtype(gen_fun.byref_arg_types[3])) + gpu_out = pnlvm.jit_engine.pycuda.driver.Out(out) + + def f(): + gen_fun.cuda_call(gpu_state, gpu_n, gpu_p, gpu_out) + return out[0] + else: + assert False, "Unknown mode: {}".format(mode) + + res = [f() for i in range(32)] + res = int(''.join(str(x) for x in res), 2) + assert res == exp[n - 1] + + benchmark(f) From 586cae671b30c8109079f5e7ce0d7f8b2b103fa4 Mon Sep 17 00:00:00 2001 From: Katherine Mantel Date: Tue, 6 Dec 2022 01:45:13 -0500 Subject: [PATCH 109/127] utilities: safe_equals: handle dict-like objects in np arrays (#2551) --- psyneulink/core/globals/utilities.py | 32 +++++++++++++--------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/psyneulink/core/globals/utilities.py b/psyneulink/core/globals/utilities.py index 5d4eb532f27..9d43215225b 100644 --- a/psyneulink/core/globals/utilities.py +++ b/psyneulink/core/globals/utilities.py @@ -1647,7 +1647,6 @@ def safe_equals(x, y): An == comparison that handles numpy's new behavior of returning an array of booleans instead of a single boolean for == """ - from collections import defaultdict with warnings.catch_warnings(): warnings.simplefilter('error') try: @@ -1660,25 +1659,24 @@ def safe_equals(x, y): try: return np.array_equal(x, y) except (DeprecationWarning, FutureWarning): + # both should have len because non-len objects would not + # have triggered the warnings on == or array_equal len_x = len(x) - try: - # IMPLEMENTATION NOTE: - # Handles case in which an element being compared is a defaultdict - # (makes copy to prevent indexing it from adding and entry to source) - if len_x != len(y): + if len_x != len(y): + return False + + if hasattr(x, 'keys') and hasattr(y, 'keys'): + # dictionary-like + if x.keys() != y.keys(): return False - for i in range(len_x): - if isinstance(x[i],defaultdict) or isinstance(y[i],defaultdict): - copy_x = x[i].copy() - copy_y = y[i].copy() - if not safe_equals(copy_x, copy_y): - return False - else: - if not safe_equals(x[i],y[i]): - return False - return True - except KeyError: + subelements = x.keys() + elif hasattr(x, 'keys') or hasattr(y, 'keys'): return False + else: + # list-like + subelements = range(len_x) + + return all([safe_equals(x[i], y[i]) for i in subelements]) @tc.typecheck From 9834789fddf7a26bc6fa720186f5f749b4ee258d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Dec 2022 19:22:58 +0000 Subject: [PATCH 110/127] requirements: update pytest-xdist requirement from <3.1.0 to <3.2.0 (#2554) --- dev_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index 223bc004f4a..43da3586fda 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -6,4 +6,4 @@ pytest-helpers-namespace<2021.12.30 pytest-profiling<=1.7.0 pytest-pycodestyle<2.4.0 pytest-pydocstyle<2.4.0 -pytest-xdist<3.1.0 +pytest-xdist<3.2.0 From 815aefeffa1f48f6eff71617d8126fc3e5ee9d17 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Fri, 9 Dec 2022 00:35:29 -0500 Subject: [PATCH 111/127] github-actions/install-pnl: Restrict jupyter-server to <2 (#2555) jupyter pulls in jupyter-server-terminals which requires pywinpty>=2.0.3 on windows [0] [0] https://github.com/jupyter-server/jupyter_server_terminals/blob/main/pyproject.toml Signed-off-by: Jan Vesely --- .github/actions/install-pnl/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/install-pnl/action.yml b/.github/actions/install-pnl/action.yml index 98787c8710b..3cab42eaa0e 100644 --- a/.github/actions/install-pnl/action.yml +++ b/.github/actions/install-pnl/action.yml @@ -51,7 +51,7 @@ runs: # terminado >= 0.10.0 pulls in pywinpty >= 1.1.0 # scipy >=1.9.2 doesn't provide win32 wheel and GA doesn't have working fortran on windows # scikit-learn >= 1.1.3 doesn't provide win32 wheel - [[ ${{ runner.os }} = Windows* ]] && pip install "pywinpty<1" "terminado<0.10" "scipy<1.9.2" "scikit-learn<1.1.3" "statsmodels<0.13.3" -c requirements.txt + [[ ${{ runner.os }} = Windows* ]] && pip install "pywinpty<1" "terminado<0.10" "scipy<1.9.2" "scikit-learn<1.1.3" "statsmodels<0.13.3" "jupyter-server<2" -c requirements.txt fi - name: Install updated package From a831154b24db4f710f80dc36c669396a0b287af9 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Tue, 20 Dec 2022 13:21:07 -0500 Subject: [PATCH 112/127] github-actions/codeql: Skip CI job on pushes that only update documentation (#2560) Signed-off-by: Jan Vesely --- .github/workflows/codeql.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 4520507bb21..9bcf97d976f 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -3,6 +3,8 @@ name: "CodeQL" on: push: branches: [ "master", "devel" ] + paths-ignore: + - 'docs/**' pull_request: branches: [ "master", "devel" ] schedule: From b2893c94ba8a03133556cc42e5752b778fa788a8 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Tue, 20 Dec 2022 21:06:07 -0500 Subject: [PATCH 113/127] requirements: Use upper and lower bound for fastkde (#2561) Signed-off-by: Jan Vesely --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ad37fb4cd73..36cb28bdaae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,4 +18,4 @@ typecheck-decorator<=1.2 leabra-psyneulink<=0.3.2 rich>=10.1, <10.13 pandas<1.5.3 -fastkde==1.0.19 +fastkde<1.0.20, >=1.0.19 From 77e91523a76a0186706f86f3a06357b89d4d1f8a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Jan 2023 00:58:16 +0000 Subject: [PATCH 114/127] requirements: update pillow requirement from <9.4.0 to <9.5.0 (#2565) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 36cb28bdaae..7947d153677 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ matplotlib<3.6.3 modeci_mdf<0.5, >=0.3.4; (platform_machine == 'AMD64' or platform_machine == 'x86_64') and platform_python_implementation == 'CPython' and implementation_name == 'cpython' networkx<2.9 numpy<1.21.7, >=1.17.0 -pillow<9.4.0 +pillow<9.5.0 pint<0.21.0 toposort<1.8 torch>=1.8.0, <1.14.0; (platform_machine == 'AMD64' or platform_machine == 'x86_64') and platform_python_implementation == 'CPython' and implementation_name == 'cpython' From 7c97f94b6694aa6933f2a9dc84771cdce601447a Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Thu, 5 Jan 2023 21:31:05 -0500 Subject: [PATCH 115/127] library/AutodiffComposition: Drop empty docstring (#2567) Signed-off-by: Jan Vesely --- psyneulink/library/compositions/autodiffcomposition.py | 1 - 1 file changed, 1 deletion(-) diff --git a/psyneulink/library/compositions/autodiffcomposition.py b/psyneulink/library/compositions/autodiffcomposition.py index ee5f37ecb31..36ed8c4eeae 100644 --- a/psyneulink/library/compositions/autodiffcomposition.py +++ b/psyneulink/library/compositions/autodiffcomposition.py @@ -376,7 +376,6 @@ class AutodiffComposition(Composition): componentCategory = AUTODIFF_COMPOSITION class Parameters(Composition.Parameters): - """""" optimizer = None learning_rate = Parameter(.001, fallback_default=True) losses = Parameter([]) From 930f1d776290cc3824ab8c6651e03bbf117481ef Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Fri, 6 Jan 2023 01:13:37 -0500 Subject: [PATCH 116/127] github-actions: Generalize package name extraction from dependabot branch name (#2566) Signed-off-by: Jan Vesely --- .github/actions/install-pnl/action.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/actions/install-pnl/action.yml b/.github/actions/install-pnl/action.yml index 3cab42eaa0e..a0a6bdde249 100644 --- a/.github/actions/install-pnl/action.yml +++ b/.github/actions/install-pnl/action.yml @@ -59,7 +59,9 @@ runs: shell: bash id: new_package run: | - export NEW_PACKAGE=$(echo '${{ github.head_ref }}' | cut -f 4 -d/ | sed 's/-gt.*//' | sed 's/-lt.*//') + # The branch name pattern is: "dependabot/pip/$TARGET/$PACKAGE{-gt-$MINVERSION,,}{-lt-$MAXVERSION,}{-$VERSION,} + # The expression below extracts just the $PACKAGE part + export NEW_PACKAGE=$(echo '${{ github.head_ref }}' | cut -f 4 -d/ | sed 's/-gt.*//' | sed 's/-lt.*//' | sed 's/-[0-9\.]*$//' ) if grep "$NEW_PACKAGE" *requirements.txt; then echo "new_package=$NEW_PACKAGE" >> $GITHUB_OUTPUT # save a list of all installed packages (including pip, wheel; it's never empty) From 1fe007dae926316804853fb236e193bb19e65efc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Jan 2023 17:54:24 +0000 Subject: [PATCH 117/127] requirements: bump fastkde from 1.0.19 to 1.0.20 (#2559) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7947d153677..b6f4087e48c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,4 +18,4 @@ typecheck-decorator<=1.2 leabra-psyneulink<=0.3.2 rich>=10.1, <10.13 pandas<1.5.3 -fastkde<1.0.20, >=1.0.19 +fastkde>=1.0.19, <1.0.21 From 44a2f2ee734342a58a0ecc53a79a78ab7ef3cdf7 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Mon, 9 Jan 2023 15:11:09 -0500 Subject: [PATCH 118/127] tests/mechanism: Do not check for string value of numpy error embedded in MechanismError (#2569) string values of errors are not stable and can change between versions. Fixes 'input_list_of_strings' tests when using numpy>=1.22.x. Refactor handling of input port execution exceptions to only check calls to 'execute', parameter set/get do not throw exceptions. Signed-off-by: Jan Vesely --- psyneulink/core/components/mechanisms/mechanism.py | 9 +++++---- tests/mechanisms/test_kwta.py | 2 +- tests/mechanisms/test_recurrent_transfer_mechanism.py | 3 +-- tests/mechanisms/test_transfer_mechanism.py | 3 +-- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/psyneulink/core/components/mechanisms/mechanism.py b/psyneulink/core/components/mechanisms/mechanism.py index eca1d31bd5e..b439846e79f 100644 --- a/psyneulink/core/components/mechanisms/mechanism.py +++ b/psyneulink/core/components/mechanisms/mechanism.py @@ -2634,14 +2634,15 @@ def _get_variable_from_input(self, input, context=None): # Call input_port._execute with newly assigned variable and assign result to input_port.value base_error_msg = f"Input to '{self.name}' ({input_item}) is incompatible " \ f"with its corresponding {InputPort.__name__} ({input_port.full_name})" + variable = input_port.parameters.variable.get(context) try: - input_port.parameters.value._set( - input_port._execute(input_port.parameters.variable.get(context), context), - context) - except (RunError,TypeError) as error: + value = input_port._execute(variable, context) + except (RunError, TypeError) as error: raise MechanismError(f"{base_error_msg}: '{error.args[0]}.'") except: raise MechanismError(f"{base_error_msg}.") + else: + input_port.parameters.value._set(value, context) else: raise MechanismError(f"Length ({len(input_item)}) of input ({input_item}) does not match " f"required length ({input_port.default_input_shape.size}) for input " diff --git a/tests/mechanisms/test_kwta.py b/tests/mechanisms/test_kwta.py index 8154995671b..cd1a999d1f1 100644 --- a/tests/mechanisms/test_kwta.py +++ b/tests/mechanisms/test_kwta.py @@ -58,7 +58,7 @@ def test_kwta_inputs_list_of_strings(self): ) K.execute(["one", "two", "three", "four"]) assert ('"Input to \'K\' ([\'one\' \'two\' \'three\' \'four\']) is incompatible with its corresponding ' - 'InputPort (K[InputPort-0]): \'cannot perform reduce with flexible type.\'"' in str(error_text.value)) + 'InputPort (K[InputPort-0]):' in str(error_text.value)) def test_kwta_var_list_of_strings(self): with pytest.raises(ParameterError) as error_text: diff --git a/tests/mechanisms/test_recurrent_transfer_mechanism.py b/tests/mechanisms/test_recurrent_transfer_mechanism.py index bb6e431f05e..18d7eb883e0 100644 --- a/tests/mechanisms/test_recurrent_transfer_mechanism.py +++ b/tests/mechanisms/test_recurrent_transfer_mechanism.py @@ -190,8 +190,7 @@ def test_recurrent_mech_inputs_list_of_strings(self): ) R.execute(["one", "two", "three", "four"]) assert '"Input to \'R\' ([\'one\' \'two\' \'three\' \'four\']) is incompatible ' \ - 'with its corresponding InputPort (R[InputPort-0]): ' \ - '\'cannot perform reduce with flexible type.\'"' in str(error_text.value) + 'with its corresponding InputPort (R[InputPort-0]): ' in str(error_text.value) def test_recurrent_mech_var_list_of_strings(self): with pytest.raises(ParameterError) as error_text: diff --git a/tests/mechanisms/test_transfer_mechanism.py b/tests/mechanisms/test_transfer_mechanism.py index 5c0203a96b7..fcbe979feca 100644 --- a/tests/mechanisms/test_transfer_mechanism.py +++ b/tests/mechanisms/test_transfer_mechanism.py @@ -108,8 +108,7 @@ def test_transfer_mech_inputs_list_of_strings(self): ) T.execute(["one", "two", "three", "four"]) assert '"Input to \'T\' ([\'one\' \'two\' \'three\' \'four\']) is incompatible ' \ - 'with its corresponding InputPort (T[InputPort-0]): ' \ - '\'cannot perform reduce with flexible type.\'"' in str(error_text.value) + 'with its corresponding InputPort (T[InputPort-0]): ' in str(error_text.value) @pytest.mark.mechanism @pytest.mark.transfer_mechanism From c47306da483273a3ad113bd4d0228bedf78a841a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 Jan 2023 01:50:44 +0000 Subject: [PATCH 119/127] requirements: update networkx requirement from <2.9 to <3.1 (#2570) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b6f4087e48c..6153855c68e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ grpcio-tools<1.43.0 llvmlite<0.40 matplotlib<3.6.3 modeci_mdf<0.5, >=0.3.4; (platform_machine == 'AMD64' or platform_machine == 'x86_64') and platform_python_implementation == 'CPython' and implementation_name == 'cpython' -networkx<2.9 +networkx<3.1 numpy<1.21.7, >=1.17.0 pillow<9.5.0 pint<0.21.0 From 5a57b861a3ea02da7963a073b3485646a3d1bb76 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 Jan 2023 04:10:31 +0000 Subject: [PATCH 120/127] requirements: update toposort requirement from <1.8 to <1.9 (#2573) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6153855c68e..976e87a8541 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ networkx<3.1 numpy<1.21.7, >=1.17.0 pillow<9.5.0 pint<0.21.0 -toposort<1.8 +toposort<1.9 torch>=1.8.0, <1.14.0; (platform_machine == 'AMD64' or platform_machine == 'x86_64') and platform_python_implementation == 'CPython' and implementation_name == 'cpython' typecheck-decorator<=1.2 leabra-psyneulink<=0.3.2 From 8185333cd3530c4e4710a8227b7af9966e4a02af Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Jan 2023 20:25:40 +0000 Subject: [PATCH 121/127] requirements: update toposort requirement from <1.9 to <1.10 (#2575) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 976e87a8541..816e66a9afd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ networkx<3.1 numpy<1.21.7, >=1.17.0 pillow<9.5.0 pint<0.21.0 -toposort<1.9 +toposort<1.10 torch>=1.8.0, <1.14.0; (platform_machine == 'AMD64' or platform_machine == 'x86_64') and platform_python_implementation == 'CPython' and implementation_name == 'cpython' typecheck-decorator<=1.2 leabra-psyneulink<=0.3.2 From f567015535922e4e66ca6411a654b67db0f4fc07 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Jan 2023 02:23:54 +0000 Subject: [PATCH 122/127] requirements: update matplotlib requirement from <3.6.3 to <3.6.4 (#2576) --- requirements.txt | 2 +- tutorial_requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 816e66a9afd..0410992ad91 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ graphviz<0.21.0 grpcio<1.43.0 grpcio-tools<1.43.0 llvmlite<0.40 -matplotlib<3.6.3 +matplotlib<3.6.4 modeci_mdf<0.5, >=0.3.4; (platform_machine == 'AMD64' or platform_machine == 'x86_64') and platform_python_implementation == 'CPython' and implementation_name == 'cpython' networkx<3.1 numpy<1.21.7, >=1.17.0 diff --git a/tutorial_requirements.txt b/tutorial_requirements.txt index ebda6d54f3a..8e08358b08f 100644 --- a/tutorial_requirements.txt +++ b/tutorial_requirements.txt @@ -1,3 +1,3 @@ graphviz<0.21.0 jupyter<=1.0.0 -matplotlib<3.6.3 +matplotlib<3.6.4 From f9324fc6da49fba67091d7e8342ebaefe7f76d77 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Thu, 12 Jan 2023 15:55:27 -0500 Subject: [PATCH 123/127] functions/DriftDiffusionAnalytical: Re-associate floating point operations Individual terms of the sum are close in magnitude. Instead of accmulating all the postive terms, alternate between positive and negative terms. This keeps the intermadiate/partial results small in magnitude improving accuracy of the floating point expression. This is enough to make the "test_drift_difussion_analytical_shenhav_compat_mode" when using numpy-1.22 with its new AVX512 based floating point operations [0] [0] https://github.com/numpy/numpy/commit/1eff1c543a8f1e9d7ea29182b8c76db5a2efc3c2 Signed-off-by: Jan Vesely --- .../nonstateful/distributionfunctions.py | 28 +++++++++++++------ tests/functions/test_distribution.py | 9 +++--- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/psyneulink/core/components/functions/nonstateful/distributionfunctions.py b/psyneulink/core/components/functions/nonstateful/distributionfunctions.py index b8a64bc1510..10122cb8ddc 100644 --- a/psyneulink/core/components/functions/nonstateful/distributionfunctions.py +++ b/psyneulink/core/components/functions/nonstateful/distributionfunctions.py @@ -1378,20 +1378,32 @@ def csch(x): moments["mean_rt_minus"] = noise**2 / (drift_rate**2) * (2 * Z * coth(2 * Z) - (-X + Z) * coth(-X + Z)) moments["var_rt_plus"] = noise**4 / (drift_rate**4) * \ - ((2 * Z)**2 * (csch(2 * Z))**2 + (2 * Z) * coth(2 * Z) - - (Z + X)**2 * (csch(Z + X))**2 - (Z + X) * coth(Z + X)) + (((2 * Z)**2 * csch(2 * Z)**2 - + (Z + X)**2 * csch(Z + X)**2) + + ((2 * Z) * coth(2 * Z) - + (Z + X) * coth(Z + X))) moments["var_rt_minus"] = noise**4 / (drift_rate**4) * \ - ((2 * Z)**2 * (csch(2 * Z))**2 + (2 * Z) * coth(2 * Z) - - (Z - X)**2 * (csch(Z - X))**2 - (Z - X) * coth(Z - X)) + (((2 * Z)**2 * csch(2 * Z)**2 - + (Z - X)**2 * csch(Z - X)**2) + + ((2 * Z) * coth(2 * Z) - + (Z - X) * coth(Z - X))) moments["skew_rt_plus"] = noise**6 / (drift_rate**6) * \ - (3 * (2 * Z)**2 * (csch(2 * Z))**2 + 2 * (2 * Z)**3 * coth(2 * Z) * (csch(2 * Z))**2 + 3 * (2 * Z) * coth(2 * Z) - - 3 * (Z + X)**2 * (csch(Z + X))**2 - 2 * (Z + X)**3 * coth(Z + X) * (csch(Z + X))**2 - 3 * (Z + X) * coth(Z + X)) + (3 * ((2 * Z)**2 * csch(2 * Z)**2 - + (Z + X)**2 * csch(Z + X)**2) + + 2 * ((2 * Z)**3 * coth(2 * Z) * csch(2 * Z)**2 - + (Z + X)**3 * coth(Z + X) * csch(Z + X)**2) + + 3 * ((2 * Z) * coth(2 * Z) - + (Z + X) * coth(Z + X))) moments["skew_rt_minus"] = noise**6 / (drift_rate**6) * \ - (3 * (2 * Z)**2 * (csch(2 * Z))**2 + 2 * (2 * Z)**3 * coth(2 * Z) * (csch(2 * Z))**2 + 3 * (2 * Z) * coth(2 * Z) - - 3 * (Z - X)**2 * (csch(Z - X))**2 - 2 * (Z - X)**3 * coth(Z - X) * (csch(Z - X))**2 - 3 * (Z - X) * coth(Z - X)) + (3 * ((2 * Z)**2 * csch(2 * Z)**2 - + (Z - X)**2 * csch(Z - X)**2) + + 2 * ((2 * Z)**3 * coth(2 * Z) * csch(2 * Z)**2 - + (Z - X)**3 * coth(Z - X) * csch(Z - X)**2) + + 3 * ((2 * Z) * coth(2 * Z) - + (Z - X) * coth(Z - X))) # divide third central moment by var_rt**1.5 to get skewness moments['skew_rt_plus'] /= moments['var_rt_plus']**1.5 diff --git a/tests/functions/test_distribution.py b/tests/functions/test_distribution.py index 1075490b6b4..da896ddf8c2 100644 --- a/tests/functions/test_distribution.py +++ b/tests/functions/test_distribution.py @@ -25,20 +25,19 @@ 0.5173675420165031, 0.06942854144616283, 6.302631815990666, 1.4934079600147951, 0.4288991185241868, 1.7740760781361433) dda_expected_small = (0.5828813465336954, 0.04801236718458773, - 0.532471083815943, 0.09633801362499317, 6.111833139205608, - 1.5821207676710864, 0.5392724012504414, 1.8065252817609618) + 0.532471083815943, 0.09633801555720854, 6.1142591416669765, + 1.5821207676710864, 0.5392724051148722, 1.806647390875747) # Different libm implementations produce slightly different results if sys.platform.startswith("win") or sys.platform.startswith("darwin"): dda_expected_small = (0.5828813465336954, 0.04801236718458773, - 0.5324710838150166, 0.09633802135385469, 6.119380538293901, - 1.58212076767016, 0.5392724012504414, 1.8065252817609618) + 0.5324710838150166, 0.09633802135385469, 6.117763080882898, + 1.58212076767016, 0.5392724012504414, 1.8064031532265) normal_expected_mt = (1.0890232855122397) uniform_expected_mt = (0.6879771504250405) normal_expected_philox = (0.5910357654927911) uniform_expected_philox = (0.6043448764869507) -llvm_expected = {} llvm_expected = {'fp64': {}, 'fp32': {}} llvm_expected['fp64'][dda_expected_small] = (0.5828813465336954, 0.04801236718458773, 0.5324710838085324, 0.09633787836991654, 6.0158766570416775, From 43c26abe874472d1b3b1a54357ec2063715059d5 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Thu, 12 Jan 2023 19:43:53 -0500 Subject: [PATCH 124/127] llvm, functions/DriftDiffusionAnalytical: Re-associate floating point operations Individual terms of the sum are close in magnitude. Instead of accmulating all the postive terms, alternate between positive and negative terms. This keeps the intermadiate/partial results small in magnitude improving accuracy of the floating point expression. This matches the updated Python version. Fix use of incorrect terms. Skew calculation needs cubic terms. Signed-off-by: Jan Vesely --- .../nonstateful/distributionfunctions.py | 69 ++++++++++--------- tests/functions/test_distribution.py | 16 ++--- 2 files changed, 45 insertions(+), 40 deletions(-) diff --git a/psyneulink/core/components/functions/nonstateful/distributionfunctions.py b/psyneulink/core/components/functions/nonstateful/distributionfunctions.py index 10122cb8ddc..904751a9cb8 100644 --- a/psyneulink/core/components/functions/nonstateful/distributionfunctions.py +++ b/psyneulink/core/components/functions/nonstateful/distributionfunctions.py @@ -1505,7 +1505,7 @@ def _get_arg_out_ptr(idx): x0tilde = builder.fdiv(y0tilde, drift_rate_normed) exp_f = ctx.get_builtin("exp", [bias_adj.type]) - # Precompute the same values as Python above + # Pre-compute the same values as Python above neg2_x0tilde_atilde = builder.fmul(x0tilde.type(-2), x0tilde) neg2_x0tilde_atilde = builder.fmul(neg2_x0tilde_atilde, atilde) exp_neg2_x0tilde_atilde = builder.call(exp_f, [neg2_x0tilde_atilde]) @@ -1627,9 +1627,9 @@ def _get_arg_out_ptr(idx): ZmX_sqr_csch_ZmX_sqr = builder.fmul(ZmX_sqr, csch_ZmX_sqr) # Variance plus - v_rt_p = builder.fadd(Z2_sqr_csch_Z2_sqr, Z2_coth_Z2) - v_rt_p = builder.fsub(v_rt_p, ZpX_sqr_csch_ZpX_sqr) - v_rt_p = builder.fsub(v_rt_p, ZpX_coth_ZpX) + v_rt_pA = builder.fsub(Z2_sqr_csch_Z2_sqr, ZpX_sqr_csch_ZpX_sqr) + v_rt_pB = builder.fsub(Z2_coth_Z2, ZpX_coth_ZpX) + v_rt_p = builder.fadd(v_rt_pA, v_rt_pB) v_rt_p = builder.fmul(noise_q_drift_q, v_rt_p) builder.store(v_rt_p, var_rt_plus_ptr) @@ -1637,9 +1637,9 @@ def _get_arg_out_ptr(idx): v_rt_p_1_5 = builder.call(pow_f, [v_rt_p, v_rt_p.type(1.5)]) # Variance minus - v_rt_m = builder.fadd(Z2_sqr_csch_Z2_sqr, Z2_coth_Z2) - v_rt_m = builder.fsub(v_rt_m, ZmX_sqr_csch_ZmX_sqr) - v_rt_m = builder.fsub(v_rt_m, ZmX_coth_ZmX) + v_rt_mA = builder.fsub(Z2_sqr_csch_Z2_sqr, ZmX_sqr_csch_ZmX_sqr) + v_rt_mB = builder.fsub(Z2_coth_Z2, ZmX_coth_ZmX) + v_rt_m = builder.fadd(v_rt_mA, v_rt_mB) v_rt_m = builder.fmul(noise_q_drift_q, v_rt_m) builder.store(v_rt_m, var_rt_minus_ptr) @@ -1651,38 +1651,43 @@ def _get_arg_out_ptr(idx): drift_rate_6 = builder.fmul(drift_rate_q, drift_rate_sqr) srt_tmp0 = builder.fdiv(noise_6, drift_rate_6) - srt_tmp1a = builder.fmul(Z2_sqr_csch_Z2_sqr.type(3), - Z2_sqr_csch_Z2_sqr) - srt_tmp2a = builder.fmul(Z2_coth_Z2, Z2_sqr_csch_Z2_sqr) - srt_tmp2a = builder.fmul(srt_tmp2a.type(2), srt_tmp2a) - srt_tmp3a = builder.fmul(Z2_coth_Z2.type(3), Z2_coth_Z2) - s_rt = builder.fadd(srt_tmp1a, srt_tmp2a) - s_rt = builder.fadd(s_rt, srt_tmp3a) + + Z2_cub_coth_Z2_csch_Z2_sqr = builder.fmul(Z2_coth_Z2, Z2_sqr_csch_Z2_sqr) + ZpX_cub_coth_ZpX_csch_Z2_sqr = builder.fmul(ZpX_coth_ZpX, ZpX_sqr_csch_ZpX_sqr) + ZmX_cub_coth_ZmX_csch_Z2_sqr = builder.fmul(ZmX_coth_ZmX, ZmX_sqr_csch_ZmX_sqr) # Skew plus - srtp_tmp1b = builder.fmul(ZpX_sqr_csch_ZpX_sqr.type(3), - ZpX_sqr_csch_ZpX_sqr) - srtp_tmp2b = builder.fmul(ZpX_coth_ZpX, ZpX_sqr_csch_ZpX_sqr) - srtp_tmp2b = builder.fmul(srtp_tmp2b.type(2), srtp_tmp2b) - srtp_tmp3b = builder.fmul(ZpX_coth_ZpX.type(3), ZpX_coth_ZpX) - - s_rt_p = builder.fsub(s_rt, srtp_tmp1b) - s_rt_p = builder.fsub(s_rt_p, srtp_tmp2b) - s_rt_p = builder.fsub(s_rt_p, srtp_tmp3b) + s_rt_p_tmpA = builder.fsub(Z2_sqr_csch_Z2_sqr, ZpX_sqr_csch_ZpX_sqr) + s_rt_p_tmpA = builder.fmul(s_rt_p_tmpA, s_rt_p_tmpA.type(3)) + + s_rt_p_tmpB = builder.fsub(Z2_cub_coth_Z2_csch_Z2_sqr, + ZpX_cub_coth_ZpX_csch_Z2_sqr) + s_rt_p_tmpB = builder.fadd(s_rt_p_tmpB, s_rt_p_tmpB) + + s_rt_p_tmpC = builder.fsub(Z2_coth_Z2, ZpX_coth_ZpX) + s_rt_p_tmpC = builder.fmul(s_rt_p_tmpC, s_rt_p_tmpC.type(3)) + + s_rt_p = builder.fadd(s_rt_p_tmpA, s_rt_p_tmpB) + s_rt_p = builder.fadd(s_rt_p, s_rt_p_tmpC) + s_rt_p = builder.fmul(srt_tmp0, s_rt_p) s_rt_p = builder.fdiv(s_rt_p, v_rt_p_1_5) builder.store(s_rt_p, skew_rt_plus_ptr) # Skew minus - srtm_tmp1b = builder.fmul(ZmX_sqr_csch_ZmX_sqr.type(3), - ZmX_sqr_csch_ZmX_sqr) - srtm_tmp2b = builder.fmul(ZmX_coth_ZmX, ZmX_sqr_csch_ZmX_sqr) - srtm_tmp2b = builder.fmul(srtm_tmp2b.type(2), srtm_tmp2b) - srtm_tmp3b = builder.fmul(ZmX_coth_ZmX.type(3), ZmX_coth_ZmX) - - s_rt_m = builder.fsub(s_rt, srtm_tmp1b) - s_rt_m = builder.fsub(s_rt_m, srtm_tmp2b) - s_rt_m = builder.fsub(s_rt_m, srtm_tmp3b) + s_rt_m_tmpA = builder.fsub(Z2_sqr_csch_Z2_sqr, ZmX_sqr_csch_ZmX_sqr) + s_rt_m_tmpA = builder.fmul(s_rt_m_tmpA, s_rt_m_tmpA.type(3)) + + s_rt_m_tmpB = builder.fsub(Z2_cub_coth_Z2_csch_Z2_sqr, + ZmX_cub_coth_ZmX_csch_Z2_sqr) + s_rt_m_tmpB = builder.fadd(s_rt_m_tmpB, s_rt_m_tmpB) + + s_rt_m_tmpC = builder.fsub(Z2_coth_Z2, ZmX_coth_ZmX) + s_rt_m_tmpC = builder.fmul(s_rt_m_tmpC, s_rt_m_tmpC.type(3)) + + s_rt_m = builder.fadd(s_rt_m_tmpA, s_rt_m_tmpB) + s_rt_m = builder.fadd(s_rt_m, s_rt_m_tmpC) + s_rt_m = builder.fmul(srt_tmp0, s_rt_m) s_rt_m = builder.fdiv(s_rt_m, v_rt_m_1_5) builder.store(s_rt_m, skew_rt_minus_ptr) diff --git a/tests/functions/test_distribution.py b/tests/functions/test_distribution.py index da896ddf8c2..95dc59afae8 100644 --- a/tests/functions/test_distribution.py +++ b/tests/functions/test_distribution.py @@ -40,16 +40,16 @@ llvm_expected = {'fp64': {}, 'fp32': {}} llvm_expected['fp64'][dda_expected_small] = (0.5828813465336954, 0.04801236718458773, - 0.5324710838085324, 0.09633787836991654, 6.0158766570416775, - 1.5821207675877176, 0.5392731045768397, 1.8434859117411773) + 0.5324710838085324, 0.09633788030213193, 6.0183026674990625, + 1.5821207675877176, 0.5392731084412705, 1.843608020219776) # add fp32 results llvm_expected['fp32'][dda_expected_random] = (0.42365485429763794, 0.0, - 0.5173675417900085, 0.06942801177501678, 6.302331447601318, - 1.4934077262878418, 0.428894966840744, 1.7738982439041138) + 0.5173675417900085, 0.069428451359272, 6.302595138549805, + 1.4934077262878418, 0.42889538407325745, 1.7739042043685913) llvm_expected['fp32'][dda_expected_negative] = (0.4236549735069275, 5.960464477539063e-08, - 0.5173678398132324, 0.06942889094352722, 6.303247451782227, - 1.4934080839157104, 0.42889583110809326, 1.7739603519439697) + 0.5173678398132324, 0.06942932307720184, 6.302994251251221, + 1.4934080839157104, 0.4288962781429291, 1.7739406824111938) llvm_expected['fp32'][dda_expected_small] = None llvm_expected['fp32'][normal_expected_philox] = (0.5655658841133118) llvm_expected['fp32'][uniform_expected_philox] = (0.6180108785629272) @@ -109,8 +109,8 @@ def test_execute(func, variable, params, prng, llvm_skip, expected, benchmark, f # it to the mechanism above if func_mode == "PTX" and precision == 'fp32' and expected is dda_expected_negative: expected = (0.4236549735069275, 5.960464477539063e-08, - 0.5173678398132324, 0.06942889094352722, 6.303247451782227, - 1.4934064149856567, 0.42889145016670227, 1.7737685441970825) + 0.5173678398132324, 0.06942932307720184, 6.302994728088379, + 1.4934064149856567, 0.4288918972015381, 1.7737658023834229) expected = llvm_expected.get(precision, {}).get(expected, expected) if expected is None: From bc053a3b388bf8f7b6f6c9eb6869ad8dc6b79b17 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Thu, 12 Jan 2023 21:18:09 -0500 Subject: [PATCH 125/127] tests/DriftDiffusionAnalytical: Use mac/windows results for numpy>=1.22 and AVX512 CPUs Signed-off-by: Jan Vesely --- conftest.py | 14 ++++++++++++-- dev_requirements.txt | 1 + tests/functions/test_distribution.py | 9 ++++++++- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/conftest.py b/conftest.py index cae7d36ce73..85c6f2eea8c 100644 --- a/conftest.py +++ b/conftest.py @@ -1,8 +1,10 @@ +import contextlib import doctest +import io +import numpy as np import psyneulink import pytest -import numpy as np - +import re from psyneulink import clear_registry, primary_registries from psyneulink.core import llvm as pnlvm @@ -201,6 +203,14 @@ def mech_wrapper(x): else: assert False, "Unknown mechanism mode: {}".format(mech_mode) +@pytest.helpers.register +def numpy_uses_avx512(): + out = io.StringIO() + with contextlib.redirect_stdout(out): + np.show_config() + + return re.search(' found = .*AVX512.*', out.getvalue()) is not None + @pytest.helpers.register def expand_np_ndarray(arr): # this will fail on an input containing a float (not np.ndarray) diff --git a/dev_requirements.txt b/dev_requirements.txt index 43da3586fda..3e2f63b1c7b 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,4 +1,5 @@ jupyter<=1.0.0 +packaging<24.0 pytest<7.2.1 pytest-benchmark<4.0.1 pytest-cov<4.0.1 diff --git a/tests/functions/test_distribution.py b/tests/functions/test_distribution.py index 95dc59afae8..4bb805a095e 100644 --- a/tests/functions/test_distribution.py +++ b/tests/functions/test_distribution.py @@ -2,6 +2,8 @@ import pytest import sys +from packaging import version as pversion + import psyneulink.core.llvm as pnlvm import psyneulink.core.components.functions.nonstateful.distributionfunctions as Functions from psyneulink.core.globals.utilities import _SeededPhilox @@ -27,8 +29,13 @@ dda_expected_small = (0.5828813465336954, 0.04801236718458773, 0.532471083815943, 0.09633801555720854, 6.1142591416669765, 1.5821207676710864, 0.5392724051148722, 1.806647390875747) + # Different libm implementations produce slightly different results -if sys.platform.startswith("win") or sys.platform.startswith("darwin"): +# Numpy 1.22+ uses new/optimized implementation of FP routines +# on processors that support AVX512 since 1.22 [0] +# [0] https://github.com/numpy/numpy/commit/1eff1c543a8f1e9d7ea29182b8c76db5a2efc3c2 +if sys.platform.startswith("win") or sys.platform.startswith("darwin") or \ + ( pversion.parse(np.version.version) >= pversion.parse('1.22') and pytest.helpers.numpy_uses_avx512()): dda_expected_small = (0.5828813465336954, 0.04801236718458773, 0.5324710838150166, 0.09633802135385469, 6.117763080882898, 1.58212076767016, 0.5392724012504414, 1.8064031532265) From 88f2116b27a96c7706618442b9c1853c28179dbf Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Wed, 31 Aug 2022 12:22:44 -0400 Subject: [PATCH 126/127] requirements: update numpy requirement to <1.22.5 Updates the requirements on [numpy](https://github.com/numpy/numpy) to permit the latest 1.22.x release. - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v1.17.0...v1.22.4) Signed-off-by: Jan Vesely --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0410992ad91..41d0a3d96b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ llvmlite<0.40 matplotlib<3.6.4 modeci_mdf<0.5, >=0.3.4; (platform_machine == 'AMD64' or platform_machine == 'x86_64') and platform_python_implementation == 'CPython' and implementation_name == 'cpython' networkx<3.1 -numpy<1.21.7, >=1.17.0 +numpy<1.22.5, >=1.17.0 pillow<9.5.0 pint<0.21.0 toposort<1.10 From 4387eb6c2e3d1995bf7a74a03d53306d5ad9254d Mon Sep 17 00:00:00 2001 From: jdcpni Date: Fri, 13 Jan 2023 14:02:54 -0500 Subject: [PATCH 127/127] =?UTF-8?q?=E2=80=A2=20function.py=20(#2577)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * • function.py - Function_Base: move arguments referring to parameters in call to function to params arg (to be treated as runtime_params) • memoryfunctions.py - ContentAddressableMemory: add store and retrieve convenience methods * - --- psyneulink/core/components/functions/function.py | 13 ++++++++++++- .../functions/nonstateful/learningfunctions.py | 4 ++++ .../functions/stateful/memoryfunctions.py | 12 ++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/psyneulink/core/components/functions/function.py b/psyneulink/core/components/functions/function.py index 3b592428886..abcb8cdb5bd 100644 --- a/psyneulink/core/components/functions/function.py +++ b/psyneulink/core/components/functions/function.py @@ -676,7 +676,18 @@ def function(self, params=None, target_set=None, **kwargs): - assert True + + # IMPLEMENTATION NOTE: + # The following is a convenience feature that supports specification of params directly in call to function + # by moving the to a params dict, which treats them as runtime_params + if kwargs: + for key in kwargs.copy(): + if key in self.parameters.names(): + if not params: + params = {key: kwargs.pop(key)} + else: + params.update({key: kwargs.pop(key)}) + # Validate variable and assign to variable, and validate params variable = self._check_args(variable=variable, context=context, diff --git a/psyneulink/core/components/functions/nonstateful/learningfunctions.py b/psyneulink/core/components/functions/nonstateful/learningfunctions.py index d0c8bde085b..dd1b830a7e2 100644 --- a/psyneulink/core/components/functions/nonstateful/learningfunctions.py +++ b/psyneulink/core/components/functions/nonstateful/learningfunctions.py @@ -2118,6 +2118,10 @@ def _function(self, self._check_args(variable=variable, context=context, params=params) + # IMPLEMENTATION NOTE: if error_matrix is an arg, it must in params (put there by super.function() + if params: + error_matrix = params.pop(ERROR_MATRIX, None) + # Manage error_matrix param # During init, function is called directly from Component (i.e., not from LearningMechanism execute() method), # so need "placemarker" error_matrix for validation diff --git a/psyneulink/core/components/functions/stateful/memoryfunctions.py b/psyneulink/core/components/functions/stateful/memoryfunctions.py index 37253ab5d7e..32e58130ea0 100644 --- a/psyneulink/core/components/functions/stateful/memoryfunctions.py +++ b/psyneulink/core/components/functions/stateful/memoryfunctions.py @@ -1815,6 +1815,18 @@ def _parse_memories(self, entries, method, context=None): return memories + def store(self, entry, context=None, **kwargs): + """Store value in `memory `. + Convenience method for storing entry in memory. + """ + return self(entry, retrieval_prob=0.0, context=context, **kwargs) + + def retrieve(self, entry, context=None, **kwargs): + """Retrieve value from `memory `. + Convenience method for retrieving entry from memory. + """ + return self(entry, storage_prob=0.0, context=context, **kwargs) + @property def memory(self): """Return entries in self._memory as lists in an outer np.array;