Parallelize tests (#237)

* Remove plot visualization * Setup parallel testing
sustainable-processes · Feb 11, 2023 · 34162c8 · 34162c8
1 parent a861b91
commit 34162c8
Show file tree

Hide file tree

Showing 5 changed files with 219 additions and 32 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -43,6 +43,7 @@ jobs:
     strategy:
       matrix:
         python: ["3.8", "3.9", "3.10"]
+        group: [1, 2, 3, 4, 5]
 
     steps:
     - name: Checkout repository
@@ -64,11 +65,11 @@ jobs:
       shell: bash
       run: |
         WHL_NAME=$(ls summit-*.whl)
-        pip install ${WHL_NAME}[experiments,entmoot] pytest
+        pip install ${WHL_NAME}[experiments,entmoot] pytest pytest-split
 
     - name: Run tests
       shell: bash
-      run: summit-tests
+      run: PY_IGNORE_IMPORTMISMATCH=1 pytest --doctest-modules  --disable-warnings --ignore=experiments --splits 5 --group ${{ matrix.group }} 
 
   # Publish to pypi on version change
   # This is based on https://github.com/coveooss/pypi-publish-with-poetry

diff --git a/.test_durations b/.test_durations
@@ -0,0 +1,193 @@
+{
+    "summit/benchmarks/experimental_emulator.py::summit.benchmarks.experimental_emulator.BaumgartnerCrossCouplingEmulator": 0.0028804161120206118,
+    "summit/benchmarks/experimental_emulator.py::summit.benchmarks.experimental_emulator.ExperimentalEmulator": 0.16585987457074225,
+    "summit/benchmarks/experimental_emulator.py::summit.benchmarks.experimental_emulator.ExperimentalEmulator.load": 0.19636954087764025,
+    "summit/benchmarks/experimental_emulator.py::summit.benchmarks.experimental_emulator.ExperimentalEmulator.save": 0.2393556258175522,
+    "summit/benchmarks/experimental_emulator.py::summit.benchmarks.experimental_emulator.ExperimentalEmulator.train": 0.004319624276831746,
+    "summit/benchmarks/experimental_emulator.py::summit.benchmarks.experimental_emulator.ReizmanSuzukiEmulator": 0.0034233753103762865,
+    "summit/benchmarks/experimental_emulator.py::summit.benchmarks.experimental_emulator.get_pretrained_baumgartner_cc_emulator": 0.12654883321374655,
+    "summit/benchmarks/experimental_emulator.py::summit.benchmarks.experimental_emulator.get_pretrained_reizman_suzuki_emulator": 0.11725170980207622,
+    "summit/benchmarks/snar.py::summit.benchmarks.snar.SnarBenchmark": 0.00538808386772871,
+    "summit/benchmarks/test_functions.py::summit.benchmarks.test_functions.Hartmann3D": 0.004243040923029184,
+    "summit/benchmarks/test_functions.py::summit.benchmarks.test_functions.Himmelblau": 0.004003207897767425,
+    "summit/benchmarks/test_functions.py::summit.benchmarks.test_functions.ThreeHumpCamel": 0.003981709014624357,
+    "summit/domain.py::summit.domain.CategoricalVariable": 0.008502626325935125,
+    "summit/domain.py::summit.domain.Constraint": 0.0007488750852644444,
+    "summit/domain.py::summit.domain.ContinuousVariable": 0.0006407909095287323,
+    "summit/domain.py::summit.domain.Domain": 0.0006564159411936998,
+    "summit/run.py::summit.run.Runner": 0.08046474889852107,
+    "summit/strategies/base.py::summit.strategies.base.Chimera": 0.028825916117057204,
+    "summit/strategies/base.py::summit.strategies.base.Design": 0.0010202086996287107,
+    "summit/strategies/base.py::summit.strategies.base.LogSpaceObjectives": 0.020910751074552536,
+    "summit/strategies/base.py::summit.strategies.base.MultitoSingleObjective": 0.021709125256165862,
+    "summit/strategies/deep_reaction_optimizer.py::summit.strategies.deep_reaction_optimizer.DRO": 0.003044332144781947,
+    "summit/strategies/entmoot.py::summit.strategies.entmoot.ENTMOOT": 0.0009428758639842272,
+    "summit/strategies/factorial_doe.py::summit.strategies.factorial_doe.FullFactorial": 0.007137791020795703,
+    "summit/strategies/multitask.py::summit.strategies.multitask.MTBO": 1.3519809159915894,
+    "summit/strategies/multitask.py::summit.strategies.multitask.MTBO.suggest_experiments": 0.3304593339562416,
+    "summit/strategies/multitask.py::summit.strategies.multitask.STBO": 0.006252957973629236,
+    "summit/strategies/neldermead.py::summit.strategies.neldermead.NelderMead": 0.005805832101032138,
+    "summit/strategies/random.py::summit.strategies.random.LHS": 0.006494124885648489,
+    "summit/strategies/random.py::summit.strategies.random.Random": 0.005873373942449689,
+    "summit/strategies/random.py::summit.strategies.random.lhs": 0.003992458572611213,
+    "summit/strategies/snobfit.py::summit.strategies.snobfit.SNOBFIT": 0.020028248894959688,
+    "summit/strategies/sobo.py::summit.strategies.sobo.SOBO": 0.025062251137569547,
+    "summit/strategies/tsemo.py::summit.strategies.tsemo.TSEMO": 0.004354208009317517,
+    "summit/tests/test_benchmarks.py::test_baumgartner_CC_emulator[False-False]": 0.13930304115638137,
+    "summit/tests/test_benchmarks.py::test_baumgartner_CC_emulator[False-True]": 0.1584901250898838,
+    "summit/tests/test_benchmarks.py::test_baumgartner_CC_emulator[True-False]": 0.13903316715732217,
+    "summit/tests/test_benchmarks.py::test_baumgartner_CC_emulator[True-True]": 0.15710000111721456,
+    "summit/tests/test_benchmarks.py::test_dltz2_benchmark[6]": 0.006508958991616964,
+    "summit/tests/test_benchmarks.py::test_no_objectives": 0.0011860420927405357,
+    "summit/tests/test_benchmarks.py::test_reizman_emulator": 0.20699545880779624,
+    "summit/tests/test_benchmarks.py::test_snar_benchmark[0.0]": 0.006868125405162573,
+    "summit/tests/test_benchmarks.py::test_snar_benchmark[2.5]": 0.006350207142531872,
+    "summit/tests/test_benchmarks.py::test_train_experimental_emulator": 18.459753915900365,
+    "summit/tests/test_domain.py::test_categorical_variable": 0.0015435838140547276,
+    "summit/tests/test_domain.py::test_constraint": 0.00025299983099102974,
+    "summit/tests/test_domain.py::test_continuous_variable": 0.00030824984423816204,
+    "summit/tests/test_domain.py::test_domain": 0.00664795795455575,
+    "summit/tests/test_runner.py::test_runner_mo_integration[DTLZ2-LHS]": 0.022482083179056644,
+    "summit/tests/test_runner.py::test_runner_mo_integration[DTLZ2-NelderMead]": 0.04572600033134222,
+    "summit/tests/test_runner.py::test_runner_mo_integration[DTLZ2-Random]": 0.020428999327123165,
+    "summit/tests/test_runner.py::test_runner_mo_integration[DTLZ2-SNOBFIT]": 0.03506545792333782,
+    "summit/tests/test_runner.py::test_runner_mo_integration[DTLZ2-SOBO]": 0.10477929189801216,
+    "summit/tests/test_runner.py::test_runner_mo_integration[DTLZ2-TSEMO]": 66.37859041709453,
+    "summit/tests/test_runner.py::test_runner_mo_integration[SnarBenchmark-LHS]": 0.024271040922030807,
+    "summit/tests/test_runner.py::test_runner_mo_integration[SnarBenchmark-NelderMead]": 0.05117479106411338,
+    "summit/tests/test_runner.py::test_runner_mo_integration[SnarBenchmark-Random]": 0.023896666709333658,
+    "summit/tests/test_runner.py::test_runner_mo_integration[SnarBenchmark-SNOBFIT]": 0.03792516514658928,
+    "summit/tests/test_runner.py::test_runner_mo_integration[SnarBenchmark-SOBO]": 0.11144283413887024,
+    "summit/tests/test_runner.py::test_runner_mo_integration[SnarBenchmark-TSEMO]": 61.634299458004534,
+    "summit/tests/test_runner.py::test_runner_mo_integration[VLMOP2-LHS]": 0.016731292009353638,
+    "summit/tests/test_runner.py::test_runner_mo_integration[VLMOP2-NelderMead]": 0.032330458983778954,
+    "summit/tests/test_runner.py::test_runner_mo_integration[VLMOP2-Random]": 0.016654375242069364,
+    "summit/tests/test_runner.py::test_runner_mo_integration[VLMOP2-SNOBFIT]": 0.03339012525975704,
+    "summit/tests/test_runner.py::test_runner_mo_integration[VLMOP2-SOBO]": 0.09863324905745685,
+    "summit/tests/test_runner.py::test_runner_mo_integration[VLMOP2-TSEMO]": 62.96583229186945,
+    "summit/tests/test_runner.py::test_runner_mo_integration[experiment0-LHS]": 0.00027466798201203346,
+    "summit/tests/test_runner.py::test_runner_mo_integration[experiment0-NelderMead]": 0.00029020803049206734,
+    "summit/tests/test_runner.py::test_runner_mo_integration[experiment0-Random]": 0.00028912490233778954,
+    "summit/tests/test_runner.py::test_runner_mo_integration[experiment0-SNOBFIT]": 0.0003561251796782017,
+    "summit/tests/test_runner.py::test_runner_mo_integration[experiment0-SOBO]": 0.1487670410424471,
+    "summit/tests/test_runner.py::test_runner_mo_integration[experiment0-TSEMO]": 210.52943404088728,
+    "summit/tests/test_runner.py::test_runner_mo_integration[experiment1-LHS]": 0.04513049987144768,
+    "summit/tests/test_runner.py::test_runner_mo_integration[experiment1-NelderMead]": 0.15109241614118218,
+    "summit/tests/test_runner.py::test_runner_mo_integration[experiment1-Random]": 0.04452341725118458,
+    "summit/tests/test_runner.py::test_runner_mo_integration[experiment1-SNOBFIT]": 0.0843225410208106,
+    "summit/tests/test_runner.py::test_runner_mo_integration[experiment1-SOBO]": 0.1489123331848532,
+    "summit/tests/test_runner.py::test_runner_mo_integration[experiment1-TSEMO]": 293.86193904210813,
+    "summit/tests/test_runner.py::test_runner_so_integration[Hartmann3D-LHS]": 0.008857500040903687,
+    "summit/tests/test_runner.py::test_runner_so_integration[Hartmann3D-NelderMead]": 0.01735212467610836,
+    "summit/tests/test_runner.py::test_runner_so_integration[Hartmann3D-Random]": 0.008746706880629063,
+    "summit/tests/test_runner.py::test_runner_so_integration[Hartmann3D-SNOBFIT]": 0.010238625342026353,
+    "summit/tests/test_runner.py::test_runner_so_integration[Hartmann3D-SOBO]": 0.012094789883121848,
+    "summit/tests/test_runner.py::test_runner_so_integration[Himmelblau-LHS]": 0.008289917139336467,
+    "summit/tests/test_runner.py::test_runner_so_integration[Himmelblau-NelderMead]": 0.014168542111292481,
+    "summit/tests/test_runner.py::test_runner_so_integration[Himmelblau-Random]": 0.008111250819638371,
+    "summit/tests/test_runner.py::test_runner_so_integration[Himmelblau-SNOBFIT]": 0.010005625197663903,
+    "summit/tests/test_runner.py::test_runner_so_integration[Himmelblau-SOBO]": 0.012370293028652668,
+    "summit/tests/test_runner.py::test_runner_so_integration[ThreeHumpCamel-LHS]": 0.00829183286987245,
+    "summit/tests/test_runner.py::test_runner_so_integration[ThreeHumpCamel-NelderMead]": 0.014060416026040912,
+    "summit/tests/test_runner.py::test_runner_so_integration[ThreeHumpCamel-Random]": 0.008123415522277355,
+    "summit/tests/test_runner.py::test_runner_so_integration[ThreeHumpCamel-SNOBFIT]": 0.009760623099282384,
+    "summit/tests/test_runner.py::test_runner_so_integration[ThreeHumpCamel-SOBO]": 0.011465708957985044,
+    "summit/tests/test_runner.py::test_runner_so_integration[experiment3-LHS]": 0.06344574992544949,
+    "summit/tests/test_runner.py::test_runner_so_integration[experiment3-NelderMead]": 0.1532965840306133,
+    "summit/tests/test_runner.py::test_runner_so_integration[experiment3-Random]": 0.06305954279378057,
+    "summit/tests/test_runner.py::test_runner_so_integration[experiment3-SNOBFIT]": 0.07175970892421901,
+    "summit/tests/test_runner.py::test_runner_so_integration[experiment3-SOBO]": 0.06435650098137558,
+    "summit/tests/test_runner.py::test_runner_unit[NeptuneRunner-1-5-1-10]": 0.0014508331660181284,
+    "summit/tests/test_runner.py::test_runner_unit[NeptuneRunner-1-5-1-1]": 0.001511541660875082,
+    "summit/tests/test_runner.py::test_runner_unit[NeptuneRunner-1-5-5-10]": 0.0014402922242879868,
+    "summit/tests/test_runner.py::test_runner_unit[NeptuneRunner-1-5-5-1]": 0.001495875185355544,
+    "summit/tests/test_runner.py::test_runner_unit[NeptuneRunner-1-None-1-10]": 0.0017517071682959795,
+    "summit/tests/test_runner.py::test_runner_unit[NeptuneRunner-1-None-1-1]": 0.0015471240039914846,
+    "summit/tests/test_runner.py::test_runner_unit[NeptuneRunner-1-None-5-10]": 0.0014542089775204659,
+    "summit/tests/test_runner.py::test_runner_unit[NeptuneRunner-1-None-5-1]": 0.0014819579664617777,
+    "summit/tests/test_runner.py::test_runner_unit[NeptuneRunner-5-5-1-10]": 0.0014309999532997608,
+    "summit/tests/test_runner.py::test_runner_unit[NeptuneRunner-5-5-1-1]": 0.0015229990240186453,
+    "summit/tests/test_runner.py::test_runner_unit[NeptuneRunner-5-5-5-10]": 0.0014275838620960712,
+    "summit/tests/test_runner.py::test_runner_unit[NeptuneRunner-5-5-5-1]": 0.0015219161286950111,
+    "summit/tests/test_runner.py::test_runner_unit[NeptuneRunner-5-None-1-10]": 0.0014336253516376019,
+    "summit/tests/test_runner.py::test_runner_unit[NeptuneRunner-5-None-1-1]": 0.001511499984189868,
+    "summit/tests/test_runner.py::test_runner_unit[NeptuneRunner-5-None-5-10]": 0.0014310397673398256,
+    "summit/tests/test_runner.py::test_runner_unit[NeptuneRunner-5-None-5-1]": 0.0014997071120887995,
+    "summit/tests/test_runner.py::test_runner_unit[Runner-1-5-1-10]": 0.06810687482357025,
+    "summit/tests/test_runner.py::test_runner_unit[Runner-1-5-1-1]": 0.025623291730880737,
+    "summit/tests/test_runner.py::test_runner_unit[Runner-1-5-5-10]": 0.20943637494929135,
+    "summit/tests/test_runner.py::test_runner_unit[Runner-1-5-5-1]": 0.03984716720879078,
+    "summit/tests/test_runner.py::test_runner_unit[Runner-1-None-1-10]": 0.06902904203161597,
+    "summit/tests/test_runner.py::test_runner_unit[Runner-1-None-1-1]": 0.028223583009094,
+    "summit/tests/test_runner.py::test_runner_unit[Runner-1-None-5-10]": 0.21114112506620586,
+    "summit/tests/test_runner.py::test_runner_unit[Runner-1-None-5-1]": 0.04036216717213392,
+    "summit/tests/test_runner.py::test_runner_unit[Runner-5-5-1-10]": 0.06941070780158043,
+    "summit/tests/test_runner.py::test_runner_unit[Runner-5-5-1-1]": 0.025573250139132142,
+    "summit/tests/test_runner.py::test_runner_unit[Runner-5-5-5-10]": 0.20944154192693532,
+    "summit/tests/test_runner.py::test_runner_unit[Runner-5-5-5-1]": 0.04083641618490219,
+    "summit/tests/test_runner.py::test_runner_unit[Runner-5-None-1-10]": 0.06794670759700239,
+    "summit/tests/test_runner.py::test_runner_unit[Runner-5-None-1-1]": 0.02559074992313981,
+    "summit/tests/test_runner.py::test_runner_unit[Runner-5-None-5-10]": 0.20797933381982148,
+    "summit/tests/test_runner.py::test_runner_unit[Runner-5-None-5-1]": 0.039779959013685584,
+    "summit/tests/test_strategies.py::test_doe": 0.0021241263020783663,
+    "summit/tests/test_strategies.py::test_entmoot[1-1-True-False]": 0.003944917116314173,
+    "summit/tests/test_strategies.py::test_entmoot[1-200-True-False]": 0.0020879581570625305,
+    "summit/tests/test_strategies.py::test_entmoot[4-200-False-False]": 0.0021403736900538206,
+    "summit/tests/test_strategies.py::test_lhs": 0.00583337526768446,
+    "summit/tests/test_strategies.py::test_logspaceobjectives_transform": 0.004721831995993853,
+    "summit/tests/test_strategies.py::test_mtbo[20-False-False]": 29.888648875057697,
+    "summit/tests/test_strategies.py::test_mtbo[20-True-False]": 21.17057845904492,
+    "summit/tests/test_strategies.py::test_multitosingleobjective_transform": 0.00519745796918869,
+    "summit/tests/test_strategies.py::test_nm2D[False-False-x_start0]": 0.5504108327440917,
+    "summit/tests/test_strategies.py::test_nm2D[False-False-x_start1]": 0.5291824999731034,
+    "summit/tests/test_strategies.py::test_nm2D[False-False-x_start2]": 0.5367747510317713,
+    "summit/tests/test_strategies.py::test_nm2D[False-False-x_start3]": 0.19781883317045867,
+    "summit/tests/test_strategies.py::test_nm2D[False-False-x_start4]": 0.4797362079843879,
+    "summit/tests/test_strategies.py::test_nm2D[False-True-x_start0]": 0.711228916188702,
+    "summit/tests/test_strategies.py::test_nm2D[False-True-x_start1]": 0.7951799172442406,
+    "summit/tests/test_strategies.py::test_nm2D[False-True-x_start2]": 0.7501638350076973,
+    "summit/tests/test_strategies.py::test_nm2D[False-True-x_start3]": 0.23069662600755692,
+    "summit/tests/test_strategies.py::test_nm2D[False-True-x_start4]": 0.6294994160998613,
+    "summit/tests/test_strategies.py::test_nm2D[True-False-x_start0]": 1.3766254158690572,
+    "summit/tests/test_strategies.py::test_nm2D[True-False-x_start1]": 1.1558928748127073,
+    "summit/tests/test_strategies.py::test_nm2D[True-False-x_start2]": 0.8857064172625542,
+    "summit/tests/test_strategies.py::test_nm2D[True-False-x_start3]": 0.2579910010099411,
+    "summit/tests/test_strategies.py::test_nm2D[True-False-x_start4]": 0.7630787908565253,
+    "summit/tests/test_strategies.py::test_nm2D[True-True-x_start0]": 0.7319635418243706,
+    "summit/tests/test_strategies.py::test_nm2D[True-True-x_start1]": 0.6921850829385221,
+    "summit/tests/test_strategies.py::test_nm2D[True-True-x_start2]": 0.6975888339802623,
+    "summit/tests/test_strategies.py::test_nm2D[True-True-x_start3]": 0.24832783290185034,
+    "summit/tests/test_strategies.py::test_nm2D[True-True-x_start4]": 0.6559557500295341,
+    "summit/tests/test_strategies.py::test_nm3D[x_start0-True-True]": 1.465338874841109,
+    "summit/tests/test_strategies.py::test_nm3D[x_start1-True-False]": 1.1909232093021274,
+    "summit/tests/test_strategies.py::test_nm3D[x_start10-True-True]": 0.541453291894868,
+    "summit/tests/test_strategies.py::test_nm3D[x_start11-True-False]": 0.46658570785075426,
+    "summit/tests/test_strategies.py::test_nm3D[x_start12-False-True]": 0.5665520001202822,
+    "summit/tests/test_strategies.py::test_nm3D[x_start13-False-False]": 0.454463625093922,
+    "summit/tests/test_strategies.py::test_nm3D[x_start2-False-True]": 1.3782533747144043,
+    "summit/tests/test_strategies.py::test_nm3D[x_start3-False-False]": 1.175968041876331,
+    "summit/tests/test_strategies.py::test_nm3D[x_start4-True-False]": 0.5950960838235915,
+    "summit/tests/test_strategies.py::test_nm3D[x_start5-False-False]": 0.6289865840226412,
+    "summit/tests/test_strategies.py::test_nm3D[x_start6-True-True]": 0.537621249910444,
+    "summit/tests/test_strategies.py::test_nm3D[x_start7-True-False]": 0.750423084013164,
+    "summit/tests/test_strategies.py::test_nm3D[x_start8-False-True]": 0.5330263751093298,
+    "summit/tests/test_strategies.py::test_nm3D[x_start9-False-False]": 0.7244379580952227,
+    "summit/tests/test_strategies.py::test_random": 0.005873125279322267,
+    "summit/tests/test_strategies.py::test_snobfit[False-False-1]": 2.494571916060522,
+    "summit/tests/test_strategies.py::test_snobfit[False-False-2]": 1.3886395012959838,
+    "summit/tests/test_strategies.py::test_snobfit[False-False-4]": 2.160413125064224,
+    "summit/tests/test_strategies.py::test_snobfit[False-True-1]": 2.737286083167419,
+    "summit/tests/test_strategies.py::test_snobfit[False-True-2]": 1.6200418742373586,
+    "summit/tests/test_strategies.py::test_snobfit[False-True-4]": 2.0966434159781784,
+    "summit/tests/test_strategies.py::test_sobo[1-1-True-True]": 0.013362958328798413,
+    "summit/tests/test_strategies.py::test_sobo[1-200-True-False]": 0.1966916259843856,
+    "summit/tests/test_strategies.py::test_sobo[1-200-True-True]": 0.1432115000206977,
+    "summit/tests/test_strategies.py::test_sobo[4-200-False-False]": 0.43735779193229973,
+    "summit/tests/test_strategies.py::test_sobo[4-200-False-True]": 2.1351462081074715,
+    "summit/tests/test_strategies.py::test_strategy": 0.00029858388006687164,
+    "summit/tests/test_strategies.py::test_tsemo[10]": 19.006217750022188,
+    "summit/tests/test_strategies.py::test_tsemo[1]": 53.60011858283542,
+    "summit/tests/test_strategies.py::test_tsemo[2]": 32.46155295916833,
+    "summit/tests/test_utils.py::test_dataset": 0.0023060000967234373,
+    "summit/utils/dataset.py::summit.utils.dataset.DataSet": 0.003824874060228467
+}