update test07 with sorting

nicolas-kuechler · Apr 19, 2024 · 01cb2e2 · 01cb2e2
1 parent dfcc57a
commit 01cb2e2
Show file tree

Hide file tree

Showing 9 changed files with 145 additions and 104 deletions.
diff --git a/demo_project/doe-suite-config/designs/etl_templates/config_template.yml b/demo_project/doe-suite-config/designs/etl_templates/config_template.yml
@@ -19,6 +19,7 @@ $ETL$:
       ErrorExtractor: {} # etl-error if stderr.log is non-empty
       IgnoreExtractor: {} # ignore stdout.log
     transformers:
+      - df.sort_values: { by: [ exp_name, run, rep ], ignore_index: True }
       - df.filter: {regex: "\\$CMD\\$"}
     loaders:
       CsvSummaryLoader: # write the transformed dataframe across the whole experiment as a csv file

diff --git a/demo_project/doe-suite-config/designs/example07-etl.yml b/demo_project/doe-suite-config/designs/example07-etl.yml
@@ -24,8 +24,11 @@ $ETL$:                                                         # Visualization o
     extractors:                                                #--------------------------------------------------
       YamlExtractor: {} # by default loads all .yaml files     #| YamlExtractor  ErrorExtractor  IgnoreExtractor | Extractors:
       ErrorExtractor: {} # error stderr.log if non-empty       #--------------------------------------------------    result files to pandas df
-      IgnoreExtractor: {} # ignore stdout.log                  #                        |                          Transformers:
-    transformers:                                              #                        V                             transform df
+      IgnoreExtractor: {} # ignore stdout.log                  #                                                   Transformers:
+    transformers:                                              #                        |                             transform df
+      - df.sort_values:                                        #                        |
+          by: [ exp_name, run, rep ]                           #                        V
+          ignore_index: True                                   #
       - df.filter: {items: ["exp_name", "x", "y"]}             #                    df.filter
       - df.eval: {expr: "color = 'black'"}                     #                        |
     loaders:                                                   #                        V
@@ -64,6 +67,7 @@ $ETL$:                                                         # Visualization o
     # by including all we have a more complex color assignment -> we only include extractors + loaders and provide a custom transformer stage
         # (2) use df.X syntax to directly use pandas df transformations: https://pandas.pydata.org/docs/reference/frame.html
         #       (note there are a few limitations: some functions on df require indexes which cannot be defined here -> e.g., see Conditional Transformer)
+        - df.sort_values: { by: [ exp_name, run, rep ], ignore_index: True }
         - df.filter: {items: ["exp_name", "x", "y"]}
         - {name: ConditionalTransformer, col: "exp_name", dest: "color", value: {plus: black, square: green, triangle1: blue, triangle2: blue}}
     loaders:

diff --git a/demo_project/doe-suite-results/example07-etl_$expected/etl_results/commands/commands.csv b/demo_project/doe-suite-results/example07-etl_$expected/etl_results/commands/commands.csv
@@ -1,27 +1,27 @@
 ,$CMD$.small,$CMD$.small_v2
-0,"[{'main': ""printf 'x: 1\\ny: 1' > results/coordinates.yaml""}]",
-1,"[{'main': ""printf 'x: 2\\ny: 2' > results/coordinates.yaml""}]",
-2,"[{'main': ""printf 'x: 1\\ny: 0' > results/coordinates.yaml""}]",
-3,"[{'main': ""printf 'x: 2\\ny: 1' > results/coordinates.yaml""}]",
-4,"[{'main': ""printf 'x: 0\\ny: 1' > results/coordinates.yaml""}]",
-5,"[{'main': ""printf 'x: 0\\ny: 0' > results/coordinates.yaml""}]",
-6,"[{'main': ""printf 'x: 2\\ny: 0' > results/coordinates.yaml""}]",
-7,"[{'main': ""printf 'x: 0\\ny: 2' > results/coordinates.yaml""}]",
-8,"[{'main': ""printf 'x: 1\\ny: 2' > results/coordinates.yaml""}]",
-0,"[{'main': ""printf 'x: 8 \\ny: 7' > results/coordinates.yaml""}]",
-1,"[{'main': ""printf 'x: 7 \\ny: 5' > results/coordinates.yaml""}]",
-2,"[{'main': ""printf 'x: 6 \\ny: 5' > results/coordinates.yaml""}]",
-3,"[{'main': ""printf 'x: 9 \\ny: 5' > results/coordinates.yaml""}]",
-4,"[{'main': ""printf 'x: 8 \\ny: 6' > results/coordinates.yaml""}]",
-5,"[{'main': ""printf 'x: 8 \\ny: 3' > results/coordinates.yaml""}]",
-6,"[{'main': ""printf 'x: 8 \\ny: 4' > results/coordinates.yaml""}]",
-7,"[{'main': ""printf 'x: 10 \\ny: 5' > results/coordinates.yaml""}]",
-0,,"[{'main': ""printf 'x: 2\\ny: 3' > results/coordinates.yaml""}]"
-1,,"[{'main': ""printf 'x: 1\\ny: 4' > results/coordinates.yaml""}]"
-2,,"[{'main': ""printf 'x: 0\\ny: 4' > results/coordinates.yaml""}]"
-3,,"[{'main': ""printf 'x: 0\\ny: 3' > results/coordinates.yaml""}]"
-4,,"[{'main': ""printf 'x: 1\\ny: 3' > results/coordinates.yaml""}]"
-5,,"[{'main': ""printf 'x: 2\\ny: 4' > results/coordinates.yaml""}]"
-0,,"[{'main': ""printf 'x: 3\\ny: 3' > results/coordinates.yaml""}]"
-1,,"[{'main': ""printf 'x: -1\\ny: 3' > results/coordinates.yaml""}]"
-2,,"[{'main': ""printf 'x: 1\\ny: 5' > results/coordinates.yaml""}]"
+0,"[{'main': ""printf 'x: 8 \\ny: 6' > results/coordinates.yaml""}]",
+1,"[{'main': ""printf 'x: 9 \\ny: 5' > results/coordinates.yaml""}]",
+2,"[{'main': ""printf 'x: 8 \\ny: 4' > results/coordinates.yaml""}]",
+3,"[{'main': ""printf 'x: 7 \\ny: 5' > results/coordinates.yaml""}]",
+4,"[{'main': ""printf 'x: 8 \\ny: 7' > results/coordinates.yaml""}]",
+5,"[{'main': ""printf 'x: 10 \\ny: 5' > results/coordinates.yaml""}]",
+6,"[{'main': ""printf 'x: 8 \\ny: 3' > results/coordinates.yaml""}]",
+7,"[{'main': ""printf 'x: 6 \\ny: 5' > results/coordinates.yaml""}]",
+8,"[{'main': ""printf 'x: 0\\ny: 0' > results/coordinates.yaml""}]",
+9,"[{'main': ""printf 'x: 0\\ny: 1' > results/coordinates.yaml""}]",
+10,"[{'main': ""printf 'x: 0\\ny: 2' > results/coordinates.yaml""}]",
+11,"[{'main': ""printf 'x: 1\\ny: 0' > results/coordinates.yaml""}]",
+12,"[{'main': ""printf 'x: 1\\ny: 1' > results/coordinates.yaml""}]",
+13,"[{'main': ""printf 'x: 1\\ny: 2' > results/coordinates.yaml""}]",
+14,"[{'main': ""printf 'x: 2\\ny: 0' > results/coordinates.yaml""}]",
+15,"[{'main': ""printf 'x: 2\\ny: 1' > results/coordinates.yaml""}]",
+16,"[{'main': ""printf 'x: 2\\ny: 2' > results/coordinates.yaml""}]",
+17,,"[{'main': ""printf 'x: 0\\ny: 3' > results/coordinates.yaml""}]"
+18,,"[{'main': ""printf 'x: 0\\ny: 4' > results/coordinates.yaml""}]"
+19,,"[{'main': ""printf 'x: 1\\ny: 3' > results/coordinates.yaml""}]"
+20,,"[{'main': ""printf 'x: 1\\ny: 4' > results/coordinates.yaml""}]"
+21,,"[{'main': ""printf 'x: 2\\ny: 3' > results/coordinates.yaml""}]"
+22,,"[{'main': ""printf 'x: 2\\ny: 4' > results/coordinates.yaml""}]"
+23,,"[{'main': ""printf 'x: -1\\ny: 3' > results/coordinates.yaml""}]"
+24,,"[{'main': ""printf 'x: 3\\ny: 3' > results/coordinates.yaml""}]"
+25,,"[{'main': ""printf 'x: 1\\ny: 5' > results/coordinates.yaml""}]"
diff --git a/...t/doe-suite-results/example07-etl_$expected/etl_results/commands_stage/commands_stage.csv b/...t/doe-suite-results/example07-etl_$expected/etl_results/commands_stage/commands_stage.csv
@@ -1,27 +1,27 @@
 ,$CMD$.small,$CMD$.small_v2
-0,"[{'main': ""printf 'x: 1\\ny: 1' > results/coordinates.yaml""}]",
-1,"[{'main': ""printf 'x: 2\\ny: 2' > results/coordinates.yaml""}]",
-2,"[{'main': ""printf 'x: 1\\ny: 0' > results/coordinates.yaml""}]",
-3,"[{'main': ""printf 'x: 2\\ny: 1' > results/coordinates.yaml""}]",
-4,"[{'main': ""printf 'x: 0\\ny: 1' > results/coordinates.yaml""}]",
-5,"[{'main': ""printf 'x: 0\\ny: 0' > results/coordinates.yaml""}]",
-6,"[{'main': ""printf 'x: 2\\ny: 0' > results/coordinates.yaml""}]",
-7,"[{'main': ""printf 'x: 0\\ny: 2' > results/coordinates.yaml""}]",
-8,"[{'main': ""printf 'x: 1\\ny: 2' > results/coordinates.yaml""}]",
-0,"[{'main': ""printf 'x: 8 \\ny: 7' > results/coordinates.yaml""}]",
-1,"[{'main': ""printf 'x: 7 \\ny: 5' > results/coordinates.yaml""}]",
-2,"[{'main': ""printf 'x: 6 \\ny: 5' > results/coordinates.yaml""}]",
-3,"[{'main': ""printf 'x: 9 \\ny: 5' > results/coordinates.yaml""}]",
-4,"[{'main': ""printf 'x: 8 \\ny: 6' > results/coordinates.yaml""}]",
-5,"[{'main': ""printf 'x: 8 \\ny: 3' > results/coordinates.yaml""}]",
-6,"[{'main': ""printf 'x: 8 \\ny: 4' > results/coordinates.yaml""}]",
-7,"[{'main': ""printf 'x: 10 \\ny: 5' > results/coordinates.yaml""}]",
-0,,"[{'main': ""printf 'x: 2\\ny: 3' > results/coordinates.yaml""}]"
-1,,"[{'main': ""printf 'x: 1\\ny: 4' > results/coordinates.yaml""}]"
-2,,"[{'main': ""printf 'x: 0\\ny: 4' > results/coordinates.yaml""}]"
-3,,"[{'main': ""printf 'x: 0\\ny: 3' > results/coordinates.yaml""}]"
-4,,"[{'main': ""printf 'x: 1\\ny: 3' > results/coordinates.yaml""}]"
-5,,"[{'main': ""printf 'x: 2\\ny: 4' > results/coordinates.yaml""}]"
-0,,"[{'main': ""printf 'x: 3\\ny: 3' > results/coordinates.yaml""}]"
-1,,"[{'main': ""printf 'x: -1\\ny: 3' > results/coordinates.yaml""}]"
-2,,"[{'main': ""printf 'x: 1\\ny: 5' > results/coordinates.yaml""}]"
+0,"[{'main': ""printf 'x: 8 \\ny: 6' > results/coordinates.yaml""}]",
+1,"[{'main': ""printf 'x: 9 \\ny: 5' > results/coordinates.yaml""}]",
+2,"[{'main': ""printf 'x: 8 \\ny: 4' > results/coordinates.yaml""}]",
+3,"[{'main': ""printf 'x: 7 \\ny: 5' > results/coordinates.yaml""}]",
+4,"[{'main': ""printf 'x: 8 \\ny: 7' > results/coordinates.yaml""}]",
+5,"[{'main': ""printf 'x: 10 \\ny: 5' > results/coordinates.yaml""}]",
+6,"[{'main': ""printf 'x: 8 \\ny: 3' > results/coordinates.yaml""}]",
+7,"[{'main': ""printf 'x: 6 \\ny: 5' > results/coordinates.yaml""}]",
+8,"[{'main': ""printf 'x: 0\\ny: 0' > results/coordinates.yaml""}]",
+9,"[{'main': ""printf 'x: 0\\ny: 1' > results/coordinates.yaml""}]",
+10,"[{'main': ""printf 'x: 0\\ny: 2' > results/coordinates.yaml""}]",
+11,"[{'main': ""printf 'x: 1\\ny: 0' > results/coordinates.yaml""}]",
+12,"[{'main': ""printf 'x: 1\\ny: 1' > results/coordinates.yaml""}]",
+13,"[{'main': ""printf 'x: 1\\ny: 2' > results/coordinates.yaml""}]",
+14,"[{'main': ""printf 'x: 2\\ny: 0' > results/coordinates.yaml""}]",
+15,"[{'main': ""printf 'x: 2\\ny: 1' > results/coordinates.yaml""}]",
+16,"[{'main': ""printf 'x: 2\\ny: 2' > results/coordinates.yaml""}]",
+17,,"[{'main': ""printf 'x: 0\\ny: 3' > results/coordinates.yaml""}]"
+18,,"[{'main': ""printf 'x: 0\\ny: 4' > results/coordinates.yaml""}]"
+19,,"[{'main': ""printf 'x: 1\\ny: 3' > results/coordinates.yaml""}]"
+20,,"[{'main': ""printf 'x: 1\\ny: 4' > results/coordinates.yaml""}]"
+21,,"[{'main': ""printf 'x: 2\\ny: 3' > results/coordinates.yaml""}]"
+22,,"[{'main': ""printf 'x: 2\\ny: 4' > results/coordinates.yaml""}]"
+23,,"[{'main': ""printf 'x: -1\\ny: 3' > results/coordinates.yaml""}]"
+24,,"[{'main': ""printf 'x: 3\\ny: 3' > results/coordinates.yaml""}]"
+25,,"[{'main': ""printf 'x: 1\\ny: 5' > results/coordinates.yaml""}]"
diff --git a/demo_project/doe-suite-results/example07-etl_$expected/etl_results/coord_all/coord_all.csv b/demo_project/doe-suite-results/example07-etl_$expected/etl_results/coord_all/coord_all.csv
@@ -1,27 +1,27 @@
 ,exp_name,x,y,color
-0,square,1,1,green
-1,square,2,2,green
-2,square,1,0,green
-3,square,2,1,green
-4,square,0,1,green
-5,square,0,0,green
-6,square,2,0,green
-7,square,0,2,green
-8,square,1,2,green
-0,plus,8,7,black
-1,plus,7,5,black
-2,plus,6,5,black
-3,plus,9,5,black
-4,plus,8,6,black
-5,plus,8,3,black
-6,plus,8,4,black
-7,plus,10,5,black
-0,triangle1,2,3,blue
-1,triangle1,1,4,blue
-2,triangle1,0,4,blue
-3,triangle1,0,3,blue
-4,triangle1,1,3,blue
-5,triangle1,2,4,blue
-0,triangle2,3,3,blue
-1,triangle2,-1,3,blue
-2,triangle2,1,5,blue
+0,plus,8,6,black
+1,plus,9,5,black
+2,plus,8,4,black
+3,plus,7,5,black
+4,plus,8,7,black
+5,plus,10,5,black
+6,plus,8,3,black
+7,plus,6,5,black
+8,square,0,0,green
+9,square,0,1,green
+10,square,0,2,green
+11,square,1,0,green
+12,square,1,1,green
+13,square,1,2,green
+14,square,2,0,green
+15,square,2,1,green
+16,square,2,2,green
+17,triangle1,0,3,blue
+18,triangle1,0,4,blue
+19,triangle1,1,3,blue
+20,triangle1,1,4,blue
+21,triangle1,2,3,blue
+22,triangle1,2,4,blue
+23,triangle2,-1,3,blue
+24,triangle2,3,3,blue
+25,triangle2,1,5,blue
diff --git a/demo_project/doe-suite-results/example07-etl_$expected/etl_results/coord_plus/coord_plus.csv b/demo_project/doe-suite-results/example07-etl_$expected/etl_results/coord_plus/coord_plus.csv
@@ -1,9 +1,9 @@
 ,exp_name,x,y,color
-0,plus,8,7,black
-1,plus,7,5,black
-2,plus,6,5,black
-3,plus,9,5,black
-4,plus,8,6,black
-5,plus,8,3,black
-6,plus,8,4,black
-7,plus,10,5,black
+0,plus,8,6,black
+1,plus,9,5,black
+2,plus,8,4,black
+3,plus,7,5,black
+4,plus,8,7,black
+5,plus,10,5,black
+6,plus,8,3,black
+7,plus,6,5,black
diff --git a/...oject/doe-suite-results/example07-etl_$expected/etl_results/coord_square/coord_square.csv b/...oject/doe-suite-results/example07-etl_$expected/etl_results/coord_square/coord_square.csv
@@ -1,10 +1,10 @@
 ,exp_name,x,y,color
-0,square,1,1,black
-1,square,2,2,black
-2,square,1,0,black
-3,square,2,1,black
-4,square,0,1,black
-5,square,0,0,black
+0,square,0,0,black
+1,square,0,1,black
+2,square,0,2,black
+3,square,1,0,black
+4,square,1,1,black
+5,square,1,2,black
 6,square,2,0,black
-7,square,0,2,black
-8,square,1,2,black
+7,square,2,1,black
+8,square,2,2,black
diff --git a/...t/doe-suite-results/example07-etl_$expected/etl_results/coord_triangle/coord_triangle.csv b/...t/doe-suite-results/example07-etl_$expected/etl_results/coord_triangle/coord_triangle.csv
@@ -1,10 +1,10 @@
 ,exp_name,x,y,color
-0,triangle1,2,3,black
-1,triangle1,1,4,black
-2,triangle1,0,4,black
-3,triangle1,0,3,black
-4,triangle1,1,3,black
+0,triangle1,0,3,black
+1,triangle1,0,4,black
+2,triangle1,1,3,black
+3,triangle1,1,4,black
+4,triangle1,2,3,black
 5,triangle1,2,4,black
-0,triangle2,3,3,black
-1,triangle2,-1,3,black
-2,triangle2,1,5,black
+6,triangle2,-1,3,black
+7,triangle2,3,3,black
+8,triangle2,1,5,black
diff --git a/demo_project/doe-suite-results/example07-etl_$expected/suite_design.yml b/demo_project/doe-suite-results/example07-etl_$expected/suite_design.yml
@@ -112,6 +112,12 @@ $ETL$:
         file_regex:
         - ^stdout.log$
     transformers:
+    - df.sort_values:
+        by:
+        - exp_name
+        - run
+        - rep
+        ignore_index: true
     - df.filter:
         items:
         - exp_name
@@ -138,6 +144,12 @@ $ETL$:
         file_regex:
         - ^stdout.log$
     transformers:
+    - df.sort_values:
+        by:
+        - exp_name
+        - run
+        - rep
+        ignore_index: true
     - df.filter:
         items:
         - exp_name
@@ -165,6 +177,12 @@ $ETL$:
         file_regex:
         - ^stdout.log$
     transformers:
+    - df.sort_values:
+        by:
+        - exp_name
+        - run
+        - rep
+        ignore_index: true
     - df.filter:
         items:
         - exp_name
@@ -194,6 +212,12 @@ $ETL$:
         file_regex:
         - ^stdout.log$
     transformers:
+    - df.sort_values:
+        by:
+        - exp_name
+        - run
+        - rep
+        ignore_index: true
     - df.filter:
         items:
         - exp_name
@@ -229,6 +253,12 @@ $ETL$:
         file_regex:
         - ^stdout.log$
     transformers:
+    - df.sort_values:
+        by:
+        - exp_name
+        - run
+        - rep
+        ignore_index: true
     - df.filter:
         regex: \$CMD\$
     loaders:
@@ -252,6 +282,12 @@ $ETL$:
         file_regex:
         - ^stdout.log$
     transformers:
+    - df.sort_values:
+        by:
+        - exp_name
+        - run
+        - rep
+        ignore_index: true
     - df.filter:
         regex: \$CMD\$
     loaders: