diff --git a/demo_project/doe-suite-config/designs/etl_templates/config_template.yml b/demo_project/doe-suite-config/designs/etl_templates/config_template.yml index 370ed914..76920bcd 100644 --- a/demo_project/doe-suite-config/designs/etl_templates/config_template.yml +++ b/demo_project/doe-suite-config/designs/etl_templates/config_template.yml @@ -19,6 +19,7 @@ $ETL$: ErrorExtractor: {} # etl-error if stderr.log is non-empty IgnoreExtractor: {} # ignore stdout.log transformers: + - df.sort_values: { by: [ exp_name, run, rep ], ignore_index: True } - df.filter: {regex: "\\$CMD\\$"} loaders: CsvSummaryLoader: # write the transformed dataframe across the whole experiment as a csv file diff --git a/demo_project/doe-suite-config/designs/example07-etl.yml b/demo_project/doe-suite-config/designs/example07-etl.yml index f66b4227..c122ebaf 100644 --- a/demo_project/doe-suite-config/designs/example07-etl.yml +++ b/demo_project/doe-suite-config/designs/example07-etl.yml @@ -24,8 +24,11 @@ $ETL$: # Visualization o extractors: #-------------------------------------------------- YamlExtractor: {} # by default loads all .yaml files #| YamlExtractor ErrorExtractor IgnoreExtractor | Extractors: ErrorExtractor: {} # error stderr.log if non-empty #-------------------------------------------------- result files to pandas df - IgnoreExtractor: {} # ignore stdout.log # | Transformers: - transformers: # V transform df + IgnoreExtractor: {} # ignore stdout.log # Transformers: + transformers: # | transform df + - df.sort_values: # | + by: [ exp_name, run, rep ] # V + ignore_index: True # - df.filter: {items: ["exp_name", "x", "y"]} # df.filter - df.eval: {expr: "color = 'black'"} # | loaders: # V @@ -64,6 +67,7 @@ $ETL$: # Visualization o # by including all we have a more complex color assignment -> we only include extractors + loaders and provide a custom transformer stage # (2) use df.X syntax to directly use pandas df transformations: https://pandas.pydata.org/docs/reference/frame.html # (note there are a few limitations: some functions on df require indexes which cannot be defined here -> e.g., see Conditional Transformer) + - df.sort_values: { by: [ exp_name, run, rep ], ignore_index: True } - df.filter: {items: ["exp_name", "x", "y"]} - {name: ConditionalTransformer, col: "exp_name", dest: "color", value: {plus: black, square: green, triangle1: blue, triangle2: blue}} loaders: diff --git a/demo_project/doe-suite-results/example07-etl_$expected/etl_results/commands/commands.csv b/demo_project/doe-suite-results/example07-etl_$expected/etl_results/commands/commands.csv index d94475a7..eddb0a70 100644 --- a/demo_project/doe-suite-results/example07-etl_$expected/etl_results/commands/commands.csv +++ b/demo_project/doe-suite-results/example07-etl_$expected/etl_results/commands/commands.csv @@ -1,27 +1,27 @@ ,$CMD$.small,$CMD$.small_v2 -0,"[{'main': ""printf 'x: 1\\ny: 1' > results/coordinates.yaml""}]", -1,"[{'main': ""printf 'x: 2\\ny: 2' > results/coordinates.yaml""}]", -2,"[{'main': ""printf 'x: 1\\ny: 0' > results/coordinates.yaml""}]", -3,"[{'main': ""printf 'x: 2\\ny: 1' > results/coordinates.yaml""}]", -4,"[{'main': ""printf 'x: 0\\ny: 1' > results/coordinates.yaml""}]", -5,"[{'main': ""printf 'x: 0\\ny: 0' > results/coordinates.yaml""}]", -6,"[{'main': ""printf 'x: 2\\ny: 0' > results/coordinates.yaml""}]", -7,"[{'main': ""printf 'x: 0\\ny: 2' > results/coordinates.yaml""}]", -8,"[{'main': ""printf 'x: 1\\ny: 2' > results/coordinates.yaml""}]", -0,"[{'main': ""printf 'x: 8 \\ny: 7' > results/coordinates.yaml""}]", -1,"[{'main': ""printf 'x: 7 \\ny: 5' > results/coordinates.yaml""}]", -2,"[{'main': ""printf 'x: 6 \\ny: 5' > results/coordinates.yaml""}]", -3,"[{'main': ""printf 'x: 9 \\ny: 5' > results/coordinates.yaml""}]", -4,"[{'main': ""printf 'x: 8 \\ny: 6' > results/coordinates.yaml""}]", -5,"[{'main': ""printf 'x: 8 \\ny: 3' > results/coordinates.yaml""}]", -6,"[{'main': ""printf 'x: 8 \\ny: 4' > results/coordinates.yaml""}]", -7,"[{'main': ""printf 'x: 10 \\ny: 5' > results/coordinates.yaml""}]", -0,,"[{'main': ""printf 'x: 2\\ny: 3' > results/coordinates.yaml""}]" -1,,"[{'main': ""printf 'x: 1\\ny: 4' > results/coordinates.yaml""}]" -2,,"[{'main': ""printf 'x: 0\\ny: 4' > results/coordinates.yaml""}]" -3,,"[{'main': ""printf 'x: 0\\ny: 3' > results/coordinates.yaml""}]" -4,,"[{'main': ""printf 'x: 1\\ny: 3' > results/coordinates.yaml""}]" -5,,"[{'main': ""printf 'x: 2\\ny: 4' > results/coordinates.yaml""}]" -0,,"[{'main': ""printf 'x: 3\\ny: 3' > results/coordinates.yaml""}]" -1,,"[{'main': ""printf 'x: -1\\ny: 3' > results/coordinates.yaml""}]" -2,,"[{'main': ""printf 'x: 1\\ny: 5' > results/coordinates.yaml""}]" +0,"[{'main': ""printf 'x: 8 \\ny: 6' > results/coordinates.yaml""}]", +1,"[{'main': ""printf 'x: 9 \\ny: 5' > results/coordinates.yaml""}]", +2,"[{'main': ""printf 'x: 8 \\ny: 4' > results/coordinates.yaml""}]", +3,"[{'main': ""printf 'x: 7 \\ny: 5' > results/coordinates.yaml""}]", +4,"[{'main': ""printf 'x: 8 \\ny: 7' > results/coordinates.yaml""}]", +5,"[{'main': ""printf 'x: 10 \\ny: 5' > results/coordinates.yaml""}]", +6,"[{'main': ""printf 'x: 8 \\ny: 3' > results/coordinates.yaml""}]", +7,"[{'main': ""printf 'x: 6 \\ny: 5' > results/coordinates.yaml""}]", +8,"[{'main': ""printf 'x: 0\\ny: 0' > results/coordinates.yaml""}]", +9,"[{'main': ""printf 'x: 0\\ny: 1' > results/coordinates.yaml""}]", +10,"[{'main': ""printf 'x: 0\\ny: 2' > results/coordinates.yaml""}]", +11,"[{'main': ""printf 'x: 1\\ny: 0' > results/coordinates.yaml""}]", +12,"[{'main': ""printf 'x: 1\\ny: 1' > results/coordinates.yaml""}]", +13,"[{'main': ""printf 'x: 1\\ny: 2' > results/coordinates.yaml""}]", +14,"[{'main': ""printf 'x: 2\\ny: 0' > results/coordinates.yaml""}]", +15,"[{'main': ""printf 'x: 2\\ny: 1' > results/coordinates.yaml""}]", +16,"[{'main': ""printf 'x: 2\\ny: 2' > results/coordinates.yaml""}]", +17,,"[{'main': ""printf 'x: 0\\ny: 3' > results/coordinates.yaml""}]" +18,,"[{'main': ""printf 'x: 0\\ny: 4' > results/coordinates.yaml""}]" +19,,"[{'main': ""printf 'x: 1\\ny: 3' > results/coordinates.yaml""}]" +20,,"[{'main': ""printf 'x: 1\\ny: 4' > results/coordinates.yaml""}]" +21,,"[{'main': ""printf 'x: 2\\ny: 3' > results/coordinates.yaml""}]" +22,,"[{'main': ""printf 'x: 2\\ny: 4' > results/coordinates.yaml""}]" +23,,"[{'main': ""printf 'x: -1\\ny: 3' > results/coordinates.yaml""}]" +24,,"[{'main': ""printf 'x: 3\\ny: 3' > results/coordinates.yaml""}]" +25,,"[{'main': ""printf 'x: 1\\ny: 5' > results/coordinates.yaml""}]" diff --git a/demo_project/doe-suite-results/example07-etl_$expected/etl_results/commands_stage/commands_stage.csv b/demo_project/doe-suite-results/example07-etl_$expected/etl_results/commands_stage/commands_stage.csv index d94475a7..eddb0a70 100644 --- a/demo_project/doe-suite-results/example07-etl_$expected/etl_results/commands_stage/commands_stage.csv +++ b/demo_project/doe-suite-results/example07-etl_$expected/etl_results/commands_stage/commands_stage.csv @@ -1,27 +1,27 @@ ,$CMD$.small,$CMD$.small_v2 -0,"[{'main': ""printf 'x: 1\\ny: 1' > results/coordinates.yaml""}]", -1,"[{'main': ""printf 'x: 2\\ny: 2' > results/coordinates.yaml""}]", -2,"[{'main': ""printf 'x: 1\\ny: 0' > results/coordinates.yaml""}]", -3,"[{'main': ""printf 'x: 2\\ny: 1' > results/coordinates.yaml""}]", -4,"[{'main': ""printf 'x: 0\\ny: 1' > results/coordinates.yaml""}]", -5,"[{'main': ""printf 'x: 0\\ny: 0' > results/coordinates.yaml""}]", -6,"[{'main': ""printf 'x: 2\\ny: 0' > results/coordinates.yaml""}]", -7,"[{'main': ""printf 'x: 0\\ny: 2' > results/coordinates.yaml""}]", -8,"[{'main': ""printf 'x: 1\\ny: 2' > results/coordinates.yaml""}]", -0,"[{'main': ""printf 'x: 8 \\ny: 7' > results/coordinates.yaml""}]", -1,"[{'main': ""printf 'x: 7 \\ny: 5' > results/coordinates.yaml""}]", -2,"[{'main': ""printf 'x: 6 \\ny: 5' > results/coordinates.yaml""}]", -3,"[{'main': ""printf 'x: 9 \\ny: 5' > results/coordinates.yaml""}]", -4,"[{'main': ""printf 'x: 8 \\ny: 6' > results/coordinates.yaml""}]", -5,"[{'main': ""printf 'x: 8 \\ny: 3' > results/coordinates.yaml""}]", -6,"[{'main': ""printf 'x: 8 \\ny: 4' > results/coordinates.yaml""}]", -7,"[{'main': ""printf 'x: 10 \\ny: 5' > results/coordinates.yaml""}]", -0,,"[{'main': ""printf 'x: 2\\ny: 3' > results/coordinates.yaml""}]" -1,,"[{'main': ""printf 'x: 1\\ny: 4' > results/coordinates.yaml""}]" -2,,"[{'main': ""printf 'x: 0\\ny: 4' > results/coordinates.yaml""}]" -3,,"[{'main': ""printf 'x: 0\\ny: 3' > results/coordinates.yaml""}]" -4,,"[{'main': ""printf 'x: 1\\ny: 3' > results/coordinates.yaml""}]" -5,,"[{'main': ""printf 'x: 2\\ny: 4' > results/coordinates.yaml""}]" -0,,"[{'main': ""printf 'x: 3\\ny: 3' > results/coordinates.yaml""}]" -1,,"[{'main': ""printf 'x: -1\\ny: 3' > results/coordinates.yaml""}]" -2,,"[{'main': ""printf 'x: 1\\ny: 5' > results/coordinates.yaml""}]" +0,"[{'main': ""printf 'x: 8 \\ny: 6' > results/coordinates.yaml""}]", +1,"[{'main': ""printf 'x: 9 \\ny: 5' > results/coordinates.yaml""}]", +2,"[{'main': ""printf 'x: 8 \\ny: 4' > results/coordinates.yaml""}]", +3,"[{'main': ""printf 'x: 7 \\ny: 5' > results/coordinates.yaml""}]", +4,"[{'main': ""printf 'x: 8 \\ny: 7' > results/coordinates.yaml""}]", +5,"[{'main': ""printf 'x: 10 \\ny: 5' > results/coordinates.yaml""}]", +6,"[{'main': ""printf 'x: 8 \\ny: 3' > results/coordinates.yaml""}]", +7,"[{'main': ""printf 'x: 6 \\ny: 5' > results/coordinates.yaml""}]", +8,"[{'main': ""printf 'x: 0\\ny: 0' > results/coordinates.yaml""}]", +9,"[{'main': ""printf 'x: 0\\ny: 1' > results/coordinates.yaml""}]", +10,"[{'main': ""printf 'x: 0\\ny: 2' > results/coordinates.yaml""}]", +11,"[{'main': ""printf 'x: 1\\ny: 0' > results/coordinates.yaml""}]", +12,"[{'main': ""printf 'x: 1\\ny: 1' > results/coordinates.yaml""}]", +13,"[{'main': ""printf 'x: 1\\ny: 2' > results/coordinates.yaml""}]", +14,"[{'main': ""printf 'x: 2\\ny: 0' > results/coordinates.yaml""}]", +15,"[{'main': ""printf 'x: 2\\ny: 1' > results/coordinates.yaml""}]", +16,"[{'main': ""printf 'x: 2\\ny: 2' > results/coordinates.yaml""}]", +17,,"[{'main': ""printf 'x: 0\\ny: 3' > results/coordinates.yaml""}]" +18,,"[{'main': ""printf 'x: 0\\ny: 4' > results/coordinates.yaml""}]" +19,,"[{'main': ""printf 'x: 1\\ny: 3' > results/coordinates.yaml""}]" +20,,"[{'main': ""printf 'x: 1\\ny: 4' > results/coordinates.yaml""}]" +21,,"[{'main': ""printf 'x: 2\\ny: 3' > results/coordinates.yaml""}]" +22,,"[{'main': ""printf 'x: 2\\ny: 4' > results/coordinates.yaml""}]" +23,,"[{'main': ""printf 'x: -1\\ny: 3' > results/coordinates.yaml""}]" +24,,"[{'main': ""printf 'x: 3\\ny: 3' > results/coordinates.yaml""}]" +25,,"[{'main': ""printf 'x: 1\\ny: 5' > results/coordinates.yaml""}]" diff --git a/demo_project/doe-suite-results/example07-etl_$expected/etl_results/coord_all/coord_all.csv b/demo_project/doe-suite-results/example07-etl_$expected/etl_results/coord_all/coord_all.csv index 520690a5..74b6ff16 100644 --- a/demo_project/doe-suite-results/example07-etl_$expected/etl_results/coord_all/coord_all.csv +++ b/demo_project/doe-suite-results/example07-etl_$expected/etl_results/coord_all/coord_all.csv @@ -1,27 +1,27 @@ ,exp_name,x,y,color -0,square,1,1,green -1,square,2,2,green -2,square,1,0,green -3,square,2,1,green -4,square,0,1,green -5,square,0,0,green -6,square,2,0,green -7,square,0,2,green -8,square,1,2,green -0,plus,8,7,black -1,plus,7,5,black -2,plus,6,5,black -3,plus,9,5,black -4,plus,8,6,black -5,plus,8,3,black -6,plus,8,4,black -7,plus,10,5,black -0,triangle1,2,3,blue -1,triangle1,1,4,blue -2,triangle1,0,4,blue -3,triangle1,0,3,blue -4,triangle1,1,3,blue -5,triangle1,2,4,blue -0,triangle2,3,3,blue -1,triangle2,-1,3,blue -2,triangle2,1,5,blue +0,plus,8,6,black +1,plus,9,5,black +2,plus,8,4,black +3,plus,7,5,black +4,plus,8,7,black +5,plus,10,5,black +6,plus,8,3,black +7,plus,6,5,black +8,square,0,0,green +9,square,0,1,green +10,square,0,2,green +11,square,1,0,green +12,square,1,1,green +13,square,1,2,green +14,square,2,0,green +15,square,2,1,green +16,square,2,2,green +17,triangle1,0,3,blue +18,triangle1,0,4,blue +19,triangle1,1,3,blue +20,triangle1,1,4,blue +21,triangle1,2,3,blue +22,triangle1,2,4,blue +23,triangle2,-1,3,blue +24,triangle2,3,3,blue +25,triangle2,1,5,blue diff --git a/demo_project/doe-suite-results/example07-etl_$expected/etl_results/coord_plus/coord_plus.csv b/demo_project/doe-suite-results/example07-etl_$expected/etl_results/coord_plus/coord_plus.csv index 4b2cbc68..740cbbe5 100644 --- a/demo_project/doe-suite-results/example07-etl_$expected/etl_results/coord_plus/coord_plus.csv +++ b/demo_project/doe-suite-results/example07-etl_$expected/etl_results/coord_plus/coord_plus.csv @@ -1,9 +1,9 @@ ,exp_name,x,y,color -0,plus,8,7,black -1,plus,7,5,black -2,plus,6,5,black -3,plus,9,5,black -4,plus,8,6,black -5,plus,8,3,black -6,plus,8,4,black -7,plus,10,5,black +0,plus,8,6,black +1,plus,9,5,black +2,plus,8,4,black +3,plus,7,5,black +4,plus,8,7,black +5,plus,10,5,black +6,plus,8,3,black +7,plus,6,5,black diff --git a/demo_project/doe-suite-results/example07-etl_$expected/etl_results/coord_square/coord_square.csv b/demo_project/doe-suite-results/example07-etl_$expected/etl_results/coord_square/coord_square.csv index 5b0aa293..8327248a 100644 --- a/demo_project/doe-suite-results/example07-etl_$expected/etl_results/coord_square/coord_square.csv +++ b/demo_project/doe-suite-results/example07-etl_$expected/etl_results/coord_square/coord_square.csv @@ -1,10 +1,10 @@ ,exp_name,x,y,color -0,square,1,1,black -1,square,2,2,black -2,square,1,0,black -3,square,2,1,black -4,square,0,1,black -5,square,0,0,black +0,square,0,0,black +1,square,0,1,black +2,square,0,2,black +3,square,1,0,black +4,square,1,1,black +5,square,1,2,black 6,square,2,0,black -7,square,0,2,black -8,square,1,2,black +7,square,2,1,black +8,square,2,2,black diff --git a/demo_project/doe-suite-results/example07-etl_$expected/etl_results/coord_triangle/coord_triangle.csv b/demo_project/doe-suite-results/example07-etl_$expected/etl_results/coord_triangle/coord_triangle.csv index 857d54de..ff359a6f 100644 --- a/demo_project/doe-suite-results/example07-etl_$expected/etl_results/coord_triangle/coord_triangle.csv +++ b/demo_project/doe-suite-results/example07-etl_$expected/etl_results/coord_triangle/coord_triangle.csv @@ -1,10 +1,10 @@ ,exp_name,x,y,color -0,triangle1,2,3,black -1,triangle1,1,4,black -2,triangle1,0,4,black -3,triangle1,0,3,black -4,triangle1,1,3,black +0,triangle1,0,3,black +1,triangle1,0,4,black +2,triangle1,1,3,black +3,triangle1,1,4,black +4,triangle1,2,3,black 5,triangle1,2,4,black -0,triangle2,3,3,black -1,triangle2,-1,3,black -2,triangle2,1,5,black +6,triangle2,-1,3,black +7,triangle2,3,3,black +8,triangle2,1,5,black diff --git a/demo_project/doe-suite-results/example07-etl_$expected/suite_design.yml b/demo_project/doe-suite-results/example07-etl_$expected/suite_design.yml index 8b3dbfad..8f9212b0 100644 --- a/demo_project/doe-suite-results/example07-etl_$expected/suite_design.yml +++ b/demo_project/doe-suite-results/example07-etl_$expected/suite_design.yml @@ -112,6 +112,12 @@ $ETL$: file_regex: - ^stdout.log$ transformers: + - df.sort_values: + by: + - exp_name + - run + - rep + ignore_index: true - df.filter: items: - exp_name @@ -138,6 +144,12 @@ $ETL$: file_regex: - ^stdout.log$ transformers: + - df.sort_values: + by: + - exp_name + - run + - rep + ignore_index: true - df.filter: items: - exp_name @@ -165,6 +177,12 @@ $ETL$: file_regex: - ^stdout.log$ transformers: + - df.sort_values: + by: + - exp_name + - run + - rep + ignore_index: true - df.filter: items: - exp_name @@ -194,6 +212,12 @@ $ETL$: file_regex: - ^stdout.log$ transformers: + - df.sort_values: + by: + - exp_name + - run + - rep + ignore_index: true - df.filter: items: - exp_name @@ -229,6 +253,12 @@ $ETL$: file_regex: - ^stdout.log$ transformers: + - df.sort_values: + by: + - exp_name + - run + - rep + ignore_index: true - df.filter: regex: \$CMD\$ loaders: @@ -252,6 +282,12 @@ $ETL$: file_regex: - ^stdout.log$ transformers: + - df.sort_values: + by: + - exp_name + - run + - rep + ignore_index: true - df.filter: regex: \$CMD\$ loaders: