From 10e68ea74a7f73d2cfa57c13bee8075d9d2086a6 Mon Sep 17 00:00:00 2001 From: Max Berrendorf Date: Sun, 21 Jun 2020 12:46:50 +0200 Subject: [PATCH 1/7] Update configs --- ablation/test_skyline.py | 13 +++++ .../complex/trouillon2016_complex_fb15k.json | 14 ++++- .../complex/trouillon2016_complex_wn18.json | 16 +++++- .../conve/dettmers2018_conve_fb15k.json | 22 ++++++-- .../conve/dettmers2018_conve_fb15k237.json | 22 ++++++-- .../conve/dettmers2018_conve_wn18.json | 22 ++++++-- .../conve/dettmers2018_conve_wn18rr.json | 24 +++++++-- .../convkb/nguyen2018_convkb_fb15k237.json | 16 ++++-- .../convkb/nguyen2018_convkb_wn18rr.json | 16 ++++-- .../convkb/nguyen2018_transe_fb15k237.json | 3 -- .../convkb/nguyen2018_transe_wn18rr.json | 5 +- .../distmult/yang2014_distmult_fb15k.json | 10 ++++ .../distmult/yang2014_distmult_wn18.json | 10 ++++ .../configs/hole/nickel2016_hole_fb15k.json | 18 +++++-- .../configs/hole/nickel2016_hole_wn18.json | 16 +++++- .../configs/kg2e/he2015_kg2e_fb15k.json | 10 ++++ .../configs/kg2e/he2015_kg2e_wn18.json | 10 ++++ .../rgcn/schlichtkrull2018_rgcn_fb15k.json | 3 +- .../rgcn/schlichtkrull2018_rgcn_wn18.json | 3 +- .../configs/rotate/sun2019_rotate_fb15k.json | 19 ++++++- .../rotate/sun2019_rotate_fb15k237.json | 50 +++++++++++++++++ .../configs/rotate/sun2019_rotate_wn18.json | 21 ++++++-- .../configs/rotate/sun2019_rotate_wn18rr.json | 50 +++++++++++++++++ .../simple/kazemi2018_simple_fb15k.json | 17 +++++- .../simple/kazemi2018_simple_wn18.json | 19 +++++-- .../configs/transd/ji2015_transd_fb15k.json | 10 ++++ .../configs/transd/ji2015_transd_wn18.json | 10 ++++ .../transe/bordes2013_transe_fb15k.json | 10 ++++ .../transe/bordes2013_transe_wn18.json | 10 ++++ .../configs/transh/wang2014_transh_fb15k.json | 10 ++++ .../configs/transh/wang2014_transh_wn18.json | 10 ++++ .../configs/transr/li2015_transr_fb15k.json | 12 ++++- .../configs/transr/li2015_transr_wn18.json | 12 ++++- .../tucker/balazevic2019_tucker_fb15k.json | 17 +++++- .../tucker/balazevic2019_tucker_fb15k237.json | 51 ++++++++++++++++++ .../tucker/balazevic2019_tucker_wn18.json | 17 +++++- .../tucker/balazevic2019_tucker_wn18rr.json | 51 ++++++++++++++++++ reproducibility/generate_summary_table.py | 7 +++ reproducibility/summaries/results.pdf | Bin 53441 -> 53443 bytes 39 files changed, 598 insertions(+), 58 deletions(-) create mode 100644 ablation/test_skyline.py create mode 100644 reproducibility/configs/rotate/sun2019_rotate_fb15k237.json create mode 100644 reproducibility/configs/rotate/sun2019_rotate_wn18rr.json create mode 100644 reproducibility/configs/tucker/balazevic2019_tucker_fb15k237.json create mode 100644 reproducibility/configs/tucker/balazevic2019_tucker_wn18rr.json diff --git a/ablation/test_skyline.py b/ablation/test_skyline.py new file mode 100644 index 000000000..eeabe3c45 --- /dev/null +++ b/ablation/test_skyline.py @@ -0,0 +1,13 @@ +from collate import read_collation + +from pykeen_report.plot import make_sizeplots_trellised + +if __name__ == '__main__': + df = read_collation() + + make_sizeplots_trellised( + df=df, + target_x_header='model_bytes', + target_y_header='hits@10', + output_directory='/tmp/plot_tmp', + ) diff --git a/reproducibility/configs/complex/trouillon2016_complex_fb15k.json b/reproducibility/configs/complex/trouillon2016_complex_fb15k.json index a9a1faeb1..152129510 100644 --- a/reproducibility/configs/complex/trouillon2016_complex_fb15k.json +++ b/reproducibility/configs/complex/trouillon2016_complex_fb15k.json @@ -1,7 +1,7 @@ { "metadata": { "title": "Learn FB15K Data Set with ComplEx as described by Trouillon et al., 2016", - "comments": "In the paper it is not mentioned that they compute the mean of the vector norms as done in the published code base." + "comments": "In the paper it is not mentioned that they compute the mean of the vector norms as done in the published code base. The rank type can be found at https://github.com/ttrouill/complex/blob/67fef2324d5a7695f7b932b5606d2e4b6de09a41/efe/evaluation.py#L295." }, "pipeline": { "dataset": "fb15k", @@ -36,5 +36,17 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "mean_reciprocal_rank": { + "best": 0.692 + }, + "hits_at_k": { + "best": { + "1": 0.599, + "3": 0.759, + "10": 0.840 + } + } } } \ No newline at end of file diff --git a/reproducibility/configs/complex/trouillon2016_complex_wn18.json b/reproducibility/configs/complex/trouillon2016_complex_wn18.json index 2d4d2e8e1..a5586d14d 100644 --- a/reproducibility/configs/complex/trouillon2016_complex_wn18.json +++ b/reproducibility/configs/complex/trouillon2016_complex_wn18.json @@ -1,7 +1,7 @@ { "metadata": { "title": "Learn WN18 Data Set with ComplEx as described by Trouillon et al., 2016", - "comments": "In the paper it is not mentioned that they compute the mean of the vector norms as done in the published code base." + "comments": "In the paper it is not mentioned that they compute the mean of the vector norms as done in the published code base. The rank type can be found at https://github.com/ttrouill/complex/blob/67fef2324d5a7695f7b932b5606d2e4b6de09a41/efe/evaluation.py#L295." }, "pipeline": { "dataset": "wn18", @@ -36,5 +36,17 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "mean_reciprocal_rank": { + "best": 0.941 + }, + "hits_at_k": { + "best": { + "1": 0.936, + "3": 0.945, + "10": 0.947 + } + } } -} +} \ No newline at end of file diff --git a/reproducibility/configs/conve/dettmers2018_conve_fb15k.json b/reproducibility/configs/conve/dettmers2018_conve_fb15k.json index 71889876e..a2a0013df 100644 --- a/reproducibility/configs/conve/dettmers2018_conve_fb15k.json +++ b/reproducibility/configs/conve/dettmers2018_conve_fb15k.json @@ -1,6 +1,7 @@ { "metadata": { - "title": "Learn FB15K Data Set with ConvE as described by Dettmers et al., 2018" + "title": "Learn FB15K Data Set with ConvE as described by Dettmers et al., 2018", + "comment": "For evaluation, the non-deterministic rank definition is used, cf. https://github.com/TimDettmers/ConvE/blob/5feb358eb7dbd1f534978cdc4c20ee0bf919148a/evaluation.py#L67." }, "pipeline": { "dataset": "fb15k", @@ -18,7 +19,8 @@ "kernel_width": 3, "input_dropout": 0.2, "feature_map_dropout": 0.2, - "output_dropout": 0.3 + "output_dropout": 0.3, + "apply_batch_normalization": true }, "optimizer": "Adam", "optimizer_kwargs": { @@ -36,9 +38,21 @@ }, "evaluator_kwargs": { "filtered": true + } + }, + "results": { + "mean_rank": { + "nondeterministic": 51 + }, + "hits_at_k": { + "nondeterministic": { + "1": 0.558, + "3": 0.723, + "10": 0.831 + } }, - "evaluation_kwargs": { - "batch_size": 16 + "mean_reciprocal_rank": { + "nondeterministic": 0.657 } } } \ No newline at end of file diff --git a/reproducibility/configs/conve/dettmers2018_conve_fb15k237.json b/reproducibility/configs/conve/dettmers2018_conve_fb15k237.json index 9d232c5dc..45286cafb 100644 --- a/reproducibility/configs/conve/dettmers2018_conve_fb15k237.json +++ b/reproducibility/configs/conve/dettmers2018_conve_fb15k237.json @@ -1,6 +1,7 @@ { "metadata": { - "title": "Learn FB15K-237 Data Set with ConvE as described by Dettmers et al., 2018" + "title": "Learn FB15K-237 Data Set with ConvE as described by Dettmers et al., 2018", + "comment": "For evaluation, the non-deterministic rank definition is used, cf. https://github.com/TimDettmers/ConvE/blob/5feb358eb7dbd1f534978cdc4c20ee0bf919148a/evaluation.py#L67." }, "pipeline": { "dataset": "fb15k237", @@ -18,7 +19,8 @@ "kernel_width": 3, "input_dropout": 0.2, "feature_map_dropout": 0.2, - "output_dropout": 0.3 + "output_dropout": 0.3, + "apply_batch_normalization": true }, "optimizer": "Adam", "optimizer_kwargs": { @@ -36,9 +38,21 @@ }, "evaluator_kwargs": { "filtered": true + } + }, + "results": { + "mean_rank": { + "nondeterministic": 244 + }, + "hits_at_k": { + "nondeterministic": { + "1": 0.237, + "3": 0.356, + "10": 0.501 + } }, - "evaluation_kwargs": { - "batch_size": 16 + "mean_reciprocal_rank": { + "nondeterministic": 0.325 } } } \ No newline at end of file diff --git a/reproducibility/configs/conve/dettmers2018_conve_wn18.json b/reproducibility/configs/conve/dettmers2018_conve_wn18.json index aafd08a6b..89182e8fe 100644 --- a/reproducibility/configs/conve/dettmers2018_conve_wn18.json +++ b/reproducibility/configs/conve/dettmers2018_conve_wn18.json @@ -1,6 +1,7 @@ { "metadata": { - "title": "Learn WN18 Data Set with ConvE as described by Dettmers et al., 2018" + "title": "Learn WN18 Data Set with ConvE as described by Dettmers et al., 2018", + "comment": "For evaluation, the non-deterministic rank definition is used, cf. https://github.com/TimDettmers/ConvE/blob/5feb358eb7dbd1f534978cdc4c20ee0bf919148a/evaluation.py#L67." }, "pipeline": { "dataset": "wn18", @@ -18,7 +19,8 @@ "kernel_width": 3, "input_dropout": 0.2, "feature_map_dropout": 0.2, - "output_dropout": 0.3 + "output_dropout": 0.3, + "apply_batch_normalization": true }, "optimizer": "Adam", "optimizer_kwargs": { @@ -36,9 +38,21 @@ }, "evaluator_kwargs": { "filtered": true + } + }, + "results": { + "mean_rank": { + "nondeterministic": 374 + }, + "hits_at_k": { + "nondeterministic": { + "1": 0.935, + "3": 0.946, + "10": 0.956 + } }, - "evaluation_kwargs": { - "batch_size": 16 + "mean_reciprocal_rank": { + "nondeterministic": 0.943 } } } \ No newline at end of file diff --git a/reproducibility/configs/conve/dettmers2018_conve_wn18rr.json b/reproducibility/configs/conve/dettmers2018_conve_wn18rr.json index 1ae46122e..b79984a66 100644 --- a/reproducibility/configs/conve/dettmers2018_conve_wn18rr.json +++ b/reproducibility/configs/conve/dettmers2018_conve_wn18rr.json @@ -1,13 +1,14 @@ { "metadata": { - "title": "Learn WN18RR Data Set with ConvE as described by Dettmers et al., 2018" + "title": "Learn WN18RR Data Set with ConvE as described by Dettmers et al., 2018", + "comment": "For evaluation, the non-deterministic rank definition is used, cf. https://github.com/TimDettmers/ConvE/blob/5feb358eb7dbd1f534978cdc4c20ee0bf919148a/evaluation.py#L67." }, "pipeline": { "dataset": "wn18rr", + "model": "ConvE", "dataset_kwargs": { "create_inverse_triples": true }, - "model": "ConvE", "model_kwargs": { "embedding_dim": 200, "input_channels": 1, @@ -18,7 +19,8 @@ "kernel_width": 3, "input_dropout": 0.2, "feature_map_dropout": 0.2, - "output_dropout": 0.3 + "output_dropout": 0.3, + "apply_batch_normalization": true }, "optimizer": "Adam", "optimizer_kwargs": { @@ -36,9 +38,21 @@ }, "evaluator_kwargs": { "filtered": true + } + }, + "results": { + "mean_rank": { + "nondeterministic": 4187 + }, + "hits_at_k": { + "nondeterministic": { + "1": 0.40, + "3": 0.44, + "10": 0.52 + } }, - "evaluation_kwargs": { - "batch_size": 16 + "mean_reciprocal_rank": { + "nondeterministic": 0.43 } } } \ No newline at end of file diff --git a/reproducibility/configs/convkb/nguyen2018_convkb_fb15k237.json b/reproducibility/configs/convkb/nguyen2018_convkb_fb15k237.json index 42bf78078..a32d7a110 100644 --- a/reproducibility/configs/convkb/nguyen2018_convkb_fb15k237.json +++ b/reproducibility/configs/convkb/nguyen2018_convkb_fb15k237.json @@ -1,7 +1,7 @@ { "metadata": { "title": "Learn FB15k-237 Data Set with ConvKB as described by Nguyen et al., 2018", - "comments": "regularization weight is set to 0.0005, because in the paper the regularization term is multiplied with (regularization weight)/2" + "comments": "regularization weight is set to 0.0005, because in the paper the regularization term is multiplied with (regularization weight)/2. The evaluation is done with the optimistic rank, cf. https://github.com/daiquocnguyen/ConvKB/blob/ba02c0665a80751676289a8d5570dc420465a9ff/eval.py#L207-L236." }, "pipeline": { "dataset": "fb15k237", @@ -37,9 +37,19 @@ }, "evaluator_kwargs": { "filtered": true + } + }, + "results": { + "mean_rank": { + "best": 257 + }, + "hits_at_k": { + "best": { + "10": 0.517 + } }, - "evaluation_kwargs": { - "batch_size": 16 + "mean_reciprocal_rank": { + "best": 0.396 } } } \ No newline at end of file diff --git a/reproducibility/configs/convkb/nguyen2018_convkb_wn18rr.json b/reproducibility/configs/convkb/nguyen2018_convkb_wn18rr.json index 70c4c8da1..3acac58d5 100644 --- a/reproducibility/configs/convkb/nguyen2018_convkb_wn18rr.json +++ b/reproducibility/configs/convkb/nguyen2018_convkb_wn18rr.json @@ -1,7 +1,7 @@ { "metadata": { "title": "Learn WN18RR Data Set with ConvKB as described by Nguyen et al., 2018", - "comments": "regularization weight is set to 0.0005, because in the paper the regularization term is multiplied with (regularization weight)/2" + "comments": "regularization weight is set to 0.0005, because in the paper the regularization term is multiplied with (regularization weight)/2. The evaluation is done with the optimistic rank, cf. https://github.com/daiquocnguyen/ConvKB/blob/ba02c0665a80751676289a8d5570dc420465a9ff/eval.py#L207-L236." }, "pipeline": { "dataset": "wn18rr", @@ -37,9 +37,19 @@ }, "evaluator_kwargs": { "filtered": true + } + }, + "results": { + "mean_rank": { + "best": 2554 + }, + "hits_at_k": { + "best": { + "10": 0.525 + } }, - "evaluation_kwargs": { - "batch_size": 16 + "mean_reciprocal_rank": { + "best": 0.248 } } } \ No newline at end of file diff --git a/reproducibility/configs/convkb/nguyen2018_transe_fb15k237.json b/reproducibility/configs/convkb/nguyen2018_transe_fb15k237.json index 6510e22cc..a94f9b08e 100644 --- a/reproducibility/configs/convkb/nguyen2018_transe_fb15k237.json +++ b/reproducibility/configs/convkb/nguyen2018_transe_fb15k237.json @@ -30,9 +30,6 @@ }, "evaluator_kwargs": { "filtered": true - }, - "evaluation_kwargs": { - "batch_size": 16 } } } \ No newline at end of file diff --git a/reproducibility/configs/convkb/nguyen2018_transe_wn18rr.json b/reproducibility/configs/convkb/nguyen2018_transe_wn18rr.json index 41955e583..59ddd7d02 100644 --- a/reproducibility/configs/convkb/nguyen2018_transe_wn18rr.json +++ b/reproducibility/configs/convkb/nguyen2018_transe_wn18rr.json @@ -20,7 +20,7 @@ "margin": 5 }, "training_loop": "OWA", - "negative_sampler": "bernoulli", + "negative_sampler_cls": "bernoulli", "negative_sampler_kwargs": { "num_negs_per_pos": 1 }, @@ -30,9 +30,6 @@ }, "evaluator_kwargs": { "filtered": true - }, - "evaluation_kwargs": { - "batch_size": 16 } } } \ No newline at end of file diff --git a/reproducibility/configs/distmult/yang2014_distmult_fb15k.json b/reproducibility/configs/distmult/yang2014_distmult_fb15k.json index cf51683bf..3acfa0a9a 100644 --- a/reproducibility/configs/distmult/yang2014_distmult_fb15k.json +++ b/reproducibility/configs/distmult/yang2014_distmult_fb15k.json @@ -37,5 +37,15 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "hits_at_k": { + "unknown": { + "10": 0.577 + } + }, + "mean_reciprocal_rank": { + "unknown": 0.35 + } } } \ No newline at end of file diff --git a/reproducibility/configs/distmult/yang2014_distmult_wn18.json b/reproducibility/configs/distmult/yang2014_distmult_wn18.json index a0851c439..6bc6d1fc4 100644 --- a/reproducibility/configs/distmult/yang2014_distmult_wn18.json +++ b/reproducibility/configs/distmult/yang2014_distmult_wn18.json @@ -37,5 +37,15 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "hits_at_k": { + "unknown": { + "10": 0.942 + } + }, + "mean_reciprocal_rank": { + "unknown": 0.83 + } } } \ No newline at end of file diff --git a/reproducibility/configs/hole/nickel2016_hole_fb15k.json b/reproducibility/configs/hole/nickel2016_hole_fb15k.json index 5daf99b30..100c87d6d 100644 --- a/reproducibility/configs/hole/nickel2016_hole_fb15k.json +++ b/reproducibility/configs/hole/nickel2016_hole_fb15k.json @@ -1,14 +1,13 @@ { "metadata": { "title": "Learn FB15K Data Set with HolE as described by Nickel et al., 2016", - "comments": " We use same setting as for WN18 (based on https://github.com/mnick/holographic-embeddings/blob/master/run_hole_wn18.sh, however it is not sure whether this is the correct setting) since we couln't find the setting for FB15k." + "comments": " We use same setting as for WN18 (based on https://github.com/mnick/holographic-embeddings/blob/c2db6e1554e671ab8e6acace78ec1fd91d6a4b90/run_hole_wn18.sh, however it is not sure whether this is the correct setting) since we couldn't find the setting for FB15k. The evaluation uses the non-deterministic rank, as seen from https://github.com/mnick/holographic-embeddings/blob/c2db6e1554e671ab8e6acace78ec1fd91d6a4b90/kg/base.py#L198." }, "pipeline": { "dataset": "fb15k", "model": "HolE", "model_kwargs": { - "embedding_dim": 150, - "automatic_memory_optimization": false + "embedding_dim": 150 }, "optimizer": "AdaGrad", "optimizer_kwargs": { @@ -30,9 +29,18 @@ }, "evaluator_kwargs": { "filtered": true + } + }, + "results": { + "hits_at_k": { + "nondeterministic": { + "1": 0.402, + "3": 0.613, + "10": 0.739 + } }, - "evaluation_kwargs": { - "batch_size": 32 + "mean_reciprocal_rank": { + "nondeterministic": 0.524 } } } \ No newline at end of file diff --git a/reproducibility/configs/hole/nickel2016_hole_wn18.json b/reproducibility/configs/hole/nickel2016_hole_wn18.json index b6240090d..636aaf466 100644 --- a/reproducibility/configs/hole/nickel2016_hole_wn18.json +++ b/reproducibility/configs/hole/nickel2016_hole_wn18.json @@ -1,7 +1,7 @@ { "metadata": { "title": "Learn WN18 Data Set with HolE as described by Nickel et al., 2016", - "comments": "Based on https://github.com/mnick/holographic-embeddings/blob/master/run_hole_wn18.sh, however it is not sure whether this is the correct setting" + "comments": "Based on https://github.com/mnick/holographic-embeddings/blob/master/run_hole_wn18.sh, however it is not sure whether this is the correct setting. The evaluation uses the non-deterministic rank, as seen from https://github.com/mnick/holographic-embeddings/blob/c2db6e1554e671ab8e6acace78ec1fd91d6a4b90/kg/base.py#L198." }, "pipeline": { "dataset": "wn18", @@ -25,10 +25,22 @@ }, "training_kwargs": { "num_epochs": 500, - "batch_size": 4831 + "batch_size": 1414 }, "evaluator_kwargs": { "filtered": true } + }, + "results": { + "hits_at_k": { + "nondeterministic": { + "1": 0.930, + "3": 0.945, + "10": 0.949 + } + }, + "mean_reciprocal_rank": { + "nondeterministic": 0.938 + } } } \ No newline at end of file diff --git a/reproducibility/configs/kg2e/he2015_kg2e_fb15k.json b/reproducibility/configs/kg2e/he2015_kg2e_fb15k.json index 45a3b770f..b286b75f7 100644 --- a/reproducibility/configs/kg2e/he2015_kg2e_fb15k.json +++ b/reproducibility/configs/kg2e/he2015_kg2e_fb15k.json @@ -31,5 +31,15 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "hits_at_k": { + "unknown": { + "10": 0.715 + } + }, + "mean_rank": { + "unknown": 59 + } } } \ No newline at end of file diff --git a/reproducibility/configs/kg2e/he2015_kg2e_wn18.json b/reproducibility/configs/kg2e/he2015_kg2e_wn18.json index 0e465b897..2dee3e03c 100644 --- a/reproducibility/configs/kg2e/he2015_kg2e_wn18.json +++ b/reproducibility/configs/kg2e/he2015_kg2e_wn18.json @@ -31,5 +31,15 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "hits_at_k": { + "unknown": { + "10": 0.928 + } + }, + "mean_rank": { + "unknown": 331 + } } } \ No newline at end of file diff --git a/reproducibility/configs/rgcn/schlichtkrull2018_rgcn_fb15k.json b/reproducibility/configs/rgcn/schlichtkrull2018_rgcn_fb15k.json index 1788a92a1..97c7111a6 100644 --- a/reproducibility/configs/rgcn/schlichtkrull2018_rgcn_fb15k.json +++ b/reproducibility/configs/rgcn/schlichtkrull2018_rgcn_fb15k.json @@ -26,7 +26,8 @@ }, "training_kwargs": { "num_epochs": 500, - "batch_size": 30000 + "batch_size": 30000, + "sub_batch_size": 800 }, "evaluator_kwargs": { "filtered": true diff --git a/reproducibility/configs/rgcn/schlichtkrull2018_rgcn_wn18.json b/reproducibility/configs/rgcn/schlichtkrull2018_rgcn_wn18.json index ba4cbb93e..a65e734be 100644 --- a/reproducibility/configs/rgcn/schlichtkrull2018_rgcn_wn18.json +++ b/reproducibility/configs/rgcn/schlichtkrull2018_rgcn_wn18.json @@ -26,7 +26,8 @@ }, "training_kwargs": { "num_epochs": 500, - "batch_size": 30000 + "batch_size": 30000, + "sub_batch_size": 1000 }, "evaluator_kwargs": { "filtered": true diff --git a/reproducibility/configs/rotate/sun2019_rotate_fb15k.json b/reproducibility/configs/rotate/sun2019_rotate_fb15k.json index 3dc2781f7..0808064c5 100644 --- a/reproducibility/configs/rotate/sun2019_rotate_fb15k.json +++ b/reproducibility/configs/rotate/sun2019_rotate_fb15k.json @@ -1,7 +1,7 @@ { "metadata": { "title": "Learn FB15K Data Set with RotatE as described by Sun et al., 2019", - "comments": "Could not find initial learning rate; could not find number of epochs" + "comments": "Could not find number of epochs since early stopping has been applied; https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/master/best_config.sh. The evaluation uses the non-deterministic rank, cf. https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/a0a3cf75e8e324ef113472489a16927c39c997b2/codes/model.py#L406 ." }, "pipeline": { "dataset": "fb15k", @@ -11,7 +11,7 @@ }, "optimizer": "Adam", "optimizer_kwargs": { - "lr": 0.001 + "lr": 0.0001 }, "loss": "nssa", "loss_kwargs": { @@ -32,5 +32,20 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "mean_rank": { + "nondeterministic": 40 + }, + "hits_at_k": { + "nondeterministic": { + "1": 0.746, + "3": 0.830, + "10": 0.884 + } + }, + "mean_reciprocal_rank": { + "nondeterministic": 0.797 + } } } \ No newline at end of file diff --git a/reproducibility/configs/rotate/sun2019_rotate_fb15k237.json b/reproducibility/configs/rotate/sun2019_rotate_fb15k237.json new file mode 100644 index 000000000..350fff450 --- /dev/null +++ b/reproducibility/configs/rotate/sun2019_rotate_fb15k237.json @@ -0,0 +1,50 @@ +{ + "metadata": { + "title": "Learn FB15K-237 Data Set with RotatE as described by Sun et al., 2019", + "comments": "Could not find number of epochs since early stopping has been applied; https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/master/best_config.sh. The evaluation uses the non-deterministic rank, cf. https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/a0a3cf75e8e324ef113472489a16927c39c997b2/codes/model.py#L406 ." + }, + "pipeline": { + "dataset": "fb15k237", + "model": "RotatE", + "model_kwargs": { + "embedding_dim": 1000 + }, + "optimizer": "Adam", + "optimizer_kwargs": { + "lr": 0.00005 + }, + "loss": "nssa", + "loss_kwargs": { + "reduction": "mean", + "adversarial_temperature": 1.0, + "margin": 9 + }, + "training_loop": "OWA", + "negative_sampler": "basic", + "negative_sampler_kwargs": { + "num_negs_per_pos": 256 + }, + "training_kwargs": { + "num_epochs": 1000, + "batch_size": 1024 + }, + "evaluator_kwargs": { + "filtered": true + } + }, + "results": { + "mean_rank": { + "nondeterministic": 177 + }, + "hits_at_k": { + "nondeterministic": { + "1": 0.241, + "3": 0.375, + "10": 0.533 + } + }, + "mean_reciprocal_rank": { + "nondeterministic": 0.338 + } + } +} \ No newline at end of file diff --git a/reproducibility/configs/rotate/sun2019_rotate_wn18.json b/reproducibility/configs/rotate/sun2019_rotate_wn18.json index 03af53176..e5e4eb439 100644 --- a/reproducibility/configs/rotate/sun2019_rotate_wn18.json +++ b/reproducibility/configs/rotate/sun2019_rotate_wn18.json @@ -1,7 +1,7 @@ { "metadata": { - "title": "Learn WN18 Data Set with RotatE as described by Sun et al., 2019", - "comments": "Could not find initial learning rate; could not find number of epochs" + "title": "Learn WN18 Data Set with RotatE as described by Sun et al., 2019;", + "comments": "Could not find number of epochs since early stopping has been applied; https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/master/best_config.sh. The evaluation uses the non-deterministic rank, cf. https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/a0a3cf75e8e324ef113472489a16927c39c997b2/codes/model.py#L406 ." }, "pipeline": { "dataset": "wn18", @@ -11,7 +11,7 @@ }, "optimizer": "Adam", "optimizer_kwargs": { - "lr": 0.001 + "lr": 0.0001 }, "loss": "nssa", "loss_kwargs": { @@ -32,5 +32,20 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "mean_rank": { + "nondeterministic": 309 + }, + "hits_at_k": { + "nondeterministic": { + "1": 0.944, + "3": 0.952, + "10": 0.959 + } + }, + "mean_reciprocal_rank": { + "nondeterministic": 0.949 + } } } diff --git a/reproducibility/configs/rotate/sun2019_rotate_wn18rr.json b/reproducibility/configs/rotate/sun2019_rotate_wn18rr.json new file mode 100644 index 000000000..55d34e9b6 --- /dev/null +++ b/reproducibility/configs/rotate/sun2019_rotate_wn18rr.json @@ -0,0 +1,50 @@ +{ + "metadata": { + "title": "Learn WN18RR Data Set with RotatE as described by Sun et al., 2019; https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/master/best_config.sh", + "comments": "Could not find number of epochs since early stopping has been applied. The evaluation uses the non-deterministic rank, cf. https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/a0a3cf75e8e324ef113472489a16927c39c997b2/codes/model.py#L406 ." + }, + "pipeline": { + "dataset": "wn18rr", + "model": "RotatE", + "model_kwargs": { + "embedding_dim": 500 + }, + "optimizer": "Adam", + "optimizer_kwargs": { + "lr": 0.00005 + }, + "loss": "nssa", + "loss_kwargs": { + "reduction": "mean", + "adversarial_temperature": 0.5, + "margin": 6 + }, + "training_loop": "OWA", + "negative_sampler": "basic", + "negative_sampler_kwargs": { + "num_negs_per_pos": 1024 + }, + "training_kwargs": { + "num_epochs": 1000, + "batch_size": 512 + }, + "evaluator_kwargs": { + "filtered": true + } + }, + "results": { + "mean_rank": { + "nondeterministic": 3340 + }, + "hits_at_k": { + "nondeterministic": { + "1": 0.428, + "3": 0.492, + "10": 0.571 + } + }, + "mean_reciprocal_rank": { + "nondeterministic": 0.476 + } + } +} diff --git a/reproducibility/configs/simple/kazemi2018_simple_fb15k.json b/reproducibility/configs/simple/kazemi2018_simple_fb15k.json index 8ed33cdcd..4480add89 100644 --- a/reproducibility/configs/simple/kazemi2018_simple_fb15k.json +++ b/reproducibility/configs/simple/kazemi2018_simple_fb15k.json @@ -1,13 +1,14 @@ { "metadata": { "title": "Learn FB15k Data Set with SimplE as described by Kazemi et al., 2018", - "comments": "They make use of inverse relations. Normalization argument of regularizer is set to false." + "comments": "They make use of inverse relations. Normalization argument of regularizer is set to false. SimplE uses the optimistic rank definition for evaluation, cf. https://github.com/Mehran-k/SimplE/blob/29108230b63920afa38067b1aff8b8d53d07ed01/reader.py#L148." }, "pipeline": { "dataset": "fb15k", "model": "SimplE", "model_kwargs": { - "embedding_dim": 200 + "embedding_dim": 200, + "clamp_score": 20.0 }, "regularizer": "PowerSum", "regularizer_kwargs": { @@ -36,5 +37,17 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "mean_reciprocal_rank": { + "best": 0.727 + }, + "hits_at_k": { + "best": { + "1": 0.660, + "3": 0.773, + "10": 0.838 + } + } } } \ No newline at end of file diff --git a/reproducibility/configs/simple/kazemi2018_simple_wn18.json b/reproducibility/configs/simple/kazemi2018_simple_wn18.json index 75d88394a..1cfb200f1 100644 --- a/reproducibility/configs/simple/kazemi2018_simple_wn18.json +++ b/reproducibility/configs/simple/kazemi2018_simple_wn18.json @@ -1,13 +1,14 @@ { "metadata": { "title": "Learn WN18 Data Set with SimplE as described by Kazemi et al., 2018", - "comments": "They make use of inverse relations. Normalization argument of regularizer is set to false." + "comments": "They make use of inverse relations. Normalization argument of regularizer is set to false. SimplE uses the optimistic rank definition for evaluation, cf. https://github.com/Mehran-k/SimplE/blob/29108230b63920afa38067b1aff8b8d53d07ed01/reader.py#L148." }, "pipeline": { "dataset": "wn18", "model": "SimplE", "model_kwargs": { - "embedding_dim": 200 + "embedding_dim": 200, + "clamp_score": 20.0 }, "regularizer": "PowerSum", "regularizer_kwargs": { @@ -31,10 +32,22 @@ }, "training_kwargs": { "num_epochs": 1000, - "batch_size": 4831 + "batch_size": 1415 }, "evaluator_kwargs": { "filtered": true } + }, + "results": { + "mean_reciprocal_rank": { + "best": 0.942 + }, + "hits_at_k": { + "best": { + "1": 0.939, + "3": 0.944, + "10": 0.947 + } + } } } \ No newline at end of file diff --git a/reproducibility/configs/transd/ji2015_transd_fb15k.json b/reproducibility/configs/transd/ji2015_transd_fb15k.json index 431f814a6..40b3f40eb 100644 --- a/reproducibility/configs/transd/ji2015_transd_fb15k.json +++ b/reproducibility/configs/transd/ji2015_transd_fb15k.json @@ -31,5 +31,15 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "hits_at_k": { + "unknown": { + "10": 0.773 + } + }, + "mean_rank": { + "unknown": 91 + } } } \ No newline at end of file diff --git a/reproducibility/configs/transd/ji2015_transd_wn18.json b/reproducibility/configs/transd/ji2015_transd_wn18.json index f4b40e5b3..7d0aedfea 100644 --- a/reproducibility/configs/transd/ji2015_transd_wn18.json +++ b/reproducibility/configs/transd/ji2015_transd_wn18.json @@ -31,5 +31,15 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "hits_at_k": { + "unknown": { + "10": 0.922 + } + }, + "mean_rank": { + "unknown": 212 + } } } \ No newline at end of file diff --git a/reproducibility/configs/transe/bordes2013_transe_fb15k.json b/reproducibility/configs/transe/bordes2013_transe_fb15k.json index 0b3fcaf17..09fc759a1 100644 --- a/reproducibility/configs/transe/bordes2013_transe_fb15k.json +++ b/reproducibility/configs/transe/bordes2013_transe_fb15k.json @@ -31,5 +31,15 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "hits_at_k": { + "unknown": { + "10": 0.471 + } + }, + "mean_rank": { + "unknown": 125 + } } } \ No newline at end of file diff --git a/reproducibility/configs/transe/bordes2013_transe_wn18.json b/reproducibility/configs/transe/bordes2013_transe_wn18.json index aea82a354..79f9f578f 100644 --- a/reproducibility/configs/transe/bordes2013_transe_wn18.json +++ b/reproducibility/configs/transe/bordes2013_transe_wn18.json @@ -31,5 +31,15 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "hits_at_k": { + "unknown": { + "10": 0.892 + } + }, + "mean_rank": { + "unknown": 251 + } } } \ No newline at end of file diff --git a/reproducibility/configs/transh/wang2014_transh_fb15k.json b/reproducibility/configs/transh/wang2014_transh_fb15k.json index 8bac4ce88..7e1f271a1 100644 --- a/reproducibility/configs/transh/wang2014_transh_fb15k.json +++ b/reproducibility/configs/transh/wang2014_transh_fb15k.json @@ -35,5 +35,15 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "hits_at_k": { + "unknown": { + "10": 0.644 + } + }, + "mean_rank": { + "unknown": 87 + } } } diff --git a/reproducibility/configs/transh/wang2014_transh_wn18.json b/reproducibility/configs/transh/wang2014_transh_wn18.json index e841665d8..82e9692b5 100644 --- a/reproducibility/configs/transh/wang2014_transh_wn18.json +++ b/reproducibility/configs/transh/wang2014_transh_wn18.json @@ -35,5 +35,15 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "hits_at_k": { + "unknown": { + "10": 0.823 + } + }, + "mean_rank": { + "unknown": 388 + } } } diff --git a/reproducibility/configs/transr/li2015_transr_fb15k.json b/reproducibility/configs/transr/li2015_transr_fb15k.json index 790ea20d9..23bb8eb18 100644 --- a/reproducibility/configs/transr/li2015_transr_fb15k.json +++ b/reproducibility/configs/transr/li2015_transr_fb15k.json @@ -1,6 +1,6 @@ { "metadata": { - "title": "Learn FB15K Data Set with TransR as described by Li et al., 2015" + "title": "Learn FB15K Data Set with TransR as described by Li et al., 2015. For evaluation, the non-deterministic rank is used, cf. https://github.com/Mrlyk423/Relation_Extraction/blob/9f311285abd7e29273be7e4fd8c680dd761f40eb/TransR/Test_TransR.cpp#L177-L206." }, "pipeline": { "dataset": "fb15k", @@ -31,5 +31,15 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "mean_rank": { + "nondeterministic": 77 + }, + "hits_at_k": { + "nondeterministic": { + "10": 0.687 + } + } } } \ No newline at end of file diff --git a/reproducibility/configs/transr/li2015_transr_wn18.json b/reproducibility/configs/transr/li2015_transr_wn18.json index 257703e3d..d5afeaf91 100644 --- a/reproducibility/configs/transr/li2015_transr_wn18.json +++ b/reproducibility/configs/transr/li2015_transr_wn18.json @@ -1,6 +1,6 @@ { "metadata": { - "title": "Learn WN18 Data Set with TransR as described by Li et al., 2015" + "title": "Learn WN18 Data Set with TransR as described by Li et al., 2015. For evaluation, the non-deterministic rank is used, cf. https://github.com/Mrlyk423/Relation_Extraction/blob/9f311285abd7e29273be7e4fd8c680dd761f40eb/TransR/Test_TransR.cpp#L177-L206." }, "pipeline": { "dataset": "wn18", @@ -31,5 +31,15 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "mean_rank": { + "nondeterministic": 225 + }, + "hits_at_k": { + "nondeterministic": { + "10": 0.920 + } + } } } \ No newline at end of file diff --git a/reproducibility/configs/tucker/balazevic2019_tucker_fb15k.json b/reproducibility/configs/tucker/balazevic2019_tucker_fb15k.json index 42f897bac..fa07e435e 100644 --- a/reproducibility/configs/tucker/balazevic2019_tucker_fb15k.json +++ b/reproducibility/configs/tucker/balazevic2019_tucker_fb15k.json @@ -1,7 +1,7 @@ { "metadata": { "title": "Learn FB15K Data Set with TuckER as described by Balazevic et al., 2019", - "comments": "No label smoothing has been used for FB15k. Check whether dropouts are used correctly. They make use of inverse relations. Could not find number of epochs. In the code 500 was the default value for all datasets, that's why we assume that this is the number of epochs." + "comments": "No label smoothing has been used for FB15k. Check whether dropouts are used correctly. They make use of inverse relations. Could not find number of epochs. In the code 500 was the default value for all datasets, that's why we assume that this is the number of epochs. For evaluation, the non-deterministic rank is used, cf. https://github.com/ibalazevic/TuckER/blob/63dbba8751670db0d807579303679c0a2c266130/main.py#L78 ." }, "pipeline": { "dataset": "fb15k", @@ -14,7 +14,8 @@ "relation_dim": 200, "dropout_0": 0.2, "dropout_1": 0.2, - "dropout_2": 0.3 + "dropout_2": 0.3, + "apply_batch_normalization": true }, "optimizer": "Adam", "optimizer_kwargs": { @@ -34,5 +35,17 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "hits_at_k": { + "nondeterministic": { + "1": 0.741, + "3": 0.833, + "10": 0.892 + } + }, + "mean_reciprocal_rank": { + "nondeterministic": 0.795 + } } } \ No newline at end of file diff --git a/reproducibility/configs/tucker/balazevic2019_tucker_fb15k237.json b/reproducibility/configs/tucker/balazevic2019_tucker_fb15k237.json new file mode 100644 index 000000000..feeb77ab7 --- /dev/null +++ b/reproducibility/configs/tucker/balazevic2019_tucker_fb15k237.json @@ -0,0 +1,51 @@ +{ + "metadata": { + "title": "Learn FB15K-237 Data Set with TuckER as described by Balazevic et al., 2019", + "comments": "No label smoothing has been used for FB15k. Check whether dropouts are used correctly. They make use of inverse relations. Could not find number of epochs. In the code 500 was the default value for all datasets, that's why we assume that this is the number of epochs. For evaluation, the non-deterministic rank is used, cf. https://github.com/ibalazevic/TuckER/blob/63dbba8751670db0d807579303679c0a2c266130/main.py#L78 ." + }, + "pipeline": { + "dataset": "fb15k", + "dataset_kwargs": { + "create_inverse_triples": true + }, + "model": "TuckER", + "model_kwargs": { + "embedding_dim": 200, + "relation_dim": 200, + "dropout_0": 0.3, + "dropout_1": 0.4, + "dropout_2": 0.5, + "apply_batch_normalization": true + }, + "optimizer": "Adam", + "optimizer_kwargs": { + "lr": 0.0005, + "weight_decay": 1.0 + }, + "loss": "BCEAfterSigmoid", + "loss_kwargs": { + "reduction": "mean" + }, + "training_loop": "LCWA", + "training_kwargs": { + "num_epochs": 500, + "batch_size": 128, + "label_smoothing": 0.1 + }, + "evaluator_kwargs": { + "filtered": true + } + }, + "results": { + "hits_at_k": { + "nondeterministic": { + "1": 0.266, + "3": 0.394, + "10": 0.544 + } + }, + "mean_reciprocal_rank": { + "nondeterministic": 0.358 + } + } +} \ No newline at end of file diff --git a/reproducibility/configs/tucker/balazevic2019_tucker_wn18.json b/reproducibility/configs/tucker/balazevic2019_tucker_wn18.json index 0784d25d2..6b8357909 100644 --- a/reproducibility/configs/tucker/balazevic2019_tucker_wn18.json +++ b/reproducibility/configs/tucker/balazevic2019_tucker_wn18.json @@ -1,7 +1,7 @@ { "metadata": { "title": "Learn WN18 Data Set with TuckER as described by Balazevic et al., 2019", - "comments": "Check whether dropouts are used correctly. They make use of inverse relations. Could not find number of epochs.In the code 500 was the default value for all datasets, that's we assume that this is the number of epochs." + "comments": "Check whether dropouts are used correctly. They make use of inverse relations. Could not find number of epochs.In the code 500 was the default value for all datasets, that's we assume that this is the number of epochs. For evaluation, the non-deterministic rank is used, cf. https://github.com/ibalazevic/TuckER/blob/63dbba8751670db0d807579303679c0a2c266130/main.py#L78 ." }, "pipeline": { "dataset": "wn18", @@ -14,7 +14,8 @@ "relation_dim": 30, "dropout_0": 0.2, "dropout_1": 0.1, - "dropout_2": 0.2 + "dropout_2": 0.2, + "apply_batch_normalization": true }, "optimizer": "Adam", "optimizer_kwargs": { @@ -34,5 +35,17 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "hits_at_k": { + "nondeterministic": { + "1": 0.949, + "3": 0.955, + "10": 0.958 + } + }, + "mean_reciprocal_rank": { + "nondeterministic": 0.953 + } } } \ No newline at end of file diff --git a/reproducibility/configs/tucker/balazevic2019_tucker_wn18rr.json b/reproducibility/configs/tucker/balazevic2019_tucker_wn18rr.json new file mode 100644 index 000000000..9a6e0880c --- /dev/null +++ b/reproducibility/configs/tucker/balazevic2019_tucker_wn18rr.json @@ -0,0 +1,51 @@ +{ + "metadata": { + "title": "Learn WN18RR Data Set with TuckER as described by Balazevic et al., 2019", + "comments": "Check whether dropouts are used correctly. They make use of inverse relations. Could not find number of epochs.In the code 500 was the default value for all datasets, that's we assume that this is the number of epochs. For evaluation, the non-deterministic rank is used, cf. https://github.com/ibalazevic/TuckER/blob/63dbba8751670db0d807579303679c0a2c266130/main.py#L78 ." + }, + "pipeline": { + "dataset": "wn18", + "dataset_kwargs": { + "create_inverse_triples": true + }, + "model": "TuckER", + "model_kwargs": { + "embedding_dim": 200, + "relation_dim": 30, + "dropout_0": 0.2, + "dropout_1": 0.2, + "dropout_2": 0.3, + "apply_batch_normalization": true + }, + "optimizer": "Adam", + "optimizer_kwargs": { + "lr": 0.01, + "weight_decay": 1.0 + }, + "loss": "BCEAfterSigmoid", + "loss_kwargs": { + "reduction": "mean" + }, + "training_loop": "LCWA", + "training_kwargs": { + "num_epochs": 500, + "batch_size": 128, + "label_smoothing": 0.1 + }, + "evaluator_kwargs": { + "filtered": true + } + }, + "results": { + "hits_at_k": { + "nondeterministic": { + "1": 0.443, + "3": 0.482, + "10": 0.526 + } + }, + "mean_reciprocal_rank": { + "nondeterministic": 0.470 + } + } +} \ No newline at end of file diff --git a/reproducibility/generate_summary_table.py b/reproducibility/generate_summary_table.py index d5f6a4cc1..f33a3df5b 100644 --- a/reproducibility/generate_summary_table.py +++ b/reproducibility/generate_summary_table.py @@ -229,8 +229,15 @@ def generate_size_table(): return rv +def get_published_results( + +): + return None + + def main(): size_table = generate_size_table() + published_results = get_published_results() all_tables = generate_results_table() write_pdfs(all_tables=all_tables, size_table=size_table) diff --git a/reproducibility/summaries/results.pdf b/reproducibility/summaries/results.pdf index 44fc2eec0058a050732fb1b86c431b32700482c3..97dd4072194f79abb39c79b4b345a596f9fbfb1e 100644 GIT binary patch delta 6172 zcmai&MNk_6qpgdyxDa{LH*mc6?0U$Wb|FKIkuQU1{R>&MA^)Q0+VWIIlqj3D!09Lznrstyqwk6^r0@wU5h#^!!?s zey&k}pYe1F_*6dnV)&?XHbCq_N^a76=c+cWR1h`t?wYM7^QaJ%wrw73A)n#>_}4q z=GTt9mr1Qknp^gDe~e|6yO%7n*gHd-p`DM-6feAfx;L@S40bA9d$4b8oW8}^nBFAfh{>DAMRz#dc+GObs9C)bz5Ob8i4RKO;wW65h+a9`Y_ za2@&hU@Q423b$8SnrIQ`(&W4ER%dOqr{>fv9P3l4+0#2W&?2zpEdf1UwtH{9ph9Wz zz!w(c-@At%R3@a=_7=+rNxw&kYVyEX^}khkO#f)QDtL=iAQk!YUE7G?C*Z^2*K_Z% zCO}A%Px42C)DnE(>GRx%{{>XAX638Z>n^@`;9Y{Cl6Pl+-V6r}Jb2YUyO~<8F z`qCMEcIMbx>A1Pw;hZkJ7kvT2oQ_;IdR#T0mZszb3VMP>2#pvL96SD|JrA_pjTM!2tEDFG44pL z-n8w(ztN|syQV*J|J77%{)Z0{_i$Q~jWfBdNyp@H+UURcWxuO|@}k-jrg#W#EGbPq z;qPQ~%++EHx5vJ7HG z*CqDFcaI=wcY}-%PR(3XpVR!1{HqJI)AyWDTMQ7o&?#6TArGHEIgi4edCtVLlk4$* z@^?i<3CHOG zW$iMsFL+sL%orIJm`W>=@y1xU9KT|0HPpojMSo<|Wl^L9U%H0#Tid33$--dEM7sLW zwymcwAeq8XQO+?t2`?J)yIW*s>J$&nnW*c~i@WXT`3-wQ+GOi^M<$wAu*!LAbj;l* z5ij=k57Zi7>4APYP8$!s1L+^W9_*oiX}ajSC_fsm(9tlVAH^NQ)k<4)_=YW?WneEi zV}ar&Ui6f*Cgw>f#1=^f{+)DakgV%R&x?u8vtU?LT$rcnl6lhGFS+Y)XgKHWp|ndS zW9r?-u(GFLu@A0)HdOp}SB#WhnK95pN#6Hgg3fn(V^Dj3%}BK{6K+8gwOzGEaO^OM zmEAge=KPePZzztN%ZLI3jUo@c9*e>@1!B$PaWm>&M?+&-I_w;7vC7Q|STMYT^$r$L zL?Kvx^Ok2F&5?usQ>=k~oDLu6&y#6&tp%gIRyGX28~MYNLH22Dp~O}m@FgV#r`^?M z_zLVkP{$RC@yp$U{Sg_TXsyhH=QjU|hi|ygm1q=-P;JSe1K8PNew;07)X6GXd8JwD z%EN*$>aKK|j>Np~W}^1ZQ9TzIbq~lNuv*wRI}zJ*g8Sp`)naI($TJO<#7Y$0Oo>Ja zkly)EVNFe-J?1xFaSg}|_YMc-nQuKNTW4=&@TUj&u%f8XZ-D7Yf|FA2QNxVyBXL{& zn>NIAySd$x66LWiSz7M$$Xdt?c4|^#^H~c0%a%fMu-#I{emnMt%HoM@7Oj0LnYHTx zf~A?L7qsh)^W=NI$pnQc8c_qLLmI@GyT{xi2?O?p($v zAxK9OQ9Cumq0rLd6-?#n9d>cE#{sPZZ*^5cjdD=(<0vW{@|kAO=1!*AIj$Ede8-}y zug_4MW$mwE77=_>g7#NxSECL4Y3}el3!&Qe{j{vf$4%i{K`4)3ek0!K!wO;(d0W@! zG=Xrtz2P{%HGdKa68990$DRS0SmfqjIGPI8&UCgxYMgSd!32h7v$B<7h7^Ipl=mp)*Y#NI@9jc$xqgtyIhP^wOo_EmK=YTUKVHF?wP`_0KCel`^+8|9K9tf79B19J zNk0`;THKGwx)%DoTNd}O!~MSS z=&VEb4n^t0RO{8A*^2MWXM&y_;%-{5DHI1F%qe?rgQJS1u zmqm$kFwkLstP?oh;I~;InoeK05n5t+zwG#WhAJjMo>I`6oJ#mKNVcqas2Q>M4zKf-HqrIf6L5xw5W)$(uoX;H?(!6%6zFwdnShrqSsjjxg3cn;~)muklU5Sb zisp7}>HBRqcvh;i=X28yU35kLTf#72o$8&mcz$lMlO0wNcR3oZ<#&wwZtDwRP}%M?X3T2=z*nJ&U<^7!KFm~?vj|?A>e0s zYn9n?wx>iAB->@G|K{hf%wJbEW)$d!$x!@QG&6ZoCI7ddxfF@8cwysNwhN1GjnP`S zcVB+ykCu|{y&dPQ^o~v1t2cssu=%L!S>J6tVfYb5u^E!8t|NNG(D5S)%BzngPxv_9O#Zj z8?1rpP-vxt;5?&A?$Cwv8?9KwFB>c|MjqzhFfm4-?+8ZvYTeMRnbCq-W!JPXe^Qu( z9mY~j)@7bR0Zcx%Mp+yT9dx!=QmW^smjU*eV?c0Mc2!MG$3n zh~RE?{;I2Rgd9iS>gqsPJiL&t@eD zV+*%U;pa?AS>9C1k!^G+kEyy19}bnC+cCdH>CP8ep6gUN6$7Jr=MzoMUqj1v1t)y7 z!R{wO$rc20fW6qv#qWy+`LGqKs*}6#;cjhz77>vP_y6t{eF+=Q1&2Bh4F3ohxU*wM z=n%N5Sm~To&M1N~5G3m_xd`Ew(qC@R|E}EW^Cmgx1jYdHnp+pdFZ8|gNtSJNW{VQ1qnj!ZJ!zqOQWRXL9AT&ty$g2=$lu~$arBE4it@* zwzg)Kq{v_;JPDggs_|0~KWfJTdA69MFJztmUY?t|9K&`lKt@ZK*-2zF*FnHC2^BqcG+_r9&=gBPAeTSLDsbIp8aN%mK)M zcL*mT_!jhjcrPsAvWhxw`IslXv_B{-pkBlfeVVm*@liP*N*|3hB90m!C>52XjJo_w zGM|N@%3?X<>5%yZ-Df~VVm0COBV)b_KJu5e9HD~~(Okc?f2x2eBMvnnc^64o4fH)uxf+Aa%-I}POPYg+Nb(TWA;)6!8%5^d3JOk z>m_cske?u57S?PZq3?{Q{(%!iQIq4Qlqy`G1QtNLtl%T)EjB*ckY+jmkR*>4?!!u1 zND^cG#W3N}jGW$S9~BwrVqI%#u~l0XzzFZNGz=CqmaRkChLtgcN zfeE>}eoRV$0){T9n1{xDpJW`jz3*s(LvYgXN6;@7QH4K>86cg~&Jn#u24bcR+eu7> z9^;~3|B_QR?$TSIIyH!PFaK$^hO~T}lzNjoKsS2v$>?ri+3MfOt6&DRQ71En`6p_* z=N%8emrN!9-?pUok~C-DL1T*69rS%VypY^|vO9^%$pKTSJFAj^`EVN#^h{AsAXo&J zh6_D%Gu>Y5vtioA6Sw}!F5eC}{`PiPrMA3;vh<*v;PP2U{j-<$ebYdkPnk{^mY?x`6w~Ir&AJIZPY#=WPph zm%<~0%>wqDKU*a-JFgue(0Z=hW?M(btbts;Q}o8fyoOyK9%)7fy;6`W;RVhV>UxSy z>M;JFyn!Kg2u8W&h-ZwA7jimV z9hw~pdA@M7?!=oh4|w=4xLw6K;or6q2-!^Q>YQna%q+$zIQ_;3jxXsMoN8Dfascz) zDsdhLI7xb1nv`Fu(ydbVMg35RQig7-OTpcN5zn5J*VQpMH4+A*N^^iRg!s+)uO^hxmj zJz#xTZcH^?s`;?;=VMra$s`c!LwxFS>hbn{mKib$-5YD9jqId9;QOY?RAzm`YHrgK zqyD@cB$q+N*{V5+L-+`>G0FV0Vw|0RC!=NDm?kHI@|*W$p2jhE=j%S*O|AohagW(? zluYln?)<%5%rND3KKZH0I{4c@6B=EM#*UwZ6GM zK};Lyt`*u(OV$q)+$xEwcJ{Qjhs{pU-apX=GAya&@4fFrNA6neIlO*+f z{N6`f=aA7_tSHZY!jXm^gF%si^#?Ns6%>s)I?dRrv^`EQlN5>yGU?XqpqGhu~11sN<%Wn}i>8;{tU{+;-+mQ`MO;C^mCa zwzQ8djeir#>N4?Jj&i_dTaybL7#OS43e|n<)-*mFi$(SmWnV+du0H4|7W5AY&Acr* z;pf5gk?WG!)YPqTm`6?P*HleC03Mz0-_9fEiAPO5_fnVhl$7BInOjA}R;;N+rEo8i zl^u`AoRcG@X6byQXcz4pc+LX8?NYD{d$&p8pYM&wG%wn03MjH5(3YxI%r&C1_Ea%Pd*ovC{O#A6QSS+d}A zEjouhxcxVt!E;%1=-PU9FwMVcGnW$s*mez%G+tF)Kmh>(A&`JD7eA1VpP!8bn@`Q% zR>8{0j)7f449E}U=Lh{iCGE?oBL<&>owbvdTm6_b3Y1;eq5NY7psJcxb(J&m<5fL{ zMfTUt@9F-`Z}#)0c`oucXpi-+E4U$CDg!0DBBe%VS(?Gw>7~! z?l^tW>aV;arWMW`e}gi!cYqwQA`5q4|K3@-dd$8uhbxqZD>IeWy{coascvg?-E$$) zRac>+8fxFJLbK~5n$QUAc<9#3iTj|j)+YHfQ_`@tCZkfzNNf{#h@H}BYb)s_>FawU zQ0^#M@I$v$Dghf|*zUip#1nMBzGFpk8=v~>Gc3BOGwBVRi4bDZ{d~Xirm?!?*$sc^ z;=wrj(Rd*zVw3+|7)~njQ->fs2(P~IWOj0J=hbR)c8$3VSuws5K+;xu zO`gn~p3-1TCUb~RfLfpSM;A)Nof2Ze>k<8cr^8}L@bWPdy`h}|Nk^ZR$poB$zRizO zN*c;;tA!goz2j3;W;*&xPE6pPA$_AM-Je&&9aJ^HT=(n|5OT2-g!2LQisu*A&g{6& z$j8o%vx^dwbKJNc%6MAOwbI1aKk$KfARq^4uuSPleZn9kN8Vf^fl@R)9G`sGI+d@4 z9o*9cZY8+Z_vRvMAy$Q%_+x88bvuO8$~SQ;VPvkEEhcVFb4$W=R2b~2Us6c`)LJpC z69n9ukbf)IViEC9o8PoHuuycKk{E|y2EB4q{`_zA1qg?zVPulJ!#+4lqtbD zi*9Z6w{nu%onh^2rL_eHT)`a5c+{DvAn$z_RiXhyqARK^`Ovl81;2NHIwKuZkHqpQ@dkgO4MF0KXu^|2gLE zNJrYVU#Nde*ZJRDfO=3x;g6fLguO;h6^p=Jz6=Z5F4SLs^ zedfaNTJ_`QyZA!W92(eQg1`r)X)@i{5uCp4wHnzxlYRKv)l@Qv+=^O`#?WrX=1 Qp~n*C$7W$s)PP|B7bOoSr2qf` delta 6174 zcmai%MNpgpm$gHJySuy7IKgRxy99!}yEPVggL`lY9^5TxAXtFlZUGwC#)CD^d^J`7 z)L%7=**uF=&+eSXz5Rvn`U>AAF}~&JOEn#31-d?|c&!TIb|33+DpJQ=B5+aSIu721lM%gnzS`A`fBg zZh)%3rX(-rPAWGruqWvI%Q#4>MKM_QVawX#Ge#5?GraTZ*GOFtPZPEP!_T75Xj?;b zcU;c8x{xkIE|xCo0rz)`o)Wl+r}dtfoMPE^Kl~S!h}6nZ@X*s(;nL=_xTGWJw17cG z$lD1*+jU?v;!a#Nw^Wbi_&#%DuSw)b`0;_7$43t)(F8LDf+yKS{BcvEs(~;8B`!ykX=qH;IZ( zHSBHxldVcSh74a_b=YmF3oLVH1c3XRYza+b5uJ%R8Uizk0)%MaQxp57uG zR|g+i+~_=N@0a4*jBMUxR748plg9gg4{hBxY}+PB{&SO$zcSs$eRW><26N##pFChv z0xM0NiE?Q(j+HP-PIqbMD`yd-xpQnkrSk`5{TUYbJ%%xRvM#r*n8_#Y8e+oyH8@VxLYnntTa!w9i!D zjB%_;ZXrcQlbbhOa-9|MEMH(qoJ%X2AuD5?DK5M#T7n_bd6C0BiGt(%Q7r~MRdk=c zmXoL=@G2BnsB$3CSq}+y;G6fi?=NfV#4pOV%BCf|OS@DFNl!}o$=Bf*c3c0BK`+Fn z>mR&Y{l|U(%wc(M?a)2Qb#u^dD~b7ibGE#+YQ%mOR+M3U-R-B3)*D!8j38~(PbA5y z5%LK%bkpdX==W=xtaTIB=ooGp>c|R>)j|CSbrks!U07E+--#Ki3y@y57IgoLW6(wm zm1I$hB|K7Fn-EO7B!05b*G17N4I%#65mx01p)u0)PH#vYWPIiqPqesv4DmLUNYpsR9=B3XId4i@D8U;AruBfq%Suc*UPQAq5{&U%7U0{xA z(u9DLcUvDhT%rK$FNLmg>>-(%On%&L8b?c1sfUBZnry2X(9BOij3BlJNh>#+p0sWH zKI&9SR%y86So_^h9G}EboKr|@JX?jHv6p3-gk=B5O$0Gz`?R4sSP50G3F9HRIV8;2 zX$$e-M7w+|5MDqAHlJ@Je&9?UvS!TBZVDy*z4WAnGA9GjNpB;VjJ95+JE@DZm=9Sj z)|-}yB4|gZ5l))Lvss?b zGwdFyKPGkx8{57;6e)S5=Nb6w;1P&Ne#Y(~5~L*X8B4y{UL@F1z&Mhx({NPP4BzJL zU5Q#E+#E~dPU^m(lj%P-5~Oxx#{x_ia_j+^3%BR#URv)dI5Z6XX{t}_7jP)VpB)}Q zdKDgE2+ot_li|en>f0R>7Ok56lt^fQ-*fkHnI^TEb8foWQ#1KSQ!akNKr`9i7v^J^ zMi}52FS^&K7vPJqO55b$U*AhoK-P3$cE4+IHPSZm$dkR~ay2rx|9qCclm!rrX|6vTD7- z!#Sif-tdX)gWJOg>;gl<>~gf(#K|1u4S~FN)&BP<%2x$f$UL#QP$2jiu9nla+P5@uT-}%8TOeK!1bhCJxL;C|VC)D*m*${Q z+iTImw!9g8Or;5)U8sNB(S@6mT*V#G{?du4A zs`uQ``(v0e^0x3&r_H}&$b-hyZ%vT-_5R@MY9bH<^=){>X6g1S06wG4FVDk!C{HCl zHfSRs*V>+Dyy0u#AK#St)vM& z3885zi|2yy&iCaK;^KtM{w0ODe&cnFu^T<*C3i|Hb!vWVp8(eU6HnHHkw6aVqMg^a zGAcgFvr2MwAlDiD3JfbHRJzrZy@g?p{OjG!i%w3uKB7lqGWC;?=j3aSpR!o-WL2|Vp__L;sS;x@>c<+sXAG* zHiEfU@NpI}OjP#zD1Q>osb;yTom%}>miV(~h!c4BFK5NG0oO_ZVx45acN-Nk37P%L zG0Ts&lLpMNxWr%cYJ1>aKXaYf%a?+4?$=Bsc^~S$8Ndm)t zOdP+sg_-dp_szISzCy98(7{%Kg`3K`{}MN5sForSH4su)NLkVuJ(x}(gGkc9r^0u1 zWNi4G^)XP@VtQ96?lRgYec|W8kmquZT04Pg8GB>kMf1DPs6K)k^s-ZkxzyXJ-Ye5< zJ3smR(B0MDEJWGxClYDniTL^O$=XK>z#n3t4+=OLSiD>`$Vl&DwLt#Y_KeOS_} zqEWuA$Hbc%kaQw|2EVtv=7|WhY85T-X`b8 zxp7T-QEfK?DZH`sL&7O5y~hx=+sGUJDK%msOWtAayq51tH`lUmihZj-FngibPQj>&dLvy*Mwk zr-cc_dn-|dN*gNSn1}Q^VUXBUVCpt|RmV;E_2s@TamtnmMrxCyrXSb}aSOei-^X_k zP3D;`WSK0|O>j*Sqo+n?a~EA=X#f}_)Z@b5upnt*3|{0mMtz_&d*uEr`#a!`k>8)} zl(a7_Cs1RfQiVQ=66E|w+XNekNGF!Sc3~=5TNPtn>}R{HteF{EB^TqsPmiv!gSf6n zsyhWb8o^O3vG4#X;Rq~L?Z&mTzw3tv;cv(zEcZdJIHEimKJZ2p?$owCwRpfY`@8mm zf1W})os_iQckxVqj67eeyu}++4dG)0#$^jSK?0?+sPY-`e@5r$$mU1Qpmsu6pO>a~ zl8GmN_wslvkUeqU^R(kBCGNkNFyeGW+&gE{r6O)X$r)-#uzB>%_@@U4d{-0GXSF>? zSr{+l|2Y;m21yu8xy~@5{Q!6+LHfcCQAqE=6Inr|bF1E=6hk3GTf_;Rwf6VbH6W#vvMjq ze@yWu${OT9xcPlxP<3?q@!V6)dGJCpB#EFXXUDi8^g`-L!JeAq4Fn>gU|3b?lJ4YZ zfRKZnb-Raf|J+%jTGryaTbODlwrk@H!nl4%szkRB6n+Fi3SvR{G;O95M& zo*Y|Bp?`~7x}UBXX#1|+6%l9{5N9kvv0wyEup91P8X4d(lDsvq@MlBNx6aAy$WzO` ztF|<0s4;*39;BDzQEoc9JB!A^4s2J2u7EZpm^R6~OdB*uO9!(E$HXss$4*4mUd|*sjMU~i6GFB#fj;96s4n!){ME)~Sf^?E&&R<* z#f^KRfQQ@?+&1TqqN|7y+`X{ep|XHGnc-;~L87#g1$NX~8jrbRV|Pl^-%DMj*`MnEPJ@xSlK zV6rd3AXx~ZvA`8ili>rmS8r17WGT>aqB?>DBt%1|Xc4#i+0g8Nhz#Rr<*^Ac05 zTg#$4cVet+B+nM=t8m_%mrvrQDDDI-7NWP)lBLt(QaDo*%2tSebXxGYbn!{ihg^<6 zI=40D9o%$1eNH*`ptE6?rZKwyxD_PztJanb=s<&rK-!~A$GGnlLVqIe8pqvOh|joE z+Nmu6ah@_f#;tasW6;2*YylZ{rDb13&i0c5=UUa#!0)VFD@t_vZ#*(@9IyqZ+||4i z1E!i#YB#uVWN6I;s$iene(zugOk+TDPjen_X3joeTj|0PvBUxObyc-rp;?%rU zdtMdZ7^1E=>tq5qs>glhIbn1dh@b&(C@5r7T%NgcOW&07gGHvb-T)kFZ-<%x-PF=g6@}Ny&uQnA_|Q zh(IZ3PVGJAiF^C=&n$t#7CL}B1G##704%i=L0C>DR+GO)r0=TS0Ksq)H- zOztK=u1^=5^<@YW{vW(jz*OkoRDw`sbg^sM0kiSY7+gn0QmKzuA9 z5DObRkD8nH2Mcc-TGkJud>}p$Na+75sp%v3O^Bn82!LW?)4c3Q{6#&2GH0Tpet(1b zdesS6tc!F`6Cj45~w_{ zoX_Ys=&6uCya6e!lViS)A!RZrG$eC#%M-3!4!$Z$YnA#+q%!hVc+uq4@bCHAsfgo{ z5Ig;X9>8zH^wpyySKhGSBxs{TeXxb)6q*m%&n>&T(3UPG#qi2`M+}?u{m5_5@94Ps zrrVr9{i#Z)Kt2M(xz_1XLfi#Ey#Qeb$}4@5lLc$9D}<$TW#1mP`FgDctSp)U!ciA~ zL0sSuANW%im7_v!b-EKMi0ZL~+AOU5iL?tDfbKtsNJ$vvB+ezR7)7mTppJXuN|F+) z2S~n#F8dnZP1v}}Gwn%p+l_VHhI;YT0LG{oPtJ57U{~dl#dBQbjNfMh74KdKBQ~#`&6riH290BxHr|_B z030M{T$}~>5gHo`jDwAvV+a{9*uIb{AzqZ`F(D~1;20=X?z4QF)LRfYtEnoWz$>OK zm!F-o-0*^(%nfeM%;YqEe)0KeLVVQ(E;!N5hMv*rv#sK#2oK3ryPtla4MKsndV$b= zHu|Q24%r;umj}F@_9OfXlO^rLqwH4zL?Zax!#Ua`DTd842|(s7zk^=vX`?nQki9-h zBMkoT#aFKQ&X;J5AUKax%B%UR)r(HORT-t)g}gj-bGX0l{^ev=RiB48PCJS-h585M zDP7#p#Q17AS{19rZ&=k||E_;o4lIFK{tODe%2=zvAiEEe2{l+l@k>lpQNbhI2T-TM zB?>#^&?U9n=HkpU#$@k_xR|J)qXhvK1zQ?A#1e}zr%!;0c=w%qSA<~5J+WKD#o-?B zuI#!p+;6Vd7kTgk^Uqh2Qym54OSNqViY&r(iSIhr%YU|$(llT*D2%_=q@&_Ict=Qd z(sDL7D4-|-)jx@1R?(RIYsDg1fGU&hk@>TTkFs$>W2;T18h^xG1)3RRL*;I{!3sFXYuP^jmLq;E-bCA zYcrhn1?!PLrU;FXb=eoWVLhG+kB(K91zGvkssHeKW-ge=Aqr}6#v0Egb_iw*#MWh$ zCDIP$n?6@pBovfgE1O%Iu1@btD?uK;*34227`-#hxGsxA~88-%nES9rO#>(!w?}7xxvs00(IpwUy z3l^5)0oWlxvEQOcQ#x{`3p`D> z8hH#BwykjQ(NQJ#-qf4je ROw*u=2%$4E$!W-={} Date: Sun, 21 Jun 2020 12:47:44 +0200 Subject: [PATCH 2/7] revert un-related changes --- ablation/test_skyline.py | 13 ------------- reproducibility/generate_summary_table.py | 7 ------- reproducibility/summaries/results.pdf | Bin 53443 -> 53441 bytes 3 files changed, 20 deletions(-) delete mode 100644 ablation/test_skyline.py diff --git a/ablation/test_skyline.py b/ablation/test_skyline.py deleted file mode 100644 index eeabe3c45..000000000 --- a/ablation/test_skyline.py +++ /dev/null @@ -1,13 +0,0 @@ -from collate import read_collation - -from pykeen_report.plot import make_sizeplots_trellised - -if __name__ == '__main__': - df = read_collation() - - make_sizeplots_trellised( - df=df, - target_x_header='model_bytes', - target_y_header='hits@10', - output_directory='/tmp/plot_tmp', - ) diff --git a/reproducibility/generate_summary_table.py b/reproducibility/generate_summary_table.py index f33a3df5b..d5f6a4cc1 100644 --- a/reproducibility/generate_summary_table.py +++ b/reproducibility/generate_summary_table.py @@ -229,15 +229,8 @@ def generate_size_table(): return rv -def get_published_results( - -): - return None - - def main(): size_table = generate_size_table() - published_results = get_published_results() all_tables = generate_results_table() write_pdfs(all_tables=all_tables, size_table=size_table) diff --git a/reproducibility/summaries/results.pdf b/reproducibility/summaries/results.pdf index 97dd4072194f79abb39c79b4b345a596f9fbfb1e..44fc2eec0058a050732fb1b86c431b32700482c3 100644 GIT binary patch delta 6174 zcmai%MNpgpm$gHJySuy7IKgRxy99!}yEPVggL`lY9^5TxAXtFlZUGwC#)CD^d^J`7 z)L%7=**uF=&+eSXz5Rvn`U>AAF}~&JOEn#31-d?|c&!TIb|33+DpJQ=B5+aSIu721lM%gnzS`A`fBg zZh)%3rX(-rPAWGruqWvI%Q#4>MKM_QVawX#Ge#5?GraTZ*GOFtPZPEP!_T75Xj?;b zcU;c8x{xkIE|xCo0rz)`o)Wl+r}dtfoMPE^Kl~S!h}6nZ@X*s(;nL=_xTGWJw17cG z$lD1*+jU?v;!a#Nw^Wbi_&#%DuSw)b`0;_7$43t)(F8LDf+yKS{BcvEs(~;8B`!ykX=qH;IZ( zHSBHxldVcSh74a_b=YmF3oLVH1c3XRYza+b5uJ%R8Uizk0)%MaQxp57uG zR|g+i+~_=N@0a4*jBMUxR748plg9gg4{hBxY}+PB{&SO$zcSs$eRW><26N##pFChv z0xM0NiE?Q(j+HP-PIqbMD`yd-xpQnkrSk`5{TUYbJ%%xRvM#r*n8_#Y8e+oyH8@VxLYnntTa!w9i!D zjB%_;ZXrcQlbbhOa-9|MEMH(qoJ%X2AuD5?DK5M#T7n_bd6C0BiGt(%Q7r~MRdk=c zmXoL=@G2BnsB$3CSq}+y;G6fi?=NfV#4pOV%BCf|OS@DFNl!}o$=Bf*c3c0BK`+Fn z>mR&Y{l|U(%wc(M?a)2Qb#u^dD~b7ibGE#+YQ%mOR+M3U-R-B3)*D!8j38~(PbA5y z5%LK%bkpdX==W=xtaTIB=ooGp>c|R>)j|CSbrks!U07E+--#Ki3y@y57IgoLW6(wm zm1I$hB|K7Fn-EO7B!05b*G17N4I%#65mx01p)u0)PH#vYWPIiqPqesv4DmLUNYpsR9=B3XId4i@D8U;AruBfq%Suc*UPQAq5{&U%7U0{xA z(u9DLcUvDhT%rK$FNLmg>>-(%On%&L8b?c1sfUBZnry2X(9BOij3BlJNh>#+p0sWH zKI&9SR%y86So_^h9G}EboKr|@JX?jHv6p3-gk=B5O$0Gz`?R4sSP50G3F9HRIV8;2 zX$$e-M7w+|5MDqAHlJ@Je&9?UvS!TBZVDy*z4WAnGA9GjNpB;VjJ95+JE@DZm=9Sj z)|-}yB4|gZ5l))Lvss?b zGwdFyKPGkx8{57;6e)S5=Nb6w;1P&Ne#Y(~5~L*X8B4y{UL@F1z&Mhx({NPP4BzJL zU5Q#E+#E~dPU^m(lj%P-5~Oxx#{x_ia_j+^3%BR#URv)dI5Z6XX{t}_7jP)VpB)}Q zdKDgE2+ot_li|en>f0R>7Ok56lt^fQ-*fkHnI^TEb8foWQ#1KSQ!akNKr`9i7v^J^ zMi}52FS^&K7vPJqO55b$U*AhoK-P3$cE4+IHPSZm$dkR~ay2rx|9qCclm!rrX|6vTD7- z!#Sif-tdX)gWJOg>;gl<>~gf(#K|1u4S~FN)&BP<%2x$f$UL#QP$2jiu9nla+P5@uT-}%8TOeK!1bhCJxL;C|VC)D*m*${Q z+iTImw!9g8Or;5)U8sNB(S@6mT*V#G{?du4A zs`uQ``(v0e^0x3&r_H}&$b-hyZ%vT-_5R@MY9bH<^=){>X6g1S06wG4FVDk!C{HCl zHfSRs*V>+Dyy0u#AK#St)vM& z3885zi|2yy&iCaK;^KtM{w0ODe&cnFu^T<*C3i|Hb!vWVp8(eU6HnHHkw6aVqMg^a zGAcgFvr2MwAlDiD3JfbHRJzrZy@g?p{OjG!i%w3uKB7lqGWC;?=j3aSpR!o-WL2|Vp__L;sS;x@>c<+sXAG* zHiEfU@NpI}OjP#zD1Q>osb;yTom%}>miV(~h!c4BFK5NG0oO_ZVx45acN-Nk37P%L zG0Ts&lLpMNxWr%cYJ1>aKXaYf%a?+4?$=Bsc^~S$8Ndm)t zOdP+sg_-dp_szISzCy98(7{%Kg`3K`{}MN5sForSH4su)NLkVuJ(x}(gGkc9r^0u1 zWNi4G^)XP@VtQ96?lRgYec|W8kmquZT04Pg8GB>kMf1DPs6K)k^s-ZkxzyXJ-Ye5< zJ3smR(B0MDEJWGxClYDniTL^O$=XK>z#n3t4+=OLSiD>`$Vl&DwLt#Y_KeOS_} zqEWuA$Hbc%kaQw|2EVtv=7|WhY85T-X`b8 zxp7T-QEfK?DZH`sL&7O5y~hx=+sGUJDK%msOWtAayq51tH`lUmihZj-FngibPQj>&dLvy*Mwk zr-cc_dn-|dN*gNSn1}Q^VUXBUVCpt|RmV;E_2s@TamtnmMrxCyrXSb}aSOei-^X_k zP3D;`WSK0|O>j*Sqo+n?a~EA=X#f}_)Z@b5upnt*3|{0mMtz_&d*uEr`#a!`k>8)} zl(a7_Cs1RfQiVQ=66E|w+XNekNGF!Sc3~=5TNPtn>}R{HteF{EB^TqsPmiv!gSf6n zsyhWb8o^O3vG4#X;Rq~L?Z&mTzw3tv;cv(zEcZdJIHEimKJZ2p?$owCwRpfY`@8mm zf1W})os_iQckxVqj67eeyu}++4dG)0#$^jSK?0?+sPY-`e@5r$$mU1Qpmsu6pO>a~ zl8GmN_wslvkUeqU^R(kBCGNkNFyeGW+&gE{r6O)X$r)-#uzB>%_@@U4d{-0GXSF>? zSr{+l|2Y;m21yu8xy~@5{Q!6+LHfcCQAqE=6Inr|bF1E=6hk3GTf_;Rwf6VbH6W#vvMjq ze@yWu${OT9xcPlxP<3?q@!V6)dGJCpB#EFXXUDi8^g`-L!JeAq4Fn>gU|3b?lJ4YZ zfRKZnb-Raf|J+%jTGryaTbODlwrk@H!nl4%szkRB6n+Fi3SvR{G;O95M& zo*Y|Bp?`~7x}UBXX#1|+6%l9{5N9kvv0wyEup91P8X4d(lDsvq@MlBNx6aAy$WzO` ztF|<0s4;*39;BDzQEoc9JB!A^4s2J2u7EZpm^R6~OdB*uO9!(E$HXss$4*4mUd|*sjMU~i6GFB#fj;96s4n!){ME)~Sf^?E&&R<* z#f^KRfQQ@?+&1TqqN|7y+`X{ep|XHGnc-;~L87#g1$NX~8jrbRV|Pl^-%DMj*`MnEPJ@xSlK zV6rd3AXx~ZvA`8ili>rmS8r17WGT>aqB?>DBt%1|Xc4#i+0g8Nhz#Rr<*^Ac05 zTg#$4cVet+B+nM=t8m_%mrvrQDDDI-7NWP)lBLt(QaDo*%2tSebXxGYbn!{ihg^<6 zI=40D9o%$1eNH*`ptE6?rZKwyxD_PztJanb=s<&rK-!~A$GGnlLVqIe8pqvOh|joE z+Nmu6ah@_f#;tasW6;2*YylZ{rDb13&i0c5=UUa#!0)VFD@t_vZ#*(@9IyqZ+||4i z1E!i#YB#uVWN6I;s$iene(zugOk+TDPjen_X3joeTj|0PvBUxObyc-rp;?%rU zdtMdZ7^1E=>tq5qs>glhIbn1dh@b&(C@5r7T%NgcOW&07gGHvb-T)kFZ-<%x-PF=g6@}Ny&uQnA_|Q zh(IZ3PVGJAiF^C=&n$t#7CL}B1G##704%i=L0C>DR+GO)r0=TS0Ksq)H- zOztK=u1^=5^<@YW{vW(jz*OkoRDw`sbg^sM0kiSY7+gn0QmKzuA9 z5DObRkD8nH2Mcc-TGkJud>}p$Na+75sp%v3O^Bn82!LW?)4c3Q{6#&2GH0Tpet(1b zdesS6tc!F`6Cj45~w_{ zoX_Ys=&6uCya6e!lViS)A!RZrG$eC#%M-3!4!$Z$YnA#+q%!hVc+uq4@bCHAsfgo{ z5Ig;X9>8zH^wpyySKhGSBxs{TeXxb)6q*m%&n>&T(3UPG#qi2`M+}?u{m5_5@94Ps zrrVr9{i#Z)Kt2M(xz_1XLfi#Ey#Qeb$}4@5lLc$9D}<$TW#1mP`FgDctSp)U!ciA~ zL0sSuANW%im7_v!b-EKMi0ZL~+AOU5iL?tDfbKtsNJ$vvB+ezR7)7mTppJXuN|F+) z2S~n#F8dnZP1v}}Gwn%p+l_VHhI;YT0LG{oPtJ57U{~dl#dBQbjNfMh74KdKBQ~#`&6riH290BxHr|_B z030M{T$}~>5gHo`jDwAvV+a{9*uIb{AzqZ`F(D~1;20=X?z4QF)LRfYtEnoWz$>OK zm!F-o-0*^(%nfeM%;YqEe)0KeLVVQ(E;!N5hMv*rv#sK#2oK3ryPtla4MKsndV$b= zHu|Q24%r;umj}F@_9OfXlO^rLqwH4zL?Zax!#Ua`DTd842|(s7zk^=vX`?nQki9-h zBMkoT#aFKQ&X;J5AUKax%B%UR)r(HORT-t)g}gj-bGX0l{^ev=RiB48PCJS-h585M zDP7#p#Q17AS{19rZ&=k||E_;o4lIFK{tODe%2=zvAiEEe2{l+l@k>lpQNbhI2T-TM zB?>#^&?U9n=HkpU#$@k_xR|J)qXhvK1zQ?A#1e}zr%!;0c=w%qSA<~5J+WKD#o-?B zuI#!p+;6Vd7kTgk^Uqh2Qym54OSNqViY&r(iSIhr%YU|$(llT*D2%_=q@&_Ict=Qd z(sDL7D4-|-)jx@1R?(RIYsDg1fGU&hk@>TTkFs$>W2;T18h^xG1)3RRL*;I{!3sFXYuP^jmLq;E-bCA zYcrhn1?!PLrU;FXb=eoWVLhG+kB(K91zGvkssHeKW-ge=Aqr}6#v0Egb_iw*#MWh$ zCDIP$n?6@pBovfgE1O%Iu1@btD?uK;*34227`-#hxGsxA~88-%nES9rO#>(!w?}7xxvs00(IpwUy z3l^5)0oWlxvEQOcQ#x{`3p`D> z8hH#BwykjQ(NQJ#-qf4je ROw*u=2%$4E$!W-={}a{LH*mc6?0U$Wb|FKIkuQU1{R>&MA^)Q0+VWIIlqj3D!09Lznrstyqwk6^r0@wU5h#^!!?s zey&k}pYe1F_*6dnV)&?XHbCq_N^a76=c+cWR1h`t?wYM7^QaJ%wrw73A)n#>_}4q z=GTt9mr1Qknp^gDe~e|6yO%7n*gHd-p`DM-6feAfx;L@S40bA9d$4b8oW8}^nBFAfh{>DAMRz#dc+GObs9C)bz5Ob8i4RKO;wW65h+a9`Y_ za2@&hU@Q423b$8SnrIQ`(&W4ER%dOqr{>fv9P3l4+0#2W&?2zpEdf1UwtH{9ph9Wz zz!w(c-@At%R3@a=_7=+rNxw&kYVyEX^}khkO#f)QDtL=iAQk!YUE7G?C*Z^2*K_Z% zCO}A%Px42C)DnE(>GRx%{{>XAX638Z>n^@`;9Y{Cl6Pl+-V6r}Jb2YUyO~<8F z`qCMEcIMbx>A1Pw;hZkJ7kvT2oQ_;IdR#T0mZszb3VMP>2#pvL96SD|JrA_pjTM!2tEDFG44pL z-n8w(ztN|syQV*J|J77%{)Z0{_i$Q~jWfBdNyp@H+UURcWxuO|@}k-jrg#W#EGbPq z;qPQ~%++EHx5vJ7HG z*CqDFcaI=wcY}-%PR(3XpVR!1{HqJI)AyWDTMQ7o&?#6TArGHEIgi4edCtVLlk4$* z@^?i<3CHOG zW$iMsFL+sL%orIJm`W>=@y1xU9KT|0HPpojMSo<|Wl^L9U%H0#Tid33$--dEM7sLW zwymcwAeq8XQO+?t2`?J)yIW*s>J$&nnW*c~i@WXT`3-wQ+GOi^M<$wAu*!LAbj;l* z5ij=k57Zi7>4APYP8$!s1L+^W9_*oiX}ajSC_fsm(9tlVAH^NQ)k<4)_=YW?WneEi zV}ar&Ui6f*Cgw>f#1=^f{+)DakgV%R&x?u8vtU?LT$rcnl6lhGFS+Y)XgKHWp|ndS zW9r?-u(GFLu@A0)HdOp}SB#WhnK95pN#6Hgg3fn(V^Dj3%}BK{6K+8gwOzGEaO^OM zmEAge=KPePZzztN%ZLI3jUo@c9*e>@1!B$PaWm>&M?+&-I_w;7vC7Q|STMYT^$r$L zL?Kvx^Ok2F&5?usQ>=k~oDLu6&y#6&tp%gIRyGX28~MYNLH22Dp~O}m@FgV#r`^?M z_zLVkP{$RC@yp$U{Sg_TXsyhH=QjU|hi|ygm1q=-P;JSe1K8PNew;07)X6GXd8JwD z%EN*$>aKK|j>Np~W}^1ZQ9TzIbq~lNuv*wRI}zJ*g8Sp`)naI($TJO<#7Y$0Oo>Ja zkly)EVNFe-J?1xFaSg}|_YMc-nQuKNTW4=&@TUj&u%f8XZ-D7Yf|FA2QNxVyBXL{& zn>NIAySd$x66LWiSz7M$$Xdt?c4|^#^H~c0%a%fMu-#I{emnMt%HoM@7Oj0LnYHTx zf~A?L7qsh)^W=NI$pnQc8c_qLLmI@GyT{xi2?O?p($v zAxK9OQ9Cumq0rLd6-?#n9d>cE#{sPZZ*^5cjdD=(<0vW{@|kAO=1!*AIj$Ede8-}y zug_4MW$mwE77=_>g7#NxSECL4Y3}el3!&Qe{j{vf$4%i{K`4)3ek0!K!wO;(d0W@! zG=Xrtz2P{%HGdKa68990$DRS0SmfqjIGPI8&UCgxYMgSd!32h7v$B<7h7^Ipl=mp)*Y#NI@9jc$xqgtyIhP^wOo_EmK=YTUKVHF?wP`_0KCel`^+8|9K9tf79B19J zNk0`;THKGwx)%DoTNd}O!~MSS z=&VEb4n^t0RO{8A*^2MWXM&y_;%-{5DHI1F%qe?rgQJS1u zmqm$kFwkLstP?oh;I~;InoeK05n5t+zwG#WhAJjMo>I`6oJ#mKNVcqas2Q>M4zKf-HqrIf6L5xw5W)$(uoX;H?(!6%6zFwdnShrqSsjjxg3cn;~)muklU5Sb zisp7}>HBRqcvh;i=X28yU35kLTf#72o$8&mcz$lMlO0wNcR3oZ<#&wwZtDwRP}%M?X3T2=z*nJ&U<^7!KFm~?vj|?A>e0s zYn9n?wx>iAB->@G|K{hf%wJbEW)$d!$x!@QG&6ZoCI7ddxfF@8cwysNwhN1GjnP`S zcVB+ykCu|{y&dPQ^o~v1t2cssu=%L!S>J6tVfYb5u^E!8t|NNG(D5S)%BzngPxv_9O#Zj z8?1rpP-vxt;5?&A?$Cwv8?9KwFB>c|MjqzhFfm4-?+8ZvYTeMRnbCq-W!JPXe^Qu( z9mY~j)@7bR0Zcx%Mp+yT9dx!=QmW^smjU*eV?c0Mc2!MG$3n zh~RE?{;I2Rgd9iS>gqsPJiL&t@eD zV+*%U;pa?AS>9C1k!^G+kEyy19}bnC+cCdH>CP8ep6gUN6$7Jr=MzoMUqj1v1t)y7 z!R{wO$rc20fW6qv#qWy+`LGqKs*}6#;cjhz77>vP_y6t{eF+=Q1&2Bh4F3ohxU*wM z=n%N5Sm~To&M1N~5G3m_xd`Ew(qC@R|E}EW^Cmgx1jYdHnp+pdFZ8|gNtSJNW{VQ1qnj!ZJ!zqOQWRXL9AT&ty$g2=$lu~$arBE4it@* zwzg)Kq{v_;JPDggs_|0~KWfJTdA69MFJztmUY?t|9K&`lKt@ZK*-2zF*FnHC2^BqcG+_r9&=gBPAeTSLDsbIp8aN%mK)M zcL*mT_!jhjcrPsAvWhxw`IslXv_B{-pkBlfeVVm*@liP*N*|3hB90m!C>52XjJo_w zGM|N@%3?X<>5%yZ-Df~VVm0COBV)b_KJu5e9HD~~(Okc?f2x2eBMvnnc^64o4fH)uxf+Aa%-I}POPYg+Nb(TWA;)6!8%5^d3JOk z>m_cske?u57S?PZq3?{Q{(%!iQIq4Qlqy`G1QtNLtl%T)EjB*ckY+jmkR*>4?!!u1 zND^cG#W3N}jGW$S9~BwrVqI%#u~l0XzzFZNGz=CqmaRkChLtgcN zfeE>}eoRV$0){T9n1{xDpJW`jz3*s(LvYgXN6;@7QH4K>86cg~&Jn#u24bcR+eu7> z9^;~3|B_QR?$TSIIyH!PFaK$^hO~T}lzNjoKsS2v$>?ri+3MfOt6&DRQ71En`6p_* z=N%8emrN!9-?pUok~C-DL1T*69rS%VypY^|vO9^%$pKTSJFAj^`EVN#^h{AsAXo&J zh6_D%Gu>Y5vtioA6Sw}!F5eC}{`PiPrMA3;vh<*v;PP2U{j-<$ebYdkPnk{^mY?x`6w~Ir&AJIZPY#=WPph zm%<~0%>wqDKU*a-JFgue(0Z=hW?M(btbts;Q}o8fyoOyK9%)7fy;6`W;RVhV>UxSy z>M;JFyn!Kg2u8W&h-ZwA7jimV z9hw~pdA@M7?!=oh4|w=4xLw6K;or6q2-!^Q>YQna%q+$zIQ_;3jxXsMoN8Dfascz) zDsdhLI7xb1nv`Fu(ydbVMg35RQig7-OTpcN5zn5J*VQpMH4+A*N^^iRg!s+)uO^hxmj zJz#xTZcH^?s`;?;=VMra$s`c!LwxFS>hbn{mKib$-5YD9jqId9;QOY?RAzm`YHrgK zqyD@cB$q+N*{V5+L-+`>G0FV0Vw|0RC!=NDm?kHI@|*W$p2jhE=j%S*O|AohagW(? zluYln?)<%5%rND3KKZH0I{4c@6B=EM#*UwZ6GM zK};Lyt`*u(OV$q)+$xEwcJ{Qjhs{pU-apX=GAya&@4fFrNA6neIlO*+f z{N6`f=aA7_tSHZY!jXm^gF%si^#?Ns6%>s)I?dRrv^`EQlN5>yGU?XqpqGhu~11sN<%Wn}i>8;{tU{+;-+mQ`MO;C^mCa zwzQ8djeir#>N4?Jj&i_dTaybL7#OS43e|n<)-*mFi$(SmWnV+du0H4|7W5AY&Acr* z;pf5gk?WG!)YPqTm`6?P*HleC03Mz0-_9fEiAPO5_fnVhl$7BInOjA}R;;N+rEo8i zl^u`AoRcG@X6byQXcz4pc+LX8?NYD{d$&p8pYM&wG%wn03MjH5(3YxI%r&C1_Ea%Pd*ovC{O#A6QSS+d}A zEjouhxcxVt!E;%1=-PU9FwMVcGnW$s*mez%G+tF)Kmh>(A&`JD7eA1VpP!8bn@`Q% zR>8{0j)7f449E}U=Lh{iCGE?oBL<&>owbvdTm6_b3Y1;eq5NY7psJcxb(J&m<5fL{ zMfTUt@9F-`Z}#)0c`oucXpi-+E4U$CDg!0DBBe%VS(?Gw>7~! z?l^tW>aV;arWMW`e}gi!cYqwQA`5q4|K3@-dd$8uhbxqZD>IeWy{coascvg?-E$$) zRac>+8fxFJLbK~5n$QUAc<9#3iTj|j)+YHfQ_`@tCZkfzNNf{#h@H}BYb)s_>FawU zQ0^#M@I$v$Dghf|*zUip#1nMBzGFpk8=v~>Gc3BOGwBVRi4bDZ{d~Xirm?!?*$sc^ z;=wrj(Rd*zVw3+|7)~njQ->fs2(P~IWOj0J=hbR)c8$3VSuws5K+;xu zO`gn~p3-1TCUb~RfLfpSM;A)Nof2Ze>k<8cr^8}L@bWPdy`h}|Nk^ZR$poB$zRizO zN*c;;tA!goz2j3;W;*&xPE6pPA$_AM-Je&&9aJ^HT=(n|5OT2-g!2LQisu*A&g{6& z$j8o%vx^dwbKJNc%6MAOwbI1aKk$KfARq^4uuSPleZn9kN8Vf^fl@R)9G`sGI+d@4 z9o*9cZY8+Z_vRvMAy$Q%_+x88bvuO8$~SQ;VPvkEEhcVFb4$W=R2b~2Us6c`)LJpC z69n9ukbf)IViEC9o8PoHuuycKk{E|y2EB4q{`_zA1qg?zVPulJ!#+4lqtbD zi*9Z6w{nu%onh^2rL_eHT)`a5c+{DvAn$z_RiXhyqARK^`Ovl81;2NHIwKuZkHqpQ@dkgO4MF0KXu^|2gLE zNJrYVU#Nde*ZJRDfO=3x;g6fLguO;h6^p=Jz6=Z5F4SLs^ zedfaNTJ_`QyZA!W92(eQg1`r)X)@i{5uCp4wHnzxlYRKv)l@Qv+=^O`#?WrX=1 Qp~n*C$7W$s)PP|B7bOoSr2qf` From d3a143fa19dadd135ffe8e8b7009d8221614cccc Mon Sep 17 00:00:00 2001 From: Max Berrendorf Date: Mon, 22 Jun 2020 12:15:40 +0200 Subject: [PATCH 3/7] Some progress --- reproducibility/generate_summary_table.py | 123 +++++++++++++++++-- reproducibility/summaries/fb15k237_table.tex | 16 +-- reproducibility/summaries/fb15k_table.tex | 49 +++++--- reproducibility/summaries/results.pdf | Bin 53441 -> 53520 bytes reproducibility/summaries/results.tex | 78 ++++++------ reproducibility/summaries/wn18_table.tex | 53 +++++--- reproducibility/summaries/wn18rr_table.tex | 18 +-- 7 files changed, 240 insertions(+), 97 deletions(-) diff --git a/reproducibility/generate_summary_table.py b/reproducibility/generate_summary_table.py index d5f6a4cc1..bbf43b387 100644 --- a/reproducibility/generate_summary_table.py +++ b/reproducibility/generate_summary_table.py @@ -1,18 +1,18 @@ # -*- coding: utf-8 -*- """Generate a summary table for the reproduction.""" - +import json import logging import os +import pathlib from collections import defaultdict from typing import Iterable, List, Mapping, Tuple import humanize import pandas as pd -from jinja2 import Environment, FileSystemLoader - import pykeen.datasets import pykeen.models +from jinja2 import Environment, FileSystemLoader from utils import SKIP, read_experiment_collation logger = logging.getLogger(__name__) @@ -23,6 +23,48 @@ # can be any one-character string that you're sure won't appear in the table PHANTOM_PLACEHOLDER = '✠' +PHANTOM_PLACEHOLDER_PM = 'ᐖ' +PHANTOM_PLACEHOLDER_DOT = 'ᑅ' + + +def format_percent(v): + value = str(v) + remainder = value[4:] + return value[2:4] + '.' + remainder + + +def load_published_results_in_tall_format( + dataset: str, +) -> pd.DataFrame: + # model, columns, mean, std + data = [] + for config_path in (pathlib.Path(HERE) / 'configs').glob(f'*/*_{dataset}.json'): + with config_path.open('r') as config_file: + config = json.load(config_file) + model = config['pipeline']['model'] + dataset_ = config['pipeline']['dataset'] + if dataset_ != dataset: + logger.error(f'Config for {dataset} specifies dataset={dataset_}.') + continue + results = config.get('results') + if results is None: + logger.warning(f'No results for {dataset}-{model}') + continue + for key in [ + 'mean_rank', + 'mean_reciprocal_rank' + ]: + if key in results: + value = list(results[key].values())[0] + if key == 'mean_reciprocal_rank': + value = format_percent(value) + data.append((model, f'{key}.pub', str(value), None)) + if 'hits_at_k' in results: + hits_at_k = list(results['hits_at_k'].values())[0] + for key, value in hits_at_k.items(): + # manually reformatting, since str(100*value) leads to floating point imprecision... + data.append((model, f'hits_at_k.pub.{key}', format_percent(value), None)) + return pd.DataFrame(data=data, columns=['model', 'columns', 'mean', 'std']) def generate_results_table(): @@ -34,16 +76,20 @@ def generate_results_table(): tall_summary_df = get_tall_summary_df(dataset_df) tall_summary_df.to_csv(os.path.join(SUMMARIES, f'{dataset}.tsv'), sep='\t', index=False) - wide_summary_df = reorganize_summary_df(tall_summary_df) + published_results_tall_df = load_published_results_in_tall_format(dataset=dataset) + all_results_tall_df = pd.concat([tall_summary_df, published_results_tall_df], ignore_index=True) + wide_summary_df = reorganize_summary_df(all_results_tall_df) + wide_summary_df = wide_summary_df.applymap(lambda v: ('$' + v + '$') if v != '' else '') # Save as Latex table + pd.set_option('max_colwidth', 999) table_latex = wide_summary_df.to_latex( index=True, escape=False, - column_format='l' + ('r' * len(wide_summary_df.columns)), + column_format='ll' + ('r' * len(wide_summary_df.columns)), bold_rows=True, ) - table_latex = _process_tex(table_latex) + # table_latex = _process_tex(table_latex) with open(os.path.join(SUMMARIES, f'{dataset}_table.tex'), 'w') as file: print(table_latex, file=file) @@ -75,27 +121,80 @@ def get_tall_summary_df(df: pd.DataFrame): return pd.DataFrame(rows, columns=['model', 'columns', 'mean', 'std']) +def get_width(column): + prelen = postlen = 0 + for e in column: + if e is None: + continue + splitted = str(e).split('.') + prelen = max(prelen, len(splitted[0])) + if len(splitted) > 1: + postlen = max(postlen, len(splitted[1])) + return prelen, postlen + + +def format_values(mean, std, mean_w, std_w): + if mean is None: + return '' + # format mean + pre_len, post_len = mean_w + if '.' not in mean: + mean += '.' + pre, post = mean.split('.') + if len(pre) < pre_len: + pre = r'\phantom{' + (pre_len - len(pre)) * '0' + '}' + pre + if len(post) < post_len: + post = post + r'\phantom{' + (post_len - len(post)) * '0' + '}' + mean_str = pre + '.' + post + if std is None: + pre_len, post_len = std_w + std_str = r'\phantom{ \pm ' + '0' * pre_len + '.' + '0' * post_len + '}' + else: + pre, post = std.split('.') + pre_len, post_len = std_w + if len(pre) < pre_len: + pre = r'\phantom{' + (pre_len - len(pre)) * '0' + '}' + pre + if len(post) < post_len: + post = post + r'\phantom{' + (post_len - len(post)) * '0' + '}' + std_str = r' \pm ' + pre + '.' + post + out = mean_str + std_str + out = out.replace(r'}\phantom{', '') + return out + + def reorganize_summary_df(df: pd.DataFrame) -> pd.DataFrame: - _n = df.groupby(['columns']).aggregate(lambda q: max(map(len, q))) + df['short_columns'] = df['columns'].str.replace('.((worst)|(avg)|(best)|(pub))', '', regex=True) + _n = df.groupby(['short_columns']).aggregate({'mean': get_width, 'std': get_width}) n = _n['std'] _mean_n = _n['mean'] df['values'] = [ - (_mean_n[column] - len(mean)) * PHANTOM_PLACEHOLDER + mean + ' ± ' + ( - n[column] - len(std)) * PHANTOM_PLACEHOLDER + std - for column, mean, std in df[['columns', 'mean', 'std']].values + format_values(mean=mean, std=std, mean_w=_mean_n[column], std_w=n[column]) + for column, mean, std in df[['short_columns', 'mean', 'std']].values ] + inconsistent_rank_mask = df[~df['columns'].str.contains('.pub')].groupby(by=['model']).agg( + {'values': lambda x: len(x.unique()) > len(df['short_columns'].unique())}) + consistent_rank_models = inconsistent_rank_mask[~inconsistent_rank_mask.values].index.tolist() + for consistent_model in consistent_rank_models: + df.drop(index=df[(df['model'] == consistent_model) & df['columns'].str.contains('.((worst)|(best))', '', regex=True)].index, inplace=True) + rv = df[['model', 'columns', 'values']] rv = rv.set_index(['model', 'columns']).unstack(level=-1).reset_index().set_index('model') rv.columns = rv.columns.get_level_values(1) rv = rv[[col for col in list(rv.columns) if col not in {'training', 'evaluation'}]] rv.columns = get_renamed_columns(rv) rv = get_reordered_df(rv) + rv = rv.swaplevel(axis=1).stack().fillna('').reindex(['pub', 'avg', 'best', 'worst'], axis=0, level=1).reindex( + [r'MRR (\%)', r'Hits@1 (\%)', r'Hits@3 (\%)', r'Hits@5 (\%)', r'Hits@10 (\%)', 'MR', r'AMR (\%)', ], axis=1) return rv def _process_tex(s: str) -> str: - s = s.replace('±', '$\\pm$').replace(PHANTOM_PLACEHOLDER, '$\\phantom{5}$') + s = s.replace('±', '\\pm').replace( + PHANTOM_PLACEHOLDER, '\\phantom{5}').replace( + PHANTOM_PLACEHOLDER_PM, '\\phantom{\\pm}').replace( + PHANTOM_PLACEHOLDER_DOT, '\\phantom{.}' + ) # s = s.replace('\\begin{tabular}', '\\begin{tabular*}') # s = s.replace('\\end{tabular}', '\\end{tabular*}') return s @@ -208,7 +307,7 @@ def get_reordered_df(df: pd.DataFrame) -> pd.DataFrame: columns = [ (first_level_label, second_level_label) - for first_level_label in ['', 'avg', 'best', 'worst'] + for first_level_label in ['pub', 'avg', 'best', 'worst'] for second_level_label in r[first_level_label] ] return df[columns] diff --git a/reproducibility/summaries/fb15k237_table.tex b/reproducibility/summaries/fb15k237_table.tex index fb722a9c7..65e4feea8 100644 --- a/reproducibility/summaries/fb15k237_table.tex +++ b/reproducibility/summaries/fb15k237_table.tex @@ -1,12 +1,14 @@ -\begin{tabular}{lrrrrrrrrrrrrrrrrrrr} +\begin{tabular}{llrrrrrrr} \toprule -{} & \multicolumn{7}{l}{avg} & \multicolumn{6}{l}{best} & \multicolumn{6}{l}{worst} \\ -{} & AMR (\%) & Hits@1 (\%) & Hits@10 (\%) & Hits@3 (\%) & Hits@5 (\%) & MR & MRR (\%) & Hits@1 (\%) & Hits@10 (\%) & Hits@3 (\%) & Hits@5 (\%) & MR & MRR (\%) & Hits@1 (\%) & Hits@10 (\%) & Hits@3 (\%) & Hits@5 (\%) & MR & MRR (\%) \\ -\textbf{model } & & & & & & & & & & & & & & & & & & & \\ + & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ +\textbf{model} & {} & & & & & & & \\ \midrule -\textbf{ConvE } & $\phantom{5}$3.73 $\pm$ 0.13 & 18.22 $\pm$ 0.11 & 44.95 $\pm$ 0.17 & 29.51 $\pm$ 0.24 & 35.98 $\pm$ 0.16 & $\phantom{5}$255.46 $\pm$ $\phantom{5}$6.16 & 26.93 $\pm$ 0.11 & 18.22 $\pm$ 0.11 & 44.95 $\pm$ 0.17 & 29.51 $\pm$ 0.24 & 35.98 $\pm$ 0.16 & $\phantom{5}$255.46 $\pm$ $\phantom{5}$6.16 & 26.93 $\pm$ 0.11 & 18.22 $\pm$ 0.11 & 44.95 $\pm$ 0.17 & 29.51 $\pm$ 0.24 & 35.98 $\pm$ 0.16 & $\phantom{5}$255.46 $\pm$ $\phantom{5}$6.16 & 26.93 $\pm$ 0.11 \\ -\textbf{ConvKB} & 61.36 $\pm$ 0.65 & $\phantom{5}$3.31 $\pm$ 0.23 & $\phantom{5}$7.76 $\pm$ 0.88 & $\phantom{5}$4.04 $\pm$ 0.19 & $\phantom{5}$4.57 $\pm$ 0.22 & 4345.27 $\pm$ 46.99 & $\phantom{5}$4.71 $\pm$ 0.23 & $\phantom{5}$3.31 $\pm$ 0.23 & $\phantom{5}$7.76 $\pm$ 0.88 & $\phantom{5}$4.04 $\pm$ 0.19 & $\phantom{5}$4.57 $\pm$ 0.22 & 4345.27 $\pm$ 46.99 & $\phantom{5}$4.71 $\pm$ 0.23 & $\phantom{5}$3.31 $\pm$ 0.23 & $\phantom{5}$7.76 $\pm$ 0.88 & $\phantom{5}$4.04 $\pm$ 0.19 & $\phantom{5}$4.57 $\pm$ 0.22 & 4345.27 $\pm$ 46.99 & $\phantom{5}$4.71 $\pm$ 0.23 \\ -\textbf{RotatE} & $\phantom{5}$2.84 $\pm$ 0.00 & 17.57 $\pm$ 0.06 & 44.55 $\pm$ 0.06 & 28.97 $\pm$ 0.05 & 35.29 $\pm$ 0.09 & $\phantom{5}$191.92 $\pm$ $\phantom{5}$0.31 & 26.42 $\pm$ 0.04 & 17.57 $\pm$ 0.06 & 44.55 $\pm$ 0.06 & 28.97 $\pm$ 0.05 & 35.29 $\pm$ 0.09 & $\phantom{5}$191.92 $\pm$ $\phantom{5}$0.31 & 26.42 $\pm$ 0.04 & 17.57 $\pm$ 0.06 & 44.55 $\pm$ 0.06 & 28.97 $\pm$ 0.05 & 35.29 $\pm$ 0.09 & $\phantom{5}$191.92 $\pm$ $\phantom{5}$0.31 & 26.42 $\pm$ 0.04 \\ +\textbf{ConvE} & \textbf{pub} & $32.5\phantom{0 \pm 0.00}$ & $23.7\phantom{0 \pm 0.00}$ & $35.6\phantom{0 \pm 0.00}$ & & $50.1\phantom{0 \pm 0.00}$ & $\phantom{0}244.\phantom{00 \pm 00.00}$ & \\ + & \textbf{avg} & $26.93 \pm 0.11$ & $18.22 \pm 0.11$ & $29.51 \pm 0.24$ & $35.98 \pm 0.16$ & $44.95 \pm 0.17$ & $\phantom{0}255.46 \pm \phantom{0}6.16$ & $\phantom{0}3.73 \pm 0.13$ \\ +\textbf{ConvKB} & \textbf{pub} & $39.6\phantom{0 \pm 0.00}$ & & & & $51.7\phantom{0 \pm 0.00}$ & $\phantom{0}257.\phantom{00 \pm 00.00}$ & \\ + & \textbf{avg} & $\phantom{0}4.71 \pm 0.23$ & $\phantom{0}3.31 \pm 0.23$ & $\phantom{0}4.04 \pm 0.19$ & $\phantom{0}4.57 \pm 0.22$ & $\phantom{0}7.76 \pm 0.88$ & $4345.27 \pm 46.99$ & $61.36 \pm 0.65$ \\ +\textbf{RotatE} & \textbf{pub} & $33.8\phantom{0 \pm 0.00}$ & $24.1\phantom{0 \pm 0.00}$ & $37.5\phantom{0 \pm 0.00}$ & & $53.3\phantom{0 \pm 0.00}$ & $\phantom{0}177.\phantom{00 \pm 00.00}$ & \\ + & \textbf{avg} & $26.42 \pm 0.04$ & $17.57 \pm 0.06$ & $28.97 \pm 0.05$ & $35.29 \pm 0.09$ & $44.55 \pm 0.06$ & $\phantom{0}191.92 \pm \phantom{0}0.31$ & $\phantom{0}2.84 \pm 0.00$ \\ \bottomrule \end{tabular} diff --git a/reproducibility/summaries/fb15k_table.tex b/reproducibility/summaries/fb15k_table.tex index a279a6b5b..c616d12fc 100644 --- a/reproducibility/summaries/fb15k_table.tex +++ b/reproducibility/summaries/fb15k_table.tex @@ -1,21 +1,38 @@ -\begin{tabular}{lrrrrrrrrrrrrrrrrrrr} +\begin{tabular}{llrrrrrrr} \toprule -{} & \multicolumn{7}{l}{avg} & \multicolumn{6}{l}{best} & \multicolumn{6}{l}{worst} \\ -{} & AMR (\%) & Hits@1 (\%) & Hits@10 (\%) & Hits@3 (\%) & Hits@5 (\%) & MR & MRR (\%) & Hits@1 (\%) & Hits@10 (\%) & Hits@3 (\%) & Hits@5 (\%) & MR & MRR (\%) & Hits@1 (\%) & Hits@10 (\%) & Hits@3 (\%) & Hits@5 (\%) & MR & MRR (\%) \\ -\textbf{model } & & & & & & & & & & & & & & & & & & & \\ + & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ +\textbf{model} & {} & & & & & & & \\ \midrule -\textbf{ComplEx } & $\phantom{5}$$\phantom{5}$2.46 $\pm$ 0.28 & 10.06 $\pm$ 0.42 & 38.03 $\pm$ 1.02 & 20.82 $\pm$ 0.67 & 27.59 $\pm$ 0.94 & $\phantom{5}$171.16 $\pm$ 16.05 & 19.13 $\pm$ 0.45 & 10.06 $\pm$ 0.42 & 38.03 $\pm$ $\phantom{5}$1.02 & 20.82 $\pm$ $\phantom{5}$0.67 & 27.59 $\pm$ $\phantom{5}$0.94 & $\phantom{5}$171.16 $\pm$ 16.05 & 19.13 $\pm$ 0.45 & 10.06 $\pm$ 0.42 & 38.03 $\pm$ 1.02 & 20.82 $\pm$ 0.67 & 27.59 $\pm$ 0.94 & $\phantom{5}$$\phantom{5}$171.16 $\pm$ 16.05 & 19.13 $\pm$ 0.45 \\ -\textbf{ConvE } & $\phantom{5}$$\phantom{5}$0.73 $\pm$ 0.01 & 48.28 $\pm$ 0.12 & 79.76 $\pm$ 0.07 & 66.99 $\pm$ 0.04 & 73.27 $\pm$ 0.03 & $\phantom{5}$$\phantom{5}$50.76 $\pm$ $\phantom{5}$0.40 & 59.56 $\pm$ 0.06 & 48.28 $\pm$ 0.12 & 79.76 $\pm$ $\phantom{5}$0.07 & 66.99 $\pm$ $\phantom{5}$0.04 & 73.27 $\pm$ $\phantom{5}$0.03 & $\phantom{5}$$\phantom{5}$50.76 $\pm$ $\phantom{5}$0.40 & 59.56 $\pm$ 0.06 & 48.28 $\pm$ 0.12 & 79.76 $\pm$ 0.07 & 66.99 $\pm$ 0.04 & 73.27 $\pm$ 0.03 & $\phantom{5}$$\phantom{5}$$\phantom{5}$50.76 $\pm$ $\phantom{5}$0.40 & 59.56 $\pm$ 0.06 \\ -\textbf{DistMult} & $\phantom{5}$$\phantom{5}$1.86 $\pm$ 0.03 & 16.45 $\pm$ 0.16 & 45.00 $\pm$ 0.25 & 29.10 $\pm$ 0.17 & 35.54 $\pm$ 0.21 & $\phantom{5}$134.02 $\pm$ $\phantom{5}$1.98 & 26.06 $\pm$ 0.17 & 16.45 $\pm$ 0.16 & 45.00 $\pm$ $\phantom{5}$0.25 & 29.10 $\pm$ $\phantom{5}$0.17 & 35.54 $\pm$ $\phantom{5}$0.21 & $\phantom{5}$134.02 $\pm$ $\phantom{5}$1.98 & 26.06 $\pm$ 0.17 & 16.45 $\pm$ 0.16 & 45.00 $\pm$ 0.25 & 29.10 $\pm$ 0.17 & 35.54 $\pm$ 0.21 & $\phantom{5}$$\phantom{5}$134.02 $\pm$ $\phantom{5}$1.98 & 26.06 $\pm$ 0.17 \\ -\textbf{HolE } & $\phantom{5}$$\phantom{5}$2.71 $\pm$ 0.12 & 21.79 $\pm$ 0.19 & 58.84 $\pm$ 0.28 & 39.69 $\pm$ 0.24 & 48.06 $\pm$ 0.30 & $\phantom{5}$193.03 $\pm$ $\phantom{5}$7.61 & 34.15 $\pm$ 0.22 & 21.79 $\pm$ 0.19 & 58.84 $\pm$ $\phantom{5}$0.28 & 39.69 $\pm$ $\phantom{5}$0.24 & 48.06 $\pm$ $\phantom{5}$0.30 & $\phantom{5}$193.03 $\pm$ $\phantom{5}$7.61 & 34.15 $\pm$ 0.22 & 21.79 $\pm$ 0.19 & 58.84 $\pm$ 0.28 & 39.69 $\pm$ 0.24 & 48.06 $\pm$ 0.30 & $\phantom{5}$$\phantom{5}$193.03 $\pm$ $\phantom{5}$7.61 & 34.15 $\pm$ 0.22 \\ -\textbf{KG2E } & $\phantom{5}$78.40 $\pm$ 0.68 & $\phantom{5}$0.11 $\pm$ 0.04 & $\phantom{5}$1.01 $\pm$ 0.14 & $\phantom{5}$0.36 $\pm$ 0.08 & $\phantom{5}$0.56 $\pm$ 0.10 & 5779.07 $\pm$ 51.02 & $\phantom{5}$0.58 $\pm$ 0.07 & $\phantom{5}$0.11 $\pm$ 0.04 & $\phantom{5}$1.01 $\pm$ $\phantom{5}$0.14 & $\phantom{5}$0.36 $\pm$ $\phantom{5}$0.08 & $\phantom{5}$0.56 $\pm$ $\phantom{5}$0.10 & 5779.07 $\pm$ 51.02 & $\phantom{5}$0.58 $\pm$ 0.07 & $\phantom{5}$0.11 $\pm$ 0.04 & $\phantom{5}$1.01 $\pm$ 0.14 & $\phantom{5}$0.36 $\pm$ 0.08 & $\phantom{5}$0.56 $\pm$ 0.10 & $\phantom{5}$5779.07 $\pm$ 51.02 & $\phantom{5}$0.58 $\pm$ 0.07 \\ -\textbf{RotatE } & $\phantom{5}$$\phantom{5}$0.63 $\pm$ 0.00 & 41.53 $\pm$ 0.06 & 78.67 $\pm$ 0.08 & 64.14 $\pm$ 0.07 & 71.23 $\pm$ 0.05 & $\phantom{5}$$\phantom{5}$42.28 $\pm$ $\phantom{5}$0.13 & 55.00 $\pm$ 0.06 & 41.53 $\pm$ 0.06 & 78.67 $\pm$ $\phantom{5}$0.08 & 64.14 $\pm$ $\phantom{5}$0.07 & 71.23 $\pm$ $\phantom{5}$0.05 & $\phantom{5}$$\phantom{5}$42.28 $\pm$ $\phantom{5}$0.13 & 55.00 $\pm$ 0.06 & 41.53 $\pm$ 0.06 & 78.67 $\pm$ 0.08 & 64.14 $\pm$ 0.07 & 71.23 $\pm$ 0.05 & $\phantom{5}$$\phantom{5}$$\phantom{5}$42.28 $\pm$ $\phantom{5}$0.13 & 55.00 $\pm$ 0.06 \\ -\textbf{SimplE } & 100.02 $\pm$ 0.03 & $\phantom{5}$0.01 $\pm$ 0.00 & $\phantom{5}$0.06 $\pm$ 0.01 & $\phantom{5}$0.03 $\pm$ 0.00 & $\phantom{5}$0.04 $\pm$ 0.00 & 7395.75 $\pm$ $\phantom{5}$2.02 & $\phantom{5}$0.04 $\pm$ 0.00 & 11.58 $\pm$ 6.42 & 54.28 $\pm$ 15.80 & 24.16 $\pm$ 10.95 & 34.73 $\pm$ 13.40 & $\phantom{5}$139.34 $\pm$ 49.45 & 23.90 $\pm$ 8.79 & $\phantom{5}$0.01 $\pm$ 0.00 & $\phantom{5}$0.06 $\pm$ 0.01 & $\phantom{5}$0.03 $\pm$ 0.00 & $\phantom{5}$0.04 $\pm$ 0.00 & 14652.16 $\pm$ 45.71 & $\phantom{5}$0.03 $\pm$ 0.00 \\ -\textbf{TransD } & $\phantom{5}$$\phantom{5}$2.29 $\pm$ 0.09 & 21.22 $\pm$ 0.03 & 58.71 $\pm$ 0.14 & 40.48 $\pm$ 0.10 & 48.57 $\pm$ 0.09 & $\phantom{5}$153.37 $\pm$ $\phantom{5}$5.35 & 33.99 $\pm$ 0.03 & 21.22 $\pm$ 0.03 & 58.71 $\pm$ $\phantom{5}$0.14 & 40.48 $\pm$ $\phantom{5}$0.10 & 48.57 $\pm$ $\phantom{5}$0.09 & $\phantom{5}$153.37 $\pm$ $\phantom{5}$5.35 & 33.99 $\pm$ 0.03 & 21.22 $\pm$ 0.03 & 58.71 $\pm$ 0.14 & 40.48 $\pm$ 0.10 & 48.57 $\pm$ 0.09 & $\phantom{5}$$\phantom{5}$153.37 $\pm$ $\phantom{5}$5.35 & 33.99 $\pm$ 0.03 \\ -\textbf{TransE } & $\phantom{5}$$\phantom{5}$1.78 $\pm$ 0.01 & 15.23 $\pm$ 0.16 & 47.34 $\pm$ 0.18 & 29.85 $\pm$ 0.24 & 37.18 $\pm$ 0.24 & $\phantom{5}$127.92 $\pm$ $\phantom{5}$0.86 & 26.01 $\pm$ 0.17 & 15.23 $\pm$ 0.16 & 47.34 $\pm$ $\phantom{5}$0.18 & 29.85 $\pm$ $\phantom{5}$0.24 & 37.18 $\pm$ $\phantom{5}$0.24 & $\phantom{5}$127.92 $\pm$ $\phantom{5}$0.86 & 26.01 $\pm$ 0.17 & 15.23 $\pm$ 0.16 & 47.34 $\pm$ 0.18 & 29.85 $\pm$ 0.24 & 37.18 $\pm$ 0.24 & $\phantom{5}$$\phantom{5}$127.92 $\pm$ $\phantom{5}$0.86 & 26.01 $\pm$ 0.17 \\ -\textbf{TransH } & $\phantom{5}$85.63 $\pm$ 0.40 & $\phantom{5}$1.69 $\pm$ 0.25 & $\phantom{5}$3.74 $\pm$ 0.18 & $\phantom{5}$2.95 $\pm$ 0.20 & $\phantom{5}$3.29 $\pm$ 0.22 & 6320.02 $\pm$ 30.37 & $\phantom{5}$2.54 $\pm$ 0.20 & $\phantom{5}$1.69 $\pm$ 0.25 & $\phantom{5}$3.74 $\pm$ $\phantom{5}$0.18 & $\phantom{5}$2.95 $\pm$ $\phantom{5}$0.20 & $\phantom{5}$3.29 $\pm$ $\phantom{5}$0.22 & 6320.00 $\pm$ 30.37 & $\phantom{5}$2.54 $\pm$ 0.20 & $\phantom{5}$1.69 $\pm$ 0.25 & $\phantom{5}$3.74 $\pm$ 0.18 & $\phantom{5}$2.95 $\pm$ 0.20 & $\phantom{5}$3.29 $\pm$ 0.22 & $\phantom{5}$6320.05 $\pm$ 30.37 & $\phantom{5}$2.54 $\pm$ 0.20 \\ -\textbf{TransR } & $\phantom{5}$91.99 $\pm$ 0.22 & $\phantom{5}$0.37 $\pm$ 0.00 & $\phantom{5}$1.03 $\pm$ 0.07 & $\phantom{5}$0.63 $\pm$ 0.04 & $\phantom{5}$0.78 $\pm$ 0.06 & 6795.95 $\pm$ 16.65 & $\phantom{5}$0.65 $\pm$ 0.02 & $\phantom{5}$0.37 $\pm$ 0.00 & $\phantom{5}$1.03 $\pm$ $\phantom{5}$0.07 & $\phantom{5}$0.63 $\pm$ $\phantom{5}$0.04 & $\phantom{5}$0.78 $\pm$ $\phantom{5}$0.06 & 6795.94 $\pm$ 16.65 & $\phantom{5}$0.65 $\pm$ 0.02 & $\phantom{5}$0.37 $\pm$ 0.00 & $\phantom{5}$1.03 $\pm$ 0.07 & $\phantom{5}$0.63 $\pm$ 0.04 & $\phantom{5}$0.78 $\pm$ 0.06 & $\phantom{5}$6795.95 $\pm$ 16.65 & $\phantom{5}$0.65 $\pm$ 0.02 \\ -\textbf{TuckER } & $\phantom{5}$99.11 $\pm$ 0.39 & $\phantom{5}$0.01 $\pm$ 0.00 & $\phantom{5}$0.15 $\pm$ 0.17 & $\phantom{5}$0.02 $\pm$ 0.00 & $\phantom{5}$0.03 $\pm$ 0.01 & 7327.77 $\pm$ 29.22 & $\phantom{5}$0.07 $\pm$ 0.02 & $\phantom{5}$0.01 $\pm$ 0.00 & $\phantom{5}$0.15 $\pm$ $\phantom{5}$0.17 & $\phantom{5}$0.02 $\pm$ $\phantom{5}$0.00 & $\phantom{5}$0.03 $\pm$ $\phantom{5}$0.01 & 7327.77 $\pm$ 29.22 & $\phantom{5}$0.07 $\pm$ 0.02 & $\phantom{5}$0.01 $\pm$ 0.00 & $\phantom{5}$0.15 $\pm$ 0.17 & $\phantom{5}$0.02 $\pm$ 0.00 & $\phantom{5}$0.03 $\pm$ 0.01 & $\phantom{5}$7327.77 $\pm$ 29.22 & $\phantom{5}$0.07 $\pm$ 0.02 \\ +\textbf{ComplEx} & \textbf{pub} & $69.2\phantom{0 \pm 0.00}$ & $59.9\phantom{0 \pm 0.00}$ & $75.9\phantom{0 \pm 00.00}$ & & $84.\phantom{00 \pm 00.00}$ & & \\ + & \textbf{avg} & $19.13 \pm 0.45$ & $10.06 \pm 0.42$ & $20.82 \pm \phantom{0}0.67$ & $27.59 \pm \phantom{0}0.94$ & $38.03 \pm \phantom{0}1.02$ & $\phantom{00}171.16 \pm 16.05$ & $\phantom{00}2.46 \pm 0.28$ \\ +\textbf{ConvE} & \textbf{pub} & $65.7\phantom{0 \pm 0.00}$ & $55.8\phantom{0 \pm 0.00}$ & $72.3\phantom{0 \pm 00.00}$ & & $83.1\phantom{0 \pm 00.00}$ & $\phantom{000}51.\phantom{00 \pm 00.00}$ & \\ + & \textbf{avg} & $59.56 \pm 0.06$ & $48.28 \pm 0.12$ & $66.99 \pm \phantom{0}0.04$ & $73.27 \pm \phantom{0}0.03$ & $79.76 \pm \phantom{0}0.07$ & $\phantom{000}50.76 \pm \phantom{0}0.40$ & $\phantom{00}0.73 \pm 0.01$ \\ +\textbf{DistMult} & \textbf{pub} & $35.\phantom{00 \pm 0.00}$ & & & & $57.7\phantom{0 \pm 00.00}$ & & \\ + & \textbf{avg} & $26.06 \pm 0.17$ & $16.45 \pm 0.16$ & $29.10 \pm \phantom{0}0.17$ & $35.54 \pm \phantom{0}0.21$ & $45.00 \pm \phantom{0}0.25$ & $\phantom{00}134.02 \pm \phantom{0}1.98$ & $\phantom{00}1.86 \pm 0.03$ \\ +\textbf{HolE} & \textbf{pub} & $52.4\phantom{0 \pm 0.00}$ & $40.2\phantom{0 \pm 0.00}$ & $61.3\phantom{0 \pm 00.00}$ & & $73.9\phantom{0 \pm 00.00}$ & & \\ + & \textbf{avg} & $34.15 \pm 0.22$ & $21.79 \pm 0.19$ & $39.69 \pm \phantom{0}0.24$ & $48.06 \pm \phantom{0}0.30$ & $58.84 \pm \phantom{0}0.28$ & $\phantom{00}193.03 \pm \phantom{0}7.61$ & $\phantom{00}2.71 \pm 0.12$ \\ +\textbf{KG2E} & \textbf{pub} & & & & & $71.5\phantom{0 \pm 00.00}$ & $\phantom{000}59.\phantom{00 \pm 00.00}$ & \\ + & \textbf{avg} & $\phantom{0}0.58 \pm 0.07$ & $\phantom{0}0.11 \pm 0.04$ & $\phantom{0}0.36 \pm \phantom{0}0.08$ & $\phantom{0}0.56 \pm \phantom{0}0.10$ & $\phantom{0}1.01 \pm \phantom{0}0.14$ & $\phantom{0}5779.07 \pm 51.02$ & $\phantom{0}78.40 \pm 0.68$ \\ +\textbf{RotatE} & \textbf{pub} & $79.7\phantom{0 \pm 0.00}$ & $74.6\phantom{0 \pm 0.00}$ & $83.\phantom{00 \pm 00.00}$ & & $88.4\phantom{0 \pm 00.00}$ & $\phantom{000}40.\phantom{00 \pm 00.00}$ & \\ + & \textbf{avg} & $55.00 \pm 0.06$ & $41.53 \pm 0.06$ & $64.14 \pm \phantom{0}0.07$ & $71.23 \pm \phantom{0}0.05$ & $78.67 \pm \phantom{0}0.08$ & $\phantom{000}42.28 \pm \phantom{0}0.13$ & $\phantom{00}0.63 \pm 0.00$ \\ +\textbf{SimplE} & \textbf{pub} & $72.7\phantom{0 \pm 0.00}$ & $66.\phantom{00 \pm 0.00}$ & $77.3\phantom{0 \pm 00.00}$ & & $83.8\phantom{0 \pm 00.00}$ & & \\ + & \textbf{avg} & $\phantom{0}0.04 \pm 0.00$ & $\phantom{0}0.01 \pm 0.00$ & $\phantom{0}0.03 \pm \phantom{0}0.00$ & $\phantom{0}0.04 \pm \phantom{0}0.00$ & $\phantom{0}0.06 \pm \phantom{0}0.01$ & $\phantom{0}7395.75 \pm \phantom{0}2.02$ & $100.02 \pm 0.03$ \\ + & \textbf{best} & $23.90 \pm 8.79$ & $11.58 \pm 6.42$ & $24.16 \pm 10.95$ & $34.73 \pm 13.40$ & $54.28 \pm 15.80$ & $\phantom{00}139.34 \pm 49.45$ & \\ + & \textbf{worst} & $\phantom{0}0.03 \pm 0.00$ & $\phantom{0}0.01 \pm 0.00$ & $\phantom{0}0.03 \pm \phantom{0}0.00$ & $\phantom{0}0.04 \pm \phantom{0}0.00$ & $\phantom{0}0.06 \pm \phantom{0}0.01$ & $14652.16 \pm 45.71$ & \\ +\textbf{TransD} & \textbf{pub} & & & & & $77.3\phantom{0 \pm 00.00}$ & $\phantom{000}91.\phantom{00 \pm 00.00}$ & \\ + & \textbf{avg} & $33.99 \pm 0.03$ & $21.22 \pm 0.03$ & $40.48 \pm \phantom{0}0.10$ & $48.57 \pm \phantom{0}0.09$ & $58.71 \pm \phantom{0}0.14$ & $\phantom{00}153.37 \pm \phantom{0}5.35$ & $\phantom{00}2.29 \pm 0.09$ \\ +\textbf{TransE} & \textbf{pub} & & & & & $47.1\phantom{0 \pm 00.00}$ & $\phantom{00}125.\phantom{00 \pm 00.00}$ & \\ + & \textbf{avg} & $26.01 \pm 0.17$ & $15.23 \pm 0.16$ & $29.85 \pm \phantom{0}0.24$ & $37.18 \pm \phantom{0}0.24$ & $47.34 \pm \phantom{0}0.18$ & $\phantom{00}127.92 \pm \phantom{0}0.86$ & $\phantom{00}1.78 \pm 0.01$ \\ +\textbf{TransH} & \textbf{pub} & & & & & $64.4\phantom{0 \pm 00.00}$ & $\phantom{000}87.\phantom{00 \pm 00.00}$ & \\ + & \textbf{avg} & $\phantom{0}2.54 \pm 0.20$ & $\phantom{0}1.69 \pm 0.25$ & $\phantom{0}2.95 \pm \phantom{0}0.20$ & $\phantom{0}3.29 \pm \phantom{0}0.22$ & $\phantom{0}3.74 \pm \phantom{0}0.18$ & $\phantom{0}6320.02 \pm 30.37$ & $\phantom{0}85.63 \pm 0.40$ \\ + & \textbf{best} & $\phantom{0}2.54 \pm 0.20$ & $\phantom{0}1.69 \pm 0.25$ & $\phantom{0}2.95 \pm \phantom{0}0.20$ & $\phantom{0}3.29 \pm \phantom{0}0.22$ & $\phantom{0}3.74 \pm \phantom{0}0.18$ & $\phantom{0}6320.00 \pm 30.37$ & \\ + & \textbf{worst} & $\phantom{0}2.54 \pm 0.20$ & $\phantom{0}1.69 \pm 0.25$ & $\phantom{0}2.95 \pm \phantom{0}0.20$ & $\phantom{0}3.29 \pm \phantom{0}0.22$ & $\phantom{0}3.74 \pm \phantom{0}0.18$ & $\phantom{0}6320.05 \pm 30.37$ & \\ +\textbf{TransR} & \textbf{pub} & & & & & $68.7\phantom{0 \pm 00.00}$ & $\phantom{000}77.\phantom{00 \pm 00.00}$ & \\ + & \textbf{avg} & $\phantom{0}0.65 \pm 0.02$ & $\phantom{0}0.37 \pm 0.00$ & $\phantom{0}0.63 \pm \phantom{0}0.04$ & $\phantom{0}0.78 \pm \phantom{0}0.06$ & $\phantom{0}1.03 \pm \phantom{0}0.07$ & $\phantom{0}6795.95 \pm 16.65$ & $\phantom{0}91.99 \pm 0.22$ \\ + & \textbf{best} & $\phantom{0}0.65 \pm 0.02$ & $\phantom{0}0.37 \pm 0.00$ & $\phantom{0}0.63 \pm \phantom{0}0.04$ & $\phantom{0}0.78 \pm \phantom{0}0.06$ & $\phantom{0}1.03 \pm \phantom{0}0.07$ & $\phantom{0}6795.94 \pm 16.65$ & \\ + & \textbf{worst} & $\phantom{0}0.65 \pm 0.02$ & $\phantom{0}0.37 \pm 0.00$ & $\phantom{0}0.63 \pm \phantom{0}0.04$ & $\phantom{0}0.78 \pm \phantom{0}0.06$ & $\phantom{0}1.03 \pm \phantom{0}0.07$ & $\phantom{0}6795.95 \pm 16.65$ & \\ +\textbf{TuckER} & \textbf{pub} & $79.5\phantom{0 \pm 0.00}$ & $74.1\phantom{0 \pm 0.00}$ & $83.3\phantom{0 \pm 00.00}$ & & $89.2\phantom{0 \pm 00.00}$ & & \\ + & \textbf{avg} & $\phantom{0}0.07 \pm 0.02$ & $\phantom{0}0.01 \pm 0.00$ & $\phantom{0}0.02 \pm \phantom{0}0.00$ & $\phantom{0}0.03 \pm \phantom{0}0.01$ & $\phantom{0}0.15 \pm \phantom{0}0.17$ & $\phantom{0}7327.77 \pm 29.22$ & $\phantom{0}99.11 \pm 0.39$ \\ \bottomrule \end{tabular} diff --git a/reproducibility/summaries/results.pdf b/reproducibility/summaries/results.pdf index 44fc2eec0058a050732fb1b86c431b32700482c3..8bcbcf367c0f08dcc5bf56b1605ab2698939a8b7 100644 GIT binary patch delta 8377 zcmai%RZts%*X4_}xVyU-3lw(=?(R^aNO5R!LXpr?914XLEAB2KXrZ{f7MB3Q2@d=J zzMb8D*qzyz-`qLpJl?r;Zcjd1(r>gRR#bp^Y8W~vsKa!~XGM&#L~KYC4%&?)^E-cK9et`SXV6f9o8ET50e=I4!QN8=zlzvG#;gI<+!kApQHAWw&5 z*Pl^I5PMQ3j@AthGVE%ClS@E4fJGDhjeZCd!1I>V#dmZ^N5W>niBl}`-`~=LCD+fQ zI{Ou$kb(~KCT}N&(IPC`6a*~gvWZqhp2LQHzVj1kD)`x{^@Mv0>@%P=e_-cYue z^UP9^<+bznMPjR}?)rz?9dkLg&Usrb&i0UIQ2RqO7sqHmZjKMS(qZ$vDT;;4JHLRD1>ESVJk;^Vkzeeg9=LZS)D6vzr5Qdu#GPlr4A&XPfKUSp02K$ zY&tBVHkSRt?`Vy!mxi0u8P4steclrg%DFC4EzK`k zoskUJk8UbNb-xU3$>S)+m~XVCF2 z*N8KTcGHHJz-o_?;j;1QU6!T9Or0O`+kwd=!$=~fj$QCwfj5)Yw| zC8LY??l(CaQ~sIx(Xv+7^hR3yzIRqi<7cp%{1xYD*MVaQa8QoLitkS%YaAue%Hvxs zh@GaMdBvewU%pFK`e6QuicqK@b);Me zXP88buSo4lZo`3SH-DM#U7C5uzo&ja6Ihy^nz-YBTxSH*hmL~-33>UADR`Bpt+PiL zT-*=$8vo@4Wk;X;{4lfe&+&W5S#$KkhdY6(FVd(rkxah7)WMB%N)Q>mjhGqEW!%LxVW+=vkYi@I5Fnrl+?qAR~?c&+L zEu(4L{TmM6hO9}HpOqDyGZXQm5L=yMi{r<5Xs*N^2R?71e@?GC<5S04M_}n_KEdjz zDN)fkYs7rm8+EAFe6oGL5F98kgA>^fe;0QDQ>r042bzfRyZ_t;ZjgrW3Xr(-$Ydjh z`!H__4u=P>zF$&57E9%^6Xptz#`(^i*Duux z9|&QH$!Ku`Jq+9L8eUPBuYin|^pfL$_?rhZRIJ!kFc|MWmaKRl(nznDGOY#JS?c~X zrclLBo)`U>qhcuy1wA4Uf62~QP{3cxaVD*kV-A^14_5ETkLC*v4y@c$(lE-e?%#-P z+FpZg@YX}@Mh`zd4K>`Sf@W)Ujuuw#FDb4c7Os!{Gm*rc?xFUfJKLVk^hkvG;aJ11 z{S~siSAbh?X7T>;F#=<|J}mnF2Ou$~lyLbaFtcvBffRBC%2>iTT|l0xt+hhhI!vw~ zoM2VeF9FjluRoHZFPM!;<109O+;)mI(l;I5Kcxmucm5OJct+DSOdg$#T3mY$F@Xe| zb%CbBQs{eM!ajcrMm|uTfex^__emq67g?n8m0^J(WUo%!{g0YZ1HtBknrDmkyUu!< zQbK)$e@3DpCy*>`wKBH4s@rsckcUD|*;4etNWl>|VcoK3^SOZi>2=o`ZGjfc+2Ql+ z?ZL{Nu8xNQ{0jd3my3*m4eGV7H)HQ6A1EGx{mQz8z0ZTD&j)9Nlk0`sGK@> z(f}RV_Lm!2MVHU_Nv=ptm)Wlz4>i=uz=U zX#;oqoku60)1E)dl4no+3u_O)-iN_x#l)}t&dDb~=kX)6gDW^h_=Z}k_*@9oHbf85 zKg0A7+&;uSb+B>ywH4pciZzl-uUh`|bU885x@KcUXbTNAkDrg}15vSq2U^Ocs(jatR(c!1SNYUu zZa6U}it~Ra(x*q0$5HsX>dDA~^ytrFs_ zWiI_Vp)Z|+heei7R?Z&7pife3C&5SJ1!hloBVEud8n_!3^Z*+$%5 zXfD@AU%+bpAZ@{0;)=!m6;%=zh7f*P9T_j~&3!Hk&9A@4X~s3Q@J1>%OQzs*>WeOE z+5;Xqj+#|7DB{`Ll)O05UG9{(@)>v9;wHAvvR!Un)@*+=*RX~4BI|* zx-5#VQ`c0%JGS0I4jo{Rfmwu3Q=r7?hfbt=PF-&wIyGd-2>J_18p4`p8UhN&?g-Nq zrQk_`9qc2HT?&N`7u=4?-H+!Q+Gh@aS9J{ZS9fn=OSlsGlj?J>6sjcMc#{C9-OHPm znpZ;RT%}94O%0n4mE>6~er^SIjdwb~YTZ1-23Mtw@jqN3%oxf1$TgTa`Cb@w9rfNe z)RIJRVz3eY1@RbYljw1O%d=NyPN3M*uK31DuB^%T&$;^FsTej`9|et0v0HcbWy-tS zv?otl|1=pJJUzP-r$?h%8Wp|sCfXHEB4T+1U6qwJw?SnuO{_7SF zj3N|2beC=-5e^0=nhl&L6rm(m6QU8KyJ#sHx-SE{XQ0q78R`kw@=f9IgTlXh(e$Ea z>N5^E&szvqF`w(`S_PvAV^SGflRZN?C<9UhlZ;S&(LOb(Y;Tjt6VMPT$;M9@#W#3d z;gaQc*L{&~Yur%<4>&>KesuTzPC9GY+2$ooA${Z{c2&E zmi-pueg3O+?FLs89rW3M{}scbFoGu&@{=9Zd5$ zdWWQXjxq5|L)Fumt6dR(4;!uXg(&$r72XtMsJT%6X>a7p-f?V|LC-Tk%HpA2Ze#jp z^oJwwg-m?#sOPFH^$5&DhF^exvpn~tExB4@-?L-X15XnlL6N0uq4S! z#YM`)y-dH_bAJ;#=jU^z+>qeuZAcRa6f~r*i)FGa4qGR z5~3&lQZqiurPKoR38wb;4LiTyO`~#U|bbDRf3~wj|nlE zKulfJ$uzkN>yQ*OWQ+24x=46uf_)c%?8)x4NMo;^bquWzm4`#8U{snDWL@tc!fx)G z17GU1$2D`Ipr`p$h)8ftWDR7&B$DP-CeZrjamOe4q9!#+)$cV~xUopq@W&F`zQc?g z4%x@T3Y)u;8g6TJsf`zRzR(H9ZbH6^Sx99Cp%fD5>>vhDS#MfosPXS1r5x90FUXK?#OA zE43$Am$Y62=P^H-mAu2Z_VDoA7Y!V2H`K5KoyKdXxw;&8B&S{gVIP4Ch1*PZ$ZlMw zBxVG+g7vL8>ql*u29emwlgXp+7xSW8Z8IfflZ9h@&MpmPnh+Sd)x}Bp>uoWMhWLt2Pn;g9%HhIgXb3M`RJG ziT=l4dE6qASyo?V5xWUp%-h>M)P<A5_}|d16vb8jiN5nF6Jzm#M=~5{=b(*IddQof zfAWS($ajfGxGQ|WCGOUnLA*HpG>zWhY`9>2CWz!PCDU9%bcdnif7$Oim^f3);W@KE zd)ovzP+Wpq5C!<4R#_DMq7F!@reeWnH{{BVlX9xZnq!q(X@6W=No+Cc>Vjp1Hl?88 z2Id3dMHs>9k0yl!H|}3_5)J?Cu_TyyS+>9uOn$!+Obk_qp&64y`BQ4{sXup;S%RI) zh_U}z`gPu(d33Gg(k>JxQQdoOHqipAJ|=&=yYL_akbJe2nxw%?g17xs5vOpHB=9`C zX0<0ROY~%I`m%r^KE^w$$$`8D0ph3I)-4=w%LoTV`fAa$nwPNEVJ56(Q+RVCRGA?{ z+fjK-?(e=RaOEy7^@V(B$5dqO8Z&-Ceuijih--fLrQ16Yp@C^8t_h>Y*kXUTD&8NS zMK*<>GACyEQYS?~=}{h13>!Wks6Mx0{){x7$+tZl}NKBq@ozE(UEy`3L-TV&sfbPs8B4+QN?vy_fHd+hyx9^+oyvz6G#Qb7F z;HGY8a7s0)EP{a`U3tktylW}>=kfe>>B*2g#yu@K%nGSSx*`6d@0N|RZJ@K*6e{Zc zQZA!jWA#^(BGljTtJbjvsbOc6W1whb=a9#4irW)>Q|Lrm)y70!ziK{d5h!Z&0o8Rd8ib(In6*i2l#goz3c>NddDK6|E(?AdCDg0cTG) z2p3_v$GbkPJf_n&O7cPxjqBd(^u#wYLBEfeUM=C#AUUn~tt zWQscA-56uKY84tzyef9Rnz+teT_7xMkPw-yv;lXyqI8h>IB+wC1KjI1aPl( zI9L$~@{}(M!ntX@B8NJQl()3fOeBTYq<(W!3O(-8z0^mo9u$%MBnc5aH~HH<%9)k3 z;-^=7pD(;UuR1iIybuC73OV`Vqfdjj8;P~R!TFEGz!m~ zUi>fVs{UIJWFMqyPke)hmEXe{dJOo0IeX+cQe$I%mLN}dRsXU9C@<(lSwS#Z44jG! zf_qqQ%=g$cui;6)|HG-+1~LCdw5{G&QwVzn*XDtDrkwm=2s82eZ#nEgZ`S`Yb8+A& zCexv8YtgFy=4v-T{A~U=Ubd4?bT5B1(GIsQgjiL+vQ!kg`!(V-XGgyuwY&lcJIQf!W) z9MH}Tmwx-m<(5u(F7T4~ZLK@%-ho+Gt=9?T1Erf-Z*z|^eZbwrwBj7?G^V}v^M;MK zTLGM4EuZsxXT4Z%^R*KYRL_HKc5r6O=*uxWMsG~WZP@1Jm1SZyDiP5nJjWSFT}hTp z8NlDk?d#VDV!*B{MdTD93+idInqNcBe34e=9F4QRQ6V^b)aw#lQ3q>{?OGx;96=&`(#5zdJ#tc@h=Y1xZi?T~i`X#EK$p75P%z|$$;_iY466tIL^q6(5q4%<%g_BEDJ*ZMakwU0k ztv2_!U_Q4yI*0bgAKMz*qIqW8jhvo&W2%A_$`;?z46Sp{X4W44b&eB(d6(5; zq+Iuv;mn;!^Z?aW9>uZPis0YF z2r*$|xLojILb`r{0I4dW+1>?h3!9plx_h{OAk8h!4ZbEZJ4^nAZJM_Oxo`38Oq4e6 z`m7Q4KAW7*W>IbWD~>GaP*fy>b>-fQQ5{9oDay~#DaxUV;KYGK=uTx3~bqVvo(4k`!%b49d0a z;@$`Q=CBg#6w<;Yp(W zBLQAKKZOpdHC@AUry11LUR}+UeE|G;4>^OFAsMpp-c4D^RaJxRr>_?d*s-S&mq2{P z7B{^fvX9_rx+OCS;vIBvAlb9{4)eiooShbdJHHzbXB}I|qYmu^{F=~HwNo=4=bEZeguwDbbw%h1yswqCDO#1#s_e?FrlslX zlQcZogfgSBMjiKjbLmC2IY{9qkK$Gh{;_J|#q~2K{NuMKixZoa&UC|l03J&y@4O9< zd*LbY-s6AcS#&x>0bSpy7OZ=VHhD4Hhwa$#Kc0RfT!6-oU$yO$1j>gXc~*`*kcwmb7yU;D$6^}59EjsmBz~6vg0D6H zee7Y{U%OjHWlTGqH~uE2Rw}I7qRMPHIpw5}9~xglU`qwkmJX>B^=ZB{DAM)pu}IVSC~m4hvz#oagCY<8@;d?AZf< z^ZedC>g(4JtW`hoUyl`s(1>zd)3|x`4uW|K6;wbW@l!aA6vBLqs_qm@?rU~_m%RG? zs*+a-soYY-HkLz-OxV5rqKoIz5P4TNF!vipn!mM3e&oxz;Wof#UU`W%qQOmDx?uZq z>JAb)Q?ah^mg_hZJM^KubBtS(GF(!kzl27|el}mQ<=Du6m{*PVee7`h0DG>?tY#7O zD|1lv{pVJo(CLv|dg2ZFOlA+cTWk5)qfiRwhK8gDb^?&TK1F{_r}kG>d0?lhk*p{) zP+U8{F!yig{`6AG_uhg6QK!4dqw==5drr|ZOZmdkHQ8$YW)@b@gh=DfW%;59{Au@n zThhbM$b8?#k;zM)>&KNCv;d44fe02ca3aVH+sO;d#0yU}sfUu?wp%;u;c-e1zpEE_ zuQnVtHu?)&ny$Pf{TQliDTBIw-g7}XnbaQxg3KVi`hugWvA>(ImWnc~t>wsz@Xc7I z9n_W;$$c59OxESn`{@N~jOpq+P#SJj5q&-n==;1dn@ypMhY0kBHi9#HhKzJ(z$l0) zFIqLRKNDI5F?Uhn*HUAKXHk$4`2LW+)|KteEq)tRIkQmv>=h7lz8Qq`5%rSyANA(c zi2Yzp`;U|JVvAGU*iEW9I`8F@gw`EMpNa^8i#u4ZWUxN|FB4bpbOC{C6eJvIt1yQjBUcr;DVzniAU;6(kILdr^` z7rit}@HQRzwD?})OPtHvFTGVP6obb^rh%6}$zqMsv}ye^xak&+5vI;rNDCE1D_KQN zQ%4bSHS(zvfMj$3nGrmtv`&p&^GwSy(t7AWv&?oly4R8^W8l8GU84TU&+GE2kEAVC z1#=f79rD(*lQ><$ZCWKY`6fKUTxxh}@O(h>RoZzI;GBT-;a~De1xZLGq$~9HmsQ2K zY+iy48TY-h=~O<}hP_Nrg|6%oHk&&1HKx(EL**|_Cxye6jx6+2JMQFkw{H^oo#V`=W=_&$3yQqMW@A&< H0b>6Tl>>`1 delta 8286 zcmaiYRZyIP(q%%h5ZpDmyE}xz-66QUyYmHiC%6O~+=3;LK|*jD+y@`rf)i}--iLp; zY9IFHRQKug)>5ZBwe($I$-7iK1YVwul5!>h@>$tyRRFvDSbtODW1{6-wvR(hS^3#8 zJxBy8@kh5WaHY< zBa0jDN3FdIY@3nIhYwZJ0!74$e&52|w+%bCNs$L{if~t^ARJfc4R6rrpNmKXC&e)` z#puZwHxrofLu7Rpr<2)A>CBzu`^lYUkoBh--Ql!j_9S1PZRXH9${GMtm*!@?eYP@H zAy{}spS!%iri~HzzLV#n%FrmYD^dYHg?88Q^oof@ks75LsLlQ>$!OZ;p`(4a(q^1v zRazS{>U-Ha!$sE_KF`W|+LYPM@@Wz<-E>*Wed*#M615lUp(jCTVn3?IA5UeSC$HsH ziYS~ag%yf?-?bSJaWx>vJLlUA>?`4mQoWLC`R?K_MN;aMVo}<4eC^6LbA zf(+f%Art-nZIktGBI=#PZ9|>8;qlt2f7cv^WMWGis^_}Uqjdn{tM=mVUkS8Y@53b+ z731-bRACeR>6e60_C-1<>J?#xx}6a5GAZYE!9V*)81 zR1S~p*vA2g)H!+}BKyTR+p_<<_7Gb2cT0YvvWRquU5Vs-qd}Zh(=R_InHKdjkh85{ zRBmN(CK~*51p*dcE<+w4TAzQp0Kjx801FM(x91F-S-4w}H+`{%TW%DUoiz+dmUW17 zL~FjyTVK%MClVvhK4okz*7+eO6P5b9XVb^BQ2{Jb_OSW(cCPg^y6m{F+?uv+6xVlj zK0`C=O`7zi+#}j%;8c+>k(7qgW13yAJ(3P?D~l%Y&dhpc2u4o2sC^-l8*O{0?Hgvh zW-~VTP-ufI^u-7HiKsJdFbHWuz%+vp9jdalWdIDJG-4M(YGh=>+ z0bP50C+BwAusY%uv~`BME(MnD@8XTPOjhns@W8m+V!p$k3zQAZR);sHFiG>?gMq6K zKgw6+3aciuXwl?ph}GtIRY-vQYSj-^G|MG&f-%X{j|7G!U%#LwIvBr!Gqc6O4`c3L zgZ*xLW)S(5oKD;r{awEtt3~L3zEC($1h$J?eUXf~E$H$+m9DDZP_t2~{tDNfaeQ+{ zTqZ{7Je%4dwdJ78GeR+C5veM<#8={0sn=;eCqnS6-nr^9k@Tz_hvz8REi=S#%{KEW zQEruF;2U3jYbGRc1Fmat`Wcbn{T1 zR#$|w2^B-0{qK&TettGb+stOfQPCjqVArY6=HVXq9>>9 ziea-iko~5Tip1NZj5uzC5^DYcz!g_svm}dj<%3#}*xf^npZ;SQN%P74aoFKQkYxP0 zw4x)vD(2zG$S=7R1zUZop6=L>>+v8Odsy)K8L*=97;7`8RFmXL&~;Nqs1SLIzD1;U1MKhyEO)2W zxI;5}N){OC3~Bk8Pe1FJ0zV$%u99P|;PDo4lghyk?I{}etN|F$Di_k<^bNIwY7WKI zE@qXk7bXHi9sM?!dYj7v_1=fuP7Qwwy%MO{r%xFf=^h)lPUX>Zs2MWu2OIe07{+6> zTVx7>7i>htzS`Wf6L{qUflYB(=GvYzLHU*q6oZ}Pt6pwIuP^YF6zJ`h3aC3C6-pQN z94xbzR;;E#FIwbKiqE&x%bPUEqCQzur>DmTUwj9pYd#Uf6q}`#EVc7bR)|2SOdEmX zr5TSvr6eTpqyqn<+RHbIQAuLa6(X1DW0&)c6&M9XX(1xY#oPes(EXqc>KI$)-a7SY z4MzE2iKO6POWf2XJe|tX2qV*KqI8I!@A=`~^4#9Hr++*Ta(3! z^&guL({?(u4P=0A_jXTc0;@KP1#cEnFxVV#r{7j)FUNpT*N!vbZPISdyH&@_-FH{0 zK#o>lx%M~l?2+duwQM}Ou}=+p*6d`R1 znwoF43x@YeUaeu9qgu_{FL5eWs>sI&c1o&UGh6s4G!tgJT?v+Qdj}m?6V70bm2_3Eavc020(Rbtu z+A-~#8|L=Vl06y2bS2`X+4Q;J3UVsnm*=<^XhZZY)A0@@9LrXSVe;&eGLipj&TO)e zj67bb4yz=CS6NvtUsaxaqK1Qx7cEZiU`TJuwG3cnLrQi?+I4N&RxM%D;-OG4vEVJp=xwlB{) zim9iFWFn{KIp+>Ck!@?TtTElXr-?Nz5?4ahJk65P!OrMk_0Ywh=ZPIrT%D%>puK#n zHUfZrW*ABhmJnwTi1q6Yrcz8>+z|q#*wY7xKh%WLm2e7?%Uj(b-A*5WP@_W1X`=b5 zkETHZqicP0bDUP(NVth;^Jqw}7+ZF}rBLb$05ESS*rcPDz>I~DG9E%l7U^{`7^c-> z2GgLu=X8-yVvY(D`WHa2>jevKScUpL%IkF8{Wl()JB1l{ z6#t*fPglvs)BVop|Cf|J?}YqsApkQy6E6L)u5$iVHE?|?Px|=wi9*$?)omk2gKh0~ zO41#Ay(k-pa<+Qp6^d;7680$nOoKj_s|YRy^GBXC;vvp;ejc-9;Mv?3;v9-a=_cQK zOZgK^jvB?%{DM@V%L#eOJAU=S6MYZM`rfRupu>vgE)Ay@enZUl_XZ#i96IJU_g3to zB!aG;cH4cfJs=vgIlUncJN)U=TSDV{lA zzL}5B?dl_@G1J8{#;KPm!e5tNrg?^_77bsnwB7bcR+kv-FD0%CEMeK%^!^+j>PL$d z83zNy8qBL%YuTT_8~FmZ#i=Vd8lF@g`aWtDsaBcTViO9*8o-voD z=mn&g5+*|E^L#qqJ6H)+!vt}VI~z@`vsfhVog{|qCFrnop-^BcZW9+ zEat2SllRg;@0@Tq`6fbkq?+%Db2&FFzj6+{2kDQU9*%9_9tf4c(en&?b?^wnAw6Sp z5DHP`3&xNuvlk){HRLmn=IJsVRWZZ0IeS;GnsR53A$~9UP~1iTAEStu@g6H4ptq3y zkF%SVJwPIah1A4i_Py{XCGEj9t@E?=LdR&U^7775H9IZ?ZRd+-ml*pNT3^8HN8WIo{r zU*Qjxe&G|PtKzG)k5R7o>(*!1y1i=$Fm&*KyjgQd&tvfFFJM{C4h&rbLXYoM^C7E! zixbDSeMx@BVvRtk=TGs6MMZJC&TzC(tYoV2Hf>DHo3Y0X%Fvnl#wWNw0Vl%8i&uf{ z?tN-=)VF^h{pKe|%+xtn;Io1Y_8NVU@ugP1=Z4<85rW9uf{R@?f6ILTsXzVJ;4;77 z8(3XU0m9b&njSG3yS<8mV3fJ#xw{_nQwfg^s^~{p$J4a;9qilVn{xjKrZ1NqC!MbY zF&k?2>reYUf*oSJ?fr=!Y6gy+5nEM;a37Q~$~JH!yD#@#UjaTgR;2V2v9D_mw_}{$ z=qWd;Q)-1%D}sFzNMkUOr0qizsV{_TPRq)u?4-~t)zN`edn_3kR!pjPYa~H9L?8Lv zyIB~UmU?|egF z!Op9EF>BMBiU#0LjzOu7?{> z**Yhc&aT$my4#5hRnG?OD}jg&68+vC6of=1_9w@T-(e?B=n)Aizvfiopk04+?fA==;&YDI zY$G|Jj|*D^BBslsY(FAPb%=uQMhVK3XdmJdc*QKtj2Ad=#)XRH%TxsRw*oEPl+Obe zIncv36@ZvOVGSkZ6E z15X6Z(j0Wr`mQUc53ddleClg1i4fC!WqNJrPXi^m=l0K9xvnlbS*u4C6AoLb*Hpt0 zQ(R2MLBgxaRF^B_U5<3bEpjn^IXN9#t=-swO#<#k#{C93-lQk6n7dI!YHtC8Nz6in z^Jc`t)GQni=|XIg>D$y4y_#<2gWaiv@^(d)%i%Ogi_Uu_3l0&pVE>K{H{@g|tzq<# z9~Xw0xeIN>e}V%#P;=kQaC|&SLLT-uIWH?nXeo^8xCu<>j-MM6Oke3e_I-aFeWMRm zB?NNi9AM}5JWo2YR%Di+5mZ;5kt$%U>G22Aj`@iy-MJp1sgw4+PbHbq6K6{aUeieG z)D7F(kf5#!a0NMd;3uY+cYcxyc*vDBOqwfdfMeFn#_`dx*Dq~oOc+C<&p72vh+@q$ zzc(FVf;cW!bEwgD$2{MvMFPrpkmqO&2Xtyk_BzM&JA87BR&U>n@lqW4|MfqjmUO0E zxCzqy)M0#i9y(H>wgiX7HtDMRLG8Y7;g@rJxbER;oHHeilchRIuIZvQA5odzMHU&G0J1K1c9yGJJwH@4|l!UXsWW{iZ;unP@1#KX5hTzMly< zJ`5BpZ8D~nhU&L9-f&@9ne1=Q{=OW}&97$r4&_OaHYob%<}X95;^^}Ixu=Y6|AlNw z;;n+L9o@XZ3$Y^^OGdso5QK#C!KzA!c&8|f3pvDDr+Wx{@ZJj5%D#l<)*R!*SI5t5 zvolR=Qt5Tk1C-MJAozH7;##B|wCj{@|M_XMQgC2j$m5{?fRq0WRT!xZQZf!*XN>cB zcwsW`Hk!1he5Yucvg`k+E@=C>Db)IlcxuCLD@Uor3kLP8`;Tluxm?uaJ6g*}%sXLm zI88GwWs{kRR^2NhsA$jDvY|&1ue@t+XDv(ej;16>oV;H#8t@k6a6jJUPCJXH;I_!5 zhN-eo#(L1h^k+=IcvpV)HdDO&f1rm|=)bgNKZ?xYreqccLSS@DY+NMP$`0=8O}-SRp~^lN|oM^gfI94%CY`Az|D#Cu95cbU<;t9&lZdmRwyX>^&69v%`c>iiG5^AcS3t&h&N!*mm zb{HC;{*zYFo})=IhD(eU`8i$UnUYT`1@XWBafIXxFhCN9Z_Iba87=(w(K?(eGIjiE zv_L;JSa<`PEJcy#I24D02yzo6MOUU%E;}*J(b$(#0oT-}iF#(Yd3gaQ_DK6r6tQL2bkEiPGp=cPm8l zSG_F>(D~k1$oEHV#TduEeE3hqUE_os3$bZeay#YaL1(DpF?Ov3HLW@}c^j7zBs1^Y z_iQf>aIRMw4f@8!zM@E7^u{Cm#sQNL>aOaQ5(sTUso&tZk)kpWtT_zs_`QQ3IQ7A| z;57f=&GcFDwUy2tA_fWQ?V!037oF|ZL!fsk5Vxl!c$efUgmfj^WqO>(^yvApG46cr z%4gC9{1a~BcUwN_(qi;Aw(u&(eAi7)K+9ndzW}NIqX*L&Bg8!N5zsx)U=y9 zL1jEVZIFt+Frry0^7*todB|^;AL=*@P6)#X8qOn?9Kz9+thi<$0%53A!rJY?n+XP##GZTrfE` zTsl8H6>=OBV4<1U19(m7lRY{M<)VDqU|k;Ngpj+_3oi5^ImIus(jcvM`G^emyxXG=Kd+U* zl?4+(Fy_KPgdHR!!<)XK6ccW%-JL{6(1;<>VPV}*pjAQ(bPpaNrG6kKaxQQGP}+XR z)%ieJO;k?t&$mcjhXsan6ESY`Om))QabumZp;iX{^I=q!Gk>ZNu&eRN)t&f?{VjsgnC>D#z%yh8vzmhsiB#GiH^c zPU#5N!g*5u#%X0601jdGf5)Q}BKL{Al+rE$}B3@J!(j&>!Vi_n_?=gO!)SDMG ztE(v{!zm-Ll$(KCZg?G@%noc#Pv^G;zxe2y5MH%_icd81*3KyPnOAYr1&5?+-A}*G zgrFR@dvT%pZ1he29I`ohs0@5L?SK0#LYlY_hrC}N5Q^gc5y|=^nrzq{9goYL@ps6J zJypzx1+v%YPj3f)_u?v5ed9^7eakM?epDBs8ZdA%vDLELm$xG{BmV?TDEq{iDUuD4>FGwCjq{0ngDE=ua z%E~w-djRSbs9b($d~H#+zO^hHOqb?8kq{U2bF?_Hs(4FXn^1h=(CITEB-VZJ4vFFq zdmwa6x;TJy?@F&L-TBYf`ymfppbsVsoodS)U#f1?lI7xm75}Dfy*#*`nyG#`jY9WZ z^;1lO2loh(c4q#@2H6@)VC_$WxYhRz{q>?zj6jV^-pJfpl&*Awz}PB#hoQqUsYB8` z{I@EKXz58+zn%9`2$jo1ow*A?4Ys;YGMzYmHomz|nid+kg(6FSv_0r5x9H0+x*PvO`?`oU!9jNaESRQdV})eHTPr zoI6z+`A}zRZjhi9HxCydNLWyoUs_r~N*2Tg0`Up+aLI9T%LxlfP>KEjBpFIm!ZtrK zBHm!jOq28$>`Bn2rlK Date: Mon, 22 Jun 2020 15:36:23 +0200 Subject: [PATCH 4/7] Optionally leave out std --- reproducibility/generate_summary_table.py | 43 ++++++++++--------- .../summaries/fb15k237_table_without_std.tex | 14 ++++++ .../summaries/fb15k_table_without_std.tex | 38 ++++++++++++++++ .../summaries/wn18_table_without_std.tex | 42 ++++++++++++++++++ .../summaries/wn18rr_table_without_std.tex | 16 +++++++ 5 files changed, 133 insertions(+), 20 deletions(-) create mode 100644 reproducibility/summaries/fb15k237_table_without_std.tex create mode 100644 reproducibility/summaries/fb15k_table_without_std.tex create mode 100644 reproducibility/summaries/wn18_table_without_std.tex create mode 100644 reproducibility/summaries/wn18rr_table_without_std.tex diff --git a/reproducibility/generate_summary_table.py b/reproducibility/generate_summary_table.py index bbf43b387..d15a87d5d 100644 --- a/reproducibility/generate_summary_table.py +++ b/reproducibility/generate_summary_table.py @@ -67,7 +67,7 @@ def load_published_results_in_tall_format( return pd.DataFrame(data=data, columns=['model', 'columns', 'mean', 'std']) -def generate_results_table(): +def generate_results_table(with_std: bool): all_tables = [] for dataset, dataset_df in read_experiment_collation().groupby('dataset'): if len(dataset_df['model'].unique()) < 2: @@ -78,7 +78,7 @@ def generate_results_table(): published_results_tall_df = load_published_results_in_tall_format(dataset=dataset) all_results_tall_df = pd.concat([tall_summary_df, published_results_tall_df], ignore_index=True) - wide_summary_df = reorganize_summary_df(all_results_tall_df) + wide_summary_df = reorganize_summary_df(all_results_tall_df, with_std=with_std) wide_summary_df = wide_summary_df.applymap(lambda v: ('$' + v + '$') if v != '' else '') # Save as Latex table @@ -90,7 +90,8 @@ def generate_results_table(): bold_rows=True, ) # table_latex = _process_tex(table_latex) - with open(os.path.join(SUMMARIES, f'{dataset}_table.tex'), 'w') as file: + suffix = '' if with_std else '_without_std' + with open(os.path.join(SUMMARIES, f'{dataset}_table{suffix}.tex'), 'w') as file: print(table_latex, file=file) dataset = pykeen.datasets.datasets[dataset].__name__ @@ -133,7 +134,7 @@ def get_width(column): return prelen, postlen -def format_values(mean, std, mean_w, std_w): +def format_values(mean, std, mean_w, std_w, with_std: bool) -> str: if mean is None: return '' # format mean @@ -145,31 +146,32 @@ def format_values(mean, std, mean_w, std_w): pre = r'\phantom{' + (pre_len - len(pre)) * '0' + '}' + pre if len(post) < post_len: post = post + r'\phantom{' + (post_len - len(post)) * '0' + '}' - mean_str = pre + '.' + post - if std is None: - pre_len, post_len = std_w - std_str = r'\phantom{ \pm ' + '0' * pre_len + '.' + '0' * post_len + '}' - else: - pre, post = std.split('.') - pre_len, post_len = std_w - if len(pre) < pre_len: - pre = r'\phantom{' + (pre_len - len(pre)) * '0' + '}' + pre - if len(post) < post_len: - post = post + r'\phantom{' + (post_len - len(post)) * '0' + '}' - std_str = r' \pm ' + pre + '.' + post - out = mean_str + std_str + out = pre + '.' + post + if with_std: + if std is None: + pre_len, post_len = std_w + std_str = r'\phantom{ \pm ' + '0' * pre_len + '.' + '0' * post_len + '}' + else: + pre, post = std.split('.') + pre_len, post_len = std_w + if len(pre) < pre_len: + pre = r'\phantom{' + (pre_len - len(pre)) * '0' + '}' + pre + if len(post) < post_len: + post = post + r'\phantom{' + (post_len - len(post)) * '0' + '}' + std_str = r' \pm ' + pre + '.' + post + out += std_str out = out.replace(r'}\phantom{', '') return out -def reorganize_summary_df(df: pd.DataFrame) -> pd.DataFrame: +def reorganize_summary_df(df: pd.DataFrame, with_std: bool) -> pd.DataFrame: df['short_columns'] = df['columns'].str.replace('.((worst)|(avg)|(best)|(pub))', '', regex=True) _n = df.groupby(['short_columns']).aggregate({'mean': get_width, 'std': get_width}) n = _n['std'] _mean_n = _n['mean'] df['values'] = [ - format_values(mean=mean, std=std, mean_w=_mean_n[column], std_w=n[column]) + format_values(mean=mean, std=std, mean_w=_mean_n[column], std_w=n[column], with_std=with_std) for column, mean, std in df[['short_columns', 'mean', 'std']].values ] inconsistent_rank_mask = df[~df['columns'].str.contains('.pub')].groupby(by=['model']).agg( @@ -330,7 +332,8 @@ def generate_size_table(): def main(): size_table = generate_size_table() - all_tables = generate_results_table() + for with_std in (False, True): + all_tables = generate_results_table(with_std=with_std) write_pdfs(all_tables=all_tables, size_table=size_table) diff --git a/reproducibility/summaries/fb15k237_table_without_std.tex b/reproducibility/summaries/fb15k237_table_without_std.tex new file mode 100644 index 000000000..f281db819 --- /dev/null +++ b/reproducibility/summaries/fb15k237_table_without_std.tex @@ -0,0 +1,14 @@ +\begin{tabular}{llrrrrrrr} +\toprule + & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ +\textbf{model} & {} & & & & & & & \\ +\midrule +\textbf{ConvE} & \textbf{pub} & $32.5\phantom{0}$ & $23.7\phantom{0}$ & $35.6\phantom{0}$ & & $50.1\phantom{0}$ & $\phantom{0}244.\phantom{00}$ & \\ + & \textbf{avg} & $26.93$ & $18.22$ & $29.51$ & $35.98$ & $44.95$ & $\phantom{0}255.46$ & $\phantom{0}3.73$ \\ +\textbf{ConvKB} & \textbf{pub} & $39.6\phantom{0}$ & & & & $51.7\phantom{0}$ & $\phantom{0}257.\phantom{00}$ & \\ + & \textbf{avg} & $\phantom{0}4.71$ & $\phantom{0}3.31$ & $\phantom{0}4.04$ & $\phantom{0}4.57$ & $\phantom{0}7.76$ & $4345.27$ & $61.36$ \\ +\textbf{RotatE} & \textbf{pub} & $33.8\phantom{0}$ & $24.1\phantom{0}$ & $37.5\phantom{0}$ & & $53.3\phantom{0}$ & $\phantom{0}177.\phantom{00}$ & \\ + & \textbf{avg} & $26.42$ & $17.57$ & $28.97$ & $35.29$ & $44.55$ & $\phantom{0}191.92$ & $\phantom{0}2.84$ \\ +\bottomrule +\end{tabular} + diff --git a/reproducibility/summaries/fb15k_table_without_std.tex b/reproducibility/summaries/fb15k_table_without_std.tex new file mode 100644 index 000000000..0646a293d --- /dev/null +++ b/reproducibility/summaries/fb15k_table_without_std.tex @@ -0,0 +1,38 @@ +\begin{tabular}{llrrrrrrr} +\toprule + & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ +\textbf{model} & {} & & & & & & & \\ +\midrule +\textbf{ComplEx} & \textbf{pub} & $69.2\phantom{0}$ & $59.9\phantom{0}$ & $75.9\phantom{0}$ & & $84.\phantom{00}$ & & \\ + & \textbf{avg} & $19.13$ & $10.06$ & $20.82$ & $27.59$ & $38.03$ & $\phantom{00}171.16$ & $\phantom{00}2.46$ \\ +\textbf{ConvE} & \textbf{pub} & $65.7\phantom{0}$ & $55.8\phantom{0}$ & $72.3\phantom{0}$ & & $83.1\phantom{0}$ & $\phantom{000}51.\phantom{00}$ & \\ + & \textbf{avg} & $59.56$ & $48.28$ & $66.99$ & $73.27$ & $79.76$ & $\phantom{000}50.76$ & $\phantom{00}0.73$ \\ +\textbf{DistMult} & \textbf{pub} & $35.\phantom{00}$ & & & & $57.7\phantom{0}$ & & \\ + & \textbf{avg} & $26.06$ & $16.45$ & $29.10$ & $35.54$ & $45.00$ & $\phantom{00}134.02$ & $\phantom{00}1.86$ \\ +\textbf{HolE} & \textbf{pub} & $52.4\phantom{0}$ & $40.2\phantom{0}$ & $61.3\phantom{0}$ & & $73.9\phantom{0}$ & & \\ + & \textbf{avg} & $34.15$ & $21.79$ & $39.69$ & $48.06$ & $58.84$ & $\phantom{00}193.03$ & $\phantom{00}2.71$ \\ +\textbf{KG2E} & \textbf{pub} & & & & & $71.5\phantom{0}$ & $\phantom{000}59.\phantom{00}$ & \\ + & \textbf{avg} & $\phantom{0}0.58$ & $\phantom{0}0.11$ & $\phantom{0}0.36$ & $\phantom{0}0.56$ & $\phantom{0}1.01$ & $\phantom{0}5779.07$ & $\phantom{0}78.40$ \\ +\textbf{RotatE} & \textbf{pub} & $79.7\phantom{0}$ & $74.6\phantom{0}$ & $83.\phantom{00}$ & & $88.4\phantom{0}$ & $\phantom{000}40.\phantom{00}$ & \\ + & \textbf{avg} & $55.00$ & $41.53$ & $64.14$ & $71.23$ & $78.67$ & $\phantom{000}42.28$ & $\phantom{00}0.63$ \\ +\textbf{SimplE} & \textbf{pub} & $72.7\phantom{0}$ & $66.\phantom{00}$ & $77.3\phantom{0}$ & & $83.8\phantom{0}$ & & \\ + & \textbf{avg} & $\phantom{0}0.04$ & $\phantom{0}0.01$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $\phantom{0}0.06$ & $\phantom{0}7395.75$ & $100.02$ \\ + & \textbf{best} & $23.90$ & $11.58$ & $24.16$ & $34.73$ & $54.28$ & $\phantom{00}139.34$ & \\ + & \textbf{worst} & $\phantom{0}0.03$ & $\phantom{0}0.01$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $\phantom{0}0.06$ & $14652.16$ & \\ +\textbf{TransD} & \textbf{pub} & & & & & $77.3\phantom{0}$ & $\phantom{000}91.\phantom{00}$ & \\ + & \textbf{avg} & $33.99$ & $21.22$ & $40.48$ & $48.57$ & $58.71$ & $\phantom{00}153.37$ & $\phantom{00}2.29$ \\ +\textbf{TransE} & \textbf{pub} & & & & & $47.1\phantom{0}$ & $\phantom{00}125.\phantom{00}$ & \\ + & \textbf{avg} & $26.01$ & $15.23$ & $29.85$ & $37.18$ & $47.34$ & $\phantom{00}127.92$ & $\phantom{00}1.78$ \\ +\textbf{TransH} & \textbf{pub} & & & & & $64.4\phantom{0}$ & $\phantom{000}87.\phantom{00}$ & \\ + & \textbf{avg} & $\phantom{0}2.54$ & $\phantom{0}1.69$ & $\phantom{0}2.95$ & $\phantom{0}3.29$ & $\phantom{0}3.74$ & $\phantom{0}6320.02$ & $\phantom{0}85.63$ \\ + & \textbf{best} & $\phantom{0}2.54$ & $\phantom{0}1.69$ & $\phantom{0}2.95$ & $\phantom{0}3.29$ & $\phantom{0}3.74$ & $\phantom{0}6320.00$ & \\ + & \textbf{worst} & $\phantom{0}2.54$ & $\phantom{0}1.69$ & $\phantom{0}2.95$ & $\phantom{0}3.29$ & $\phantom{0}3.74$ & $\phantom{0}6320.05$ & \\ +\textbf{TransR} & \textbf{pub} & & & & & $68.7\phantom{0}$ & $\phantom{000}77.\phantom{00}$ & \\ + & \textbf{avg} & $\phantom{0}0.65$ & $\phantom{0}0.37$ & $\phantom{0}0.63$ & $\phantom{0}0.78$ & $\phantom{0}1.03$ & $\phantom{0}6795.95$ & $\phantom{0}91.99$ \\ + & \textbf{best} & $\phantom{0}0.65$ & $\phantom{0}0.37$ & $\phantom{0}0.63$ & $\phantom{0}0.78$ & $\phantom{0}1.03$ & $\phantom{0}6795.94$ & \\ + & \textbf{worst} & $\phantom{0}0.65$ & $\phantom{0}0.37$ & $\phantom{0}0.63$ & $\phantom{0}0.78$ & $\phantom{0}1.03$ & $\phantom{0}6795.95$ & \\ +\textbf{TuckER} & \textbf{pub} & $79.5\phantom{0}$ & $74.1\phantom{0}$ & $83.3\phantom{0}$ & & $89.2\phantom{0}$ & & \\ + & \textbf{avg} & $\phantom{0}0.07$ & $\phantom{0}0.01$ & $\phantom{0}0.02$ & $\phantom{0}0.03$ & $\phantom{0}0.15$ & $\phantom{0}7327.77$ & $\phantom{0}99.11$ \\ +\bottomrule +\end{tabular} + diff --git a/reproducibility/summaries/wn18_table_without_std.tex b/reproducibility/summaries/wn18_table_without_std.tex new file mode 100644 index 000000000..202a6d6b0 --- /dev/null +++ b/reproducibility/summaries/wn18_table_without_std.tex @@ -0,0 +1,42 @@ +\begin{tabular}{llrrrrrrr} +\toprule + & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ +\textbf{model} & {} & & & & & & & \\ +\midrule +\textbf{ComplEx} & \textbf{pub} & $94.1\phantom{0}$ & $93.6\phantom{0}$ & $94.5\phantom{0}$ & & $94.7\phantom{0}$ & & \\ + & \textbf{avg} & $19.49$ & $12.36$ & $20.66$ & $25.24$ & $32.92$ & $\phantom{00}452.67$ & $\phantom{00}2.21$ \\ +\textbf{ConvE} & \textbf{pub} & $94.3\phantom{0}$ & $93.5\phantom{0}$ & $94.6\phantom{0}$ & & $95.6\phantom{0}$ & $\phantom{00}374.\phantom{00}$ & \\ + & \textbf{avg} & $88.81$ & $85.14$ & $91.76$ & $93.29$ & $94.85$ & $\phantom{00}444.40$ & $\phantom{00}2.17$ \\ +\textbf{DistMult} & \textbf{pub} & $83.\phantom{00}$ & & & & $94.2\phantom{0}$ & & \\ + & \textbf{avg} & $77.44$ & $67.45$ & $85.94$ & $89.52$ & $92.72$ & $\phantom{00}458.64$ & $\phantom{00}2.24$ \\ +\textbf{HolE} & \textbf{pub} & $93.8\phantom{0}$ & $93.\phantom{00}$ & $94.5\phantom{0}$ & & $94.9\phantom{0}$ & & \\ + & \textbf{avg} & $70.44$ & $59.29$ & $79.29$ & $84.12$ & $88.61$ & $\phantom{00}812.64$ & $\phantom{00}3.97$ \\ + & \textbf{best} & $70.44$ & $59.29$ & $79.29$ & $84.12$ & $88.61$ & $\phantom{00}812.63$ & \\ + & \textbf{worst} & $70.44$ & $59.29$ & $79.29$ & $84.12$ & $88.61$ & $\phantom{00}812.64$ & \\ +\textbf{KG2E} & \textbf{pub} & & & & & $92.8\phantom{0}$ & $\phantom{00}331.\phantom{00}$ & \\ + & \textbf{avg} & $\phantom{0}3.61$ & $\phantom{0}1.35$ & $\phantom{0}3.21$ & $\phantom{0}4.57$ & $\phantom{0}7.02$ & $\phantom{0}2708.89$ & $\phantom{0}13.25$ \\ + & \textbf{best} & $\phantom{0}3.61$ & $\phantom{0}1.35$ & $\phantom{0}3.21$ & $\phantom{0}4.57$ & $\phantom{0}7.02$ & $\phantom{0}2708.88$ & \\ + & \textbf{worst} & $\phantom{0}3.61$ & $\phantom{0}1.35$ & $\phantom{0}3.21$ & $\phantom{0}4.57$ & $\phantom{0}7.02$ & $\phantom{0}2708.89$ & \\ +\textbf{RotatE} & \textbf{pub} & $94.9\phantom{0}$ & $94.4\phantom{0}$ & $95.2\phantom{0}$ & & $95.9\phantom{0}$ & $\phantom{00}309.\phantom{00}$ & \\ + & \textbf{avg} & $87.29$ & $82.17$ & $91.53$ & $93.44$ & $95.28$ & $\phantom{00}123.68$ & $\phantom{00}0.61$ \\ +\textbf{SimplE} & \textbf{pub} & $94.2\phantom{0}$ & $93.9\phantom{0}$ & $94.4\phantom{0}$ & & $94.7\phantom{0}$ & & \\ + & \textbf{avg} & $\phantom{0}0.04$ & $\phantom{0}0.01$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $\phantom{0}0.07$ & $20376.43$ & $\phantom{0}99.57$ \\ + & \textbf{best} & $38.48$ & $33.93$ & $39.59$ & $42.76$ & $47.01$ & $\phantom{00}384.53$ & \\ + & \textbf{worst} & $\phantom{0}0.03$ & $\phantom{0}0.01$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $\phantom{0}0.07$ & $40368.33$ & \\ +\textbf{TransD} & \textbf{pub} & & & & & $92.2\phantom{0}$ & $\phantom{00}212.\phantom{00}$ & \\ + & \textbf{avg} & $36.22$ & $\phantom{0}3.94$ & $65.63$ & $79.64$ & $87.27$ & $\phantom{00}444.39$ & $\phantom{00}2.17$ \\ +\textbf{TransE} & \textbf{pub} & & & & & $89.2\phantom{0}$ & $\phantom{00}251.\phantom{00}$ & \\ + & \textbf{avg} & $39.19$ & $\phantom{0}9.99$ & $64.74$ & $75.44$ & $84.25$ & $\phantom{00}468.24$ & $\phantom{00}2.29$ \\ +\textbf{TransH} & \textbf{pub} & & & & & $82.3\phantom{0}$ & $\phantom{00}388.\phantom{00}$ & \\ + & \textbf{avg} & $\phantom{0}0.18$ & $\phantom{0}0.04$ & $\phantom{0}0.19$ & $\phantom{0}0.29$ & $\phantom{0}0.39$ & $19678.04$ & $\phantom{0}96.16$ \\ + & \textbf{best} & $\phantom{0}0.18$ & $\phantom{0}0.04$ & $\phantom{0}0.19$ & $\phantom{0}0.29$ & $\phantom{0}0.39$ & $19678.02$ & \\ + & \textbf{worst} & $\phantom{0}0.18$ & $\phantom{0}0.04$ & $\phantom{0}0.19$ & $\phantom{0}0.29$ & $\phantom{0}0.39$ & $19678.06$ & \\ +\textbf{TransR} & \textbf{pub} & & & & & $92.\phantom{00}$ & $\phantom{00}225.\phantom{00}$ & \\ + & \textbf{avg} & $\phantom{0}0.06$ & $\phantom{0}0.00$ & $\phantom{0}0.04$ & $\phantom{0}0.05$ & $\phantom{0}0.11$ & $19686.49$ & $\phantom{0}96.20$ \\ + & \textbf{best} & $\phantom{0}0.06$ & $\phantom{0}0.00$ & $\phantom{0}0.04$ & $\phantom{0}0.05$ & $\phantom{0}0.11$ & $19686.49$ & \\ + & \textbf{worst} & $\phantom{0}0.06$ & $\phantom{0}0.00$ & $\phantom{0}0.04$ & $\phantom{0}0.05$ & $\phantom{0}0.11$ & $19686.50$ & \\ +\textbf{TuckER} & \textbf{pub} & $95.3\phantom{0}$ & $94.9\phantom{0}$ & $95.5\phantom{0}$ & & $95.8\phantom{0}$ & & \\ + & \textbf{avg} & $\phantom{0}0.03$ & $\phantom{0}0.00$ & $\phantom{0}0.02$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $20622.46$ & $100.78$ \\ +\bottomrule +\end{tabular} + diff --git a/reproducibility/summaries/wn18rr_table_without_std.tex b/reproducibility/summaries/wn18rr_table_without_std.tex new file mode 100644 index 000000000..ea176c2d7 --- /dev/null +++ b/reproducibility/summaries/wn18rr_table_without_std.tex @@ -0,0 +1,16 @@ +\begin{tabular}{llrrrrrrr} +\toprule + & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ +\textbf{model} & {} & & & & & & & \\ +\midrule +\textbf{ConvE} & \textbf{pub} & $43.\phantom{00}$ & $\phantom{0}4.\phantom{00}$ & $44.\phantom{00}$ & & $52.\phantom{00}$ & $\phantom{0}4187.\phantom{00}$ & \\ + & \textbf{avg} & $44.69$ & $40.98$ & $46.49$ & $48.92$ & $51.76$ & $\phantom{0}5369.49$ & $26.49$ \\ +\textbf{ConvKB} & \textbf{pub} & $24.8\phantom{0}$ & & & & $52.5\phantom{0}$ & $\phantom{0}2554.\phantom{00}$ & \\ + & \textbf{avg} & $\phantom{0}0.30$ & $\phantom{0}0.09$ & $\phantom{0}0.21$ & $\phantom{0}0.32$ & $\phantom{0}0.57$ & $13634.66$ & $67.27$ \\ + & \textbf{best} & $\phantom{0}0.30$ & $\phantom{0}0.09$ & $\phantom{0}0.21$ & $\phantom{0}0.32$ & $\phantom{0}0.57$ & $13634.65$ & \\ + & \textbf{worst} & $\phantom{0}0.30$ & $\phantom{0}0.09$ & $\phantom{0}0.21$ & $\phantom{0}0.32$ & $\phantom{0}0.57$ & $13634.66$ & \\ +\textbf{RotatE} & \textbf{pub} & $47.6\phantom{0}$ & $42.8\phantom{0}$ & $49.2\phantom{0}$ & & $57.1\phantom{0}$ & $\phantom{0}3340.\phantom{00}$ & \\ + & \textbf{avg} & $48.40$ & $44.02$ & $50.55$ & $52.98$ & $56.51$ & $\phantom{0}4263.32$ & $21.03$ \\ +\bottomrule +\end{tabular} + From 441ccc8f9f019f7245b668a1dd8be9f0ebcc2e6b Mon Sep 17 00:00:00 2001 From: Max Berrendorf Date: Mon, 22 Jun 2020 15:53:30 +0200 Subject: [PATCH 5/7] Add midrules --- reproducibility/generate_summary_table.py | 7 +++++- reproducibility/summaries/fb15k237_table.tex | 4 ++-- .../summaries/fb15k237_table_without_std.tex | 4 ++-- reproducibility/summaries/fb15k_table.tex | 22 +++++++++---------- .../summaries/fb15k_table_without_std.tex | 22 +++++++++---------- reproducibility/summaries/wn18_table.tex | 22 +++++++++---------- .../summaries/wn18_table_without_std.tex | 22 +++++++++---------- reproducibility/summaries/wn18rr_table.tex | 4 ++-- .../summaries/wn18rr_table_without_std.tex | 4 ++-- 9 files changed, 58 insertions(+), 53 deletions(-) diff --git a/reproducibility/generate_summary_table.py b/reproducibility/generate_summary_table.py index d15a87d5d..51c61be82 100644 --- a/reproducibility/generate_summary_table.py +++ b/reproducibility/generate_summary_table.py @@ -67,7 +67,7 @@ def load_published_results_in_tall_format( return pd.DataFrame(data=data, columns=['model', 'columns', 'mean', 'std']) -def generate_results_table(with_std: bool): +def generate_results_table(with_std: bool, midrule_between_models: bool = True): all_tables = [] for dataset, dataset_df in read_experiment_collation().groupby('dataset'): if len(dataset_df['model'].unique()) < 2: @@ -91,6 +91,11 @@ def generate_results_table(with_std: bool): ) # table_latex = _process_tex(table_latex) suffix = '' if with_std else '_without_std' + # add midrule + if midrule_between_models: + table_latex_head, table_latex_body = table_latex.split('\\midrule', maxsplit=1) + table_latex_body = table_latex_body.replace('\n\\textbf', '\\midrule\n\\textbf') + table_latex = table_latex_head + table_latex_body with open(os.path.join(SUMMARIES, f'{dataset}_table{suffix}.tex'), 'w') as file: print(table_latex, file=file) diff --git a/reproducibility/summaries/fb15k237_table.tex b/reproducibility/summaries/fb15k237_table.tex index 65e4feea8..aa724365d 100644 --- a/reproducibility/summaries/fb15k237_table.tex +++ b/reproducibility/summaries/fb15k237_table.tex @@ -4,9 +4,9 @@ \textbf{model} & {} & & & & & & & \\ \midrule \textbf{ConvE} & \textbf{pub} & $32.5\phantom{0 \pm 0.00}$ & $23.7\phantom{0 \pm 0.00}$ & $35.6\phantom{0 \pm 0.00}$ & & $50.1\phantom{0 \pm 0.00}$ & $\phantom{0}244.\phantom{00 \pm 00.00}$ & \\ - & \textbf{avg} & $26.93 \pm 0.11$ & $18.22 \pm 0.11$ & $29.51 \pm 0.24$ & $35.98 \pm 0.16$ & $44.95 \pm 0.17$ & $\phantom{0}255.46 \pm \phantom{0}6.16$ & $\phantom{0}3.73 \pm 0.13$ \\ + & \textbf{avg} & $26.93 \pm 0.11$ & $18.22 \pm 0.11$ & $29.51 \pm 0.24$ & $35.98 \pm 0.16$ & $44.95 \pm 0.17$ & $\phantom{0}255.46 \pm \phantom{0}6.16$ & $\phantom{0}3.73 \pm 0.13$ \\\midrule \textbf{ConvKB} & \textbf{pub} & $39.6\phantom{0 \pm 0.00}$ & & & & $51.7\phantom{0 \pm 0.00}$ & $\phantom{0}257.\phantom{00 \pm 00.00}$ & \\ - & \textbf{avg} & $\phantom{0}4.71 \pm 0.23$ & $\phantom{0}3.31 \pm 0.23$ & $\phantom{0}4.04 \pm 0.19$ & $\phantom{0}4.57 \pm 0.22$ & $\phantom{0}7.76 \pm 0.88$ & $4345.27 \pm 46.99$ & $61.36 \pm 0.65$ \\ + & \textbf{avg} & $\phantom{0}4.71 \pm 0.23$ & $\phantom{0}3.31 \pm 0.23$ & $\phantom{0}4.04 \pm 0.19$ & $\phantom{0}4.57 \pm 0.22$ & $\phantom{0}7.76 \pm 0.88$ & $4345.27 \pm 46.99$ & $61.36 \pm 0.65$ \\\midrule \textbf{RotatE} & \textbf{pub} & $33.8\phantom{0 \pm 0.00}$ & $24.1\phantom{0 \pm 0.00}$ & $37.5\phantom{0 \pm 0.00}$ & & $53.3\phantom{0 \pm 0.00}$ & $\phantom{0}177.\phantom{00 \pm 00.00}$ & \\ & \textbf{avg} & $26.42 \pm 0.04$ & $17.57 \pm 0.06$ & $28.97 \pm 0.05$ & $35.29 \pm 0.09$ & $44.55 \pm 0.06$ & $\phantom{0}191.92 \pm \phantom{0}0.31$ & $\phantom{0}2.84 \pm 0.00$ \\ \bottomrule diff --git a/reproducibility/summaries/fb15k237_table_without_std.tex b/reproducibility/summaries/fb15k237_table_without_std.tex index f281db819..62c86c767 100644 --- a/reproducibility/summaries/fb15k237_table_without_std.tex +++ b/reproducibility/summaries/fb15k237_table_without_std.tex @@ -4,9 +4,9 @@ \textbf{model} & {} & & & & & & & \\ \midrule \textbf{ConvE} & \textbf{pub} & $32.5\phantom{0}$ & $23.7\phantom{0}$ & $35.6\phantom{0}$ & & $50.1\phantom{0}$ & $\phantom{0}244.\phantom{00}$ & \\ - & \textbf{avg} & $26.93$ & $18.22$ & $29.51$ & $35.98$ & $44.95$ & $\phantom{0}255.46$ & $\phantom{0}3.73$ \\ + & \textbf{avg} & $26.93$ & $18.22$ & $29.51$ & $35.98$ & $44.95$ & $\phantom{0}255.46$ & $\phantom{0}3.73$ \\\midrule \textbf{ConvKB} & \textbf{pub} & $39.6\phantom{0}$ & & & & $51.7\phantom{0}$ & $\phantom{0}257.\phantom{00}$ & \\ - & \textbf{avg} & $\phantom{0}4.71$ & $\phantom{0}3.31$ & $\phantom{0}4.04$ & $\phantom{0}4.57$ & $\phantom{0}7.76$ & $4345.27$ & $61.36$ \\ + & \textbf{avg} & $\phantom{0}4.71$ & $\phantom{0}3.31$ & $\phantom{0}4.04$ & $\phantom{0}4.57$ & $\phantom{0}7.76$ & $4345.27$ & $61.36$ \\\midrule \textbf{RotatE} & \textbf{pub} & $33.8\phantom{0}$ & $24.1\phantom{0}$ & $37.5\phantom{0}$ & & $53.3\phantom{0}$ & $\phantom{0}177.\phantom{00}$ & \\ & \textbf{avg} & $26.42$ & $17.57$ & $28.97$ & $35.29$ & $44.55$ & $\phantom{0}191.92$ & $\phantom{0}2.84$ \\ \bottomrule diff --git a/reproducibility/summaries/fb15k_table.tex b/reproducibility/summaries/fb15k_table.tex index c616d12fc..54b96f9ae 100644 --- a/reproducibility/summaries/fb15k_table.tex +++ b/reproducibility/summaries/fb15k_table.tex @@ -4,33 +4,33 @@ \textbf{model} & {} & & & & & & & \\ \midrule \textbf{ComplEx} & \textbf{pub} & $69.2\phantom{0 \pm 0.00}$ & $59.9\phantom{0 \pm 0.00}$ & $75.9\phantom{0 \pm 00.00}$ & & $84.\phantom{00 \pm 00.00}$ & & \\ - & \textbf{avg} & $19.13 \pm 0.45$ & $10.06 \pm 0.42$ & $20.82 \pm \phantom{0}0.67$ & $27.59 \pm \phantom{0}0.94$ & $38.03 \pm \phantom{0}1.02$ & $\phantom{00}171.16 \pm 16.05$ & $\phantom{00}2.46 \pm 0.28$ \\ + & \textbf{avg} & $19.13 \pm 0.45$ & $10.06 \pm 0.42$ & $20.82 \pm \phantom{0}0.67$ & $27.59 \pm \phantom{0}0.94$ & $38.03 \pm \phantom{0}1.02$ & $\phantom{00}171.16 \pm 16.05$ & $\phantom{00}2.46 \pm 0.28$ \\\midrule \textbf{ConvE} & \textbf{pub} & $65.7\phantom{0 \pm 0.00}$ & $55.8\phantom{0 \pm 0.00}$ & $72.3\phantom{0 \pm 00.00}$ & & $83.1\phantom{0 \pm 00.00}$ & $\phantom{000}51.\phantom{00 \pm 00.00}$ & \\ - & \textbf{avg} & $59.56 \pm 0.06$ & $48.28 \pm 0.12$ & $66.99 \pm \phantom{0}0.04$ & $73.27 \pm \phantom{0}0.03$ & $79.76 \pm \phantom{0}0.07$ & $\phantom{000}50.76 \pm \phantom{0}0.40$ & $\phantom{00}0.73 \pm 0.01$ \\ + & \textbf{avg} & $59.56 \pm 0.06$ & $48.28 \pm 0.12$ & $66.99 \pm \phantom{0}0.04$ & $73.27 \pm \phantom{0}0.03$ & $79.76 \pm \phantom{0}0.07$ & $\phantom{000}50.76 \pm \phantom{0}0.40$ & $\phantom{00}0.73 \pm 0.01$ \\\midrule \textbf{DistMult} & \textbf{pub} & $35.\phantom{00 \pm 0.00}$ & & & & $57.7\phantom{0 \pm 00.00}$ & & \\ - & \textbf{avg} & $26.06 \pm 0.17$ & $16.45 \pm 0.16$ & $29.10 \pm \phantom{0}0.17$ & $35.54 \pm \phantom{0}0.21$ & $45.00 \pm \phantom{0}0.25$ & $\phantom{00}134.02 \pm \phantom{0}1.98$ & $\phantom{00}1.86 \pm 0.03$ \\ + & \textbf{avg} & $26.06 \pm 0.17$ & $16.45 \pm 0.16$ & $29.10 \pm \phantom{0}0.17$ & $35.54 \pm \phantom{0}0.21$ & $45.00 \pm \phantom{0}0.25$ & $\phantom{00}134.02 \pm \phantom{0}1.98$ & $\phantom{00}1.86 \pm 0.03$ \\\midrule \textbf{HolE} & \textbf{pub} & $52.4\phantom{0 \pm 0.00}$ & $40.2\phantom{0 \pm 0.00}$ & $61.3\phantom{0 \pm 00.00}$ & & $73.9\phantom{0 \pm 00.00}$ & & \\ - & \textbf{avg} & $34.15 \pm 0.22$ & $21.79 \pm 0.19$ & $39.69 \pm \phantom{0}0.24$ & $48.06 \pm \phantom{0}0.30$ & $58.84 \pm \phantom{0}0.28$ & $\phantom{00}193.03 \pm \phantom{0}7.61$ & $\phantom{00}2.71 \pm 0.12$ \\ + & \textbf{avg} & $34.15 \pm 0.22$ & $21.79 \pm 0.19$ & $39.69 \pm \phantom{0}0.24$ & $48.06 \pm \phantom{0}0.30$ & $58.84 \pm \phantom{0}0.28$ & $\phantom{00}193.03 \pm \phantom{0}7.61$ & $\phantom{00}2.71 \pm 0.12$ \\\midrule \textbf{KG2E} & \textbf{pub} & & & & & $71.5\phantom{0 \pm 00.00}$ & $\phantom{000}59.\phantom{00 \pm 00.00}$ & \\ - & \textbf{avg} & $\phantom{0}0.58 \pm 0.07$ & $\phantom{0}0.11 \pm 0.04$ & $\phantom{0}0.36 \pm \phantom{0}0.08$ & $\phantom{0}0.56 \pm \phantom{0}0.10$ & $\phantom{0}1.01 \pm \phantom{0}0.14$ & $\phantom{0}5779.07 \pm 51.02$ & $\phantom{0}78.40 \pm 0.68$ \\ + & \textbf{avg} & $\phantom{0}0.58 \pm 0.07$ & $\phantom{0}0.11 \pm 0.04$ & $\phantom{0}0.36 \pm \phantom{0}0.08$ & $\phantom{0}0.56 \pm \phantom{0}0.10$ & $\phantom{0}1.01 \pm \phantom{0}0.14$ & $\phantom{0}5779.07 \pm 51.02$ & $\phantom{0}78.40 \pm 0.68$ \\\midrule \textbf{RotatE} & \textbf{pub} & $79.7\phantom{0 \pm 0.00}$ & $74.6\phantom{0 \pm 0.00}$ & $83.\phantom{00 \pm 00.00}$ & & $88.4\phantom{0 \pm 00.00}$ & $\phantom{000}40.\phantom{00 \pm 00.00}$ & \\ - & \textbf{avg} & $55.00 \pm 0.06$ & $41.53 \pm 0.06$ & $64.14 \pm \phantom{0}0.07$ & $71.23 \pm \phantom{0}0.05$ & $78.67 \pm \phantom{0}0.08$ & $\phantom{000}42.28 \pm \phantom{0}0.13$ & $\phantom{00}0.63 \pm 0.00$ \\ + & \textbf{avg} & $55.00 \pm 0.06$ & $41.53 \pm 0.06$ & $64.14 \pm \phantom{0}0.07$ & $71.23 \pm \phantom{0}0.05$ & $78.67 \pm \phantom{0}0.08$ & $\phantom{000}42.28 \pm \phantom{0}0.13$ & $\phantom{00}0.63 \pm 0.00$ \\\midrule \textbf{SimplE} & \textbf{pub} & $72.7\phantom{0 \pm 0.00}$ & $66.\phantom{00 \pm 0.00}$ & $77.3\phantom{0 \pm 00.00}$ & & $83.8\phantom{0 \pm 00.00}$ & & \\ & \textbf{avg} & $\phantom{0}0.04 \pm 0.00$ & $\phantom{0}0.01 \pm 0.00$ & $\phantom{0}0.03 \pm \phantom{0}0.00$ & $\phantom{0}0.04 \pm \phantom{0}0.00$ & $\phantom{0}0.06 \pm \phantom{0}0.01$ & $\phantom{0}7395.75 \pm \phantom{0}2.02$ & $100.02 \pm 0.03$ \\ & \textbf{best} & $23.90 \pm 8.79$ & $11.58 \pm 6.42$ & $24.16 \pm 10.95$ & $34.73 \pm 13.40$ & $54.28 \pm 15.80$ & $\phantom{00}139.34 \pm 49.45$ & \\ - & \textbf{worst} & $\phantom{0}0.03 \pm 0.00$ & $\phantom{0}0.01 \pm 0.00$ & $\phantom{0}0.03 \pm \phantom{0}0.00$ & $\phantom{0}0.04 \pm \phantom{0}0.00$ & $\phantom{0}0.06 \pm \phantom{0}0.01$ & $14652.16 \pm 45.71$ & \\ + & \textbf{worst} & $\phantom{0}0.03 \pm 0.00$ & $\phantom{0}0.01 \pm 0.00$ & $\phantom{0}0.03 \pm \phantom{0}0.00$ & $\phantom{0}0.04 \pm \phantom{0}0.00$ & $\phantom{0}0.06 \pm \phantom{0}0.01$ & $14652.16 \pm 45.71$ & \\\midrule \textbf{TransD} & \textbf{pub} & & & & & $77.3\phantom{0 \pm 00.00}$ & $\phantom{000}91.\phantom{00 \pm 00.00}$ & \\ - & \textbf{avg} & $33.99 \pm 0.03$ & $21.22 \pm 0.03$ & $40.48 \pm \phantom{0}0.10$ & $48.57 \pm \phantom{0}0.09$ & $58.71 \pm \phantom{0}0.14$ & $\phantom{00}153.37 \pm \phantom{0}5.35$ & $\phantom{00}2.29 \pm 0.09$ \\ + & \textbf{avg} & $33.99 \pm 0.03$ & $21.22 \pm 0.03$ & $40.48 \pm \phantom{0}0.10$ & $48.57 \pm \phantom{0}0.09$ & $58.71 \pm \phantom{0}0.14$ & $\phantom{00}153.37 \pm \phantom{0}5.35$ & $\phantom{00}2.29 \pm 0.09$ \\\midrule \textbf{TransE} & \textbf{pub} & & & & & $47.1\phantom{0 \pm 00.00}$ & $\phantom{00}125.\phantom{00 \pm 00.00}$ & \\ - & \textbf{avg} & $26.01 \pm 0.17$ & $15.23 \pm 0.16$ & $29.85 \pm \phantom{0}0.24$ & $37.18 \pm \phantom{0}0.24$ & $47.34 \pm \phantom{0}0.18$ & $\phantom{00}127.92 \pm \phantom{0}0.86$ & $\phantom{00}1.78 \pm 0.01$ \\ + & \textbf{avg} & $26.01 \pm 0.17$ & $15.23 \pm 0.16$ & $29.85 \pm \phantom{0}0.24$ & $37.18 \pm \phantom{0}0.24$ & $47.34 \pm \phantom{0}0.18$ & $\phantom{00}127.92 \pm \phantom{0}0.86$ & $\phantom{00}1.78 \pm 0.01$ \\\midrule \textbf{TransH} & \textbf{pub} & & & & & $64.4\phantom{0 \pm 00.00}$ & $\phantom{000}87.\phantom{00 \pm 00.00}$ & \\ & \textbf{avg} & $\phantom{0}2.54 \pm 0.20$ & $\phantom{0}1.69 \pm 0.25$ & $\phantom{0}2.95 \pm \phantom{0}0.20$ & $\phantom{0}3.29 \pm \phantom{0}0.22$ & $\phantom{0}3.74 \pm \phantom{0}0.18$ & $\phantom{0}6320.02 \pm 30.37$ & $\phantom{0}85.63 \pm 0.40$ \\ & \textbf{best} & $\phantom{0}2.54 \pm 0.20$ & $\phantom{0}1.69 \pm 0.25$ & $\phantom{0}2.95 \pm \phantom{0}0.20$ & $\phantom{0}3.29 \pm \phantom{0}0.22$ & $\phantom{0}3.74 \pm \phantom{0}0.18$ & $\phantom{0}6320.00 \pm 30.37$ & \\ - & \textbf{worst} & $\phantom{0}2.54 \pm 0.20$ & $\phantom{0}1.69 \pm 0.25$ & $\phantom{0}2.95 \pm \phantom{0}0.20$ & $\phantom{0}3.29 \pm \phantom{0}0.22$ & $\phantom{0}3.74 \pm \phantom{0}0.18$ & $\phantom{0}6320.05 \pm 30.37$ & \\ + & \textbf{worst} & $\phantom{0}2.54 \pm 0.20$ & $\phantom{0}1.69 \pm 0.25$ & $\phantom{0}2.95 \pm \phantom{0}0.20$ & $\phantom{0}3.29 \pm \phantom{0}0.22$ & $\phantom{0}3.74 \pm \phantom{0}0.18$ & $\phantom{0}6320.05 \pm 30.37$ & \\\midrule \textbf{TransR} & \textbf{pub} & & & & & $68.7\phantom{0 \pm 00.00}$ & $\phantom{000}77.\phantom{00 \pm 00.00}$ & \\ & \textbf{avg} & $\phantom{0}0.65 \pm 0.02$ & $\phantom{0}0.37 \pm 0.00$ & $\phantom{0}0.63 \pm \phantom{0}0.04$ & $\phantom{0}0.78 \pm \phantom{0}0.06$ & $\phantom{0}1.03 \pm \phantom{0}0.07$ & $\phantom{0}6795.95 \pm 16.65$ & $\phantom{0}91.99 \pm 0.22$ \\ & \textbf{best} & $\phantom{0}0.65 \pm 0.02$ & $\phantom{0}0.37 \pm 0.00$ & $\phantom{0}0.63 \pm \phantom{0}0.04$ & $\phantom{0}0.78 \pm \phantom{0}0.06$ & $\phantom{0}1.03 \pm \phantom{0}0.07$ & $\phantom{0}6795.94 \pm 16.65$ & \\ - & \textbf{worst} & $\phantom{0}0.65 \pm 0.02$ & $\phantom{0}0.37 \pm 0.00$ & $\phantom{0}0.63 \pm \phantom{0}0.04$ & $\phantom{0}0.78 \pm \phantom{0}0.06$ & $\phantom{0}1.03 \pm \phantom{0}0.07$ & $\phantom{0}6795.95 \pm 16.65$ & \\ + & \textbf{worst} & $\phantom{0}0.65 \pm 0.02$ & $\phantom{0}0.37 \pm 0.00$ & $\phantom{0}0.63 \pm \phantom{0}0.04$ & $\phantom{0}0.78 \pm \phantom{0}0.06$ & $\phantom{0}1.03 \pm \phantom{0}0.07$ & $\phantom{0}6795.95 \pm 16.65$ & \\\midrule \textbf{TuckER} & \textbf{pub} & $79.5\phantom{0 \pm 0.00}$ & $74.1\phantom{0 \pm 0.00}$ & $83.3\phantom{0 \pm 00.00}$ & & $89.2\phantom{0 \pm 00.00}$ & & \\ & \textbf{avg} & $\phantom{0}0.07 \pm 0.02$ & $\phantom{0}0.01 \pm 0.00$ & $\phantom{0}0.02 \pm \phantom{0}0.00$ & $\phantom{0}0.03 \pm \phantom{0}0.01$ & $\phantom{0}0.15 \pm \phantom{0}0.17$ & $\phantom{0}7327.77 \pm 29.22$ & $\phantom{0}99.11 \pm 0.39$ \\ \bottomrule diff --git a/reproducibility/summaries/fb15k_table_without_std.tex b/reproducibility/summaries/fb15k_table_without_std.tex index 0646a293d..b9e90ad79 100644 --- a/reproducibility/summaries/fb15k_table_without_std.tex +++ b/reproducibility/summaries/fb15k_table_without_std.tex @@ -4,33 +4,33 @@ \textbf{model} & {} & & & & & & & \\ \midrule \textbf{ComplEx} & \textbf{pub} & $69.2\phantom{0}$ & $59.9\phantom{0}$ & $75.9\phantom{0}$ & & $84.\phantom{00}$ & & \\ - & \textbf{avg} & $19.13$ & $10.06$ & $20.82$ & $27.59$ & $38.03$ & $\phantom{00}171.16$ & $\phantom{00}2.46$ \\ + & \textbf{avg} & $19.13$ & $10.06$ & $20.82$ & $27.59$ & $38.03$ & $\phantom{00}171.16$ & $\phantom{00}2.46$ \\\midrule \textbf{ConvE} & \textbf{pub} & $65.7\phantom{0}$ & $55.8\phantom{0}$ & $72.3\phantom{0}$ & & $83.1\phantom{0}$ & $\phantom{000}51.\phantom{00}$ & \\ - & \textbf{avg} & $59.56$ & $48.28$ & $66.99$ & $73.27$ & $79.76$ & $\phantom{000}50.76$ & $\phantom{00}0.73$ \\ + & \textbf{avg} & $59.56$ & $48.28$ & $66.99$ & $73.27$ & $79.76$ & $\phantom{000}50.76$ & $\phantom{00}0.73$ \\\midrule \textbf{DistMult} & \textbf{pub} & $35.\phantom{00}$ & & & & $57.7\phantom{0}$ & & \\ - & \textbf{avg} & $26.06$ & $16.45$ & $29.10$ & $35.54$ & $45.00$ & $\phantom{00}134.02$ & $\phantom{00}1.86$ \\ + & \textbf{avg} & $26.06$ & $16.45$ & $29.10$ & $35.54$ & $45.00$ & $\phantom{00}134.02$ & $\phantom{00}1.86$ \\\midrule \textbf{HolE} & \textbf{pub} & $52.4\phantom{0}$ & $40.2\phantom{0}$ & $61.3\phantom{0}$ & & $73.9\phantom{0}$ & & \\ - & \textbf{avg} & $34.15$ & $21.79$ & $39.69$ & $48.06$ & $58.84$ & $\phantom{00}193.03$ & $\phantom{00}2.71$ \\ + & \textbf{avg} & $34.15$ & $21.79$ & $39.69$ & $48.06$ & $58.84$ & $\phantom{00}193.03$ & $\phantom{00}2.71$ \\\midrule \textbf{KG2E} & \textbf{pub} & & & & & $71.5\phantom{0}$ & $\phantom{000}59.\phantom{00}$ & \\ - & \textbf{avg} & $\phantom{0}0.58$ & $\phantom{0}0.11$ & $\phantom{0}0.36$ & $\phantom{0}0.56$ & $\phantom{0}1.01$ & $\phantom{0}5779.07$ & $\phantom{0}78.40$ \\ + & \textbf{avg} & $\phantom{0}0.58$ & $\phantom{0}0.11$ & $\phantom{0}0.36$ & $\phantom{0}0.56$ & $\phantom{0}1.01$ & $\phantom{0}5779.07$ & $\phantom{0}78.40$ \\\midrule \textbf{RotatE} & \textbf{pub} & $79.7\phantom{0}$ & $74.6\phantom{0}$ & $83.\phantom{00}$ & & $88.4\phantom{0}$ & $\phantom{000}40.\phantom{00}$ & \\ - & \textbf{avg} & $55.00$ & $41.53$ & $64.14$ & $71.23$ & $78.67$ & $\phantom{000}42.28$ & $\phantom{00}0.63$ \\ + & \textbf{avg} & $55.00$ & $41.53$ & $64.14$ & $71.23$ & $78.67$ & $\phantom{000}42.28$ & $\phantom{00}0.63$ \\\midrule \textbf{SimplE} & \textbf{pub} & $72.7\phantom{0}$ & $66.\phantom{00}$ & $77.3\phantom{0}$ & & $83.8\phantom{0}$ & & \\ & \textbf{avg} & $\phantom{0}0.04$ & $\phantom{0}0.01$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $\phantom{0}0.06$ & $\phantom{0}7395.75$ & $100.02$ \\ & \textbf{best} & $23.90$ & $11.58$ & $24.16$ & $34.73$ & $54.28$ & $\phantom{00}139.34$ & \\ - & \textbf{worst} & $\phantom{0}0.03$ & $\phantom{0}0.01$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $\phantom{0}0.06$ & $14652.16$ & \\ + & \textbf{worst} & $\phantom{0}0.03$ & $\phantom{0}0.01$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $\phantom{0}0.06$ & $14652.16$ & \\\midrule \textbf{TransD} & \textbf{pub} & & & & & $77.3\phantom{0}$ & $\phantom{000}91.\phantom{00}$ & \\ - & \textbf{avg} & $33.99$ & $21.22$ & $40.48$ & $48.57$ & $58.71$ & $\phantom{00}153.37$ & $\phantom{00}2.29$ \\ + & \textbf{avg} & $33.99$ & $21.22$ & $40.48$ & $48.57$ & $58.71$ & $\phantom{00}153.37$ & $\phantom{00}2.29$ \\\midrule \textbf{TransE} & \textbf{pub} & & & & & $47.1\phantom{0}$ & $\phantom{00}125.\phantom{00}$ & \\ - & \textbf{avg} & $26.01$ & $15.23$ & $29.85$ & $37.18$ & $47.34$ & $\phantom{00}127.92$ & $\phantom{00}1.78$ \\ + & \textbf{avg} & $26.01$ & $15.23$ & $29.85$ & $37.18$ & $47.34$ & $\phantom{00}127.92$ & $\phantom{00}1.78$ \\\midrule \textbf{TransH} & \textbf{pub} & & & & & $64.4\phantom{0}$ & $\phantom{000}87.\phantom{00}$ & \\ & \textbf{avg} & $\phantom{0}2.54$ & $\phantom{0}1.69$ & $\phantom{0}2.95$ & $\phantom{0}3.29$ & $\phantom{0}3.74$ & $\phantom{0}6320.02$ & $\phantom{0}85.63$ \\ & \textbf{best} & $\phantom{0}2.54$ & $\phantom{0}1.69$ & $\phantom{0}2.95$ & $\phantom{0}3.29$ & $\phantom{0}3.74$ & $\phantom{0}6320.00$ & \\ - & \textbf{worst} & $\phantom{0}2.54$ & $\phantom{0}1.69$ & $\phantom{0}2.95$ & $\phantom{0}3.29$ & $\phantom{0}3.74$ & $\phantom{0}6320.05$ & \\ + & \textbf{worst} & $\phantom{0}2.54$ & $\phantom{0}1.69$ & $\phantom{0}2.95$ & $\phantom{0}3.29$ & $\phantom{0}3.74$ & $\phantom{0}6320.05$ & \\\midrule \textbf{TransR} & \textbf{pub} & & & & & $68.7\phantom{0}$ & $\phantom{000}77.\phantom{00}$ & \\ & \textbf{avg} & $\phantom{0}0.65$ & $\phantom{0}0.37$ & $\phantom{0}0.63$ & $\phantom{0}0.78$ & $\phantom{0}1.03$ & $\phantom{0}6795.95$ & $\phantom{0}91.99$ \\ & \textbf{best} & $\phantom{0}0.65$ & $\phantom{0}0.37$ & $\phantom{0}0.63$ & $\phantom{0}0.78$ & $\phantom{0}1.03$ & $\phantom{0}6795.94$ & \\ - & \textbf{worst} & $\phantom{0}0.65$ & $\phantom{0}0.37$ & $\phantom{0}0.63$ & $\phantom{0}0.78$ & $\phantom{0}1.03$ & $\phantom{0}6795.95$ & \\ + & \textbf{worst} & $\phantom{0}0.65$ & $\phantom{0}0.37$ & $\phantom{0}0.63$ & $\phantom{0}0.78$ & $\phantom{0}1.03$ & $\phantom{0}6795.95$ & \\\midrule \textbf{TuckER} & \textbf{pub} & $79.5\phantom{0}$ & $74.1\phantom{0}$ & $83.3\phantom{0}$ & & $89.2\phantom{0}$ & & \\ & \textbf{avg} & $\phantom{0}0.07$ & $\phantom{0}0.01$ & $\phantom{0}0.02$ & $\phantom{0}0.03$ & $\phantom{0}0.15$ & $\phantom{0}7327.77$ & $\phantom{0}99.11$ \\ \bottomrule diff --git a/reproducibility/summaries/wn18_table.tex b/reproducibility/summaries/wn18_table.tex index a567b96d5..5f8255a58 100644 --- a/reproducibility/summaries/wn18_table.tex +++ b/reproducibility/summaries/wn18_table.tex @@ -4,37 +4,37 @@ \textbf{model} & {} & & & & & & & \\ \midrule \textbf{ComplEx} & \textbf{pub} & $94.1\phantom{0 \pm 0.00}$ & $93.6\phantom{0 \pm 0.00}$ & $94.5\phantom{0 \pm 0.00}$ & & $94.7\phantom{0 \pm 0.00}$ & & \\ - & \textbf{avg} & $19.49 \pm 2.55$ & $12.36 \pm 1.96$ & $20.66 \pm 2.75$ & $25.24 \pm 3.33$ & $32.92 \pm 4.40$ & $\phantom{00}452.67 \pm \phantom{0}63.05$ & $\phantom{00}2.21 \pm 0.31$ \\ + & \textbf{avg} & $19.49 \pm 2.55$ & $12.36 \pm 1.96$ & $20.66 \pm 2.75$ & $25.24 \pm 3.33$ & $32.92 \pm 4.40$ & $\phantom{00}452.67 \pm \phantom{0}63.05$ & $\phantom{00}2.21 \pm 0.31$ \\\midrule \textbf{ConvE} & \textbf{pub} & $94.3\phantom{0 \pm 0.00}$ & $93.5\phantom{0 \pm 0.00}$ & $94.6\phantom{0 \pm 0.00}$ & & $95.6\phantom{0 \pm 0.00}$ & $\phantom{00}374.\phantom{00 \pm 000.00}$ & \\ - & \textbf{avg} & $88.81 \pm 0.09$ & $85.14 \pm 0.10$ & $91.76 \pm 0.11$ & $93.29 \pm 0.04$ & $94.85 \pm 0.06$ & $\phantom{00}444.40 \pm \phantom{0}14.82$ & $\phantom{00}2.17 \pm 0.07$ \\ + & \textbf{avg} & $88.81 \pm 0.09$ & $85.14 \pm 0.10$ & $91.76 \pm 0.11$ & $93.29 \pm 0.04$ & $94.85 \pm 0.06$ & $\phantom{00}444.40 \pm \phantom{0}14.82$ & $\phantom{00}2.17 \pm 0.07$ \\\midrule \textbf{DistMult} & \textbf{pub} & $83.\phantom{00 \pm 0.00}$ & & & & $94.2\phantom{0 \pm 0.00}$ & & \\ - & \textbf{avg} & $77.44 \pm 0.22$ & $67.45 \pm 0.34$ & $85.94 \pm 0.21$ & $89.52 \pm 0.25$ & $92.72 \pm 0.18$ & $\phantom{00}458.64 \pm \phantom{0}23.96$ & $\phantom{00}2.24 \pm 0.12$ \\ + & \textbf{avg} & $77.44 \pm 0.22$ & $67.45 \pm 0.34$ & $85.94 \pm 0.21$ & $89.52 \pm 0.25$ & $92.72 \pm 0.18$ & $\phantom{00}458.64 \pm \phantom{0}23.96$ & $\phantom{00}2.24 \pm 0.12$ \\\midrule \textbf{HolE} & \textbf{pub} & $93.8\phantom{0 \pm 0.00}$ & $93.\phantom{00 \pm 0.00}$ & $94.5\phantom{0 \pm 0.00}$ & & $94.9\phantom{0 \pm 0.00}$ & & \\ & \textbf{avg} & $70.44 \pm 0.45$ & $59.29 \pm 0.53$ & $79.29 \pm 0.47$ & $84.12 \pm 0.36$ & $88.61 \pm 0.42$ & $\phantom{00}812.64 \pm \phantom{0}28.33$ & $\phantom{00}3.97 \pm 0.14$ \\ & \textbf{best} & $70.44 \pm 0.45$ & $59.29 \pm 0.53$ & $79.29 \pm 0.47$ & $84.12 \pm 0.36$ & $88.61 \pm 0.42$ & $\phantom{00}812.63 \pm \phantom{0}28.33$ & \\ - & \textbf{worst} & $70.44 \pm 0.45$ & $59.29 \pm 0.53$ & $79.29 \pm 0.47$ & $84.12 \pm 0.36$ & $88.61 \pm 0.42$ & $\phantom{00}812.64 \pm \phantom{0}28.34$ & \\ + & \textbf{worst} & $70.44 \pm 0.45$ & $59.29 \pm 0.53$ & $79.29 \pm 0.47$ & $84.12 \pm 0.36$ & $88.61 \pm 0.42$ & $\phantom{00}812.64 \pm \phantom{0}28.34$ & \\\midrule \textbf{KG2E} & \textbf{pub} & & & & & $92.8\phantom{0 \pm 0.00}$ & $\phantom{00}331.\phantom{00 \pm 000.00}$ & \\ & \textbf{avg} & $\phantom{0}3.61 \pm 0.26$ & $\phantom{0}1.35 \pm 0.22$ & $\phantom{0}3.21 \pm 0.31$ & $\phantom{0}4.57 \pm 0.34$ & $\phantom{0}7.02 \pm 0.43$ & $\phantom{0}2708.89 \pm \phantom{0}44.57$ & $\phantom{0}13.25 \pm 0.22$ \\ & \textbf{best} & $\phantom{0}3.61 \pm 0.26$ & $\phantom{0}1.35 \pm 0.22$ & $\phantom{0}3.21 \pm 0.31$ & $\phantom{0}4.57 \pm 0.34$ & $\phantom{0}7.02 \pm 0.43$ & $\phantom{0}2708.88 \pm \phantom{0}44.57$ & \\ - & \textbf{worst} & $\phantom{0}3.61 \pm 0.26$ & $\phantom{0}1.35 \pm 0.22$ & $\phantom{0}3.21 \pm 0.31$ & $\phantom{0}4.57 \pm 0.34$ & $\phantom{0}7.02 \pm 0.43$ & $\phantom{0}2708.89 \pm \phantom{0}44.57$ & \\ + & \textbf{worst} & $\phantom{0}3.61 \pm 0.26$ & $\phantom{0}1.35 \pm 0.22$ & $\phantom{0}3.21 \pm 0.31$ & $\phantom{0}4.57 \pm 0.34$ & $\phantom{0}7.02 \pm 0.43$ & $\phantom{0}2708.89 \pm \phantom{0}44.57$ & \\\midrule \textbf{RotatE} & \textbf{pub} & $94.9\phantom{0 \pm 0.00}$ & $94.4\phantom{0 \pm 0.00}$ & $95.2\phantom{0 \pm 0.00}$ & & $95.9\phantom{0 \pm 0.00}$ & $\phantom{00}309.\phantom{00 \pm 000.00}$ & \\ - & \textbf{avg} & $87.29 \pm 0.12$ & $82.17 \pm 0.20$ & $91.53 \pm 0.12$ & $93.44 \pm 0.07$ & $95.28 \pm 0.08$ & $\phantom{00}123.68 \pm \phantom{00}1.71$ & $\phantom{00}0.61 \pm 0.01$ \\ + & \textbf{avg} & $87.29 \pm 0.12$ & $82.17 \pm 0.20$ & $91.53 \pm 0.12$ & $93.44 \pm 0.07$ & $95.28 \pm 0.08$ & $\phantom{00}123.68 \pm \phantom{00}1.71$ & $\phantom{00}0.61 \pm 0.01$ \\\midrule \textbf{SimplE} & \textbf{pub} & $94.2\phantom{0 \pm 0.00}$ & $93.9\phantom{0 \pm 0.00}$ & $94.4\phantom{0 \pm 0.00}$ & & $94.7\phantom{0 \pm 0.00}$ & & \\ & \textbf{avg} & $\phantom{0}0.04 \pm 0.01$ & $\phantom{0}0.01 \pm 0.01$ & $\phantom{0}0.03 \pm 0.00$ & $\phantom{0}0.04 \pm 0.01$ & $\phantom{0}0.07 \pm 0.03$ & $20376.43 \pm \phantom{0}42.30$ & $\phantom{0}99.57 \pm 0.21$ \\ & \textbf{best} & $38.48 \pm 4.00$ & $33.93 \pm 4.32$ & $39.59 \pm 4.67$ & $42.76 \pm 3.73$ & $47.01 \pm 2.66$ & $\phantom{00}384.53 \pm \phantom{0}66.45$ & \\ - & \textbf{worst} & $\phantom{0}0.03 \pm 0.01$ & $\phantom{0}0.01 \pm 0.01$ & $\phantom{0}0.03 \pm 0.00$ & $\phantom{0}0.04 \pm 0.01$ & $\phantom{0}0.07 \pm 0.03$ & $40368.33 \pm 114.95$ & \\ + & \textbf{worst} & $\phantom{0}0.03 \pm 0.01$ & $\phantom{0}0.01 \pm 0.01$ & $\phantom{0}0.03 \pm 0.00$ & $\phantom{0}0.04 \pm 0.01$ & $\phantom{0}0.07 \pm 0.03$ & $40368.33 \pm 114.95$ & \\\midrule \textbf{TransD} & \textbf{pub} & & & & & $92.2\phantom{0 \pm 0.00}$ & $\phantom{00}212.\phantom{00 \pm 000.00}$ & \\ - & \textbf{avg} & $36.22 \pm 0.12$ & $\phantom{0}3.94 \pm 0.27$ & $65.63 \pm 0.55$ & $79.64 \pm 0.43$ & $87.27 \pm 0.41$ & $\phantom{00}444.39 \pm \phantom{0}25.61$ & $\phantom{00}2.17 \pm 0.13$ \\ + & \textbf{avg} & $36.22 \pm 0.12$ & $\phantom{0}3.94 \pm 0.27$ & $65.63 \pm 0.55$ & $79.64 \pm 0.43$ & $87.27 \pm 0.41$ & $\phantom{00}444.39 \pm \phantom{0}25.61$ & $\phantom{00}2.17 \pm 0.13$ \\\midrule \textbf{TransE} & \textbf{pub} & & & & & $89.2\phantom{0 \pm 0.00}$ & $\phantom{00}251.\phantom{00 \pm 000.00}$ & \\ - & \textbf{avg} & $39.19 \pm 1.21$ & $\phantom{0}9.99 \pm 1.82$ & $64.74 \pm 0.91$ & $75.44 \pm 0.48$ & $84.25 \pm 0.33$ & $\phantom{00}468.24 \pm \phantom{0}13.64$ & $\phantom{00}2.29 \pm 0.07$ \\ + & \textbf{avg} & $39.19 \pm 1.21$ & $\phantom{0}9.99 \pm 1.82$ & $64.74 \pm 0.91$ & $75.44 \pm 0.48$ & $84.25 \pm 0.33$ & $\phantom{00}468.24 \pm \phantom{0}13.64$ & $\phantom{00}2.29 \pm 0.07$ \\\midrule \textbf{TransH} & \textbf{pub} & & & & & $82.3\phantom{0 \pm 0.00}$ & $\phantom{00}388.\phantom{00 \pm 000.00}$ & \\ & \textbf{avg} & $\phantom{0}0.18 \pm 0.04$ & $\phantom{0}0.04 \pm 0.02$ & $\phantom{0}0.19 \pm 0.10$ & $\phantom{0}0.29 \pm 0.14$ & $\phantom{0}0.39 \pm 0.11$ & $19678.04 \pm \phantom{0}18.92$ & $\phantom{0}96.16 \pm 0.09$ \\ & \textbf{best} & $\phantom{0}0.18 \pm 0.04$ & $\phantom{0}0.04 \pm 0.02$ & $\phantom{0}0.19 \pm 0.10$ & $\phantom{0}0.29 \pm 0.14$ & $\phantom{0}0.39 \pm 0.11$ & $19678.02 \pm \phantom{0}18.92$ & \\ - & \textbf{worst} & $\phantom{0}0.18 \pm 0.04$ & $\phantom{0}0.04 \pm 0.02$ & $\phantom{0}0.19 \pm 0.10$ & $\phantom{0}0.29 \pm 0.14$ & $\phantom{0}0.39 \pm 0.11$ & $19678.06 \pm \phantom{0}18.92$ & \\ + & \textbf{worst} & $\phantom{0}0.18 \pm 0.04$ & $\phantom{0}0.04 \pm 0.02$ & $\phantom{0}0.19 \pm 0.10$ & $\phantom{0}0.29 \pm 0.14$ & $\phantom{0}0.39 \pm 0.11$ & $19678.06 \pm \phantom{0}18.92$ & \\\midrule \textbf{TransR} & \textbf{pub} & & & & & $92.\phantom{00 \pm 0.00}$ & $\phantom{00}225.\phantom{00 \pm 000.00}$ & \\ & \textbf{avg} & $\phantom{0}0.06 \pm 0.02$ & $\phantom{0}0.00 \pm 0.00$ & $\phantom{0}0.04 \pm 0.02$ & $\phantom{0}0.05 \pm 0.03$ & $\phantom{0}0.11 \pm 0.06$ & $19686.49 \pm 100.97$ & $\phantom{0}96.20 \pm 0.49$ \\ & \textbf{best} & $\phantom{0}0.06 \pm 0.02$ & $\phantom{0}0.00 \pm 0.00$ & $\phantom{0}0.04 \pm 0.02$ & $\phantom{0}0.05 \pm 0.03$ & $\phantom{0}0.11 \pm 0.06$ & $19686.49 \pm 100.97$ & \\ - & \textbf{worst} & $\phantom{0}0.06 \pm 0.02$ & $\phantom{0}0.00 \pm 0.00$ & $\phantom{0}0.04 \pm 0.02$ & $\phantom{0}0.05 \pm 0.03$ & $\phantom{0}0.11 \pm 0.06$ & $19686.50 \pm 100.97$ & \\ + & \textbf{worst} & $\phantom{0}0.06 \pm 0.02$ & $\phantom{0}0.00 \pm 0.00$ & $\phantom{0}0.04 \pm 0.02$ & $\phantom{0}0.05 \pm 0.03$ & $\phantom{0}0.11 \pm 0.06$ & $19686.50 \pm 100.97$ & \\\midrule \textbf{TuckER} & \textbf{pub} & $95.3\phantom{0 \pm 0.00}$ & $94.9\phantom{0 \pm 0.00}$ & $95.5\phantom{0 \pm 0.00}$ & & $95.8\phantom{0 \pm 0.00}$ & & \\ & \textbf{avg} & $\phantom{0}0.03 \pm 0.01$ & $\phantom{0}0.00 \pm 0.00$ & $\phantom{0}0.02 \pm 0.01$ & $\phantom{0}0.03 \pm 0.03$ & $\phantom{0}0.04 \pm 0.03$ & $20622.46 \pm 153.52$ & $100.78 \pm 0.75$ \\ \bottomrule diff --git a/reproducibility/summaries/wn18_table_without_std.tex b/reproducibility/summaries/wn18_table_without_std.tex index 202a6d6b0..da10ebf38 100644 --- a/reproducibility/summaries/wn18_table_without_std.tex +++ b/reproducibility/summaries/wn18_table_without_std.tex @@ -4,37 +4,37 @@ \textbf{model} & {} & & & & & & & \\ \midrule \textbf{ComplEx} & \textbf{pub} & $94.1\phantom{0}$ & $93.6\phantom{0}$ & $94.5\phantom{0}$ & & $94.7\phantom{0}$ & & \\ - & \textbf{avg} & $19.49$ & $12.36$ & $20.66$ & $25.24$ & $32.92$ & $\phantom{00}452.67$ & $\phantom{00}2.21$ \\ + & \textbf{avg} & $19.49$ & $12.36$ & $20.66$ & $25.24$ & $32.92$ & $\phantom{00}452.67$ & $\phantom{00}2.21$ \\\midrule \textbf{ConvE} & \textbf{pub} & $94.3\phantom{0}$ & $93.5\phantom{0}$ & $94.6\phantom{0}$ & & $95.6\phantom{0}$ & $\phantom{00}374.\phantom{00}$ & \\ - & \textbf{avg} & $88.81$ & $85.14$ & $91.76$ & $93.29$ & $94.85$ & $\phantom{00}444.40$ & $\phantom{00}2.17$ \\ + & \textbf{avg} & $88.81$ & $85.14$ & $91.76$ & $93.29$ & $94.85$ & $\phantom{00}444.40$ & $\phantom{00}2.17$ \\\midrule \textbf{DistMult} & \textbf{pub} & $83.\phantom{00}$ & & & & $94.2\phantom{0}$ & & \\ - & \textbf{avg} & $77.44$ & $67.45$ & $85.94$ & $89.52$ & $92.72$ & $\phantom{00}458.64$ & $\phantom{00}2.24$ \\ + & \textbf{avg} & $77.44$ & $67.45$ & $85.94$ & $89.52$ & $92.72$ & $\phantom{00}458.64$ & $\phantom{00}2.24$ \\\midrule \textbf{HolE} & \textbf{pub} & $93.8\phantom{0}$ & $93.\phantom{00}$ & $94.5\phantom{0}$ & & $94.9\phantom{0}$ & & \\ & \textbf{avg} & $70.44$ & $59.29$ & $79.29$ & $84.12$ & $88.61$ & $\phantom{00}812.64$ & $\phantom{00}3.97$ \\ & \textbf{best} & $70.44$ & $59.29$ & $79.29$ & $84.12$ & $88.61$ & $\phantom{00}812.63$ & \\ - & \textbf{worst} & $70.44$ & $59.29$ & $79.29$ & $84.12$ & $88.61$ & $\phantom{00}812.64$ & \\ + & \textbf{worst} & $70.44$ & $59.29$ & $79.29$ & $84.12$ & $88.61$ & $\phantom{00}812.64$ & \\\midrule \textbf{KG2E} & \textbf{pub} & & & & & $92.8\phantom{0}$ & $\phantom{00}331.\phantom{00}$ & \\ & \textbf{avg} & $\phantom{0}3.61$ & $\phantom{0}1.35$ & $\phantom{0}3.21$ & $\phantom{0}4.57$ & $\phantom{0}7.02$ & $\phantom{0}2708.89$ & $\phantom{0}13.25$ \\ & \textbf{best} & $\phantom{0}3.61$ & $\phantom{0}1.35$ & $\phantom{0}3.21$ & $\phantom{0}4.57$ & $\phantom{0}7.02$ & $\phantom{0}2708.88$ & \\ - & \textbf{worst} & $\phantom{0}3.61$ & $\phantom{0}1.35$ & $\phantom{0}3.21$ & $\phantom{0}4.57$ & $\phantom{0}7.02$ & $\phantom{0}2708.89$ & \\ + & \textbf{worst} & $\phantom{0}3.61$ & $\phantom{0}1.35$ & $\phantom{0}3.21$ & $\phantom{0}4.57$ & $\phantom{0}7.02$ & $\phantom{0}2708.89$ & \\\midrule \textbf{RotatE} & \textbf{pub} & $94.9\phantom{0}$ & $94.4\phantom{0}$ & $95.2\phantom{0}$ & & $95.9\phantom{0}$ & $\phantom{00}309.\phantom{00}$ & \\ - & \textbf{avg} & $87.29$ & $82.17$ & $91.53$ & $93.44$ & $95.28$ & $\phantom{00}123.68$ & $\phantom{00}0.61$ \\ + & \textbf{avg} & $87.29$ & $82.17$ & $91.53$ & $93.44$ & $95.28$ & $\phantom{00}123.68$ & $\phantom{00}0.61$ \\\midrule \textbf{SimplE} & \textbf{pub} & $94.2\phantom{0}$ & $93.9\phantom{0}$ & $94.4\phantom{0}$ & & $94.7\phantom{0}$ & & \\ & \textbf{avg} & $\phantom{0}0.04$ & $\phantom{0}0.01$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $\phantom{0}0.07$ & $20376.43$ & $\phantom{0}99.57$ \\ & \textbf{best} & $38.48$ & $33.93$ & $39.59$ & $42.76$ & $47.01$ & $\phantom{00}384.53$ & \\ - & \textbf{worst} & $\phantom{0}0.03$ & $\phantom{0}0.01$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $\phantom{0}0.07$ & $40368.33$ & \\ + & \textbf{worst} & $\phantom{0}0.03$ & $\phantom{0}0.01$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $\phantom{0}0.07$ & $40368.33$ & \\\midrule \textbf{TransD} & \textbf{pub} & & & & & $92.2\phantom{0}$ & $\phantom{00}212.\phantom{00}$ & \\ - & \textbf{avg} & $36.22$ & $\phantom{0}3.94$ & $65.63$ & $79.64$ & $87.27$ & $\phantom{00}444.39$ & $\phantom{00}2.17$ \\ + & \textbf{avg} & $36.22$ & $\phantom{0}3.94$ & $65.63$ & $79.64$ & $87.27$ & $\phantom{00}444.39$ & $\phantom{00}2.17$ \\\midrule \textbf{TransE} & \textbf{pub} & & & & & $89.2\phantom{0}$ & $\phantom{00}251.\phantom{00}$ & \\ - & \textbf{avg} & $39.19$ & $\phantom{0}9.99$ & $64.74$ & $75.44$ & $84.25$ & $\phantom{00}468.24$ & $\phantom{00}2.29$ \\ + & \textbf{avg} & $39.19$ & $\phantom{0}9.99$ & $64.74$ & $75.44$ & $84.25$ & $\phantom{00}468.24$ & $\phantom{00}2.29$ \\\midrule \textbf{TransH} & \textbf{pub} & & & & & $82.3\phantom{0}$ & $\phantom{00}388.\phantom{00}$ & \\ & \textbf{avg} & $\phantom{0}0.18$ & $\phantom{0}0.04$ & $\phantom{0}0.19$ & $\phantom{0}0.29$ & $\phantom{0}0.39$ & $19678.04$ & $\phantom{0}96.16$ \\ & \textbf{best} & $\phantom{0}0.18$ & $\phantom{0}0.04$ & $\phantom{0}0.19$ & $\phantom{0}0.29$ & $\phantom{0}0.39$ & $19678.02$ & \\ - & \textbf{worst} & $\phantom{0}0.18$ & $\phantom{0}0.04$ & $\phantom{0}0.19$ & $\phantom{0}0.29$ & $\phantom{0}0.39$ & $19678.06$ & \\ + & \textbf{worst} & $\phantom{0}0.18$ & $\phantom{0}0.04$ & $\phantom{0}0.19$ & $\phantom{0}0.29$ & $\phantom{0}0.39$ & $19678.06$ & \\\midrule \textbf{TransR} & \textbf{pub} & & & & & $92.\phantom{00}$ & $\phantom{00}225.\phantom{00}$ & \\ & \textbf{avg} & $\phantom{0}0.06$ & $\phantom{0}0.00$ & $\phantom{0}0.04$ & $\phantom{0}0.05$ & $\phantom{0}0.11$ & $19686.49$ & $\phantom{0}96.20$ \\ & \textbf{best} & $\phantom{0}0.06$ & $\phantom{0}0.00$ & $\phantom{0}0.04$ & $\phantom{0}0.05$ & $\phantom{0}0.11$ & $19686.49$ & \\ - & \textbf{worst} & $\phantom{0}0.06$ & $\phantom{0}0.00$ & $\phantom{0}0.04$ & $\phantom{0}0.05$ & $\phantom{0}0.11$ & $19686.50$ & \\ + & \textbf{worst} & $\phantom{0}0.06$ & $\phantom{0}0.00$ & $\phantom{0}0.04$ & $\phantom{0}0.05$ & $\phantom{0}0.11$ & $19686.50$ & \\\midrule \textbf{TuckER} & \textbf{pub} & $95.3\phantom{0}$ & $94.9\phantom{0}$ & $95.5\phantom{0}$ & & $95.8\phantom{0}$ & & \\ & \textbf{avg} & $\phantom{0}0.03$ & $\phantom{0}0.00$ & $\phantom{0}0.02$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $20622.46$ & $100.78$ \\ \bottomrule diff --git a/reproducibility/summaries/wn18rr_table.tex b/reproducibility/summaries/wn18rr_table.tex index 6cce260fb..9ed5c208a 100644 --- a/reproducibility/summaries/wn18rr_table.tex +++ b/reproducibility/summaries/wn18rr_table.tex @@ -4,11 +4,11 @@ \textbf{model} & {} & & & & & & & \\ \midrule \textbf{ConvE} & \textbf{pub} & $43.\phantom{00 \pm 0.00}$ & $\phantom{0}4.\phantom{00 \pm 0.00}$ & $44.\phantom{00 \pm 0.00}$ & & $52.\phantom{00 \pm 0.00}$ & $\phantom{0}4187.\phantom{00 \pm 000.00}$ & \\ - & \textbf{avg} & $44.69 \pm 0.21$ & $40.98 \pm 0.22$ & $46.49 \pm 0.14$ & $48.92 \pm 0.23$ & $51.76 \pm 0.13$ & $\phantom{0}5369.49 \pm \phantom{0}50.92$ & $26.49 \pm 0.25$ \\ + & \textbf{avg} & $44.69 \pm 0.21$ & $40.98 \pm 0.22$ & $46.49 \pm 0.14$ & $48.92 \pm 0.23$ & $51.76 \pm 0.13$ & $\phantom{0}5369.49 \pm \phantom{0}50.92$ & $26.49 \pm 0.25$ \\\midrule \textbf{ConvKB} & \textbf{pub} & $24.8\phantom{0 \pm 0.00}$ & & & & $52.5\phantom{0 \pm 0.00}$ & $\phantom{0}2554.\phantom{00 \pm 000.00}$ & \\ & \textbf{avg} & $\phantom{0}0.30 \pm 0.07$ & $\phantom{0}0.09 \pm 0.03$ & $\phantom{0}0.21 \pm 0.07$ & $\phantom{0}0.32 \pm 0.10$ & $\phantom{0}0.57 \pm 0.16$ & $13634.66 \pm 714.24$ & $67.27 \pm 3.52$ \\ & \textbf{best} & $\phantom{0}0.30 \pm 0.07$ & $\phantom{0}0.09 \pm 0.03$ & $\phantom{0}0.21 \pm 0.07$ & $\phantom{0}0.32 \pm 0.10$ & $\phantom{0}0.57 \pm 0.16$ & $13634.65 \pm 714.24$ & \\ - & \textbf{worst} & $\phantom{0}0.30 \pm 0.07$ & $\phantom{0}0.09 \pm 0.03$ & $\phantom{0}0.21 \pm 0.07$ & $\phantom{0}0.32 \pm 0.10$ & $\phantom{0}0.57 \pm 0.16$ & $13634.66 \pm 714.24$ & \\ + & \textbf{worst} & $\phantom{0}0.30 \pm 0.07$ & $\phantom{0}0.09 \pm 0.03$ & $\phantom{0}0.21 \pm 0.07$ & $\phantom{0}0.32 \pm 0.10$ & $\phantom{0}0.57 \pm 0.16$ & $13634.66 \pm 714.24$ & \\\midrule \textbf{RotatE} & \textbf{pub} & $47.6\phantom{0 \pm 0.00}$ & $42.8\phantom{0 \pm 0.00}$ & $49.2\phantom{0 \pm 0.00}$ & & $57.1\phantom{0 \pm 0.00}$ & $\phantom{0}3340.\phantom{00 \pm 000.00}$ & \\ & \textbf{avg} & $48.40 \pm 0.09$ & $44.02 \pm 0.15$ & $50.55 \pm 0.12$ & $52.98 \pm 0.11$ & $56.51 \pm 0.26$ & $\phantom{0}4263.32 \pm \phantom{0}90.33$ & $21.03 \pm 0.45$ \\ \bottomrule diff --git a/reproducibility/summaries/wn18rr_table_without_std.tex b/reproducibility/summaries/wn18rr_table_without_std.tex index ea176c2d7..7f15cabdd 100644 --- a/reproducibility/summaries/wn18rr_table_without_std.tex +++ b/reproducibility/summaries/wn18rr_table_without_std.tex @@ -4,11 +4,11 @@ \textbf{model} & {} & & & & & & & \\ \midrule \textbf{ConvE} & \textbf{pub} & $43.\phantom{00}$ & $\phantom{0}4.\phantom{00}$ & $44.\phantom{00}$ & & $52.\phantom{00}$ & $\phantom{0}4187.\phantom{00}$ & \\ - & \textbf{avg} & $44.69$ & $40.98$ & $46.49$ & $48.92$ & $51.76$ & $\phantom{0}5369.49$ & $26.49$ \\ + & \textbf{avg} & $44.69$ & $40.98$ & $46.49$ & $48.92$ & $51.76$ & $\phantom{0}5369.49$ & $26.49$ \\\midrule \textbf{ConvKB} & \textbf{pub} & $24.8\phantom{0}$ & & & & $52.5\phantom{0}$ & $\phantom{0}2554.\phantom{00}$ & \\ & \textbf{avg} & $\phantom{0}0.30$ & $\phantom{0}0.09$ & $\phantom{0}0.21$ & $\phantom{0}0.32$ & $\phantom{0}0.57$ & $13634.66$ & $67.27$ \\ & \textbf{best} & $\phantom{0}0.30$ & $\phantom{0}0.09$ & $\phantom{0}0.21$ & $\phantom{0}0.32$ & $\phantom{0}0.57$ & $13634.65$ & \\ - & \textbf{worst} & $\phantom{0}0.30$ & $\phantom{0}0.09$ & $\phantom{0}0.21$ & $\phantom{0}0.32$ & $\phantom{0}0.57$ & $13634.66$ & \\ + & \textbf{worst} & $\phantom{0}0.30$ & $\phantom{0}0.09$ & $\phantom{0}0.21$ & $\phantom{0}0.32$ & $\phantom{0}0.57$ & $13634.66$ & \\\midrule \textbf{RotatE} & \textbf{pub} & $47.6\phantom{0}$ & $42.8\phantom{0}$ & $49.2\phantom{0}$ & & $57.1\phantom{0}$ & $\phantom{0}3340.\phantom{00}$ & \\ & \textbf{avg} & $48.40$ & $44.02$ & $50.55$ & $52.98$ & $56.51$ & $\phantom{0}4263.32$ & $21.03$ \\ \bottomrule From c184d5ec8049e2004c845b3f06bfa43063c66364 Mon Sep 17 00:00:00 2001 From: Max Berrendorf Date: Mon, 22 Jun 2020 16:01:31 +0200 Subject: [PATCH 6/7] Rename index levels --- reproducibility/generate_summary_table.py | 8 +++ reproducibility/summaries/fb15k237_table.tex | 16 ++--- .../summaries/fb15k237_table_without_std.tex | 16 ++--- reproducibility/summaries/fb15k_table.tex | 64 ++++++++--------- .../summaries/fb15k_table_without_std.tex | 64 ++++++++--------- reproducibility/summaries/wn18_table.tex | 72 +++++++++---------- .../summaries/wn18_table_without_std.tex | 72 +++++++++---------- reproducibility/summaries/wn18rr_table.tex | 20 +++--- .../summaries/wn18rr_table_without_std.tex | 20 +++--- 9 files changed, 180 insertions(+), 172 deletions(-) diff --git a/reproducibility/generate_summary_table.py b/reproducibility/generate_summary_table.py index 51c61be82..4ab8d2bc5 100644 --- a/reproducibility/generate_summary_table.py +++ b/reproducibility/generate_summary_table.py @@ -81,6 +81,14 @@ def generate_results_table(with_std: bool, midrule_between_models: bool = True): wide_summary_df = reorganize_summary_df(all_results_tall_df, with_std=with_std) wide_summary_df = wide_summary_df.applymap(lambda v: ('$' + v + '$') if v != '' else '') + translation = dict( + pub='pub', + avg='R', + best='O', + worst='P', + ) + wide_summary_df = wide_summary_df.reindex(wide_summary_df.index.set_levels([translation[k] for k in wide_summary_df.index.levels[1]], level=1)) + # Save as Latex table pd.set_option('max_colwidth', 999) table_latex = wide_summary_df.to_latex( diff --git a/reproducibility/summaries/fb15k237_table.tex b/reproducibility/summaries/fb15k237_table.tex index aa724365d..f3f5a0ab9 100644 --- a/reproducibility/summaries/fb15k237_table.tex +++ b/reproducibility/summaries/fb15k237_table.tex @@ -1,14 +1,14 @@ \begin{tabular}{llrrrrrrr} \toprule - & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ -\textbf{model} & {} & & & & & & & \\ + & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ +\textbf{model} & {} & & & & & & & \\ \midrule -\textbf{ConvE} & \textbf{pub} & $32.5\phantom{0 \pm 0.00}$ & $23.7\phantom{0 \pm 0.00}$ & $35.6\phantom{0 \pm 0.00}$ & & $50.1\phantom{0 \pm 0.00}$ & $\phantom{0}244.\phantom{00 \pm 00.00}$ & \\ - & \textbf{avg} & $26.93 \pm 0.11$ & $18.22 \pm 0.11$ & $29.51 \pm 0.24$ & $35.98 \pm 0.16$ & $44.95 \pm 0.17$ & $\phantom{0}255.46 \pm \phantom{0}6.16$ & $\phantom{0}3.73 \pm 0.13$ \\\midrule -\textbf{ConvKB} & \textbf{pub} & $39.6\phantom{0 \pm 0.00}$ & & & & $51.7\phantom{0 \pm 0.00}$ & $\phantom{0}257.\phantom{00 \pm 00.00}$ & \\ - & \textbf{avg} & $\phantom{0}4.71 \pm 0.23$ & $\phantom{0}3.31 \pm 0.23$ & $\phantom{0}4.04 \pm 0.19$ & $\phantom{0}4.57 \pm 0.22$ & $\phantom{0}7.76 \pm 0.88$ & $4345.27 \pm 46.99$ & $61.36 \pm 0.65$ \\\midrule -\textbf{RotatE} & \textbf{pub} & $33.8\phantom{0 \pm 0.00}$ & $24.1\phantom{0 \pm 0.00}$ & $37.5\phantom{0 \pm 0.00}$ & & $53.3\phantom{0 \pm 0.00}$ & $\phantom{0}177.\phantom{00 \pm 00.00}$ & \\ - & \textbf{avg} & $26.42 \pm 0.04$ & $17.57 \pm 0.06$ & $28.97 \pm 0.05$ & $35.29 \pm 0.09$ & $44.55 \pm 0.06$ & $\phantom{0}191.92 \pm \phantom{0}0.31$ & $\phantom{0}2.84 \pm 0.00$ \\ +\textbf{ConvE} & \textbf{pub} & $32.5\phantom{0 \pm 0.00}$ & $23.7\phantom{0 \pm 0.00}$ & $35.6\phantom{0 \pm 0.00}$ & & $50.1\phantom{0 \pm 0.00}$ & $\phantom{0}244.\phantom{00 \pm 00.00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{ConvKB} & \textbf{pub} & $39.6\phantom{0 \pm 0.00}$ & & & & $51.7\phantom{0 \pm 0.00}$ & $\phantom{0}257.\phantom{00 \pm 00.00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{RotatE} & \textbf{pub} & $33.8\phantom{0 \pm 0.00}$ & $24.1\phantom{0 \pm 0.00}$ & $37.5\phantom{0 \pm 0.00}$ & & $53.3\phantom{0 \pm 0.00}$ & $\phantom{0}177.\phantom{00 \pm 00.00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ \bottomrule \end{tabular} diff --git a/reproducibility/summaries/fb15k237_table_without_std.tex b/reproducibility/summaries/fb15k237_table_without_std.tex index 62c86c767..2abdcef87 100644 --- a/reproducibility/summaries/fb15k237_table_without_std.tex +++ b/reproducibility/summaries/fb15k237_table_without_std.tex @@ -1,14 +1,14 @@ \begin{tabular}{llrrrrrrr} \toprule - & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ -\textbf{model} & {} & & & & & & & \\ + & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ +\textbf{model} & {} & & & & & & & \\ \midrule -\textbf{ConvE} & \textbf{pub} & $32.5\phantom{0}$ & $23.7\phantom{0}$ & $35.6\phantom{0}$ & & $50.1\phantom{0}$ & $\phantom{0}244.\phantom{00}$ & \\ - & \textbf{avg} & $26.93$ & $18.22$ & $29.51$ & $35.98$ & $44.95$ & $\phantom{0}255.46$ & $\phantom{0}3.73$ \\\midrule -\textbf{ConvKB} & \textbf{pub} & $39.6\phantom{0}$ & & & & $51.7\phantom{0}$ & $\phantom{0}257.\phantom{00}$ & \\ - & \textbf{avg} & $\phantom{0}4.71$ & $\phantom{0}3.31$ & $\phantom{0}4.04$ & $\phantom{0}4.57$ & $\phantom{0}7.76$ & $4345.27$ & $61.36$ \\\midrule -\textbf{RotatE} & \textbf{pub} & $33.8\phantom{0}$ & $24.1\phantom{0}$ & $37.5\phantom{0}$ & & $53.3\phantom{0}$ & $\phantom{0}177.\phantom{00}$ & \\ - & \textbf{avg} & $26.42$ & $17.57$ & $28.97$ & $35.29$ & $44.55$ & $\phantom{0}191.92$ & $\phantom{0}2.84$ \\ +\textbf{ConvE} & \textbf{pub} & $32.5\phantom{0}$ & $23.7\phantom{0}$ & $35.6\phantom{0}$ & & $50.1\phantom{0}$ & $\phantom{0}244.\phantom{00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{ConvKB} & \textbf{pub} & $39.6\phantom{0}$ & & & & $51.7\phantom{0}$ & $\phantom{0}257.\phantom{00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{RotatE} & \textbf{pub} & $33.8\phantom{0}$ & $24.1\phantom{0}$ & $37.5\phantom{0}$ & & $53.3\phantom{0}$ & $\phantom{0}177.\phantom{00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ \bottomrule \end{tabular} diff --git a/reproducibility/summaries/fb15k_table.tex b/reproducibility/summaries/fb15k_table.tex index 54b96f9ae..a7c61d6ca 100644 --- a/reproducibility/summaries/fb15k_table.tex +++ b/reproducibility/summaries/fb15k_table.tex @@ -1,38 +1,38 @@ \begin{tabular}{llrrrrrrr} \toprule - & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ -\textbf{model} & {} & & & & & & & \\ + & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ +\textbf{model} & {} & & & & & & & \\ \midrule -\textbf{ComplEx} & \textbf{pub} & $69.2\phantom{0 \pm 0.00}$ & $59.9\phantom{0 \pm 0.00}$ & $75.9\phantom{0 \pm 00.00}$ & & $84.\phantom{00 \pm 00.00}$ & & \\ - & \textbf{avg} & $19.13 \pm 0.45$ & $10.06 \pm 0.42$ & $20.82 \pm \phantom{0}0.67$ & $27.59 \pm \phantom{0}0.94$ & $38.03 \pm \phantom{0}1.02$ & $\phantom{00}171.16 \pm 16.05$ & $\phantom{00}2.46 \pm 0.28$ \\\midrule -\textbf{ConvE} & \textbf{pub} & $65.7\phantom{0 \pm 0.00}$ & $55.8\phantom{0 \pm 0.00}$ & $72.3\phantom{0 \pm 00.00}$ & & $83.1\phantom{0 \pm 00.00}$ & $\phantom{000}51.\phantom{00 \pm 00.00}$ & \\ - & \textbf{avg} & $59.56 \pm 0.06$ & $48.28 \pm 0.12$ & $66.99 \pm \phantom{0}0.04$ & $73.27 \pm \phantom{0}0.03$ & $79.76 \pm \phantom{0}0.07$ & $\phantom{000}50.76 \pm \phantom{0}0.40$ & $\phantom{00}0.73 \pm 0.01$ \\\midrule -\textbf{DistMult} & \textbf{pub} & $35.\phantom{00 \pm 0.00}$ & & & & $57.7\phantom{0 \pm 00.00}$ & & \\ - & \textbf{avg} & $26.06 \pm 0.17$ & $16.45 \pm 0.16$ & $29.10 \pm \phantom{0}0.17$ & $35.54 \pm \phantom{0}0.21$ & $45.00 \pm \phantom{0}0.25$ & $\phantom{00}134.02 \pm \phantom{0}1.98$ & $\phantom{00}1.86 \pm 0.03$ \\\midrule -\textbf{HolE} & \textbf{pub} & $52.4\phantom{0 \pm 0.00}$ & $40.2\phantom{0 \pm 0.00}$ & $61.3\phantom{0 \pm 00.00}$ & & $73.9\phantom{0 \pm 00.00}$ & & \\ - & \textbf{avg} & $34.15 \pm 0.22$ & $21.79 \pm 0.19$ & $39.69 \pm \phantom{0}0.24$ & $48.06 \pm \phantom{0}0.30$ & $58.84 \pm \phantom{0}0.28$ & $\phantom{00}193.03 \pm \phantom{0}7.61$ & $\phantom{00}2.71 \pm 0.12$ \\\midrule -\textbf{KG2E} & \textbf{pub} & & & & & $71.5\phantom{0 \pm 00.00}$ & $\phantom{000}59.\phantom{00 \pm 00.00}$ & \\ - & \textbf{avg} & $\phantom{0}0.58 \pm 0.07$ & $\phantom{0}0.11 \pm 0.04$ & $\phantom{0}0.36 \pm \phantom{0}0.08$ & $\phantom{0}0.56 \pm \phantom{0}0.10$ & $\phantom{0}1.01 \pm \phantom{0}0.14$ & $\phantom{0}5779.07 \pm 51.02$ & $\phantom{0}78.40 \pm 0.68$ \\\midrule -\textbf{RotatE} & \textbf{pub} & $79.7\phantom{0 \pm 0.00}$ & $74.6\phantom{0 \pm 0.00}$ & $83.\phantom{00 \pm 00.00}$ & & $88.4\phantom{0 \pm 00.00}$ & $\phantom{000}40.\phantom{00 \pm 00.00}$ & \\ - & \textbf{avg} & $55.00 \pm 0.06$ & $41.53 \pm 0.06$ & $64.14 \pm \phantom{0}0.07$ & $71.23 \pm \phantom{0}0.05$ & $78.67 \pm \phantom{0}0.08$ & $\phantom{000}42.28 \pm \phantom{0}0.13$ & $\phantom{00}0.63 \pm 0.00$ \\\midrule -\textbf{SimplE} & \textbf{pub} & $72.7\phantom{0 \pm 0.00}$ & $66.\phantom{00 \pm 0.00}$ & $77.3\phantom{0 \pm 00.00}$ & & $83.8\phantom{0 \pm 00.00}$ & & \\ - & \textbf{avg} & $\phantom{0}0.04 \pm 0.00$ & $\phantom{0}0.01 \pm 0.00$ & $\phantom{0}0.03 \pm \phantom{0}0.00$ & $\phantom{0}0.04 \pm \phantom{0}0.00$ & $\phantom{0}0.06 \pm \phantom{0}0.01$ & $\phantom{0}7395.75 \pm \phantom{0}2.02$ & $100.02 \pm 0.03$ \\ - & \textbf{best} & $23.90 \pm 8.79$ & $11.58 \pm 6.42$ & $24.16 \pm 10.95$ & $34.73 \pm 13.40$ & $54.28 \pm 15.80$ & $\phantom{00}139.34 \pm 49.45$ & \\ - & \textbf{worst} & $\phantom{0}0.03 \pm 0.00$ & $\phantom{0}0.01 \pm 0.00$ & $\phantom{0}0.03 \pm \phantom{0}0.00$ & $\phantom{0}0.04 \pm \phantom{0}0.00$ & $\phantom{0}0.06 \pm \phantom{0}0.01$ & $14652.16 \pm 45.71$ & \\\midrule -\textbf{TransD} & \textbf{pub} & & & & & $77.3\phantom{0 \pm 00.00}$ & $\phantom{000}91.\phantom{00 \pm 00.00}$ & \\ - & \textbf{avg} & $33.99 \pm 0.03$ & $21.22 \pm 0.03$ & $40.48 \pm \phantom{0}0.10$ & $48.57 \pm \phantom{0}0.09$ & $58.71 \pm \phantom{0}0.14$ & $\phantom{00}153.37 \pm \phantom{0}5.35$ & $\phantom{00}2.29 \pm 0.09$ \\\midrule -\textbf{TransE} & \textbf{pub} & & & & & $47.1\phantom{0 \pm 00.00}$ & $\phantom{00}125.\phantom{00 \pm 00.00}$ & \\ - & \textbf{avg} & $26.01 \pm 0.17$ & $15.23 \pm 0.16$ & $29.85 \pm \phantom{0}0.24$ & $37.18 \pm \phantom{0}0.24$ & $47.34 \pm \phantom{0}0.18$ & $\phantom{00}127.92 \pm \phantom{0}0.86$ & $\phantom{00}1.78 \pm 0.01$ \\\midrule -\textbf{TransH} & \textbf{pub} & & & & & $64.4\phantom{0 \pm 00.00}$ & $\phantom{000}87.\phantom{00 \pm 00.00}$ & \\ - & \textbf{avg} & $\phantom{0}2.54 \pm 0.20$ & $\phantom{0}1.69 \pm 0.25$ & $\phantom{0}2.95 \pm \phantom{0}0.20$ & $\phantom{0}3.29 \pm \phantom{0}0.22$ & $\phantom{0}3.74 \pm \phantom{0}0.18$ & $\phantom{0}6320.02 \pm 30.37$ & $\phantom{0}85.63 \pm 0.40$ \\ - & \textbf{best} & $\phantom{0}2.54 \pm 0.20$ & $\phantom{0}1.69 \pm 0.25$ & $\phantom{0}2.95 \pm \phantom{0}0.20$ & $\phantom{0}3.29 \pm \phantom{0}0.22$ & $\phantom{0}3.74 \pm \phantom{0}0.18$ & $\phantom{0}6320.00 \pm 30.37$ & \\ - & \textbf{worst} & $\phantom{0}2.54 \pm 0.20$ & $\phantom{0}1.69 \pm 0.25$ & $\phantom{0}2.95 \pm \phantom{0}0.20$ & $\phantom{0}3.29 \pm \phantom{0}0.22$ & $\phantom{0}3.74 \pm \phantom{0}0.18$ & $\phantom{0}6320.05 \pm 30.37$ & \\\midrule -\textbf{TransR} & \textbf{pub} & & & & & $68.7\phantom{0 \pm 00.00}$ & $\phantom{000}77.\phantom{00 \pm 00.00}$ & \\ - & \textbf{avg} & $\phantom{0}0.65 \pm 0.02$ & $\phantom{0}0.37 \pm 0.00$ & $\phantom{0}0.63 \pm \phantom{0}0.04$ & $\phantom{0}0.78 \pm \phantom{0}0.06$ & $\phantom{0}1.03 \pm \phantom{0}0.07$ & $\phantom{0}6795.95 \pm 16.65$ & $\phantom{0}91.99 \pm 0.22$ \\ - & \textbf{best} & $\phantom{0}0.65 \pm 0.02$ & $\phantom{0}0.37 \pm 0.00$ & $\phantom{0}0.63 \pm \phantom{0}0.04$ & $\phantom{0}0.78 \pm \phantom{0}0.06$ & $\phantom{0}1.03 \pm \phantom{0}0.07$ & $\phantom{0}6795.94 \pm 16.65$ & \\ - & \textbf{worst} & $\phantom{0}0.65 \pm 0.02$ & $\phantom{0}0.37 \pm 0.00$ & $\phantom{0}0.63 \pm \phantom{0}0.04$ & $\phantom{0}0.78 \pm \phantom{0}0.06$ & $\phantom{0}1.03 \pm \phantom{0}0.07$ & $\phantom{0}6795.95 \pm 16.65$ & \\\midrule -\textbf{TuckER} & \textbf{pub} & $79.5\phantom{0 \pm 0.00}$ & $74.1\phantom{0 \pm 0.00}$ & $83.3\phantom{0 \pm 00.00}$ & & $89.2\phantom{0 \pm 00.00}$ & & \\ - & \textbf{avg} & $\phantom{0}0.07 \pm 0.02$ & $\phantom{0}0.01 \pm 0.00$ & $\phantom{0}0.02 \pm \phantom{0}0.00$ & $\phantom{0}0.03 \pm \phantom{0}0.01$ & $\phantom{0}0.15 \pm \phantom{0}0.17$ & $\phantom{0}7327.77 \pm 29.22$ & $\phantom{0}99.11 \pm 0.39$ \\ +\textbf{ComplEx} & \textbf{pub} & $69.2\phantom{0 \pm 0.00}$ & $59.9\phantom{0 \pm 0.00}$ & $75.9\phantom{0 \pm 00.00}$ & & $84.\phantom{00 \pm 00.00}$ & & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{ConvE} & \textbf{pub} & $65.7\phantom{0 \pm 0.00}$ & $55.8\phantom{0 \pm 0.00}$ & $72.3\phantom{0 \pm 00.00}$ & & $83.1\phantom{0 \pm 00.00}$ & $\phantom{000}51.\phantom{00 \pm 00.00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{DistMult} & \textbf{pub} & $35.\phantom{00 \pm 0.00}$ & & & & $57.7\phantom{0 \pm 00.00}$ & & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{HolE} & \textbf{pub} & $52.4\phantom{0 \pm 0.00}$ & $40.2\phantom{0 \pm 0.00}$ & $61.3\phantom{0 \pm 00.00}$ & & $73.9\phantom{0 \pm 00.00}$ & & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{KG2E} & \textbf{pub} & & & & & $71.5\phantom{0 \pm 00.00}$ & $\phantom{000}59.\phantom{00 \pm 00.00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{RotatE} & \textbf{pub} & $79.7\phantom{0 \pm 0.00}$ & $74.6\phantom{0 \pm 0.00}$ & $83.\phantom{00 \pm 00.00}$ & & $88.4\phantom{0 \pm 00.00}$ & $\phantom{000}40.\phantom{00 \pm 00.00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{SimplE} & \textbf{pub} & $72.7\phantom{0 \pm 0.00}$ & $66.\phantom{00 \pm 0.00}$ & $77.3\phantom{0 \pm 00.00}$ & & $83.8\phantom{0 \pm 00.00}$ & & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{TransD} & \textbf{pub} & & & & & $77.3\phantom{0 \pm 00.00}$ & $\phantom{000}91.\phantom{00 \pm 00.00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{TransE} & \textbf{pub} & & & & & $47.1\phantom{0 \pm 00.00}$ & $\phantom{00}125.\phantom{00 \pm 00.00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{TransH} & \textbf{pub} & & & & & $64.4\phantom{0 \pm 00.00}$ & $\phantom{000}87.\phantom{00 \pm 00.00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{TransR} & \textbf{pub} & & & & & $68.7\phantom{0 \pm 00.00}$ & $\phantom{000}77.\phantom{00 \pm 00.00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{TuckER} & \textbf{pub} & $79.5\phantom{0 \pm 0.00}$ & $74.1\phantom{0 \pm 0.00}$ & $83.3\phantom{0 \pm 00.00}$ & & $89.2\phantom{0 \pm 00.00}$ & & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ \bottomrule \end{tabular} diff --git a/reproducibility/summaries/fb15k_table_without_std.tex b/reproducibility/summaries/fb15k_table_without_std.tex index b9e90ad79..1aee51964 100644 --- a/reproducibility/summaries/fb15k_table_without_std.tex +++ b/reproducibility/summaries/fb15k_table_without_std.tex @@ -1,38 +1,38 @@ \begin{tabular}{llrrrrrrr} \toprule - & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ -\textbf{model} & {} & & & & & & & \\ + & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ +\textbf{model} & {} & & & & & & & \\ \midrule -\textbf{ComplEx} & \textbf{pub} & $69.2\phantom{0}$ & $59.9\phantom{0}$ & $75.9\phantom{0}$ & & $84.\phantom{00}$ & & \\ - & \textbf{avg} & $19.13$ & $10.06$ & $20.82$ & $27.59$ & $38.03$ & $\phantom{00}171.16$ & $\phantom{00}2.46$ \\\midrule -\textbf{ConvE} & \textbf{pub} & $65.7\phantom{0}$ & $55.8\phantom{0}$ & $72.3\phantom{0}$ & & $83.1\phantom{0}$ & $\phantom{000}51.\phantom{00}$ & \\ - & \textbf{avg} & $59.56$ & $48.28$ & $66.99$ & $73.27$ & $79.76$ & $\phantom{000}50.76$ & $\phantom{00}0.73$ \\\midrule -\textbf{DistMult} & \textbf{pub} & $35.\phantom{00}$ & & & & $57.7\phantom{0}$ & & \\ - & \textbf{avg} & $26.06$ & $16.45$ & $29.10$ & $35.54$ & $45.00$ & $\phantom{00}134.02$ & $\phantom{00}1.86$ \\\midrule -\textbf{HolE} & \textbf{pub} & $52.4\phantom{0}$ & $40.2\phantom{0}$ & $61.3\phantom{0}$ & & $73.9\phantom{0}$ & & \\ - & \textbf{avg} & $34.15$ & $21.79$ & $39.69$ & $48.06$ & $58.84$ & $\phantom{00}193.03$ & $\phantom{00}2.71$ \\\midrule -\textbf{KG2E} & \textbf{pub} & & & & & $71.5\phantom{0}$ & $\phantom{000}59.\phantom{00}$ & \\ - & \textbf{avg} & $\phantom{0}0.58$ & $\phantom{0}0.11$ & $\phantom{0}0.36$ & $\phantom{0}0.56$ & $\phantom{0}1.01$ & $\phantom{0}5779.07$ & $\phantom{0}78.40$ \\\midrule -\textbf{RotatE} & \textbf{pub} & $79.7\phantom{0}$ & $74.6\phantom{0}$ & $83.\phantom{00}$ & & $88.4\phantom{0}$ & $\phantom{000}40.\phantom{00}$ & \\ - & \textbf{avg} & $55.00$ & $41.53$ & $64.14$ & $71.23$ & $78.67$ & $\phantom{000}42.28$ & $\phantom{00}0.63$ \\\midrule -\textbf{SimplE} & \textbf{pub} & $72.7\phantom{0}$ & $66.\phantom{00}$ & $77.3\phantom{0}$ & & $83.8\phantom{0}$ & & \\ - & \textbf{avg} & $\phantom{0}0.04$ & $\phantom{0}0.01$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $\phantom{0}0.06$ & $\phantom{0}7395.75$ & $100.02$ \\ - & \textbf{best} & $23.90$ & $11.58$ & $24.16$ & $34.73$ & $54.28$ & $\phantom{00}139.34$ & \\ - & \textbf{worst} & $\phantom{0}0.03$ & $\phantom{0}0.01$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $\phantom{0}0.06$ & $14652.16$ & \\\midrule -\textbf{TransD} & \textbf{pub} & & & & & $77.3\phantom{0}$ & $\phantom{000}91.\phantom{00}$ & \\ - & \textbf{avg} & $33.99$ & $21.22$ & $40.48$ & $48.57$ & $58.71$ & $\phantom{00}153.37$ & $\phantom{00}2.29$ \\\midrule -\textbf{TransE} & \textbf{pub} & & & & & $47.1\phantom{0}$ & $\phantom{00}125.\phantom{00}$ & \\ - & \textbf{avg} & $26.01$ & $15.23$ & $29.85$ & $37.18$ & $47.34$ & $\phantom{00}127.92$ & $\phantom{00}1.78$ \\\midrule -\textbf{TransH} & \textbf{pub} & & & & & $64.4\phantom{0}$ & $\phantom{000}87.\phantom{00}$ & \\ - & \textbf{avg} & $\phantom{0}2.54$ & $\phantom{0}1.69$ & $\phantom{0}2.95$ & $\phantom{0}3.29$ & $\phantom{0}3.74$ & $\phantom{0}6320.02$ & $\phantom{0}85.63$ \\ - & \textbf{best} & $\phantom{0}2.54$ & $\phantom{0}1.69$ & $\phantom{0}2.95$ & $\phantom{0}3.29$ & $\phantom{0}3.74$ & $\phantom{0}6320.00$ & \\ - & \textbf{worst} & $\phantom{0}2.54$ & $\phantom{0}1.69$ & $\phantom{0}2.95$ & $\phantom{0}3.29$ & $\phantom{0}3.74$ & $\phantom{0}6320.05$ & \\\midrule -\textbf{TransR} & \textbf{pub} & & & & & $68.7\phantom{0}$ & $\phantom{000}77.\phantom{00}$ & \\ - & \textbf{avg} & $\phantom{0}0.65$ & $\phantom{0}0.37$ & $\phantom{0}0.63$ & $\phantom{0}0.78$ & $\phantom{0}1.03$ & $\phantom{0}6795.95$ & $\phantom{0}91.99$ \\ - & \textbf{best} & $\phantom{0}0.65$ & $\phantom{0}0.37$ & $\phantom{0}0.63$ & $\phantom{0}0.78$ & $\phantom{0}1.03$ & $\phantom{0}6795.94$ & \\ - & \textbf{worst} & $\phantom{0}0.65$ & $\phantom{0}0.37$ & $\phantom{0}0.63$ & $\phantom{0}0.78$ & $\phantom{0}1.03$ & $\phantom{0}6795.95$ & \\\midrule -\textbf{TuckER} & \textbf{pub} & $79.5\phantom{0}$ & $74.1\phantom{0}$ & $83.3\phantom{0}$ & & $89.2\phantom{0}$ & & \\ - & \textbf{avg} & $\phantom{0}0.07$ & $\phantom{0}0.01$ & $\phantom{0}0.02$ & $\phantom{0}0.03$ & $\phantom{0}0.15$ & $\phantom{0}7327.77$ & $\phantom{0}99.11$ \\ +\textbf{ComplEx} & \textbf{pub} & $69.2\phantom{0}$ & $59.9\phantom{0}$ & $75.9\phantom{0}$ & & $84.\phantom{00}$ & & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{ConvE} & \textbf{pub} & $65.7\phantom{0}$ & $55.8\phantom{0}$ & $72.3\phantom{0}$ & & $83.1\phantom{0}$ & $\phantom{000}51.\phantom{00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{DistMult} & \textbf{pub} & $35.\phantom{00}$ & & & & $57.7\phantom{0}$ & & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{HolE} & \textbf{pub} & $52.4\phantom{0}$ & $40.2\phantom{0}$ & $61.3\phantom{0}$ & & $73.9\phantom{0}$ & & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{KG2E} & \textbf{pub} & & & & & $71.5\phantom{0}$ & $\phantom{000}59.\phantom{00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{RotatE} & \textbf{pub} & $79.7\phantom{0}$ & $74.6\phantom{0}$ & $83.\phantom{00}$ & & $88.4\phantom{0}$ & $\phantom{000}40.\phantom{00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{SimplE} & \textbf{pub} & $72.7\phantom{0}$ & $66.\phantom{00}$ & $77.3\phantom{0}$ & & $83.8\phantom{0}$ & & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{TransD} & \textbf{pub} & & & & & $77.3\phantom{0}$ & $\phantom{000}91.\phantom{00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{TransE} & \textbf{pub} & & & & & $47.1\phantom{0}$ & $\phantom{00}125.\phantom{00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{TransH} & \textbf{pub} & & & & & $64.4\phantom{0}$ & $\phantom{000}87.\phantom{00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{TransR} & \textbf{pub} & & & & & $68.7\phantom{0}$ & $\phantom{000}77.\phantom{00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{TuckER} & \textbf{pub} & $79.5\phantom{0}$ & $74.1\phantom{0}$ & $83.3\phantom{0}$ & & $89.2\phantom{0}$ & & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ \bottomrule \end{tabular} diff --git a/reproducibility/summaries/wn18_table.tex b/reproducibility/summaries/wn18_table.tex index 5f8255a58..6c9890da0 100644 --- a/reproducibility/summaries/wn18_table.tex +++ b/reproducibility/summaries/wn18_table.tex @@ -1,42 +1,42 @@ \begin{tabular}{llrrrrrrr} \toprule - & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ -\textbf{model} & {} & & & & & & & \\ + & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ +\textbf{model} & {} & & & & & & & \\ \midrule -\textbf{ComplEx} & \textbf{pub} & $94.1\phantom{0 \pm 0.00}$ & $93.6\phantom{0 \pm 0.00}$ & $94.5\phantom{0 \pm 0.00}$ & & $94.7\phantom{0 \pm 0.00}$ & & \\ - & \textbf{avg} & $19.49 \pm 2.55$ & $12.36 \pm 1.96$ & $20.66 \pm 2.75$ & $25.24 \pm 3.33$ & $32.92 \pm 4.40$ & $\phantom{00}452.67 \pm \phantom{0}63.05$ & $\phantom{00}2.21 \pm 0.31$ \\\midrule -\textbf{ConvE} & \textbf{pub} & $94.3\phantom{0 \pm 0.00}$ & $93.5\phantom{0 \pm 0.00}$ & $94.6\phantom{0 \pm 0.00}$ & & $95.6\phantom{0 \pm 0.00}$ & $\phantom{00}374.\phantom{00 \pm 000.00}$ & \\ - & \textbf{avg} & $88.81 \pm 0.09$ & $85.14 \pm 0.10$ & $91.76 \pm 0.11$ & $93.29 \pm 0.04$ & $94.85 \pm 0.06$ & $\phantom{00}444.40 \pm \phantom{0}14.82$ & $\phantom{00}2.17 \pm 0.07$ \\\midrule -\textbf{DistMult} & \textbf{pub} & $83.\phantom{00 \pm 0.00}$ & & & & $94.2\phantom{0 \pm 0.00}$ & & \\ - & \textbf{avg} & $77.44 \pm 0.22$ & $67.45 \pm 0.34$ & $85.94 \pm 0.21$ & $89.52 \pm 0.25$ & $92.72 \pm 0.18$ & $\phantom{00}458.64 \pm \phantom{0}23.96$ & $\phantom{00}2.24 \pm 0.12$ \\\midrule -\textbf{HolE} & \textbf{pub} & $93.8\phantom{0 \pm 0.00}$ & $93.\phantom{00 \pm 0.00}$ & $94.5\phantom{0 \pm 0.00}$ & & $94.9\phantom{0 \pm 0.00}$ & & \\ - & \textbf{avg} & $70.44 \pm 0.45$ & $59.29 \pm 0.53$ & $79.29 \pm 0.47$ & $84.12 \pm 0.36$ & $88.61 \pm 0.42$ & $\phantom{00}812.64 \pm \phantom{0}28.33$ & $\phantom{00}3.97 \pm 0.14$ \\ - & \textbf{best} & $70.44 \pm 0.45$ & $59.29 \pm 0.53$ & $79.29 \pm 0.47$ & $84.12 \pm 0.36$ & $88.61 \pm 0.42$ & $\phantom{00}812.63 \pm \phantom{0}28.33$ & \\ - & \textbf{worst} & $70.44 \pm 0.45$ & $59.29 \pm 0.53$ & $79.29 \pm 0.47$ & $84.12 \pm 0.36$ & $88.61 \pm 0.42$ & $\phantom{00}812.64 \pm \phantom{0}28.34$ & \\\midrule -\textbf{KG2E} & \textbf{pub} & & & & & $92.8\phantom{0 \pm 0.00}$ & $\phantom{00}331.\phantom{00 \pm 000.00}$ & \\ - & \textbf{avg} & $\phantom{0}3.61 \pm 0.26$ & $\phantom{0}1.35 \pm 0.22$ & $\phantom{0}3.21 \pm 0.31$ & $\phantom{0}4.57 \pm 0.34$ & $\phantom{0}7.02 \pm 0.43$ & $\phantom{0}2708.89 \pm \phantom{0}44.57$ & $\phantom{0}13.25 \pm 0.22$ \\ - & \textbf{best} & $\phantom{0}3.61 \pm 0.26$ & $\phantom{0}1.35 \pm 0.22$ & $\phantom{0}3.21 \pm 0.31$ & $\phantom{0}4.57 \pm 0.34$ & $\phantom{0}7.02 \pm 0.43$ & $\phantom{0}2708.88 \pm \phantom{0}44.57$ & \\ - & \textbf{worst} & $\phantom{0}3.61 \pm 0.26$ & $\phantom{0}1.35 \pm 0.22$ & $\phantom{0}3.21 \pm 0.31$ & $\phantom{0}4.57 \pm 0.34$ & $\phantom{0}7.02 \pm 0.43$ & $\phantom{0}2708.89 \pm \phantom{0}44.57$ & \\\midrule -\textbf{RotatE} & \textbf{pub} & $94.9\phantom{0 \pm 0.00}$ & $94.4\phantom{0 \pm 0.00}$ & $95.2\phantom{0 \pm 0.00}$ & & $95.9\phantom{0 \pm 0.00}$ & $\phantom{00}309.\phantom{00 \pm 000.00}$ & \\ - & \textbf{avg} & $87.29 \pm 0.12$ & $82.17 \pm 0.20$ & $91.53 \pm 0.12$ & $93.44 \pm 0.07$ & $95.28 \pm 0.08$ & $\phantom{00}123.68 \pm \phantom{00}1.71$ & $\phantom{00}0.61 \pm 0.01$ \\\midrule -\textbf{SimplE} & \textbf{pub} & $94.2\phantom{0 \pm 0.00}$ & $93.9\phantom{0 \pm 0.00}$ & $94.4\phantom{0 \pm 0.00}$ & & $94.7\phantom{0 \pm 0.00}$ & & \\ - & \textbf{avg} & $\phantom{0}0.04 \pm 0.01$ & $\phantom{0}0.01 \pm 0.01$ & $\phantom{0}0.03 \pm 0.00$ & $\phantom{0}0.04 \pm 0.01$ & $\phantom{0}0.07 \pm 0.03$ & $20376.43 \pm \phantom{0}42.30$ & $\phantom{0}99.57 \pm 0.21$ \\ - & \textbf{best} & $38.48 \pm 4.00$ & $33.93 \pm 4.32$ & $39.59 \pm 4.67$ & $42.76 \pm 3.73$ & $47.01 \pm 2.66$ & $\phantom{00}384.53 \pm \phantom{0}66.45$ & \\ - & \textbf{worst} & $\phantom{0}0.03 \pm 0.01$ & $\phantom{0}0.01 \pm 0.01$ & $\phantom{0}0.03 \pm 0.00$ & $\phantom{0}0.04 \pm 0.01$ & $\phantom{0}0.07 \pm 0.03$ & $40368.33 \pm 114.95$ & \\\midrule -\textbf{TransD} & \textbf{pub} & & & & & $92.2\phantom{0 \pm 0.00}$ & $\phantom{00}212.\phantom{00 \pm 000.00}$ & \\ - & \textbf{avg} & $36.22 \pm 0.12$ & $\phantom{0}3.94 \pm 0.27$ & $65.63 \pm 0.55$ & $79.64 \pm 0.43$ & $87.27 \pm 0.41$ & $\phantom{00}444.39 \pm \phantom{0}25.61$ & $\phantom{00}2.17 \pm 0.13$ \\\midrule -\textbf{TransE} & \textbf{pub} & & & & & $89.2\phantom{0 \pm 0.00}$ & $\phantom{00}251.\phantom{00 \pm 000.00}$ & \\ - & \textbf{avg} & $39.19 \pm 1.21$ & $\phantom{0}9.99 \pm 1.82$ & $64.74 \pm 0.91$ & $75.44 \pm 0.48$ & $84.25 \pm 0.33$ & $\phantom{00}468.24 \pm \phantom{0}13.64$ & $\phantom{00}2.29 \pm 0.07$ \\\midrule -\textbf{TransH} & \textbf{pub} & & & & & $82.3\phantom{0 \pm 0.00}$ & $\phantom{00}388.\phantom{00 \pm 000.00}$ & \\ - & \textbf{avg} & $\phantom{0}0.18 \pm 0.04$ & $\phantom{0}0.04 \pm 0.02$ & $\phantom{0}0.19 \pm 0.10$ & $\phantom{0}0.29 \pm 0.14$ & $\phantom{0}0.39 \pm 0.11$ & $19678.04 \pm \phantom{0}18.92$ & $\phantom{0}96.16 \pm 0.09$ \\ - & \textbf{best} & $\phantom{0}0.18 \pm 0.04$ & $\phantom{0}0.04 \pm 0.02$ & $\phantom{0}0.19 \pm 0.10$ & $\phantom{0}0.29 \pm 0.14$ & $\phantom{0}0.39 \pm 0.11$ & $19678.02 \pm \phantom{0}18.92$ & \\ - & \textbf{worst} & $\phantom{0}0.18 \pm 0.04$ & $\phantom{0}0.04 \pm 0.02$ & $\phantom{0}0.19 \pm 0.10$ & $\phantom{0}0.29 \pm 0.14$ & $\phantom{0}0.39 \pm 0.11$ & $19678.06 \pm \phantom{0}18.92$ & \\\midrule -\textbf{TransR} & \textbf{pub} & & & & & $92.\phantom{00 \pm 0.00}$ & $\phantom{00}225.\phantom{00 \pm 000.00}$ & \\ - & \textbf{avg} & $\phantom{0}0.06 \pm 0.02$ & $\phantom{0}0.00 \pm 0.00$ & $\phantom{0}0.04 \pm 0.02$ & $\phantom{0}0.05 \pm 0.03$ & $\phantom{0}0.11 \pm 0.06$ & $19686.49 \pm 100.97$ & $\phantom{0}96.20 \pm 0.49$ \\ - & \textbf{best} & $\phantom{0}0.06 \pm 0.02$ & $\phantom{0}0.00 \pm 0.00$ & $\phantom{0}0.04 \pm 0.02$ & $\phantom{0}0.05 \pm 0.03$ & $\phantom{0}0.11 \pm 0.06$ & $19686.49 \pm 100.97$ & \\ - & \textbf{worst} & $\phantom{0}0.06 \pm 0.02$ & $\phantom{0}0.00 \pm 0.00$ & $\phantom{0}0.04 \pm 0.02$ & $\phantom{0}0.05 \pm 0.03$ & $\phantom{0}0.11 \pm 0.06$ & $19686.50 \pm 100.97$ & \\\midrule -\textbf{TuckER} & \textbf{pub} & $95.3\phantom{0 \pm 0.00}$ & $94.9\phantom{0 \pm 0.00}$ & $95.5\phantom{0 \pm 0.00}$ & & $95.8\phantom{0 \pm 0.00}$ & & \\ - & \textbf{avg} & $\phantom{0}0.03 \pm 0.01$ & $\phantom{0}0.00 \pm 0.00$ & $\phantom{0}0.02 \pm 0.01$ & $\phantom{0}0.03 \pm 0.03$ & $\phantom{0}0.04 \pm 0.03$ & $20622.46 \pm 153.52$ & $100.78 \pm 0.75$ \\ +\textbf{ComplEx} & \textbf{pub} & $94.1\phantom{0 \pm 0.00}$ & $93.6\phantom{0 \pm 0.00}$ & $94.5\phantom{0 \pm 0.00}$ & & $94.7\phantom{0 \pm 0.00}$ & & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{ConvE} & \textbf{pub} & $94.3\phantom{0 \pm 0.00}$ & $93.5\phantom{0 \pm 0.00}$ & $94.6\phantom{0 \pm 0.00}$ & & $95.6\phantom{0 \pm 0.00}$ & $\phantom{00}374.\phantom{00 \pm 000.00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{DistMult} & \textbf{pub} & $83.\phantom{00 \pm 0.00}$ & & & & $94.2\phantom{0 \pm 0.00}$ & & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{HolE} & \textbf{pub} & $93.8\phantom{0 \pm 0.00}$ & $93.\phantom{00 \pm 0.00}$ & $94.5\phantom{0 \pm 0.00}$ & & $94.9\phantom{0 \pm 0.00}$ & & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{KG2E} & \textbf{pub} & & & & & $92.8\phantom{0 \pm 0.00}$ & $\phantom{00}331.\phantom{00 \pm 000.00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{RotatE} & \textbf{pub} & $94.9\phantom{0 \pm 0.00}$ & $94.4\phantom{0 \pm 0.00}$ & $95.2\phantom{0 \pm 0.00}$ & & $95.9\phantom{0 \pm 0.00}$ & $\phantom{00}309.\phantom{00 \pm 000.00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{SimplE} & \textbf{pub} & $94.2\phantom{0 \pm 0.00}$ & $93.9\phantom{0 \pm 0.00}$ & $94.4\phantom{0 \pm 0.00}$ & & $94.7\phantom{0 \pm 0.00}$ & & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{TransD} & \textbf{pub} & & & & & $92.2\phantom{0 \pm 0.00}$ & $\phantom{00}212.\phantom{00 \pm 000.00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{TransE} & \textbf{pub} & & & & & $89.2\phantom{0 \pm 0.00}$ & $\phantom{00}251.\phantom{00 \pm 000.00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{TransH} & \textbf{pub} & & & & & $82.3\phantom{0 \pm 0.00}$ & $\phantom{00}388.\phantom{00 \pm 000.00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{TransR} & \textbf{pub} & & & & & $92.\phantom{00 \pm 0.00}$ & $\phantom{00}225.\phantom{00 \pm 000.00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{TuckER} & \textbf{pub} & $95.3\phantom{0 \pm 0.00}$ & $94.9\phantom{0 \pm 0.00}$ & $95.5\phantom{0 \pm 0.00}$ & & $95.8\phantom{0 \pm 0.00}$ & & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ \bottomrule \end{tabular} diff --git a/reproducibility/summaries/wn18_table_without_std.tex b/reproducibility/summaries/wn18_table_without_std.tex index da10ebf38..a321a0e8f 100644 --- a/reproducibility/summaries/wn18_table_without_std.tex +++ b/reproducibility/summaries/wn18_table_without_std.tex @@ -1,42 +1,42 @@ \begin{tabular}{llrrrrrrr} \toprule - & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ -\textbf{model} & {} & & & & & & & \\ + & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ +\textbf{model} & {} & & & & & & & \\ \midrule -\textbf{ComplEx} & \textbf{pub} & $94.1\phantom{0}$ & $93.6\phantom{0}$ & $94.5\phantom{0}$ & & $94.7\phantom{0}$ & & \\ - & \textbf{avg} & $19.49$ & $12.36$ & $20.66$ & $25.24$ & $32.92$ & $\phantom{00}452.67$ & $\phantom{00}2.21$ \\\midrule -\textbf{ConvE} & \textbf{pub} & $94.3\phantom{0}$ & $93.5\phantom{0}$ & $94.6\phantom{0}$ & & $95.6\phantom{0}$ & $\phantom{00}374.\phantom{00}$ & \\ - & \textbf{avg} & $88.81$ & $85.14$ & $91.76$ & $93.29$ & $94.85$ & $\phantom{00}444.40$ & $\phantom{00}2.17$ \\\midrule -\textbf{DistMult} & \textbf{pub} & $83.\phantom{00}$ & & & & $94.2\phantom{0}$ & & \\ - & \textbf{avg} & $77.44$ & $67.45$ & $85.94$ & $89.52$ & $92.72$ & $\phantom{00}458.64$ & $\phantom{00}2.24$ \\\midrule -\textbf{HolE} & \textbf{pub} & $93.8\phantom{0}$ & $93.\phantom{00}$ & $94.5\phantom{0}$ & & $94.9\phantom{0}$ & & \\ - & \textbf{avg} & $70.44$ & $59.29$ & $79.29$ & $84.12$ & $88.61$ & $\phantom{00}812.64$ & $\phantom{00}3.97$ \\ - & \textbf{best} & $70.44$ & $59.29$ & $79.29$ & $84.12$ & $88.61$ & $\phantom{00}812.63$ & \\ - & \textbf{worst} & $70.44$ & $59.29$ & $79.29$ & $84.12$ & $88.61$ & $\phantom{00}812.64$ & \\\midrule -\textbf{KG2E} & \textbf{pub} & & & & & $92.8\phantom{0}$ & $\phantom{00}331.\phantom{00}$ & \\ - & \textbf{avg} & $\phantom{0}3.61$ & $\phantom{0}1.35$ & $\phantom{0}3.21$ & $\phantom{0}4.57$ & $\phantom{0}7.02$ & $\phantom{0}2708.89$ & $\phantom{0}13.25$ \\ - & \textbf{best} & $\phantom{0}3.61$ & $\phantom{0}1.35$ & $\phantom{0}3.21$ & $\phantom{0}4.57$ & $\phantom{0}7.02$ & $\phantom{0}2708.88$ & \\ - & \textbf{worst} & $\phantom{0}3.61$ & $\phantom{0}1.35$ & $\phantom{0}3.21$ & $\phantom{0}4.57$ & $\phantom{0}7.02$ & $\phantom{0}2708.89$ & \\\midrule -\textbf{RotatE} & \textbf{pub} & $94.9\phantom{0}$ & $94.4\phantom{0}$ & $95.2\phantom{0}$ & & $95.9\phantom{0}$ & $\phantom{00}309.\phantom{00}$ & \\ - & \textbf{avg} & $87.29$ & $82.17$ & $91.53$ & $93.44$ & $95.28$ & $\phantom{00}123.68$ & $\phantom{00}0.61$ \\\midrule -\textbf{SimplE} & \textbf{pub} & $94.2\phantom{0}$ & $93.9\phantom{0}$ & $94.4\phantom{0}$ & & $94.7\phantom{0}$ & & \\ - & \textbf{avg} & $\phantom{0}0.04$ & $\phantom{0}0.01$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $\phantom{0}0.07$ & $20376.43$ & $\phantom{0}99.57$ \\ - & \textbf{best} & $38.48$ & $33.93$ & $39.59$ & $42.76$ & $47.01$ & $\phantom{00}384.53$ & \\ - & \textbf{worst} & $\phantom{0}0.03$ & $\phantom{0}0.01$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $\phantom{0}0.07$ & $40368.33$ & \\\midrule -\textbf{TransD} & \textbf{pub} & & & & & $92.2\phantom{0}$ & $\phantom{00}212.\phantom{00}$ & \\ - & \textbf{avg} & $36.22$ & $\phantom{0}3.94$ & $65.63$ & $79.64$ & $87.27$ & $\phantom{00}444.39$ & $\phantom{00}2.17$ \\\midrule -\textbf{TransE} & \textbf{pub} & & & & & $89.2\phantom{0}$ & $\phantom{00}251.\phantom{00}$ & \\ - & \textbf{avg} & $39.19$ & $\phantom{0}9.99$ & $64.74$ & $75.44$ & $84.25$ & $\phantom{00}468.24$ & $\phantom{00}2.29$ \\\midrule -\textbf{TransH} & \textbf{pub} & & & & & $82.3\phantom{0}$ & $\phantom{00}388.\phantom{00}$ & \\ - & \textbf{avg} & $\phantom{0}0.18$ & $\phantom{0}0.04$ & $\phantom{0}0.19$ & $\phantom{0}0.29$ & $\phantom{0}0.39$ & $19678.04$ & $\phantom{0}96.16$ \\ - & \textbf{best} & $\phantom{0}0.18$ & $\phantom{0}0.04$ & $\phantom{0}0.19$ & $\phantom{0}0.29$ & $\phantom{0}0.39$ & $19678.02$ & \\ - & \textbf{worst} & $\phantom{0}0.18$ & $\phantom{0}0.04$ & $\phantom{0}0.19$ & $\phantom{0}0.29$ & $\phantom{0}0.39$ & $19678.06$ & \\\midrule -\textbf{TransR} & \textbf{pub} & & & & & $92.\phantom{00}$ & $\phantom{00}225.\phantom{00}$ & \\ - & \textbf{avg} & $\phantom{0}0.06$ & $\phantom{0}0.00$ & $\phantom{0}0.04$ & $\phantom{0}0.05$ & $\phantom{0}0.11$ & $19686.49$ & $\phantom{0}96.20$ \\ - & \textbf{best} & $\phantom{0}0.06$ & $\phantom{0}0.00$ & $\phantom{0}0.04$ & $\phantom{0}0.05$ & $\phantom{0}0.11$ & $19686.49$ & \\ - & \textbf{worst} & $\phantom{0}0.06$ & $\phantom{0}0.00$ & $\phantom{0}0.04$ & $\phantom{0}0.05$ & $\phantom{0}0.11$ & $19686.50$ & \\\midrule -\textbf{TuckER} & \textbf{pub} & $95.3\phantom{0}$ & $94.9\phantom{0}$ & $95.5\phantom{0}$ & & $95.8\phantom{0}$ & & \\ - & \textbf{avg} & $\phantom{0}0.03$ & $\phantom{0}0.00$ & $\phantom{0}0.02$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $20622.46$ & $100.78$ \\ +\textbf{ComplEx} & \textbf{pub} & $94.1\phantom{0}$ & $93.6\phantom{0}$ & $94.5\phantom{0}$ & & $94.7\phantom{0}$ & & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{ConvE} & \textbf{pub} & $94.3\phantom{0}$ & $93.5\phantom{0}$ & $94.6\phantom{0}$ & & $95.6\phantom{0}$ & $\phantom{00}374.\phantom{00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{DistMult} & \textbf{pub} & $83.\phantom{00}$ & & & & $94.2\phantom{0}$ & & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{HolE} & \textbf{pub} & $93.8\phantom{0}$ & $93.\phantom{00}$ & $94.5\phantom{0}$ & & $94.9\phantom{0}$ & & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{KG2E} & \textbf{pub} & & & & & $92.8\phantom{0}$ & $\phantom{00}331.\phantom{00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{RotatE} & \textbf{pub} & $94.9\phantom{0}$ & $94.4\phantom{0}$ & $95.2\phantom{0}$ & & $95.9\phantom{0}$ & $\phantom{00}309.\phantom{00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{SimplE} & \textbf{pub} & $94.2\phantom{0}$ & $93.9\phantom{0}$ & $94.4\phantom{0}$ & & $94.7\phantom{0}$ & & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{TransD} & \textbf{pub} & & & & & $92.2\phantom{0}$ & $\phantom{00}212.\phantom{00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{TransE} & \textbf{pub} & & & & & $89.2\phantom{0}$ & $\phantom{00}251.\phantom{00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{TransH} & \textbf{pub} & & & & & $82.3\phantom{0}$ & $\phantom{00}388.\phantom{00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{TransR} & \textbf{pub} & & & & & $92.\phantom{00}$ & $\phantom{00}225.\phantom{00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{TuckER} & \textbf{pub} & $95.3\phantom{0}$ & $94.9\phantom{0}$ & $95.5\phantom{0}$ & & $95.8\phantom{0}$ & & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ \bottomrule \end{tabular} diff --git a/reproducibility/summaries/wn18rr_table.tex b/reproducibility/summaries/wn18rr_table.tex index 9ed5c208a..fc2715392 100644 --- a/reproducibility/summaries/wn18rr_table.tex +++ b/reproducibility/summaries/wn18rr_table.tex @@ -1,16 +1,16 @@ \begin{tabular}{llrrrrrrr} \toprule - & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ -\textbf{model} & {} & & & & & & & \\ + & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ +\textbf{model} & {} & & & & & & & \\ \midrule -\textbf{ConvE} & \textbf{pub} & $43.\phantom{00 \pm 0.00}$ & $\phantom{0}4.\phantom{00 \pm 0.00}$ & $44.\phantom{00 \pm 0.00}$ & & $52.\phantom{00 \pm 0.00}$ & $\phantom{0}4187.\phantom{00 \pm 000.00}$ & \\ - & \textbf{avg} & $44.69 \pm 0.21$ & $40.98 \pm 0.22$ & $46.49 \pm 0.14$ & $48.92 \pm 0.23$ & $51.76 \pm 0.13$ & $\phantom{0}5369.49 \pm \phantom{0}50.92$ & $26.49 \pm 0.25$ \\\midrule -\textbf{ConvKB} & \textbf{pub} & $24.8\phantom{0 \pm 0.00}$ & & & & $52.5\phantom{0 \pm 0.00}$ & $\phantom{0}2554.\phantom{00 \pm 000.00}$ & \\ - & \textbf{avg} & $\phantom{0}0.30 \pm 0.07$ & $\phantom{0}0.09 \pm 0.03$ & $\phantom{0}0.21 \pm 0.07$ & $\phantom{0}0.32 \pm 0.10$ & $\phantom{0}0.57 \pm 0.16$ & $13634.66 \pm 714.24$ & $67.27 \pm 3.52$ \\ - & \textbf{best} & $\phantom{0}0.30 \pm 0.07$ & $\phantom{0}0.09 \pm 0.03$ & $\phantom{0}0.21 \pm 0.07$ & $\phantom{0}0.32 \pm 0.10$ & $\phantom{0}0.57 \pm 0.16$ & $13634.65 \pm 714.24$ & \\ - & \textbf{worst} & $\phantom{0}0.30 \pm 0.07$ & $\phantom{0}0.09 \pm 0.03$ & $\phantom{0}0.21 \pm 0.07$ & $\phantom{0}0.32 \pm 0.10$ & $\phantom{0}0.57 \pm 0.16$ & $13634.66 \pm 714.24$ & \\\midrule -\textbf{RotatE} & \textbf{pub} & $47.6\phantom{0 \pm 0.00}$ & $42.8\phantom{0 \pm 0.00}$ & $49.2\phantom{0 \pm 0.00}$ & & $57.1\phantom{0 \pm 0.00}$ & $\phantom{0}3340.\phantom{00 \pm 000.00}$ & \\ - & \textbf{avg} & $48.40 \pm 0.09$ & $44.02 \pm 0.15$ & $50.55 \pm 0.12$ & $52.98 \pm 0.11$ & $56.51 \pm 0.26$ & $\phantom{0}4263.32 \pm \phantom{0}90.33$ & $21.03 \pm 0.45$ \\ +\textbf{ConvE} & \textbf{pub} & $43.\phantom{00 \pm 0.00}$ & $\phantom{0}4.\phantom{00 \pm 0.00}$ & $44.\phantom{00 \pm 0.00}$ & & $52.\phantom{00 \pm 0.00}$ & $\phantom{0}4187.\phantom{00 \pm 000.00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{ConvKB} & \textbf{pub} & $24.8\phantom{0 \pm 0.00}$ & & & & $52.5\phantom{0 \pm 0.00}$ & $\phantom{0}2554.\phantom{00 \pm 000.00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{RotatE} & \textbf{pub} & $47.6\phantom{0 \pm 0.00}$ & $42.8\phantom{0 \pm 0.00}$ & $49.2\phantom{0 \pm 0.00}$ & & $57.1\phantom{0 \pm 0.00}$ & $\phantom{0}3340.\phantom{00 \pm 000.00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ \bottomrule \end{tabular} diff --git a/reproducibility/summaries/wn18rr_table_without_std.tex b/reproducibility/summaries/wn18rr_table_without_std.tex index 7f15cabdd..987f485f5 100644 --- a/reproducibility/summaries/wn18rr_table_without_std.tex +++ b/reproducibility/summaries/wn18rr_table_without_std.tex @@ -1,16 +1,16 @@ \begin{tabular}{llrrrrrrr} \toprule - & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ -\textbf{model} & {} & & & & & & & \\ + & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ +\textbf{model} & {} & & & & & & & \\ \midrule -\textbf{ConvE} & \textbf{pub} & $43.\phantom{00}$ & $\phantom{0}4.\phantom{00}$ & $44.\phantom{00}$ & & $52.\phantom{00}$ & $\phantom{0}4187.\phantom{00}$ & \\ - & \textbf{avg} & $44.69$ & $40.98$ & $46.49$ & $48.92$ & $51.76$ & $\phantom{0}5369.49$ & $26.49$ \\\midrule -\textbf{ConvKB} & \textbf{pub} & $24.8\phantom{0}$ & & & & $52.5\phantom{0}$ & $\phantom{0}2554.\phantom{00}$ & \\ - & \textbf{avg} & $\phantom{0}0.30$ & $\phantom{0}0.09$ & $\phantom{0}0.21$ & $\phantom{0}0.32$ & $\phantom{0}0.57$ & $13634.66$ & $67.27$ \\ - & \textbf{best} & $\phantom{0}0.30$ & $\phantom{0}0.09$ & $\phantom{0}0.21$ & $\phantom{0}0.32$ & $\phantom{0}0.57$ & $13634.65$ & \\ - & \textbf{worst} & $\phantom{0}0.30$ & $\phantom{0}0.09$ & $\phantom{0}0.21$ & $\phantom{0}0.32$ & $\phantom{0}0.57$ & $13634.66$ & \\\midrule -\textbf{RotatE} & \textbf{pub} & $47.6\phantom{0}$ & $42.8\phantom{0}$ & $49.2\phantom{0}$ & & $57.1\phantom{0}$ & $\phantom{0}3340.\phantom{00}$ & \\ - & \textbf{avg} & $48.40$ & $44.02$ & $50.55$ & $52.98$ & $56.51$ & $\phantom{0}4263.32$ & $21.03$ \\ +\textbf{ConvE} & \textbf{pub} & $43.\phantom{00}$ & $\phantom{0}4.\phantom{00}$ & $44.\phantom{00}$ & & $52.\phantom{00}$ & $\phantom{0}4187.\phantom{00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{ConvKB} & \textbf{pub} & $24.8\phantom{0}$ & & & & $52.5\phantom{0}$ & $\phantom{0}2554.\phantom{00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ + & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule +\textbf{RotatE} & \textbf{pub} & $47.6\phantom{0}$ & $42.8\phantom{0}$ & $49.2\phantom{0}$ & & $57.1\phantom{0}$ & $\phantom{0}3340.\phantom{00}$ & \\ + & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ \bottomrule \end{tabular} From ef1512de91a19000b846163669eafc6801a3265b Mon Sep 17 00:00:00 2001 From: Max Berrendorf Date: Mon, 22 Jun 2020 16:50:29 +0200 Subject: [PATCH 7/7] Fix index renaming --- reproducibility/generate_summary_table.py | 2 +- reproducibility/summaries/fb15k237_table.tex | 16 ++--- .../summaries/fb15k237_table_without_std.tex | 16 ++--- reproducibility/summaries/fb15k_table.tex | 64 ++++++++--------- .../summaries/fb15k_table_without_std.tex | 64 ++++++++--------- reproducibility/summaries/wn18_table.tex | 72 +++++++++---------- .../summaries/wn18_table_without_std.tex | 72 +++++++++---------- reproducibility/summaries/wn18rr_table.tex | 20 +++--- .../summaries/wn18rr_table_without_std.tex | 20 +++--- 9 files changed, 173 insertions(+), 173 deletions(-) diff --git a/reproducibility/generate_summary_table.py b/reproducibility/generate_summary_table.py index 4ab8d2bc5..1d07afae5 100644 --- a/reproducibility/generate_summary_table.py +++ b/reproducibility/generate_summary_table.py @@ -87,7 +87,7 @@ def generate_results_table(with_std: bool, midrule_between_models: bool = True): best='O', worst='P', ) - wide_summary_df = wide_summary_df.reindex(wide_summary_df.index.set_levels([translation[k] for k in wide_summary_df.index.levels[1]], level=1)) + wide_summary_df.index = wide_summary_df.index.set_levels([translation[k] for k in wide_summary_df.index.levels[1]], level=1) # Save as Latex table pd.set_option('max_colwidth', 999) diff --git a/reproducibility/summaries/fb15k237_table.tex b/reproducibility/summaries/fb15k237_table.tex index f3f5a0ab9..d6b0fd396 100644 --- a/reproducibility/summaries/fb15k237_table.tex +++ b/reproducibility/summaries/fb15k237_table.tex @@ -1,14 +1,14 @@ \begin{tabular}{llrrrrrrr} \toprule - & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ -\textbf{model} & {} & & & & & & & \\ + & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ +\textbf{model} & {} & & & & & & & \\ \midrule -\textbf{ConvE} & \textbf{pub} & $32.5\phantom{0 \pm 0.00}$ & $23.7\phantom{0 \pm 0.00}$ & $35.6\phantom{0 \pm 0.00}$ & & $50.1\phantom{0 \pm 0.00}$ & $\phantom{0}244.\phantom{00 \pm 00.00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{ConvKB} & \textbf{pub} & $39.6\phantom{0 \pm 0.00}$ & & & & $51.7\phantom{0 \pm 0.00}$ & $\phantom{0}257.\phantom{00 \pm 00.00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{RotatE} & \textbf{pub} & $33.8\phantom{0 \pm 0.00}$ & $24.1\phantom{0 \pm 0.00}$ & $37.5\phantom{0 \pm 0.00}$ & & $53.3\phantom{0 \pm 0.00}$ & $\phantom{0}177.\phantom{00 \pm 00.00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ +\textbf{ConvE} & \textbf{pub} & $32.5\phantom{0 \pm 0.00}$ & $23.7\phantom{0 \pm 0.00}$ & $35.6\phantom{0 \pm 0.00}$ & & $50.1\phantom{0 \pm 0.00}$ & $\phantom{0}244.\phantom{00 \pm 00.00}$ & \\ + & \textbf{R} & $26.93 \pm 0.11$ & $18.22 \pm 0.11$ & $29.51 \pm 0.24$ & $35.98 \pm 0.16$ & $44.95 \pm 0.17$ & $\phantom{0}255.46 \pm \phantom{0}6.16$ & $\phantom{0}3.73 \pm 0.13$ \\\midrule +\textbf{ConvKB} & \textbf{pub} & $39.6\phantom{0 \pm 0.00}$ & & & & $51.7\phantom{0 \pm 0.00}$ & $\phantom{0}257.\phantom{00 \pm 00.00}$ & \\ + & \textbf{R} & $\phantom{0}4.71 \pm 0.23$ & $\phantom{0}3.31 \pm 0.23$ & $\phantom{0}4.04 \pm 0.19$ & $\phantom{0}4.57 \pm 0.22$ & $\phantom{0}7.76 \pm 0.88$ & $4345.27 \pm 46.99$ & $61.36 \pm 0.65$ \\\midrule +\textbf{RotatE} & \textbf{pub} & $33.8\phantom{0 \pm 0.00}$ & $24.1\phantom{0 \pm 0.00}$ & $37.5\phantom{0 \pm 0.00}$ & & $53.3\phantom{0 \pm 0.00}$ & $\phantom{0}177.\phantom{00 \pm 00.00}$ & \\ + & \textbf{R} & $26.42 \pm 0.04$ & $17.57 \pm 0.06$ & $28.97 \pm 0.05$ & $35.29 \pm 0.09$ & $44.55 \pm 0.06$ & $\phantom{0}191.92 \pm \phantom{0}0.31$ & $\phantom{0}2.84 \pm 0.00$ \\ \bottomrule \end{tabular} diff --git a/reproducibility/summaries/fb15k237_table_without_std.tex b/reproducibility/summaries/fb15k237_table_without_std.tex index 2abdcef87..1bb8ef971 100644 --- a/reproducibility/summaries/fb15k237_table_without_std.tex +++ b/reproducibility/summaries/fb15k237_table_without_std.tex @@ -1,14 +1,14 @@ \begin{tabular}{llrrrrrrr} \toprule - & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ -\textbf{model} & {} & & & & & & & \\ + & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ +\textbf{model} & {} & & & & & & & \\ \midrule -\textbf{ConvE} & \textbf{pub} & $32.5\phantom{0}$ & $23.7\phantom{0}$ & $35.6\phantom{0}$ & & $50.1\phantom{0}$ & $\phantom{0}244.\phantom{00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{ConvKB} & \textbf{pub} & $39.6\phantom{0}$ & & & & $51.7\phantom{0}$ & $\phantom{0}257.\phantom{00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{RotatE} & \textbf{pub} & $33.8\phantom{0}$ & $24.1\phantom{0}$ & $37.5\phantom{0}$ & & $53.3\phantom{0}$ & $\phantom{0}177.\phantom{00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ +\textbf{ConvE} & \textbf{pub} & $32.5\phantom{0}$ & $23.7\phantom{0}$ & $35.6\phantom{0}$ & & $50.1\phantom{0}$ & $\phantom{0}244.\phantom{00}$ & \\ + & \textbf{R} & $26.93$ & $18.22$ & $29.51$ & $35.98$ & $44.95$ & $\phantom{0}255.46$ & $\phantom{0}3.73$ \\\midrule +\textbf{ConvKB} & \textbf{pub} & $39.6\phantom{0}$ & & & & $51.7\phantom{0}$ & $\phantom{0}257.\phantom{00}$ & \\ + & \textbf{R} & $\phantom{0}4.71$ & $\phantom{0}3.31$ & $\phantom{0}4.04$ & $\phantom{0}4.57$ & $\phantom{0}7.76$ & $4345.27$ & $61.36$ \\\midrule +\textbf{RotatE} & \textbf{pub} & $33.8\phantom{0}$ & $24.1\phantom{0}$ & $37.5\phantom{0}$ & & $53.3\phantom{0}$ & $\phantom{0}177.\phantom{00}$ & \\ + & \textbf{R} & $26.42$ & $17.57$ & $28.97$ & $35.29$ & $44.55$ & $\phantom{0}191.92$ & $\phantom{0}2.84$ \\ \bottomrule \end{tabular} diff --git a/reproducibility/summaries/fb15k_table.tex b/reproducibility/summaries/fb15k_table.tex index a7c61d6ca..dda4f721f 100644 --- a/reproducibility/summaries/fb15k_table.tex +++ b/reproducibility/summaries/fb15k_table.tex @@ -1,38 +1,38 @@ \begin{tabular}{llrrrrrrr} \toprule - & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ -\textbf{model} & {} & & & & & & & \\ + & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ +\textbf{model} & {} & & & & & & & \\ \midrule -\textbf{ComplEx} & \textbf{pub} & $69.2\phantom{0 \pm 0.00}$ & $59.9\phantom{0 \pm 0.00}$ & $75.9\phantom{0 \pm 00.00}$ & & $84.\phantom{00 \pm 00.00}$ & & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{ConvE} & \textbf{pub} & $65.7\phantom{0 \pm 0.00}$ & $55.8\phantom{0 \pm 0.00}$ & $72.3\phantom{0 \pm 00.00}$ & & $83.1\phantom{0 \pm 00.00}$ & $\phantom{000}51.\phantom{00 \pm 00.00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{DistMult} & \textbf{pub} & $35.\phantom{00 \pm 0.00}$ & & & & $57.7\phantom{0 \pm 00.00}$ & & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{HolE} & \textbf{pub} & $52.4\phantom{0 \pm 0.00}$ & $40.2\phantom{0 \pm 0.00}$ & $61.3\phantom{0 \pm 00.00}$ & & $73.9\phantom{0 \pm 00.00}$ & & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{KG2E} & \textbf{pub} & & & & & $71.5\phantom{0 \pm 00.00}$ & $\phantom{000}59.\phantom{00 \pm 00.00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{RotatE} & \textbf{pub} & $79.7\phantom{0 \pm 0.00}$ & $74.6\phantom{0 \pm 0.00}$ & $83.\phantom{00 \pm 00.00}$ & & $88.4\phantom{0 \pm 00.00}$ & $\phantom{000}40.\phantom{00 \pm 00.00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{SimplE} & \textbf{pub} & $72.7\phantom{0 \pm 0.00}$ & $66.\phantom{00 \pm 0.00}$ & $77.3\phantom{0 \pm 00.00}$ & & $83.8\phantom{0 \pm 00.00}$ & & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{TransD} & \textbf{pub} & & & & & $77.3\phantom{0 \pm 00.00}$ & $\phantom{000}91.\phantom{00 \pm 00.00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{TransE} & \textbf{pub} & & & & & $47.1\phantom{0 \pm 00.00}$ & $\phantom{00}125.\phantom{00 \pm 00.00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{TransH} & \textbf{pub} & & & & & $64.4\phantom{0 \pm 00.00}$ & $\phantom{000}87.\phantom{00 \pm 00.00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{TransR} & \textbf{pub} & & & & & $68.7\phantom{0 \pm 00.00}$ & $\phantom{000}77.\phantom{00 \pm 00.00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{TuckER} & \textbf{pub} & $79.5\phantom{0 \pm 0.00}$ & $74.1\phantom{0 \pm 0.00}$ & $83.3\phantom{0 \pm 00.00}$ & & $89.2\phantom{0 \pm 00.00}$ & & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ +\textbf{ComplEx} & \textbf{pub} & $69.2\phantom{0 \pm 0.00}$ & $59.9\phantom{0 \pm 0.00}$ & $75.9\phantom{0 \pm 00.00}$ & & $84.\phantom{00 \pm 00.00}$ & & \\ + & \textbf{R} & $19.13 \pm 0.45$ & $10.06 \pm 0.42$ & $20.82 \pm \phantom{0}0.67$ & $27.59 \pm \phantom{0}0.94$ & $38.03 \pm \phantom{0}1.02$ & $\phantom{00}171.16 \pm 16.05$ & $\phantom{00}2.46 \pm 0.28$ \\\midrule +\textbf{ConvE} & \textbf{pub} & $65.7\phantom{0 \pm 0.00}$ & $55.8\phantom{0 \pm 0.00}$ & $72.3\phantom{0 \pm 00.00}$ & & $83.1\phantom{0 \pm 00.00}$ & $\phantom{000}51.\phantom{00 \pm 00.00}$ & \\ + & \textbf{R} & $59.56 \pm 0.06$ & $48.28 \pm 0.12$ & $66.99 \pm \phantom{0}0.04$ & $73.27 \pm \phantom{0}0.03$ & $79.76 \pm \phantom{0}0.07$ & $\phantom{000}50.76 \pm \phantom{0}0.40$ & $\phantom{00}0.73 \pm 0.01$ \\\midrule +\textbf{DistMult} & \textbf{pub} & $35.\phantom{00 \pm 0.00}$ & & & & $57.7\phantom{0 \pm 00.00}$ & & \\ + & \textbf{R} & $26.06 \pm 0.17$ & $16.45 \pm 0.16$ & $29.10 \pm \phantom{0}0.17$ & $35.54 \pm \phantom{0}0.21$ & $45.00 \pm \phantom{0}0.25$ & $\phantom{00}134.02 \pm \phantom{0}1.98$ & $\phantom{00}1.86 \pm 0.03$ \\\midrule +\textbf{HolE} & \textbf{pub} & $52.4\phantom{0 \pm 0.00}$ & $40.2\phantom{0 \pm 0.00}$ & $61.3\phantom{0 \pm 00.00}$ & & $73.9\phantom{0 \pm 00.00}$ & & \\ + & \textbf{R} & $34.15 \pm 0.22$ & $21.79 \pm 0.19$ & $39.69 \pm \phantom{0}0.24$ & $48.06 \pm \phantom{0}0.30$ & $58.84 \pm \phantom{0}0.28$ & $\phantom{00}193.03 \pm \phantom{0}7.61$ & $\phantom{00}2.71 \pm 0.12$ \\\midrule +\textbf{KG2E} & \textbf{pub} & & & & & $71.5\phantom{0 \pm 00.00}$ & $\phantom{000}59.\phantom{00 \pm 00.00}$ & \\ + & \textbf{R} & $\phantom{0}0.58 \pm 0.07$ & $\phantom{0}0.11 \pm 0.04$ & $\phantom{0}0.36 \pm \phantom{0}0.08$ & $\phantom{0}0.56 \pm \phantom{0}0.10$ & $\phantom{0}1.01 \pm \phantom{0}0.14$ & $\phantom{0}5779.07 \pm 51.02$ & $\phantom{0}78.40 \pm 0.68$ \\\midrule +\textbf{RotatE} & \textbf{pub} & $79.7\phantom{0 \pm 0.00}$ & $74.6\phantom{0 \pm 0.00}$ & $83.\phantom{00 \pm 00.00}$ & & $88.4\phantom{0 \pm 00.00}$ & $\phantom{000}40.\phantom{00 \pm 00.00}$ & \\ + & \textbf{R} & $55.00 \pm 0.06$ & $41.53 \pm 0.06$ & $64.14 \pm \phantom{0}0.07$ & $71.23 \pm \phantom{0}0.05$ & $78.67 \pm \phantom{0}0.08$ & $\phantom{000}42.28 \pm \phantom{0}0.13$ & $\phantom{00}0.63 \pm 0.00$ \\\midrule +\textbf{SimplE} & \textbf{pub} & $72.7\phantom{0 \pm 0.00}$ & $66.\phantom{00 \pm 0.00}$ & $77.3\phantom{0 \pm 00.00}$ & & $83.8\phantom{0 \pm 00.00}$ & & \\ + & \textbf{R} & $\phantom{0}0.04 \pm 0.00$ & $\phantom{0}0.01 \pm 0.00$ & $\phantom{0}0.03 \pm \phantom{0}0.00$ & $\phantom{0}0.04 \pm \phantom{0}0.00$ & $\phantom{0}0.06 \pm \phantom{0}0.01$ & $\phantom{0}7395.75 \pm \phantom{0}2.02$ & $100.02 \pm 0.03$ \\ + & \textbf{O} & $23.90 \pm 8.79$ & $11.58 \pm 6.42$ & $24.16 \pm 10.95$ & $34.73 \pm 13.40$ & $54.28 \pm 15.80$ & $\phantom{00}139.34 \pm 49.45$ & \\ + & \textbf{P} & $\phantom{0}0.03 \pm 0.00$ & $\phantom{0}0.01 \pm 0.00$ & $\phantom{0}0.03 \pm \phantom{0}0.00$ & $\phantom{0}0.04 \pm \phantom{0}0.00$ & $\phantom{0}0.06 \pm \phantom{0}0.01$ & $14652.16 \pm 45.71$ & \\\midrule +\textbf{TransD} & \textbf{pub} & & & & & $77.3\phantom{0 \pm 00.00}$ & $\phantom{000}91.\phantom{00 \pm 00.00}$ & \\ + & \textbf{R} & $33.99 \pm 0.03$ & $21.22 \pm 0.03$ & $40.48 \pm \phantom{0}0.10$ & $48.57 \pm \phantom{0}0.09$ & $58.71 \pm \phantom{0}0.14$ & $\phantom{00}153.37 \pm \phantom{0}5.35$ & $\phantom{00}2.29 \pm 0.09$ \\\midrule +\textbf{TransE} & \textbf{pub} & & & & & $47.1\phantom{0 \pm 00.00}$ & $\phantom{00}125.\phantom{00 \pm 00.00}$ & \\ + & \textbf{R} & $26.01 \pm 0.17$ & $15.23 \pm 0.16$ & $29.85 \pm \phantom{0}0.24$ & $37.18 \pm \phantom{0}0.24$ & $47.34 \pm \phantom{0}0.18$ & $\phantom{00}127.92 \pm \phantom{0}0.86$ & $\phantom{00}1.78 \pm 0.01$ \\\midrule +\textbf{TransH} & \textbf{pub} & & & & & $64.4\phantom{0 \pm 00.00}$ & $\phantom{000}87.\phantom{00 \pm 00.00}$ & \\ + & \textbf{R} & $\phantom{0}2.54 \pm 0.20$ & $\phantom{0}1.69 \pm 0.25$ & $\phantom{0}2.95 \pm \phantom{0}0.20$ & $\phantom{0}3.29 \pm \phantom{0}0.22$ & $\phantom{0}3.74 \pm \phantom{0}0.18$ & $\phantom{0}6320.02 \pm 30.37$ & $\phantom{0}85.63 \pm 0.40$ \\ + & \textbf{O} & $\phantom{0}2.54 \pm 0.20$ & $\phantom{0}1.69 \pm 0.25$ & $\phantom{0}2.95 \pm \phantom{0}0.20$ & $\phantom{0}3.29 \pm \phantom{0}0.22$ & $\phantom{0}3.74 \pm \phantom{0}0.18$ & $\phantom{0}6320.00 \pm 30.37$ & \\ + & \textbf{P} & $\phantom{0}2.54 \pm 0.20$ & $\phantom{0}1.69 \pm 0.25$ & $\phantom{0}2.95 \pm \phantom{0}0.20$ & $\phantom{0}3.29 \pm \phantom{0}0.22$ & $\phantom{0}3.74 \pm \phantom{0}0.18$ & $\phantom{0}6320.05 \pm 30.37$ & \\\midrule +\textbf{TransR} & \textbf{pub} & & & & & $68.7\phantom{0 \pm 00.00}$ & $\phantom{000}77.\phantom{00 \pm 00.00}$ & \\ + & \textbf{R} & $\phantom{0}0.65 \pm 0.02$ & $\phantom{0}0.37 \pm 0.00$ & $\phantom{0}0.63 \pm \phantom{0}0.04$ & $\phantom{0}0.78 \pm \phantom{0}0.06$ & $\phantom{0}1.03 \pm \phantom{0}0.07$ & $\phantom{0}6795.95 \pm 16.65$ & $\phantom{0}91.99 \pm 0.22$ \\ + & \textbf{O} & $\phantom{0}0.65 \pm 0.02$ & $\phantom{0}0.37 \pm 0.00$ & $\phantom{0}0.63 \pm \phantom{0}0.04$ & $\phantom{0}0.78 \pm \phantom{0}0.06$ & $\phantom{0}1.03 \pm \phantom{0}0.07$ & $\phantom{0}6795.94 \pm 16.65$ & \\ + & \textbf{P} & $\phantom{0}0.65 \pm 0.02$ & $\phantom{0}0.37 \pm 0.00$ & $\phantom{0}0.63 \pm \phantom{0}0.04$ & $\phantom{0}0.78 \pm \phantom{0}0.06$ & $\phantom{0}1.03 \pm \phantom{0}0.07$ & $\phantom{0}6795.95 \pm 16.65$ & \\\midrule +\textbf{TuckER} & \textbf{pub} & $79.5\phantom{0 \pm 0.00}$ & $74.1\phantom{0 \pm 0.00}$ & $83.3\phantom{0 \pm 00.00}$ & & $89.2\phantom{0 \pm 00.00}$ & & \\ + & \textbf{R} & $\phantom{0}0.07 \pm 0.02$ & $\phantom{0}0.01 \pm 0.00$ & $\phantom{0}0.02 \pm \phantom{0}0.00$ & $\phantom{0}0.03 \pm \phantom{0}0.01$ & $\phantom{0}0.15 \pm \phantom{0}0.17$ & $\phantom{0}7327.77 \pm 29.22$ & $\phantom{0}99.11 \pm 0.39$ \\ \bottomrule \end{tabular} diff --git a/reproducibility/summaries/fb15k_table_without_std.tex b/reproducibility/summaries/fb15k_table_without_std.tex index 1aee51964..da81adbd4 100644 --- a/reproducibility/summaries/fb15k_table_without_std.tex +++ b/reproducibility/summaries/fb15k_table_without_std.tex @@ -1,38 +1,38 @@ \begin{tabular}{llrrrrrrr} \toprule - & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ -\textbf{model} & {} & & & & & & & \\ + & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ +\textbf{model} & {} & & & & & & & \\ \midrule -\textbf{ComplEx} & \textbf{pub} & $69.2\phantom{0}$ & $59.9\phantom{0}$ & $75.9\phantom{0}$ & & $84.\phantom{00}$ & & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{ConvE} & \textbf{pub} & $65.7\phantom{0}$ & $55.8\phantom{0}$ & $72.3\phantom{0}$ & & $83.1\phantom{0}$ & $\phantom{000}51.\phantom{00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{DistMult} & \textbf{pub} & $35.\phantom{00}$ & & & & $57.7\phantom{0}$ & & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{HolE} & \textbf{pub} & $52.4\phantom{0}$ & $40.2\phantom{0}$ & $61.3\phantom{0}$ & & $73.9\phantom{0}$ & & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{KG2E} & \textbf{pub} & & & & & $71.5\phantom{0}$ & $\phantom{000}59.\phantom{00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{RotatE} & \textbf{pub} & $79.7\phantom{0}$ & $74.6\phantom{0}$ & $83.\phantom{00}$ & & $88.4\phantom{0}$ & $\phantom{000}40.\phantom{00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{SimplE} & \textbf{pub} & $72.7\phantom{0}$ & $66.\phantom{00}$ & $77.3\phantom{0}$ & & $83.8\phantom{0}$ & & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{TransD} & \textbf{pub} & & & & & $77.3\phantom{0}$ & $\phantom{000}91.\phantom{00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{TransE} & \textbf{pub} & & & & & $47.1\phantom{0}$ & $\phantom{00}125.\phantom{00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{TransH} & \textbf{pub} & & & & & $64.4\phantom{0}$ & $\phantom{000}87.\phantom{00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{TransR} & \textbf{pub} & & & & & $68.7\phantom{0}$ & $\phantom{000}77.\phantom{00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{TuckER} & \textbf{pub} & $79.5\phantom{0}$ & $74.1\phantom{0}$ & $83.3\phantom{0}$ & & $89.2\phantom{0}$ & & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ +\textbf{ComplEx} & \textbf{pub} & $69.2\phantom{0}$ & $59.9\phantom{0}$ & $75.9\phantom{0}$ & & $84.\phantom{00}$ & & \\ + & \textbf{R} & $19.13$ & $10.06$ & $20.82$ & $27.59$ & $38.03$ & $\phantom{00}171.16$ & $\phantom{00}2.46$ \\\midrule +\textbf{ConvE} & \textbf{pub} & $65.7\phantom{0}$ & $55.8\phantom{0}$ & $72.3\phantom{0}$ & & $83.1\phantom{0}$ & $\phantom{000}51.\phantom{00}$ & \\ + & \textbf{R} & $59.56$ & $48.28$ & $66.99$ & $73.27$ & $79.76$ & $\phantom{000}50.76$ & $\phantom{00}0.73$ \\\midrule +\textbf{DistMult} & \textbf{pub} & $35.\phantom{00}$ & & & & $57.7\phantom{0}$ & & \\ + & \textbf{R} & $26.06$ & $16.45$ & $29.10$ & $35.54$ & $45.00$ & $\phantom{00}134.02$ & $\phantom{00}1.86$ \\\midrule +\textbf{HolE} & \textbf{pub} & $52.4\phantom{0}$ & $40.2\phantom{0}$ & $61.3\phantom{0}$ & & $73.9\phantom{0}$ & & \\ + & \textbf{R} & $34.15$ & $21.79$ & $39.69$ & $48.06$ & $58.84$ & $\phantom{00}193.03$ & $\phantom{00}2.71$ \\\midrule +\textbf{KG2E} & \textbf{pub} & & & & & $71.5\phantom{0}$ & $\phantom{000}59.\phantom{00}$ & \\ + & \textbf{R} & $\phantom{0}0.58$ & $\phantom{0}0.11$ & $\phantom{0}0.36$ & $\phantom{0}0.56$ & $\phantom{0}1.01$ & $\phantom{0}5779.07$ & $\phantom{0}78.40$ \\\midrule +\textbf{RotatE} & \textbf{pub} & $79.7\phantom{0}$ & $74.6\phantom{0}$ & $83.\phantom{00}$ & & $88.4\phantom{0}$ & $\phantom{000}40.\phantom{00}$ & \\ + & \textbf{R} & $55.00$ & $41.53$ & $64.14$ & $71.23$ & $78.67$ & $\phantom{000}42.28$ & $\phantom{00}0.63$ \\\midrule +\textbf{SimplE} & \textbf{pub} & $72.7\phantom{0}$ & $66.\phantom{00}$ & $77.3\phantom{0}$ & & $83.8\phantom{0}$ & & \\ + & \textbf{R} & $\phantom{0}0.04$ & $\phantom{0}0.01$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $\phantom{0}0.06$ & $\phantom{0}7395.75$ & $100.02$ \\ + & \textbf{O} & $23.90$ & $11.58$ & $24.16$ & $34.73$ & $54.28$ & $\phantom{00}139.34$ & \\ + & \textbf{P} & $\phantom{0}0.03$ & $\phantom{0}0.01$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $\phantom{0}0.06$ & $14652.16$ & \\\midrule +\textbf{TransD} & \textbf{pub} & & & & & $77.3\phantom{0}$ & $\phantom{000}91.\phantom{00}$ & \\ + & \textbf{R} & $33.99$ & $21.22$ & $40.48$ & $48.57$ & $58.71$ & $\phantom{00}153.37$ & $\phantom{00}2.29$ \\\midrule +\textbf{TransE} & \textbf{pub} & & & & & $47.1\phantom{0}$ & $\phantom{00}125.\phantom{00}$ & \\ + & \textbf{R} & $26.01$ & $15.23$ & $29.85$ & $37.18$ & $47.34$ & $\phantom{00}127.92$ & $\phantom{00}1.78$ \\\midrule +\textbf{TransH} & \textbf{pub} & & & & & $64.4\phantom{0}$ & $\phantom{000}87.\phantom{00}$ & \\ + & \textbf{R} & $\phantom{0}2.54$ & $\phantom{0}1.69$ & $\phantom{0}2.95$ & $\phantom{0}3.29$ & $\phantom{0}3.74$ & $\phantom{0}6320.02$ & $\phantom{0}85.63$ \\ + & \textbf{O} & $\phantom{0}2.54$ & $\phantom{0}1.69$ & $\phantom{0}2.95$ & $\phantom{0}3.29$ & $\phantom{0}3.74$ & $\phantom{0}6320.00$ & \\ + & \textbf{P} & $\phantom{0}2.54$ & $\phantom{0}1.69$ & $\phantom{0}2.95$ & $\phantom{0}3.29$ & $\phantom{0}3.74$ & $\phantom{0}6320.05$ & \\\midrule +\textbf{TransR} & \textbf{pub} & & & & & $68.7\phantom{0}$ & $\phantom{000}77.\phantom{00}$ & \\ + & \textbf{R} & $\phantom{0}0.65$ & $\phantom{0}0.37$ & $\phantom{0}0.63$ & $\phantom{0}0.78$ & $\phantom{0}1.03$ & $\phantom{0}6795.95$ & $\phantom{0}91.99$ \\ + & \textbf{O} & $\phantom{0}0.65$ & $\phantom{0}0.37$ & $\phantom{0}0.63$ & $\phantom{0}0.78$ & $\phantom{0}1.03$ & $\phantom{0}6795.94$ & \\ + & \textbf{P} & $\phantom{0}0.65$ & $\phantom{0}0.37$ & $\phantom{0}0.63$ & $\phantom{0}0.78$ & $\phantom{0}1.03$ & $\phantom{0}6795.95$ & \\\midrule +\textbf{TuckER} & \textbf{pub} & $79.5\phantom{0}$ & $74.1\phantom{0}$ & $83.3\phantom{0}$ & & $89.2\phantom{0}$ & & \\ + & \textbf{R} & $\phantom{0}0.07$ & $\phantom{0}0.01$ & $\phantom{0}0.02$ & $\phantom{0}0.03$ & $\phantom{0}0.15$ & $\phantom{0}7327.77$ & $\phantom{0}99.11$ \\ \bottomrule \end{tabular} diff --git a/reproducibility/summaries/wn18_table.tex b/reproducibility/summaries/wn18_table.tex index 6c9890da0..03462c9c0 100644 --- a/reproducibility/summaries/wn18_table.tex +++ b/reproducibility/summaries/wn18_table.tex @@ -1,42 +1,42 @@ \begin{tabular}{llrrrrrrr} \toprule - & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ -\textbf{model} & {} & & & & & & & \\ + & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ +\textbf{model} & {} & & & & & & & \\ \midrule -\textbf{ComplEx} & \textbf{pub} & $94.1\phantom{0 \pm 0.00}$ & $93.6\phantom{0 \pm 0.00}$ & $94.5\phantom{0 \pm 0.00}$ & & $94.7\phantom{0 \pm 0.00}$ & & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{ConvE} & \textbf{pub} & $94.3\phantom{0 \pm 0.00}$ & $93.5\phantom{0 \pm 0.00}$ & $94.6\phantom{0 \pm 0.00}$ & & $95.6\phantom{0 \pm 0.00}$ & $\phantom{00}374.\phantom{00 \pm 000.00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{DistMult} & \textbf{pub} & $83.\phantom{00 \pm 0.00}$ & & & & $94.2\phantom{0 \pm 0.00}$ & & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{HolE} & \textbf{pub} & $93.8\phantom{0 \pm 0.00}$ & $93.\phantom{00 \pm 0.00}$ & $94.5\phantom{0 \pm 0.00}$ & & $94.9\phantom{0 \pm 0.00}$ & & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{KG2E} & \textbf{pub} & & & & & $92.8\phantom{0 \pm 0.00}$ & $\phantom{00}331.\phantom{00 \pm 000.00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{RotatE} & \textbf{pub} & $94.9\phantom{0 \pm 0.00}$ & $94.4\phantom{0 \pm 0.00}$ & $95.2\phantom{0 \pm 0.00}$ & & $95.9\phantom{0 \pm 0.00}$ & $\phantom{00}309.\phantom{00 \pm 000.00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{SimplE} & \textbf{pub} & $94.2\phantom{0 \pm 0.00}$ & $93.9\phantom{0 \pm 0.00}$ & $94.4\phantom{0 \pm 0.00}$ & & $94.7\phantom{0 \pm 0.00}$ & & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{TransD} & \textbf{pub} & & & & & $92.2\phantom{0 \pm 0.00}$ & $\phantom{00}212.\phantom{00 \pm 000.00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{TransE} & \textbf{pub} & & & & & $89.2\phantom{0 \pm 0.00}$ & $\phantom{00}251.\phantom{00 \pm 000.00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{TransH} & \textbf{pub} & & & & & $82.3\phantom{0 \pm 0.00}$ & $\phantom{00}388.\phantom{00 \pm 000.00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{TransR} & \textbf{pub} & & & & & $92.\phantom{00 \pm 0.00}$ & $\phantom{00}225.\phantom{00 \pm 000.00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{TuckER} & \textbf{pub} & $95.3\phantom{0 \pm 0.00}$ & $94.9\phantom{0 \pm 0.00}$ & $95.5\phantom{0 \pm 0.00}$ & & $95.8\phantom{0 \pm 0.00}$ & & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ +\textbf{ComplEx} & \textbf{pub} & $94.1\phantom{0 \pm 0.00}$ & $93.6\phantom{0 \pm 0.00}$ & $94.5\phantom{0 \pm 0.00}$ & & $94.7\phantom{0 \pm 0.00}$ & & \\ + & \textbf{R} & $19.49 \pm 2.55$ & $12.36 \pm 1.96$ & $20.66 \pm 2.75$ & $25.24 \pm 3.33$ & $32.92 \pm 4.40$ & $\phantom{00}452.67 \pm \phantom{0}63.05$ & $\phantom{00}2.21 \pm 0.31$ \\\midrule +\textbf{ConvE} & \textbf{pub} & $94.3\phantom{0 \pm 0.00}$ & $93.5\phantom{0 \pm 0.00}$ & $94.6\phantom{0 \pm 0.00}$ & & $95.6\phantom{0 \pm 0.00}$ & $\phantom{00}374.\phantom{00 \pm 000.00}$ & \\ + & \textbf{R} & $88.81 \pm 0.09$ & $85.14 \pm 0.10$ & $91.76 \pm 0.11$ & $93.29 \pm 0.04$ & $94.85 \pm 0.06$ & $\phantom{00}444.40 \pm \phantom{0}14.82$ & $\phantom{00}2.17 \pm 0.07$ \\\midrule +\textbf{DistMult} & \textbf{pub} & $83.\phantom{00 \pm 0.00}$ & & & & $94.2\phantom{0 \pm 0.00}$ & & \\ + & \textbf{R} & $77.44 \pm 0.22$ & $67.45 \pm 0.34$ & $85.94 \pm 0.21$ & $89.52 \pm 0.25$ & $92.72 \pm 0.18$ & $\phantom{00}458.64 \pm \phantom{0}23.96$ & $\phantom{00}2.24 \pm 0.12$ \\\midrule +\textbf{HolE} & \textbf{pub} & $93.8\phantom{0 \pm 0.00}$ & $93.\phantom{00 \pm 0.00}$ & $94.5\phantom{0 \pm 0.00}$ & & $94.9\phantom{0 \pm 0.00}$ & & \\ + & \textbf{R} & $70.44 \pm 0.45$ & $59.29 \pm 0.53$ & $79.29 \pm 0.47$ & $84.12 \pm 0.36$ & $88.61 \pm 0.42$ & $\phantom{00}812.64 \pm \phantom{0}28.33$ & $\phantom{00}3.97 \pm 0.14$ \\ + & \textbf{O} & $70.44 \pm 0.45$ & $59.29 \pm 0.53$ & $79.29 \pm 0.47$ & $84.12 \pm 0.36$ & $88.61 \pm 0.42$ & $\phantom{00}812.63 \pm \phantom{0}28.33$ & \\ + & \textbf{P} & $70.44 \pm 0.45$ & $59.29 \pm 0.53$ & $79.29 \pm 0.47$ & $84.12 \pm 0.36$ & $88.61 \pm 0.42$ & $\phantom{00}812.64 \pm \phantom{0}28.34$ & \\\midrule +\textbf{KG2E} & \textbf{pub} & & & & & $92.8\phantom{0 \pm 0.00}$ & $\phantom{00}331.\phantom{00 \pm 000.00}$ & \\ + & \textbf{R} & $\phantom{0}3.61 \pm 0.26$ & $\phantom{0}1.35 \pm 0.22$ & $\phantom{0}3.21 \pm 0.31$ & $\phantom{0}4.57 \pm 0.34$ & $\phantom{0}7.02 \pm 0.43$ & $\phantom{0}2708.89 \pm \phantom{0}44.57$ & $\phantom{0}13.25 \pm 0.22$ \\ + & \textbf{O} & $\phantom{0}3.61 \pm 0.26$ & $\phantom{0}1.35 \pm 0.22$ & $\phantom{0}3.21 \pm 0.31$ & $\phantom{0}4.57 \pm 0.34$ & $\phantom{0}7.02 \pm 0.43$ & $\phantom{0}2708.88 \pm \phantom{0}44.57$ & \\ + & \textbf{P} & $\phantom{0}3.61 \pm 0.26$ & $\phantom{0}1.35 \pm 0.22$ & $\phantom{0}3.21 \pm 0.31$ & $\phantom{0}4.57 \pm 0.34$ & $\phantom{0}7.02 \pm 0.43$ & $\phantom{0}2708.89 \pm \phantom{0}44.57$ & \\\midrule +\textbf{RotatE} & \textbf{pub} & $94.9\phantom{0 \pm 0.00}$ & $94.4\phantom{0 \pm 0.00}$ & $95.2\phantom{0 \pm 0.00}$ & & $95.9\phantom{0 \pm 0.00}$ & $\phantom{00}309.\phantom{00 \pm 000.00}$ & \\ + & \textbf{R} & $87.29 \pm 0.12$ & $82.17 \pm 0.20$ & $91.53 \pm 0.12$ & $93.44 \pm 0.07$ & $95.28 \pm 0.08$ & $\phantom{00}123.68 \pm \phantom{00}1.71$ & $\phantom{00}0.61 \pm 0.01$ \\\midrule +\textbf{SimplE} & \textbf{pub} & $94.2\phantom{0 \pm 0.00}$ & $93.9\phantom{0 \pm 0.00}$ & $94.4\phantom{0 \pm 0.00}$ & & $94.7\phantom{0 \pm 0.00}$ & & \\ + & \textbf{R} & $\phantom{0}0.04 \pm 0.01$ & $\phantom{0}0.01 \pm 0.01$ & $\phantom{0}0.03 \pm 0.00$ & $\phantom{0}0.04 \pm 0.01$ & $\phantom{0}0.07 \pm 0.03$ & $20376.43 \pm \phantom{0}42.30$ & $\phantom{0}99.57 \pm 0.21$ \\ + & \textbf{O} & $38.48 \pm 4.00$ & $33.93 \pm 4.32$ & $39.59 \pm 4.67$ & $42.76 \pm 3.73$ & $47.01 \pm 2.66$ & $\phantom{00}384.53 \pm \phantom{0}66.45$ & \\ + & \textbf{P} & $\phantom{0}0.03 \pm 0.01$ & $\phantom{0}0.01 \pm 0.01$ & $\phantom{0}0.03 \pm 0.00$ & $\phantom{0}0.04 \pm 0.01$ & $\phantom{0}0.07 \pm 0.03$ & $40368.33 \pm 114.95$ & \\\midrule +\textbf{TransD} & \textbf{pub} & & & & & $92.2\phantom{0 \pm 0.00}$ & $\phantom{00}212.\phantom{00 \pm 000.00}$ & \\ + & \textbf{R} & $36.22 \pm 0.12$ & $\phantom{0}3.94 \pm 0.27$ & $65.63 \pm 0.55$ & $79.64 \pm 0.43$ & $87.27 \pm 0.41$ & $\phantom{00}444.39 \pm \phantom{0}25.61$ & $\phantom{00}2.17 \pm 0.13$ \\\midrule +\textbf{TransE} & \textbf{pub} & & & & & $89.2\phantom{0 \pm 0.00}$ & $\phantom{00}251.\phantom{00 \pm 000.00}$ & \\ + & \textbf{R} & $39.19 \pm 1.21$ & $\phantom{0}9.99 \pm 1.82$ & $64.74 \pm 0.91$ & $75.44 \pm 0.48$ & $84.25 \pm 0.33$ & $\phantom{00}468.24 \pm \phantom{0}13.64$ & $\phantom{00}2.29 \pm 0.07$ \\\midrule +\textbf{TransH} & \textbf{pub} & & & & & $82.3\phantom{0 \pm 0.00}$ & $\phantom{00}388.\phantom{00 \pm 000.00}$ & \\ + & \textbf{R} & $\phantom{0}0.18 \pm 0.04$ & $\phantom{0}0.04 \pm 0.02$ & $\phantom{0}0.19 \pm 0.10$ & $\phantom{0}0.29 \pm 0.14$ & $\phantom{0}0.39 \pm 0.11$ & $19678.04 \pm \phantom{0}18.92$ & $\phantom{0}96.16 \pm 0.09$ \\ + & \textbf{O} & $\phantom{0}0.18 \pm 0.04$ & $\phantom{0}0.04 \pm 0.02$ & $\phantom{0}0.19 \pm 0.10$ & $\phantom{0}0.29 \pm 0.14$ & $\phantom{0}0.39 \pm 0.11$ & $19678.02 \pm \phantom{0}18.92$ & \\ + & \textbf{P} & $\phantom{0}0.18 \pm 0.04$ & $\phantom{0}0.04 \pm 0.02$ & $\phantom{0}0.19 \pm 0.10$ & $\phantom{0}0.29 \pm 0.14$ & $\phantom{0}0.39 \pm 0.11$ & $19678.06 \pm \phantom{0}18.92$ & \\\midrule +\textbf{TransR} & \textbf{pub} & & & & & $92.\phantom{00 \pm 0.00}$ & $\phantom{00}225.\phantom{00 \pm 000.00}$ & \\ + & \textbf{R} & $\phantom{0}0.06 \pm 0.02$ & $\phantom{0}0.00 \pm 0.00$ & $\phantom{0}0.04 \pm 0.02$ & $\phantom{0}0.05 \pm 0.03$ & $\phantom{0}0.11 \pm 0.06$ & $19686.49 \pm 100.97$ & $\phantom{0}96.20 \pm 0.49$ \\ + & \textbf{O} & $\phantom{0}0.06 \pm 0.02$ & $\phantom{0}0.00 \pm 0.00$ & $\phantom{0}0.04 \pm 0.02$ & $\phantom{0}0.05 \pm 0.03$ & $\phantom{0}0.11 \pm 0.06$ & $19686.49 \pm 100.97$ & \\ + & \textbf{P} & $\phantom{0}0.06 \pm 0.02$ & $\phantom{0}0.00 \pm 0.00$ & $\phantom{0}0.04 \pm 0.02$ & $\phantom{0}0.05 \pm 0.03$ & $\phantom{0}0.11 \pm 0.06$ & $19686.50 \pm 100.97$ & \\\midrule +\textbf{TuckER} & \textbf{pub} & $95.3\phantom{0 \pm 0.00}$ & $94.9\phantom{0 \pm 0.00}$ & $95.5\phantom{0 \pm 0.00}$ & & $95.8\phantom{0 \pm 0.00}$ & & \\ + & \textbf{R} & $\phantom{0}0.03 \pm 0.01$ & $\phantom{0}0.00 \pm 0.00$ & $\phantom{0}0.02 \pm 0.01$ & $\phantom{0}0.03 \pm 0.03$ & $\phantom{0}0.04 \pm 0.03$ & $20622.46 \pm 153.52$ & $100.78 \pm 0.75$ \\ \bottomrule \end{tabular} diff --git a/reproducibility/summaries/wn18_table_without_std.tex b/reproducibility/summaries/wn18_table_without_std.tex index a321a0e8f..6ca0f1383 100644 --- a/reproducibility/summaries/wn18_table_without_std.tex +++ b/reproducibility/summaries/wn18_table_without_std.tex @@ -1,42 +1,42 @@ \begin{tabular}{llrrrrrrr} \toprule - & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ -\textbf{model} & {} & & & & & & & \\ + & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ +\textbf{model} & {} & & & & & & & \\ \midrule -\textbf{ComplEx} & \textbf{pub} & $94.1\phantom{0}$ & $93.6\phantom{0}$ & $94.5\phantom{0}$ & & $94.7\phantom{0}$ & & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{ConvE} & \textbf{pub} & $94.3\phantom{0}$ & $93.5\phantom{0}$ & $94.6\phantom{0}$ & & $95.6\phantom{0}$ & $\phantom{00}374.\phantom{00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{DistMult} & \textbf{pub} & $83.\phantom{00}$ & & & & $94.2\phantom{0}$ & & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{HolE} & \textbf{pub} & $93.8\phantom{0}$ & $93.\phantom{00}$ & $94.5\phantom{0}$ & & $94.9\phantom{0}$ & & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{KG2E} & \textbf{pub} & & & & & $92.8\phantom{0}$ & $\phantom{00}331.\phantom{00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{RotatE} & \textbf{pub} & $94.9\phantom{0}$ & $94.4\phantom{0}$ & $95.2\phantom{0}$ & & $95.9\phantom{0}$ & $\phantom{00}309.\phantom{00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{SimplE} & \textbf{pub} & $94.2\phantom{0}$ & $93.9\phantom{0}$ & $94.4\phantom{0}$ & & $94.7\phantom{0}$ & & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{TransD} & \textbf{pub} & & & & & $92.2\phantom{0}$ & $\phantom{00}212.\phantom{00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{TransE} & \textbf{pub} & & & & & $89.2\phantom{0}$ & $\phantom{00}251.\phantom{00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{TransH} & \textbf{pub} & & & & & $82.3\phantom{0}$ & $\phantom{00}388.\phantom{00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{TransR} & \textbf{pub} & & & & & $92.\phantom{00}$ & $\phantom{00}225.\phantom{00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{TuckER} & \textbf{pub} & $95.3\phantom{0}$ & $94.9\phantom{0}$ & $95.5\phantom{0}$ & & $95.8\phantom{0}$ & & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ +\textbf{ComplEx} & \textbf{pub} & $94.1\phantom{0}$ & $93.6\phantom{0}$ & $94.5\phantom{0}$ & & $94.7\phantom{0}$ & & \\ + & \textbf{R} & $19.49$ & $12.36$ & $20.66$ & $25.24$ & $32.92$ & $\phantom{00}452.67$ & $\phantom{00}2.21$ \\\midrule +\textbf{ConvE} & \textbf{pub} & $94.3\phantom{0}$ & $93.5\phantom{0}$ & $94.6\phantom{0}$ & & $95.6\phantom{0}$ & $\phantom{00}374.\phantom{00}$ & \\ + & \textbf{R} & $88.81$ & $85.14$ & $91.76$ & $93.29$ & $94.85$ & $\phantom{00}444.40$ & $\phantom{00}2.17$ \\\midrule +\textbf{DistMult} & \textbf{pub} & $83.\phantom{00}$ & & & & $94.2\phantom{0}$ & & \\ + & \textbf{R} & $77.44$ & $67.45$ & $85.94$ & $89.52$ & $92.72$ & $\phantom{00}458.64$ & $\phantom{00}2.24$ \\\midrule +\textbf{HolE} & \textbf{pub} & $93.8\phantom{0}$ & $93.\phantom{00}$ & $94.5\phantom{0}$ & & $94.9\phantom{0}$ & & \\ + & \textbf{R} & $70.44$ & $59.29$ & $79.29$ & $84.12$ & $88.61$ & $\phantom{00}812.64$ & $\phantom{00}3.97$ \\ + & \textbf{O} & $70.44$ & $59.29$ & $79.29$ & $84.12$ & $88.61$ & $\phantom{00}812.63$ & \\ + & \textbf{P} & $70.44$ & $59.29$ & $79.29$ & $84.12$ & $88.61$ & $\phantom{00}812.64$ & \\\midrule +\textbf{KG2E} & \textbf{pub} & & & & & $92.8\phantom{0}$ & $\phantom{00}331.\phantom{00}$ & \\ + & \textbf{R} & $\phantom{0}3.61$ & $\phantom{0}1.35$ & $\phantom{0}3.21$ & $\phantom{0}4.57$ & $\phantom{0}7.02$ & $\phantom{0}2708.89$ & $\phantom{0}13.25$ \\ + & \textbf{O} & $\phantom{0}3.61$ & $\phantom{0}1.35$ & $\phantom{0}3.21$ & $\phantom{0}4.57$ & $\phantom{0}7.02$ & $\phantom{0}2708.88$ & \\ + & \textbf{P} & $\phantom{0}3.61$ & $\phantom{0}1.35$ & $\phantom{0}3.21$ & $\phantom{0}4.57$ & $\phantom{0}7.02$ & $\phantom{0}2708.89$ & \\\midrule +\textbf{RotatE} & \textbf{pub} & $94.9\phantom{0}$ & $94.4\phantom{0}$ & $95.2\phantom{0}$ & & $95.9\phantom{0}$ & $\phantom{00}309.\phantom{00}$ & \\ + & \textbf{R} & $87.29$ & $82.17$ & $91.53$ & $93.44$ & $95.28$ & $\phantom{00}123.68$ & $\phantom{00}0.61$ \\\midrule +\textbf{SimplE} & \textbf{pub} & $94.2\phantom{0}$ & $93.9\phantom{0}$ & $94.4\phantom{0}$ & & $94.7\phantom{0}$ & & \\ + & \textbf{R} & $\phantom{0}0.04$ & $\phantom{0}0.01$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $\phantom{0}0.07$ & $20376.43$ & $\phantom{0}99.57$ \\ + & \textbf{O} & $38.48$ & $33.93$ & $39.59$ & $42.76$ & $47.01$ & $\phantom{00}384.53$ & \\ + & \textbf{P} & $\phantom{0}0.03$ & $\phantom{0}0.01$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $\phantom{0}0.07$ & $40368.33$ & \\\midrule +\textbf{TransD} & \textbf{pub} & & & & & $92.2\phantom{0}$ & $\phantom{00}212.\phantom{00}$ & \\ + & \textbf{R} & $36.22$ & $\phantom{0}3.94$ & $65.63$ & $79.64$ & $87.27$ & $\phantom{00}444.39$ & $\phantom{00}2.17$ \\\midrule +\textbf{TransE} & \textbf{pub} & & & & & $89.2\phantom{0}$ & $\phantom{00}251.\phantom{00}$ & \\ + & \textbf{R} & $39.19$ & $\phantom{0}9.99$ & $64.74$ & $75.44$ & $84.25$ & $\phantom{00}468.24$ & $\phantom{00}2.29$ \\\midrule +\textbf{TransH} & \textbf{pub} & & & & & $82.3\phantom{0}$ & $\phantom{00}388.\phantom{00}$ & \\ + & \textbf{R} & $\phantom{0}0.18$ & $\phantom{0}0.04$ & $\phantom{0}0.19$ & $\phantom{0}0.29$ & $\phantom{0}0.39$ & $19678.04$ & $\phantom{0}96.16$ \\ + & \textbf{O} & $\phantom{0}0.18$ & $\phantom{0}0.04$ & $\phantom{0}0.19$ & $\phantom{0}0.29$ & $\phantom{0}0.39$ & $19678.02$ & \\ + & \textbf{P} & $\phantom{0}0.18$ & $\phantom{0}0.04$ & $\phantom{0}0.19$ & $\phantom{0}0.29$ & $\phantom{0}0.39$ & $19678.06$ & \\\midrule +\textbf{TransR} & \textbf{pub} & & & & & $92.\phantom{00}$ & $\phantom{00}225.\phantom{00}$ & \\ + & \textbf{R} & $\phantom{0}0.06$ & $\phantom{0}0.00$ & $\phantom{0}0.04$ & $\phantom{0}0.05$ & $\phantom{0}0.11$ & $19686.49$ & $\phantom{0}96.20$ \\ + & \textbf{O} & $\phantom{0}0.06$ & $\phantom{0}0.00$ & $\phantom{0}0.04$ & $\phantom{0}0.05$ & $\phantom{0}0.11$ & $19686.49$ & \\ + & \textbf{P} & $\phantom{0}0.06$ & $\phantom{0}0.00$ & $\phantom{0}0.04$ & $\phantom{0}0.05$ & $\phantom{0}0.11$ & $19686.50$ & \\\midrule +\textbf{TuckER} & \textbf{pub} & $95.3\phantom{0}$ & $94.9\phantom{0}$ & $95.5\phantom{0}$ & & $95.8\phantom{0}$ & & \\ + & \textbf{R} & $\phantom{0}0.03$ & $\phantom{0}0.00$ & $\phantom{0}0.02$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $20622.46$ & $100.78$ \\ \bottomrule \end{tabular} diff --git a/reproducibility/summaries/wn18rr_table.tex b/reproducibility/summaries/wn18rr_table.tex index fc2715392..d206e1fc6 100644 --- a/reproducibility/summaries/wn18rr_table.tex +++ b/reproducibility/summaries/wn18rr_table.tex @@ -1,16 +1,16 @@ \begin{tabular}{llrrrrrrr} \toprule - & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ -\textbf{model} & {} & & & & & & & \\ + & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ +\textbf{model} & {} & & & & & & & \\ \midrule -\textbf{ConvE} & \textbf{pub} & $43.\phantom{00 \pm 0.00}$ & $\phantom{0}4.\phantom{00 \pm 0.00}$ & $44.\phantom{00 \pm 0.00}$ & & $52.\phantom{00 \pm 0.00}$ & $\phantom{0}4187.\phantom{00 \pm 000.00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{ConvKB} & \textbf{pub} & $24.8\phantom{0 \pm 0.00}$ & & & & $52.5\phantom{0 \pm 0.00}$ & $\phantom{0}2554.\phantom{00 \pm 000.00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{RotatE} & \textbf{pub} & $47.6\phantom{0 \pm 0.00}$ & $42.8\phantom{0 \pm 0.00}$ & $49.2\phantom{0 \pm 0.00}$ & & $57.1\phantom{0 \pm 0.00}$ & $\phantom{0}3340.\phantom{00 \pm 000.00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ +\textbf{ConvE} & \textbf{pub} & $43.\phantom{00 \pm 0.00}$ & $\phantom{0}4.\phantom{00 \pm 0.00}$ & $44.\phantom{00 \pm 0.00}$ & & $52.\phantom{00 \pm 0.00}$ & $\phantom{0}4187.\phantom{00 \pm 000.00}$ & \\ + & \textbf{R} & $44.69 \pm 0.21$ & $40.98 \pm 0.22$ & $46.49 \pm 0.14$ & $48.92 \pm 0.23$ & $51.76 \pm 0.13$ & $\phantom{0}5369.49 \pm \phantom{0}50.92$ & $26.49 \pm 0.25$ \\\midrule +\textbf{ConvKB} & \textbf{pub} & $24.8\phantom{0 \pm 0.00}$ & & & & $52.5\phantom{0 \pm 0.00}$ & $\phantom{0}2554.\phantom{00 \pm 000.00}$ & \\ + & \textbf{R} & $\phantom{0}0.30 \pm 0.07$ & $\phantom{0}0.09 \pm 0.03$ & $\phantom{0}0.21 \pm 0.07$ & $\phantom{0}0.32 \pm 0.10$ & $\phantom{0}0.57 \pm 0.16$ & $13634.66 \pm 714.24$ & $67.27 \pm 3.52$ \\ + & \textbf{O} & $\phantom{0}0.30 \pm 0.07$ & $\phantom{0}0.09 \pm 0.03$ & $\phantom{0}0.21 \pm 0.07$ & $\phantom{0}0.32 \pm 0.10$ & $\phantom{0}0.57 \pm 0.16$ & $13634.65 \pm 714.24$ & \\ + & \textbf{P} & $\phantom{0}0.30 \pm 0.07$ & $\phantom{0}0.09 \pm 0.03$ & $\phantom{0}0.21 \pm 0.07$ & $\phantom{0}0.32 \pm 0.10$ & $\phantom{0}0.57 \pm 0.16$ & $13634.66 \pm 714.24$ & \\\midrule +\textbf{RotatE} & \textbf{pub} & $47.6\phantom{0 \pm 0.00}$ & $42.8\phantom{0 \pm 0.00}$ & $49.2\phantom{0 \pm 0.00}$ & & $57.1\phantom{0 \pm 0.00}$ & $\phantom{0}3340.\phantom{00 \pm 000.00}$ & \\ + & \textbf{R} & $48.40 \pm 0.09$ & $44.02 \pm 0.15$ & $50.55 \pm 0.12$ & $52.98 \pm 0.11$ & $56.51 \pm 0.26$ & $\phantom{0}4263.32 \pm \phantom{0}90.33$ & $21.03 \pm 0.45$ \\ \bottomrule \end{tabular} diff --git a/reproducibility/summaries/wn18rr_table_without_std.tex b/reproducibility/summaries/wn18rr_table_without_std.tex index 987f485f5..5b2d483a1 100644 --- a/reproducibility/summaries/wn18rr_table_without_std.tex +++ b/reproducibility/summaries/wn18rr_table_without_std.tex @@ -1,16 +1,16 @@ \begin{tabular}{llrrrrrrr} \toprule - & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ -\textbf{model} & {} & & & & & & & \\ + & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ +\textbf{model} & {} & & & & & & & \\ \midrule -\textbf{ConvE} & \textbf{pub} & $43.\phantom{00}$ & $\phantom{0}4.\phantom{00}$ & $44.\phantom{00}$ & & $52.\phantom{00}$ & $\phantom{0}4187.\phantom{00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{ConvKB} & \textbf{pub} & $24.8\phantom{0}$ & & & & $52.5\phantom{0}$ & $\phantom{0}2554.\phantom{00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{O} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ - & \textbf{P} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\\midrule -\textbf{RotatE} & \textbf{pub} & $47.6\phantom{0}$ & $42.8\phantom{0}$ & $49.2\phantom{0}$ & & $57.1\phantom{0}$ & $\phantom{0}3340.\phantom{00}$ & \\ - & \textbf{R} & NaN & NaN & NaN & NaN & NaN & NaN & NaN \\ +\textbf{ConvE} & \textbf{pub} & $43.\phantom{00}$ & $\phantom{0}4.\phantom{00}$ & $44.\phantom{00}$ & & $52.\phantom{00}$ & $\phantom{0}4187.\phantom{00}$ & \\ + & \textbf{R} & $44.69$ & $40.98$ & $46.49$ & $48.92$ & $51.76$ & $\phantom{0}5369.49$ & $26.49$ \\\midrule +\textbf{ConvKB} & \textbf{pub} & $24.8\phantom{0}$ & & & & $52.5\phantom{0}$ & $\phantom{0}2554.\phantom{00}$ & \\ + & \textbf{R} & $\phantom{0}0.30$ & $\phantom{0}0.09$ & $\phantom{0}0.21$ & $\phantom{0}0.32$ & $\phantom{0}0.57$ & $13634.66$ & $67.27$ \\ + & \textbf{O} & $\phantom{0}0.30$ & $\phantom{0}0.09$ & $\phantom{0}0.21$ & $\phantom{0}0.32$ & $\phantom{0}0.57$ & $13634.65$ & \\ + & \textbf{P} & $\phantom{0}0.30$ & $\phantom{0}0.09$ & $\phantom{0}0.21$ & $\phantom{0}0.32$ & $\phantom{0}0.57$ & $13634.66$ & \\\midrule +\textbf{RotatE} & \textbf{pub} & $47.6\phantom{0}$ & $42.8\phantom{0}$ & $49.2\phantom{0}$ & & $57.1\phantom{0}$ & $\phantom{0}3340.\phantom{00}$ & \\ + & \textbf{R} & $48.40$ & $44.02$ & $50.55$ & $52.98$ & $56.51$ & $\phantom{0}4263.32$ & $21.03$ \\ \bottomrule \end{tabular}