diff --git a/reproducibility/configs/complex/trouillon2016_complex_fb15k.json b/reproducibility/configs/complex/trouillon2016_complex_fb15k.json index a9a1faeb1..152129510 100644 --- a/reproducibility/configs/complex/trouillon2016_complex_fb15k.json +++ b/reproducibility/configs/complex/trouillon2016_complex_fb15k.json @@ -1,7 +1,7 @@ { "metadata": { "title": "Learn FB15K Data Set with ComplEx as described by Trouillon et al., 2016", - "comments": "In the paper it is not mentioned that they compute the mean of the vector norms as done in the published code base." + "comments": "In the paper it is not mentioned that they compute the mean of the vector norms as done in the published code base. The rank type can be found at https://github.com/ttrouill/complex/blob/67fef2324d5a7695f7b932b5606d2e4b6de09a41/efe/evaluation.py#L295." }, "pipeline": { "dataset": "fb15k", @@ -36,5 +36,17 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "mean_reciprocal_rank": { + "best": 0.692 + }, + "hits_at_k": { + "best": { + "1": 0.599, + "3": 0.759, + "10": 0.840 + } + } } } \ No newline at end of file diff --git a/reproducibility/configs/complex/trouillon2016_complex_wn18.json b/reproducibility/configs/complex/trouillon2016_complex_wn18.json index 2d4d2e8e1..a5586d14d 100644 --- a/reproducibility/configs/complex/trouillon2016_complex_wn18.json +++ b/reproducibility/configs/complex/trouillon2016_complex_wn18.json @@ -1,7 +1,7 @@ { "metadata": { "title": "Learn WN18 Data Set with ComplEx as described by Trouillon et al., 2016", - "comments": "In the paper it is not mentioned that they compute the mean of the vector norms as done in the published code base." + "comments": "In the paper it is not mentioned that they compute the mean of the vector norms as done in the published code base. The rank type can be found at https://github.com/ttrouill/complex/blob/67fef2324d5a7695f7b932b5606d2e4b6de09a41/efe/evaluation.py#L295." }, "pipeline": { "dataset": "wn18", @@ -36,5 +36,17 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "mean_reciprocal_rank": { + "best": 0.941 + }, + "hits_at_k": { + "best": { + "1": 0.936, + "3": 0.945, + "10": 0.947 + } + } } -} +} \ No newline at end of file diff --git a/reproducibility/configs/conve/dettmers2018_conve_fb15k.json b/reproducibility/configs/conve/dettmers2018_conve_fb15k.json index 71889876e..a2a0013df 100644 --- a/reproducibility/configs/conve/dettmers2018_conve_fb15k.json +++ b/reproducibility/configs/conve/dettmers2018_conve_fb15k.json @@ -1,6 +1,7 @@ { "metadata": { - "title": "Learn FB15K Data Set with ConvE as described by Dettmers et al., 2018" + "title": "Learn FB15K Data Set with ConvE as described by Dettmers et al., 2018", + "comment": "For evaluation, the non-deterministic rank definition is used, cf. https://github.com/TimDettmers/ConvE/blob/5feb358eb7dbd1f534978cdc4c20ee0bf919148a/evaluation.py#L67." }, "pipeline": { "dataset": "fb15k", @@ -18,7 +19,8 @@ "kernel_width": 3, "input_dropout": 0.2, "feature_map_dropout": 0.2, - "output_dropout": 0.3 + "output_dropout": 0.3, + "apply_batch_normalization": true }, "optimizer": "Adam", "optimizer_kwargs": { @@ -36,9 +38,21 @@ }, "evaluator_kwargs": { "filtered": true + } + }, + "results": { + "mean_rank": { + "nondeterministic": 51 + }, + "hits_at_k": { + "nondeterministic": { + "1": 0.558, + "3": 0.723, + "10": 0.831 + } }, - "evaluation_kwargs": { - "batch_size": 16 + "mean_reciprocal_rank": { + "nondeterministic": 0.657 } } } \ No newline at end of file diff --git a/reproducibility/configs/conve/dettmers2018_conve_fb15k237.json b/reproducibility/configs/conve/dettmers2018_conve_fb15k237.json index 9d232c5dc..45286cafb 100644 --- a/reproducibility/configs/conve/dettmers2018_conve_fb15k237.json +++ b/reproducibility/configs/conve/dettmers2018_conve_fb15k237.json @@ -1,6 +1,7 @@ { "metadata": { - "title": "Learn FB15K-237 Data Set with ConvE as described by Dettmers et al., 2018" + "title": "Learn FB15K-237 Data Set with ConvE as described by Dettmers et al., 2018", + "comment": "For evaluation, the non-deterministic rank definition is used, cf. https://github.com/TimDettmers/ConvE/blob/5feb358eb7dbd1f534978cdc4c20ee0bf919148a/evaluation.py#L67." }, "pipeline": { "dataset": "fb15k237", @@ -18,7 +19,8 @@ "kernel_width": 3, "input_dropout": 0.2, "feature_map_dropout": 0.2, - "output_dropout": 0.3 + "output_dropout": 0.3, + "apply_batch_normalization": true }, "optimizer": "Adam", "optimizer_kwargs": { @@ -36,9 +38,21 @@ }, "evaluator_kwargs": { "filtered": true + } + }, + "results": { + "mean_rank": { + "nondeterministic": 244 + }, + "hits_at_k": { + "nondeterministic": { + "1": 0.237, + "3": 0.356, + "10": 0.501 + } }, - "evaluation_kwargs": { - "batch_size": 16 + "mean_reciprocal_rank": { + "nondeterministic": 0.325 } } } \ No newline at end of file diff --git a/reproducibility/configs/conve/dettmers2018_conve_wn18.json b/reproducibility/configs/conve/dettmers2018_conve_wn18.json index aafd08a6b..89182e8fe 100644 --- a/reproducibility/configs/conve/dettmers2018_conve_wn18.json +++ b/reproducibility/configs/conve/dettmers2018_conve_wn18.json @@ -1,6 +1,7 @@ { "metadata": { - "title": "Learn WN18 Data Set with ConvE as described by Dettmers et al., 2018" + "title": "Learn WN18 Data Set with ConvE as described by Dettmers et al., 2018", + "comment": "For evaluation, the non-deterministic rank definition is used, cf. https://github.com/TimDettmers/ConvE/blob/5feb358eb7dbd1f534978cdc4c20ee0bf919148a/evaluation.py#L67." }, "pipeline": { "dataset": "wn18", @@ -18,7 +19,8 @@ "kernel_width": 3, "input_dropout": 0.2, "feature_map_dropout": 0.2, - "output_dropout": 0.3 + "output_dropout": 0.3, + "apply_batch_normalization": true }, "optimizer": "Adam", "optimizer_kwargs": { @@ -36,9 +38,21 @@ }, "evaluator_kwargs": { "filtered": true + } + }, + "results": { + "mean_rank": { + "nondeterministic": 374 + }, + "hits_at_k": { + "nondeterministic": { + "1": 0.935, + "3": 0.946, + "10": 0.956 + } }, - "evaluation_kwargs": { - "batch_size": 16 + "mean_reciprocal_rank": { + "nondeterministic": 0.943 } } } \ No newline at end of file diff --git a/reproducibility/configs/conve/dettmers2018_conve_wn18rr.json b/reproducibility/configs/conve/dettmers2018_conve_wn18rr.json index 1ae46122e..b79984a66 100644 --- a/reproducibility/configs/conve/dettmers2018_conve_wn18rr.json +++ b/reproducibility/configs/conve/dettmers2018_conve_wn18rr.json @@ -1,13 +1,14 @@ { "metadata": { - "title": "Learn WN18RR Data Set with ConvE as described by Dettmers et al., 2018" + "title": "Learn WN18RR Data Set with ConvE as described by Dettmers et al., 2018", + "comment": "For evaluation, the non-deterministic rank definition is used, cf. https://github.com/TimDettmers/ConvE/blob/5feb358eb7dbd1f534978cdc4c20ee0bf919148a/evaluation.py#L67." }, "pipeline": { "dataset": "wn18rr", + "model": "ConvE", "dataset_kwargs": { "create_inverse_triples": true }, - "model": "ConvE", "model_kwargs": { "embedding_dim": 200, "input_channels": 1, @@ -18,7 +19,8 @@ "kernel_width": 3, "input_dropout": 0.2, "feature_map_dropout": 0.2, - "output_dropout": 0.3 + "output_dropout": 0.3, + "apply_batch_normalization": true }, "optimizer": "Adam", "optimizer_kwargs": { @@ -36,9 +38,21 @@ }, "evaluator_kwargs": { "filtered": true + } + }, + "results": { + "mean_rank": { + "nondeterministic": 4187 + }, + "hits_at_k": { + "nondeterministic": { + "1": 0.40, + "3": 0.44, + "10": 0.52 + } }, - "evaluation_kwargs": { - "batch_size": 16 + "mean_reciprocal_rank": { + "nondeterministic": 0.43 } } } \ No newline at end of file diff --git a/reproducibility/configs/convkb/nguyen2018_convkb_fb15k237.json b/reproducibility/configs/convkb/nguyen2018_convkb_fb15k237.json index 42bf78078..a32d7a110 100644 --- a/reproducibility/configs/convkb/nguyen2018_convkb_fb15k237.json +++ b/reproducibility/configs/convkb/nguyen2018_convkb_fb15k237.json @@ -1,7 +1,7 @@ { "metadata": { "title": "Learn FB15k-237 Data Set with ConvKB as described by Nguyen et al., 2018", - "comments": "regularization weight is set to 0.0005, because in the paper the regularization term is multiplied with (regularization weight)/2" + "comments": "regularization weight is set to 0.0005, because in the paper the regularization term is multiplied with (regularization weight)/2. The evaluation is done with the optimistic rank, cf. https://github.com/daiquocnguyen/ConvKB/blob/ba02c0665a80751676289a8d5570dc420465a9ff/eval.py#L207-L236." }, "pipeline": { "dataset": "fb15k237", @@ -37,9 +37,19 @@ }, "evaluator_kwargs": { "filtered": true + } + }, + "results": { + "mean_rank": { + "best": 257 + }, + "hits_at_k": { + "best": { + "10": 0.517 + } }, - "evaluation_kwargs": { - "batch_size": 16 + "mean_reciprocal_rank": { + "best": 0.396 } } } \ No newline at end of file diff --git a/reproducibility/configs/convkb/nguyen2018_convkb_wn18rr.json b/reproducibility/configs/convkb/nguyen2018_convkb_wn18rr.json index 70c4c8da1..3acac58d5 100644 --- a/reproducibility/configs/convkb/nguyen2018_convkb_wn18rr.json +++ b/reproducibility/configs/convkb/nguyen2018_convkb_wn18rr.json @@ -1,7 +1,7 @@ { "metadata": { "title": "Learn WN18RR Data Set with ConvKB as described by Nguyen et al., 2018", - "comments": "regularization weight is set to 0.0005, because in the paper the regularization term is multiplied with (regularization weight)/2" + "comments": "regularization weight is set to 0.0005, because in the paper the regularization term is multiplied with (regularization weight)/2. The evaluation is done with the optimistic rank, cf. https://github.com/daiquocnguyen/ConvKB/blob/ba02c0665a80751676289a8d5570dc420465a9ff/eval.py#L207-L236." }, "pipeline": { "dataset": "wn18rr", @@ -37,9 +37,19 @@ }, "evaluator_kwargs": { "filtered": true + } + }, + "results": { + "mean_rank": { + "best": 2554 + }, + "hits_at_k": { + "best": { + "10": 0.525 + } }, - "evaluation_kwargs": { - "batch_size": 16 + "mean_reciprocal_rank": { + "best": 0.248 } } } \ No newline at end of file diff --git a/reproducibility/configs/convkb/nguyen2018_transe_fb15k237.json b/reproducibility/configs/convkb/nguyen2018_transe_fb15k237.json index 6510e22cc..a94f9b08e 100644 --- a/reproducibility/configs/convkb/nguyen2018_transe_fb15k237.json +++ b/reproducibility/configs/convkb/nguyen2018_transe_fb15k237.json @@ -30,9 +30,6 @@ }, "evaluator_kwargs": { "filtered": true - }, - "evaluation_kwargs": { - "batch_size": 16 } } } \ No newline at end of file diff --git a/reproducibility/configs/convkb/nguyen2018_transe_wn18rr.json b/reproducibility/configs/convkb/nguyen2018_transe_wn18rr.json index 41955e583..59ddd7d02 100644 --- a/reproducibility/configs/convkb/nguyen2018_transe_wn18rr.json +++ b/reproducibility/configs/convkb/nguyen2018_transe_wn18rr.json @@ -20,7 +20,7 @@ "margin": 5 }, "training_loop": "OWA", - "negative_sampler": "bernoulli", + "negative_sampler_cls": "bernoulli", "negative_sampler_kwargs": { "num_negs_per_pos": 1 }, @@ -30,9 +30,6 @@ }, "evaluator_kwargs": { "filtered": true - }, - "evaluation_kwargs": { - "batch_size": 16 } } } \ No newline at end of file diff --git a/reproducibility/configs/distmult/yang2014_distmult_fb15k.json b/reproducibility/configs/distmult/yang2014_distmult_fb15k.json index cf51683bf..3acfa0a9a 100644 --- a/reproducibility/configs/distmult/yang2014_distmult_fb15k.json +++ b/reproducibility/configs/distmult/yang2014_distmult_fb15k.json @@ -37,5 +37,15 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "hits_at_k": { + "unknown": { + "10": 0.577 + } + }, + "mean_reciprocal_rank": { + "unknown": 0.35 + } } } \ No newline at end of file diff --git a/reproducibility/configs/distmult/yang2014_distmult_wn18.json b/reproducibility/configs/distmult/yang2014_distmult_wn18.json index a0851c439..6bc6d1fc4 100644 --- a/reproducibility/configs/distmult/yang2014_distmult_wn18.json +++ b/reproducibility/configs/distmult/yang2014_distmult_wn18.json @@ -37,5 +37,15 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "hits_at_k": { + "unknown": { + "10": 0.942 + } + }, + "mean_reciprocal_rank": { + "unknown": 0.83 + } } } \ No newline at end of file diff --git a/reproducibility/configs/hole/nickel2016_hole_fb15k.json b/reproducibility/configs/hole/nickel2016_hole_fb15k.json index 5daf99b30..100c87d6d 100644 --- a/reproducibility/configs/hole/nickel2016_hole_fb15k.json +++ b/reproducibility/configs/hole/nickel2016_hole_fb15k.json @@ -1,14 +1,13 @@ { "metadata": { "title": "Learn FB15K Data Set with HolE as described by Nickel et al., 2016", - "comments": " We use same setting as for WN18 (based on https://github.com/mnick/holographic-embeddings/blob/master/run_hole_wn18.sh, however it is not sure whether this is the correct setting) since we couln't find the setting for FB15k." + "comments": " We use same setting as for WN18 (based on https://github.com/mnick/holographic-embeddings/blob/c2db6e1554e671ab8e6acace78ec1fd91d6a4b90/run_hole_wn18.sh, however it is not sure whether this is the correct setting) since we couldn't find the setting for FB15k. The evaluation uses the non-deterministic rank, as seen from https://github.com/mnick/holographic-embeddings/blob/c2db6e1554e671ab8e6acace78ec1fd91d6a4b90/kg/base.py#L198." }, "pipeline": { "dataset": "fb15k", "model": "HolE", "model_kwargs": { - "embedding_dim": 150, - "automatic_memory_optimization": false + "embedding_dim": 150 }, "optimizer": "AdaGrad", "optimizer_kwargs": { @@ -30,9 +29,18 @@ }, "evaluator_kwargs": { "filtered": true + } + }, + "results": { + "hits_at_k": { + "nondeterministic": { + "1": 0.402, + "3": 0.613, + "10": 0.739 + } }, - "evaluation_kwargs": { - "batch_size": 32 + "mean_reciprocal_rank": { + "nondeterministic": 0.524 } } } \ No newline at end of file diff --git a/reproducibility/configs/hole/nickel2016_hole_wn18.json b/reproducibility/configs/hole/nickel2016_hole_wn18.json index b6240090d..636aaf466 100644 --- a/reproducibility/configs/hole/nickel2016_hole_wn18.json +++ b/reproducibility/configs/hole/nickel2016_hole_wn18.json @@ -1,7 +1,7 @@ { "metadata": { "title": "Learn WN18 Data Set with HolE as described by Nickel et al., 2016", - "comments": "Based on https://github.com/mnick/holographic-embeddings/blob/master/run_hole_wn18.sh, however it is not sure whether this is the correct setting" + "comments": "Based on https://github.com/mnick/holographic-embeddings/blob/master/run_hole_wn18.sh, however it is not sure whether this is the correct setting. The evaluation uses the non-deterministic rank, as seen from https://github.com/mnick/holographic-embeddings/blob/c2db6e1554e671ab8e6acace78ec1fd91d6a4b90/kg/base.py#L198." }, "pipeline": { "dataset": "wn18", @@ -25,10 +25,22 @@ }, "training_kwargs": { "num_epochs": 500, - "batch_size": 4831 + "batch_size": 1414 }, "evaluator_kwargs": { "filtered": true } + }, + "results": { + "hits_at_k": { + "nondeterministic": { + "1": 0.930, + "3": 0.945, + "10": 0.949 + } + }, + "mean_reciprocal_rank": { + "nondeterministic": 0.938 + } } } \ No newline at end of file diff --git a/reproducibility/configs/kg2e/he2015_kg2e_fb15k.json b/reproducibility/configs/kg2e/he2015_kg2e_fb15k.json index 45a3b770f..b286b75f7 100644 --- a/reproducibility/configs/kg2e/he2015_kg2e_fb15k.json +++ b/reproducibility/configs/kg2e/he2015_kg2e_fb15k.json @@ -31,5 +31,15 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "hits_at_k": { + "unknown": { + "10": 0.715 + } + }, + "mean_rank": { + "unknown": 59 + } } } \ No newline at end of file diff --git a/reproducibility/configs/kg2e/he2015_kg2e_wn18.json b/reproducibility/configs/kg2e/he2015_kg2e_wn18.json index 0e465b897..2dee3e03c 100644 --- a/reproducibility/configs/kg2e/he2015_kg2e_wn18.json +++ b/reproducibility/configs/kg2e/he2015_kg2e_wn18.json @@ -31,5 +31,15 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "hits_at_k": { + "unknown": { + "10": 0.928 + } + }, + "mean_rank": { + "unknown": 331 + } } } \ No newline at end of file diff --git a/reproducibility/configs/rgcn/schlichtkrull2018_rgcn_fb15k.json b/reproducibility/configs/rgcn/schlichtkrull2018_rgcn_fb15k.json index 1788a92a1..97c7111a6 100644 --- a/reproducibility/configs/rgcn/schlichtkrull2018_rgcn_fb15k.json +++ b/reproducibility/configs/rgcn/schlichtkrull2018_rgcn_fb15k.json @@ -26,7 +26,8 @@ }, "training_kwargs": { "num_epochs": 500, - "batch_size": 30000 + "batch_size": 30000, + "sub_batch_size": 800 }, "evaluator_kwargs": { "filtered": true diff --git a/reproducibility/configs/rgcn/schlichtkrull2018_rgcn_wn18.json b/reproducibility/configs/rgcn/schlichtkrull2018_rgcn_wn18.json index ba4cbb93e..a65e734be 100644 --- a/reproducibility/configs/rgcn/schlichtkrull2018_rgcn_wn18.json +++ b/reproducibility/configs/rgcn/schlichtkrull2018_rgcn_wn18.json @@ -26,7 +26,8 @@ }, "training_kwargs": { "num_epochs": 500, - "batch_size": 30000 + "batch_size": 30000, + "sub_batch_size": 1000 }, "evaluator_kwargs": { "filtered": true diff --git a/reproducibility/configs/rotate/sun2019_rotate_fb15k.json b/reproducibility/configs/rotate/sun2019_rotate_fb15k.json index 3dc2781f7..0808064c5 100644 --- a/reproducibility/configs/rotate/sun2019_rotate_fb15k.json +++ b/reproducibility/configs/rotate/sun2019_rotate_fb15k.json @@ -1,7 +1,7 @@ { "metadata": { "title": "Learn FB15K Data Set with RotatE as described by Sun et al., 2019", - "comments": "Could not find initial learning rate; could not find number of epochs" + "comments": "Could not find number of epochs since early stopping has been applied; https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/master/best_config.sh. The evaluation uses the non-deterministic rank, cf. https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/a0a3cf75e8e324ef113472489a16927c39c997b2/codes/model.py#L406 ." }, "pipeline": { "dataset": "fb15k", @@ -11,7 +11,7 @@ }, "optimizer": "Adam", "optimizer_kwargs": { - "lr": 0.001 + "lr": 0.0001 }, "loss": "nssa", "loss_kwargs": { @@ -32,5 +32,20 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "mean_rank": { + "nondeterministic": 40 + }, + "hits_at_k": { + "nondeterministic": { + "1": 0.746, + "3": 0.830, + "10": 0.884 + } + }, + "mean_reciprocal_rank": { + "nondeterministic": 0.797 + } } } \ No newline at end of file diff --git a/reproducibility/configs/rotate/sun2019_rotate_fb15k237.json b/reproducibility/configs/rotate/sun2019_rotate_fb15k237.json new file mode 100644 index 000000000..350fff450 --- /dev/null +++ b/reproducibility/configs/rotate/sun2019_rotate_fb15k237.json @@ -0,0 +1,50 @@ +{ + "metadata": { + "title": "Learn FB15K-237 Data Set with RotatE as described by Sun et al., 2019", + "comments": "Could not find number of epochs since early stopping has been applied; https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/master/best_config.sh. The evaluation uses the non-deterministic rank, cf. https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/a0a3cf75e8e324ef113472489a16927c39c997b2/codes/model.py#L406 ." + }, + "pipeline": { + "dataset": "fb15k237", + "model": "RotatE", + "model_kwargs": { + "embedding_dim": 1000 + }, + "optimizer": "Adam", + "optimizer_kwargs": { + "lr": 0.00005 + }, + "loss": "nssa", + "loss_kwargs": { + "reduction": "mean", + "adversarial_temperature": 1.0, + "margin": 9 + }, + "training_loop": "OWA", + "negative_sampler": "basic", + "negative_sampler_kwargs": { + "num_negs_per_pos": 256 + }, + "training_kwargs": { + "num_epochs": 1000, + "batch_size": 1024 + }, + "evaluator_kwargs": { + "filtered": true + } + }, + "results": { + "mean_rank": { + "nondeterministic": 177 + }, + "hits_at_k": { + "nondeterministic": { + "1": 0.241, + "3": 0.375, + "10": 0.533 + } + }, + "mean_reciprocal_rank": { + "nondeterministic": 0.338 + } + } +} \ No newline at end of file diff --git a/reproducibility/configs/rotate/sun2019_rotate_wn18.json b/reproducibility/configs/rotate/sun2019_rotate_wn18.json index 03af53176..e5e4eb439 100644 --- a/reproducibility/configs/rotate/sun2019_rotate_wn18.json +++ b/reproducibility/configs/rotate/sun2019_rotate_wn18.json @@ -1,7 +1,7 @@ { "metadata": { - "title": "Learn WN18 Data Set with RotatE as described by Sun et al., 2019", - "comments": "Could not find initial learning rate; could not find number of epochs" + "title": "Learn WN18 Data Set with RotatE as described by Sun et al., 2019;", + "comments": "Could not find number of epochs since early stopping has been applied; https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/master/best_config.sh. The evaluation uses the non-deterministic rank, cf. https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/a0a3cf75e8e324ef113472489a16927c39c997b2/codes/model.py#L406 ." }, "pipeline": { "dataset": "wn18", @@ -11,7 +11,7 @@ }, "optimizer": "Adam", "optimizer_kwargs": { - "lr": 0.001 + "lr": 0.0001 }, "loss": "nssa", "loss_kwargs": { @@ -32,5 +32,20 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "mean_rank": { + "nondeterministic": 309 + }, + "hits_at_k": { + "nondeterministic": { + "1": 0.944, + "3": 0.952, + "10": 0.959 + } + }, + "mean_reciprocal_rank": { + "nondeterministic": 0.949 + } } } diff --git a/reproducibility/configs/rotate/sun2019_rotate_wn18rr.json b/reproducibility/configs/rotate/sun2019_rotate_wn18rr.json new file mode 100644 index 000000000..55d34e9b6 --- /dev/null +++ b/reproducibility/configs/rotate/sun2019_rotate_wn18rr.json @@ -0,0 +1,50 @@ +{ + "metadata": { + "title": "Learn WN18RR Data Set with RotatE as described by Sun et al., 2019; https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/master/best_config.sh", + "comments": "Could not find number of epochs since early stopping has been applied. The evaluation uses the non-deterministic rank, cf. https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding/blob/a0a3cf75e8e324ef113472489a16927c39c997b2/codes/model.py#L406 ." + }, + "pipeline": { + "dataset": "wn18rr", + "model": "RotatE", + "model_kwargs": { + "embedding_dim": 500 + }, + "optimizer": "Adam", + "optimizer_kwargs": { + "lr": 0.00005 + }, + "loss": "nssa", + "loss_kwargs": { + "reduction": "mean", + "adversarial_temperature": 0.5, + "margin": 6 + }, + "training_loop": "OWA", + "negative_sampler": "basic", + "negative_sampler_kwargs": { + "num_negs_per_pos": 1024 + }, + "training_kwargs": { + "num_epochs": 1000, + "batch_size": 512 + }, + "evaluator_kwargs": { + "filtered": true + } + }, + "results": { + "mean_rank": { + "nondeterministic": 3340 + }, + "hits_at_k": { + "nondeterministic": { + "1": 0.428, + "3": 0.492, + "10": 0.571 + } + }, + "mean_reciprocal_rank": { + "nondeterministic": 0.476 + } + } +} diff --git a/reproducibility/configs/simple/kazemi2018_simple_fb15k.json b/reproducibility/configs/simple/kazemi2018_simple_fb15k.json index 8ed33cdcd..4480add89 100644 --- a/reproducibility/configs/simple/kazemi2018_simple_fb15k.json +++ b/reproducibility/configs/simple/kazemi2018_simple_fb15k.json @@ -1,13 +1,14 @@ { "metadata": { "title": "Learn FB15k Data Set with SimplE as described by Kazemi et al., 2018", - "comments": "They make use of inverse relations. Normalization argument of regularizer is set to false." + "comments": "They make use of inverse relations. Normalization argument of regularizer is set to false. SimplE uses the optimistic rank definition for evaluation, cf. https://github.com/Mehran-k/SimplE/blob/29108230b63920afa38067b1aff8b8d53d07ed01/reader.py#L148." }, "pipeline": { "dataset": "fb15k", "model": "SimplE", "model_kwargs": { - "embedding_dim": 200 + "embedding_dim": 200, + "clamp_score": 20.0 }, "regularizer": "PowerSum", "regularizer_kwargs": { @@ -36,5 +37,17 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "mean_reciprocal_rank": { + "best": 0.727 + }, + "hits_at_k": { + "best": { + "1": 0.660, + "3": 0.773, + "10": 0.838 + } + } } } \ No newline at end of file diff --git a/reproducibility/configs/simple/kazemi2018_simple_wn18.json b/reproducibility/configs/simple/kazemi2018_simple_wn18.json index 75d88394a..1cfb200f1 100644 --- a/reproducibility/configs/simple/kazemi2018_simple_wn18.json +++ b/reproducibility/configs/simple/kazemi2018_simple_wn18.json @@ -1,13 +1,14 @@ { "metadata": { "title": "Learn WN18 Data Set with SimplE as described by Kazemi et al., 2018", - "comments": "They make use of inverse relations. Normalization argument of regularizer is set to false." + "comments": "They make use of inverse relations. Normalization argument of regularizer is set to false. SimplE uses the optimistic rank definition for evaluation, cf. https://github.com/Mehran-k/SimplE/blob/29108230b63920afa38067b1aff8b8d53d07ed01/reader.py#L148." }, "pipeline": { "dataset": "wn18", "model": "SimplE", "model_kwargs": { - "embedding_dim": 200 + "embedding_dim": 200, + "clamp_score": 20.0 }, "regularizer": "PowerSum", "regularizer_kwargs": { @@ -31,10 +32,22 @@ }, "training_kwargs": { "num_epochs": 1000, - "batch_size": 4831 + "batch_size": 1415 }, "evaluator_kwargs": { "filtered": true } + }, + "results": { + "mean_reciprocal_rank": { + "best": 0.942 + }, + "hits_at_k": { + "best": { + "1": 0.939, + "3": 0.944, + "10": 0.947 + } + } } } \ No newline at end of file diff --git a/reproducibility/configs/transd/ji2015_transd_fb15k.json b/reproducibility/configs/transd/ji2015_transd_fb15k.json index 431f814a6..40b3f40eb 100644 --- a/reproducibility/configs/transd/ji2015_transd_fb15k.json +++ b/reproducibility/configs/transd/ji2015_transd_fb15k.json @@ -31,5 +31,15 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "hits_at_k": { + "unknown": { + "10": 0.773 + } + }, + "mean_rank": { + "unknown": 91 + } } } \ No newline at end of file diff --git a/reproducibility/configs/transd/ji2015_transd_wn18.json b/reproducibility/configs/transd/ji2015_transd_wn18.json index f4b40e5b3..7d0aedfea 100644 --- a/reproducibility/configs/transd/ji2015_transd_wn18.json +++ b/reproducibility/configs/transd/ji2015_transd_wn18.json @@ -31,5 +31,15 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "hits_at_k": { + "unknown": { + "10": 0.922 + } + }, + "mean_rank": { + "unknown": 212 + } } } \ No newline at end of file diff --git a/reproducibility/configs/transe/bordes2013_transe_fb15k.json b/reproducibility/configs/transe/bordes2013_transe_fb15k.json index 0b3fcaf17..09fc759a1 100644 --- a/reproducibility/configs/transe/bordes2013_transe_fb15k.json +++ b/reproducibility/configs/transe/bordes2013_transe_fb15k.json @@ -31,5 +31,15 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "hits_at_k": { + "unknown": { + "10": 0.471 + } + }, + "mean_rank": { + "unknown": 125 + } } } \ No newline at end of file diff --git a/reproducibility/configs/transe/bordes2013_transe_wn18.json b/reproducibility/configs/transe/bordes2013_transe_wn18.json index aea82a354..79f9f578f 100644 --- a/reproducibility/configs/transe/bordes2013_transe_wn18.json +++ b/reproducibility/configs/transe/bordes2013_transe_wn18.json @@ -31,5 +31,15 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "hits_at_k": { + "unknown": { + "10": 0.892 + } + }, + "mean_rank": { + "unknown": 251 + } } } \ No newline at end of file diff --git a/reproducibility/configs/transh/wang2014_transh_fb15k.json b/reproducibility/configs/transh/wang2014_transh_fb15k.json index 8bac4ce88..7e1f271a1 100644 --- a/reproducibility/configs/transh/wang2014_transh_fb15k.json +++ b/reproducibility/configs/transh/wang2014_transh_fb15k.json @@ -35,5 +35,15 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "hits_at_k": { + "unknown": { + "10": 0.644 + } + }, + "mean_rank": { + "unknown": 87 + } } } diff --git a/reproducibility/configs/transh/wang2014_transh_wn18.json b/reproducibility/configs/transh/wang2014_transh_wn18.json index e841665d8..82e9692b5 100644 --- a/reproducibility/configs/transh/wang2014_transh_wn18.json +++ b/reproducibility/configs/transh/wang2014_transh_wn18.json @@ -35,5 +35,15 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "hits_at_k": { + "unknown": { + "10": 0.823 + } + }, + "mean_rank": { + "unknown": 388 + } } } diff --git a/reproducibility/configs/transr/li2015_transr_fb15k.json b/reproducibility/configs/transr/li2015_transr_fb15k.json index 790ea20d9..23bb8eb18 100644 --- a/reproducibility/configs/transr/li2015_transr_fb15k.json +++ b/reproducibility/configs/transr/li2015_transr_fb15k.json @@ -1,6 +1,6 @@ { "metadata": { - "title": "Learn FB15K Data Set with TransR as described by Li et al., 2015" + "title": "Learn FB15K Data Set with TransR as described by Li et al., 2015. For evaluation, the non-deterministic rank is used, cf. https://github.com/Mrlyk423/Relation_Extraction/blob/9f311285abd7e29273be7e4fd8c680dd761f40eb/TransR/Test_TransR.cpp#L177-L206." }, "pipeline": { "dataset": "fb15k", @@ -31,5 +31,15 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "mean_rank": { + "nondeterministic": 77 + }, + "hits_at_k": { + "nondeterministic": { + "10": 0.687 + } + } } } \ No newline at end of file diff --git a/reproducibility/configs/transr/li2015_transr_wn18.json b/reproducibility/configs/transr/li2015_transr_wn18.json index 257703e3d..d5afeaf91 100644 --- a/reproducibility/configs/transr/li2015_transr_wn18.json +++ b/reproducibility/configs/transr/li2015_transr_wn18.json @@ -1,6 +1,6 @@ { "metadata": { - "title": "Learn WN18 Data Set with TransR as described by Li et al., 2015" + "title": "Learn WN18 Data Set with TransR as described by Li et al., 2015. For evaluation, the non-deterministic rank is used, cf. https://github.com/Mrlyk423/Relation_Extraction/blob/9f311285abd7e29273be7e4fd8c680dd761f40eb/TransR/Test_TransR.cpp#L177-L206." }, "pipeline": { "dataset": "wn18", @@ -31,5 +31,15 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "mean_rank": { + "nondeterministic": 225 + }, + "hits_at_k": { + "nondeterministic": { + "10": 0.920 + } + } } } \ No newline at end of file diff --git a/reproducibility/configs/tucker/balazevic2019_tucker_fb15k.json b/reproducibility/configs/tucker/balazevic2019_tucker_fb15k.json index 42f897bac..fa07e435e 100644 --- a/reproducibility/configs/tucker/balazevic2019_tucker_fb15k.json +++ b/reproducibility/configs/tucker/balazevic2019_tucker_fb15k.json @@ -1,7 +1,7 @@ { "metadata": { "title": "Learn FB15K Data Set with TuckER as described by Balazevic et al., 2019", - "comments": "No label smoothing has been used for FB15k. Check whether dropouts are used correctly. They make use of inverse relations. Could not find number of epochs. In the code 500 was the default value for all datasets, that's why we assume that this is the number of epochs." + "comments": "No label smoothing has been used for FB15k. Check whether dropouts are used correctly. They make use of inverse relations. Could not find number of epochs. In the code 500 was the default value for all datasets, that's why we assume that this is the number of epochs. For evaluation, the non-deterministic rank is used, cf. https://github.com/ibalazevic/TuckER/blob/63dbba8751670db0d807579303679c0a2c266130/main.py#L78 ." }, "pipeline": { "dataset": "fb15k", @@ -14,7 +14,8 @@ "relation_dim": 200, "dropout_0": 0.2, "dropout_1": 0.2, - "dropout_2": 0.3 + "dropout_2": 0.3, + "apply_batch_normalization": true }, "optimizer": "Adam", "optimizer_kwargs": { @@ -34,5 +35,17 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "hits_at_k": { + "nondeterministic": { + "1": 0.741, + "3": 0.833, + "10": 0.892 + } + }, + "mean_reciprocal_rank": { + "nondeterministic": 0.795 + } } } \ No newline at end of file diff --git a/reproducibility/configs/tucker/balazevic2019_tucker_fb15k237.json b/reproducibility/configs/tucker/balazevic2019_tucker_fb15k237.json new file mode 100644 index 000000000..feeb77ab7 --- /dev/null +++ b/reproducibility/configs/tucker/balazevic2019_tucker_fb15k237.json @@ -0,0 +1,51 @@ +{ + "metadata": { + "title": "Learn FB15K-237 Data Set with TuckER as described by Balazevic et al., 2019", + "comments": "No label smoothing has been used for FB15k. Check whether dropouts are used correctly. They make use of inverse relations. Could not find number of epochs. In the code 500 was the default value for all datasets, that's why we assume that this is the number of epochs. For evaluation, the non-deterministic rank is used, cf. https://github.com/ibalazevic/TuckER/blob/63dbba8751670db0d807579303679c0a2c266130/main.py#L78 ." + }, + "pipeline": { + "dataset": "fb15k", + "dataset_kwargs": { + "create_inverse_triples": true + }, + "model": "TuckER", + "model_kwargs": { + "embedding_dim": 200, + "relation_dim": 200, + "dropout_0": 0.3, + "dropout_1": 0.4, + "dropout_2": 0.5, + "apply_batch_normalization": true + }, + "optimizer": "Adam", + "optimizer_kwargs": { + "lr": 0.0005, + "weight_decay": 1.0 + }, + "loss": "BCEAfterSigmoid", + "loss_kwargs": { + "reduction": "mean" + }, + "training_loop": "LCWA", + "training_kwargs": { + "num_epochs": 500, + "batch_size": 128, + "label_smoothing": 0.1 + }, + "evaluator_kwargs": { + "filtered": true + } + }, + "results": { + "hits_at_k": { + "nondeterministic": { + "1": 0.266, + "3": 0.394, + "10": 0.544 + } + }, + "mean_reciprocal_rank": { + "nondeterministic": 0.358 + } + } +} \ No newline at end of file diff --git a/reproducibility/configs/tucker/balazevic2019_tucker_wn18.json b/reproducibility/configs/tucker/balazevic2019_tucker_wn18.json index 0784d25d2..6b8357909 100644 --- a/reproducibility/configs/tucker/balazevic2019_tucker_wn18.json +++ b/reproducibility/configs/tucker/balazevic2019_tucker_wn18.json @@ -1,7 +1,7 @@ { "metadata": { "title": "Learn WN18 Data Set with TuckER as described by Balazevic et al., 2019", - "comments": "Check whether dropouts are used correctly. They make use of inverse relations. Could not find number of epochs.In the code 500 was the default value for all datasets, that's we assume that this is the number of epochs." + "comments": "Check whether dropouts are used correctly. They make use of inverse relations. Could not find number of epochs.In the code 500 was the default value for all datasets, that's we assume that this is the number of epochs. For evaluation, the non-deterministic rank is used, cf. https://github.com/ibalazevic/TuckER/blob/63dbba8751670db0d807579303679c0a2c266130/main.py#L78 ." }, "pipeline": { "dataset": "wn18", @@ -14,7 +14,8 @@ "relation_dim": 30, "dropout_0": 0.2, "dropout_1": 0.1, - "dropout_2": 0.2 + "dropout_2": 0.2, + "apply_batch_normalization": true }, "optimizer": "Adam", "optimizer_kwargs": { @@ -34,5 +35,17 @@ "evaluator_kwargs": { "filtered": true } + }, + "results": { + "hits_at_k": { + "nondeterministic": { + "1": 0.949, + "3": 0.955, + "10": 0.958 + } + }, + "mean_reciprocal_rank": { + "nondeterministic": 0.953 + } } } \ No newline at end of file diff --git a/reproducibility/configs/tucker/balazevic2019_tucker_wn18rr.json b/reproducibility/configs/tucker/balazevic2019_tucker_wn18rr.json new file mode 100644 index 000000000..9a6e0880c --- /dev/null +++ b/reproducibility/configs/tucker/balazevic2019_tucker_wn18rr.json @@ -0,0 +1,51 @@ +{ + "metadata": { + "title": "Learn WN18RR Data Set with TuckER as described by Balazevic et al., 2019", + "comments": "Check whether dropouts are used correctly. They make use of inverse relations. Could not find number of epochs.In the code 500 was the default value for all datasets, that's we assume that this is the number of epochs. For evaluation, the non-deterministic rank is used, cf. https://github.com/ibalazevic/TuckER/blob/63dbba8751670db0d807579303679c0a2c266130/main.py#L78 ." + }, + "pipeline": { + "dataset": "wn18", + "dataset_kwargs": { + "create_inverse_triples": true + }, + "model": "TuckER", + "model_kwargs": { + "embedding_dim": 200, + "relation_dim": 30, + "dropout_0": 0.2, + "dropout_1": 0.2, + "dropout_2": 0.3, + "apply_batch_normalization": true + }, + "optimizer": "Adam", + "optimizer_kwargs": { + "lr": 0.01, + "weight_decay": 1.0 + }, + "loss": "BCEAfterSigmoid", + "loss_kwargs": { + "reduction": "mean" + }, + "training_loop": "LCWA", + "training_kwargs": { + "num_epochs": 500, + "batch_size": 128, + "label_smoothing": 0.1 + }, + "evaluator_kwargs": { + "filtered": true + } + }, + "results": { + "hits_at_k": { + "nondeterministic": { + "1": 0.443, + "3": 0.482, + "10": 0.526 + } + }, + "mean_reciprocal_rank": { + "nondeterministic": 0.470 + } + } +} \ No newline at end of file diff --git a/reproducibility/generate_summary_table.py b/reproducibility/generate_summary_table.py index d5f6a4cc1..1d07afae5 100644 --- a/reproducibility/generate_summary_table.py +++ b/reproducibility/generate_summary_table.py @@ -1,18 +1,18 @@ # -*- coding: utf-8 -*- """Generate a summary table for the reproduction.""" - +import json import logging import os +import pathlib from collections import defaultdict from typing import Iterable, List, Mapping, Tuple import humanize import pandas as pd -from jinja2 import Environment, FileSystemLoader - import pykeen.datasets import pykeen.models +from jinja2 import Environment, FileSystemLoader from utils import SKIP, read_experiment_collation logger = logging.getLogger(__name__) @@ -23,9 +23,51 @@ # can be any one-character string that you're sure won't appear in the table PHANTOM_PLACEHOLDER = '✠' - - -def generate_results_table(): +PHANTOM_PLACEHOLDER_PM = 'ᐖ' +PHANTOM_PLACEHOLDER_DOT = 'ᑅ' + + +def format_percent(v): + value = str(v) + remainder = value[4:] + return value[2:4] + '.' + remainder + + +def load_published_results_in_tall_format( + dataset: str, +) -> pd.DataFrame: + # model, columns, mean, std + data = [] + for config_path in (pathlib.Path(HERE) / 'configs').glob(f'*/*_{dataset}.json'): + with config_path.open('r') as config_file: + config = json.load(config_file) + model = config['pipeline']['model'] + dataset_ = config['pipeline']['dataset'] + if dataset_ != dataset: + logger.error(f'Config for {dataset} specifies dataset={dataset_}.') + continue + results = config.get('results') + if results is None: + logger.warning(f'No results for {dataset}-{model}') + continue + for key in [ + 'mean_rank', + 'mean_reciprocal_rank' + ]: + if key in results: + value = list(results[key].values())[0] + if key == 'mean_reciprocal_rank': + value = format_percent(value) + data.append((model, f'{key}.pub', str(value), None)) + if 'hits_at_k' in results: + hits_at_k = list(results['hits_at_k'].values())[0] + for key, value in hits_at_k.items(): + # manually reformatting, since str(100*value) leads to floating point imprecision... + data.append((model, f'hits_at_k.pub.{key}', format_percent(value), None)) + return pd.DataFrame(data=data, columns=['model', 'columns', 'mean', 'std']) + + +def generate_results_table(with_std: bool, midrule_between_models: bool = True): all_tables = [] for dataset, dataset_df in read_experiment_collation().groupby('dataset'): if len(dataset_df['model'].unique()) < 2: @@ -34,17 +76,35 @@ def generate_results_table(): tall_summary_df = get_tall_summary_df(dataset_df) tall_summary_df.to_csv(os.path.join(SUMMARIES, f'{dataset}.tsv'), sep='\t', index=False) - wide_summary_df = reorganize_summary_df(tall_summary_df) + published_results_tall_df = load_published_results_in_tall_format(dataset=dataset) + all_results_tall_df = pd.concat([tall_summary_df, published_results_tall_df], ignore_index=True) + wide_summary_df = reorganize_summary_df(all_results_tall_df, with_std=with_std) + wide_summary_df = wide_summary_df.applymap(lambda v: ('$' + v + '$') if v != '' else '') + + translation = dict( + pub='pub', + avg='R', + best='O', + worst='P', + ) + wide_summary_df.index = wide_summary_df.index.set_levels([translation[k] for k in wide_summary_df.index.levels[1]], level=1) # Save as Latex table + pd.set_option('max_colwidth', 999) table_latex = wide_summary_df.to_latex( index=True, escape=False, - column_format='l' + ('r' * len(wide_summary_df.columns)), + column_format='ll' + ('r' * len(wide_summary_df.columns)), bold_rows=True, ) - table_latex = _process_tex(table_latex) - with open(os.path.join(SUMMARIES, f'{dataset}_table.tex'), 'w') as file: + # table_latex = _process_tex(table_latex) + suffix = '' if with_std else '_without_std' + # add midrule + if midrule_between_models: + table_latex_head, table_latex_body = table_latex.split('\\midrule', maxsplit=1) + table_latex_body = table_latex_body.replace('\n\\textbf', '\\midrule\n\\textbf') + table_latex = table_latex_head + table_latex_body + with open(os.path.join(SUMMARIES, f'{dataset}_table{suffix}.tex'), 'w') as file: print(table_latex, file=file) dataset = pykeen.datasets.datasets[dataset].__name__ @@ -75,27 +135,81 @@ def get_tall_summary_df(df: pd.DataFrame): return pd.DataFrame(rows, columns=['model', 'columns', 'mean', 'std']) -def reorganize_summary_df(df: pd.DataFrame) -> pd.DataFrame: - _n = df.groupby(['columns']).aggregate(lambda q: max(map(len, q))) +def get_width(column): + prelen = postlen = 0 + for e in column: + if e is None: + continue + splitted = str(e).split('.') + prelen = max(prelen, len(splitted[0])) + if len(splitted) > 1: + postlen = max(postlen, len(splitted[1])) + return prelen, postlen + + +def format_values(mean, std, mean_w, std_w, with_std: bool) -> str: + if mean is None: + return '' + # format mean + pre_len, post_len = mean_w + if '.' not in mean: + mean += '.' + pre, post = mean.split('.') + if len(pre) < pre_len: + pre = r'\phantom{' + (pre_len - len(pre)) * '0' + '}' + pre + if len(post) < post_len: + post = post + r'\phantom{' + (post_len - len(post)) * '0' + '}' + out = pre + '.' + post + if with_std: + if std is None: + pre_len, post_len = std_w + std_str = r'\phantom{ \pm ' + '0' * pre_len + '.' + '0' * post_len + '}' + else: + pre, post = std.split('.') + pre_len, post_len = std_w + if len(pre) < pre_len: + pre = r'\phantom{' + (pre_len - len(pre)) * '0' + '}' + pre + if len(post) < post_len: + post = post + r'\phantom{' + (post_len - len(post)) * '0' + '}' + std_str = r' \pm ' + pre + '.' + post + out += std_str + out = out.replace(r'}\phantom{', '') + return out + + +def reorganize_summary_df(df: pd.DataFrame, with_std: bool) -> pd.DataFrame: + df['short_columns'] = df['columns'].str.replace('.((worst)|(avg)|(best)|(pub))', '', regex=True) + _n = df.groupby(['short_columns']).aggregate({'mean': get_width, 'std': get_width}) n = _n['std'] _mean_n = _n['mean'] df['values'] = [ - (_mean_n[column] - len(mean)) * PHANTOM_PLACEHOLDER + mean + ' ± ' + ( - n[column] - len(std)) * PHANTOM_PLACEHOLDER + std - for column, mean, std in df[['columns', 'mean', 'std']].values + format_values(mean=mean, std=std, mean_w=_mean_n[column], std_w=n[column], with_std=with_std) + for column, mean, std in df[['short_columns', 'mean', 'std']].values ] + inconsistent_rank_mask = df[~df['columns'].str.contains('.pub')].groupby(by=['model']).agg( + {'values': lambda x: len(x.unique()) > len(df['short_columns'].unique())}) + consistent_rank_models = inconsistent_rank_mask[~inconsistent_rank_mask.values].index.tolist() + for consistent_model in consistent_rank_models: + df.drop(index=df[(df['model'] == consistent_model) & df['columns'].str.contains('.((worst)|(best))', '', regex=True)].index, inplace=True) + rv = df[['model', 'columns', 'values']] rv = rv.set_index(['model', 'columns']).unstack(level=-1).reset_index().set_index('model') rv.columns = rv.columns.get_level_values(1) rv = rv[[col for col in list(rv.columns) if col not in {'training', 'evaluation'}]] rv.columns = get_renamed_columns(rv) rv = get_reordered_df(rv) + rv = rv.swaplevel(axis=1).stack().fillna('').reindex(['pub', 'avg', 'best', 'worst'], axis=0, level=1).reindex( + [r'MRR (\%)', r'Hits@1 (\%)', r'Hits@3 (\%)', r'Hits@5 (\%)', r'Hits@10 (\%)', 'MR', r'AMR (\%)', ], axis=1) return rv def _process_tex(s: str) -> str: - s = s.replace('±', '$\\pm$').replace(PHANTOM_PLACEHOLDER, '$\\phantom{5}$') + s = s.replace('±', '\\pm').replace( + PHANTOM_PLACEHOLDER, '\\phantom{5}').replace( + PHANTOM_PLACEHOLDER_PM, '\\phantom{\\pm}').replace( + PHANTOM_PLACEHOLDER_DOT, '\\phantom{.}' + ) # s = s.replace('\\begin{tabular}', '\\begin{tabular*}') # s = s.replace('\\end{tabular}', '\\end{tabular*}') return s @@ -208,7 +322,7 @@ def get_reordered_df(df: pd.DataFrame) -> pd.DataFrame: columns = [ (first_level_label, second_level_label) - for first_level_label in ['', 'avg', 'best', 'worst'] + for first_level_label in ['pub', 'avg', 'best', 'worst'] for second_level_label in r[first_level_label] ] return df[columns] @@ -231,7 +345,8 @@ def generate_size_table(): def main(): size_table = generate_size_table() - all_tables = generate_results_table() + for with_std in (False, True): + all_tables = generate_results_table(with_std=with_std) write_pdfs(all_tables=all_tables, size_table=size_table) diff --git a/reproducibility/summaries/fb15k237_table.tex b/reproducibility/summaries/fb15k237_table.tex index fb722a9c7..d6b0fd396 100644 --- a/reproducibility/summaries/fb15k237_table.tex +++ b/reproducibility/summaries/fb15k237_table.tex @@ -1,12 +1,14 @@ -\begin{tabular}{lrrrrrrrrrrrrrrrrrrr} +\begin{tabular}{llrrrrrrr} \toprule -{} & \multicolumn{7}{l}{avg} & \multicolumn{6}{l}{best} & \multicolumn{6}{l}{worst} \\ -{} & AMR (\%) & Hits@1 (\%) & Hits@10 (\%) & Hits@3 (\%) & Hits@5 (\%) & MR & MRR (\%) & Hits@1 (\%) & Hits@10 (\%) & Hits@3 (\%) & Hits@5 (\%) & MR & MRR (\%) & Hits@1 (\%) & Hits@10 (\%) & Hits@3 (\%) & Hits@5 (\%) & MR & MRR (\%) \\ -\textbf{model } & & & & & & & & & & & & & & & & & & & \\ + & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ +\textbf{model} & {} & & & & & & & \\ \midrule -\textbf{ConvE } & $\phantom{5}$3.73 $\pm$ 0.13 & 18.22 $\pm$ 0.11 & 44.95 $\pm$ 0.17 & 29.51 $\pm$ 0.24 & 35.98 $\pm$ 0.16 & $\phantom{5}$255.46 $\pm$ $\phantom{5}$6.16 & 26.93 $\pm$ 0.11 & 18.22 $\pm$ 0.11 & 44.95 $\pm$ 0.17 & 29.51 $\pm$ 0.24 & 35.98 $\pm$ 0.16 & $\phantom{5}$255.46 $\pm$ $\phantom{5}$6.16 & 26.93 $\pm$ 0.11 & 18.22 $\pm$ 0.11 & 44.95 $\pm$ 0.17 & 29.51 $\pm$ 0.24 & 35.98 $\pm$ 0.16 & $\phantom{5}$255.46 $\pm$ $\phantom{5}$6.16 & 26.93 $\pm$ 0.11 \\ -\textbf{ConvKB} & 61.36 $\pm$ 0.65 & $\phantom{5}$3.31 $\pm$ 0.23 & $\phantom{5}$7.76 $\pm$ 0.88 & $\phantom{5}$4.04 $\pm$ 0.19 & $\phantom{5}$4.57 $\pm$ 0.22 & 4345.27 $\pm$ 46.99 & $\phantom{5}$4.71 $\pm$ 0.23 & $\phantom{5}$3.31 $\pm$ 0.23 & $\phantom{5}$7.76 $\pm$ 0.88 & $\phantom{5}$4.04 $\pm$ 0.19 & $\phantom{5}$4.57 $\pm$ 0.22 & 4345.27 $\pm$ 46.99 & $\phantom{5}$4.71 $\pm$ 0.23 & $\phantom{5}$3.31 $\pm$ 0.23 & $\phantom{5}$7.76 $\pm$ 0.88 & $\phantom{5}$4.04 $\pm$ 0.19 & $\phantom{5}$4.57 $\pm$ 0.22 & 4345.27 $\pm$ 46.99 & $\phantom{5}$4.71 $\pm$ 0.23 \\ -\textbf{RotatE} & $\phantom{5}$2.84 $\pm$ 0.00 & 17.57 $\pm$ 0.06 & 44.55 $\pm$ 0.06 & 28.97 $\pm$ 0.05 & 35.29 $\pm$ 0.09 & $\phantom{5}$191.92 $\pm$ $\phantom{5}$0.31 & 26.42 $\pm$ 0.04 & 17.57 $\pm$ 0.06 & 44.55 $\pm$ 0.06 & 28.97 $\pm$ 0.05 & 35.29 $\pm$ 0.09 & $\phantom{5}$191.92 $\pm$ $\phantom{5}$0.31 & 26.42 $\pm$ 0.04 & 17.57 $\pm$ 0.06 & 44.55 $\pm$ 0.06 & 28.97 $\pm$ 0.05 & 35.29 $\pm$ 0.09 & $\phantom{5}$191.92 $\pm$ $\phantom{5}$0.31 & 26.42 $\pm$ 0.04 \\ +\textbf{ConvE} & \textbf{pub} & $32.5\phantom{0 \pm 0.00}$ & $23.7\phantom{0 \pm 0.00}$ & $35.6\phantom{0 \pm 0.00}$ & & $50.1\phantom{0 \pm 0.00}$ & $\phantom{0}244.\phantom{00 \pm 00.00}$ & \\ + & \textbf{R} & $26.93 \pm 0.11$ & $18.22 \pm 0.11$ & $29.51 \pm 0.24$ & $35.98 \pm 0.16$ & $44.95 \pm 0.17$ & $\phantom{0}255.46 \pm \phantom{0}6.16$ & $\phantom{0}3.73 \pm 0.13$ \\\midrule +\textbf{ConvKB} & \textbf{pub} & $39.6\phantom{0 \pm 0.00}$ & & & & $51.7\phantom{0 \pm 0.00}$ & $\phantom{0}257.\phantom{00 \pm 00.00}$ & \\ + & \textbf{R} & $\phantom{0}4.71 \pm 0.23$ & $\phantom{0}3.31 \pm 0.23$ & $\phantom{0}4.04 \pm 0.19$ & $\phantom{0}4.57 \pm 0.22$ & $\phantom{0}7.76 \pm 0.88$ & $4345.27 \pm 46.99$ & $61.36 \pm 0.65$ \\\midrule +\textbf{RotatE} & \textbf{pub} & $33.8\phantom{0 \pm 0.00}$ & $24.1\phantom{0 \pm 0.00}$ & $37.5\phantom{0 \pm 0.00}$ & & $53.3\phantom{0 \pm 0.00}$ & $\phantom{0}177.\phantom{00 \pm 00.00}$ & \\ + & \textbf{R} & $26.42 \pm 0.04$ & $17.57 \pm 0.06$ & $28.97 \pm 0.05$ & $35.29 \pm 0.09$ & $44.55 \pm 0.06$ & $\phantom{0}191.92 \pm \phantom{0}0.31$ & $\phantom{0}2.84 \pm 0.00$ \\ \bottomrule \end{tabular} diff --git a/reproducibility/summaries/fb15k237_table_without_std.tex b/reproducibility/summaries/fb15k237_table_without_std.tex new file mode 100644 index 000000000..1bb8ef971 --- /dev/null +++ b/reproducibility/summaries/fb15k237_table_without_std.tex @@ -0,0 +1,14 @@ +\begin{tabular}{llrrrrrrr} +\toprule + & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ +\textbf{model} & {} & & & & & & & \\ +\midrule +\textbf{ConvE} & \textbf{pub} & $32.5\phantom{0}$ & $23.7\phantom{0}$ & $35.6\phantom{0}$ & & $50.1\phantom{0}$ & $\phantom{0}244.\phantom{00}$ & \\ + & \textbf{R} & $26.93$ & $18.22$ & $29.51$ & $35.98$ & $44.95$ & $\phantom{0}255.46$ & $\phantom{0}3.73$ \\\midrule +\textbf{ConvKB} & \textbf{pub} & $39.6\phantom{0}$ & & & & $51.7\phantom{0}$ & $\phantom{0}257.\phantom{00}$ & \\ + & \textbf{R} & $\phantom{0}4.71$ & $\phantom{0}3.31$ & $\phantom{0}4.04$ & $\phantom{0}4.57$ & $\phantom{0}7.76$ & $4345.27$ & $61.36$ \\\midrule +\textbf{RotatE} & \textbf{pub} & $33.8\phantom{0}$ & $24.1\phantom{0}$ & $37.5\phantom{0}$ & & $53.3\phantom{0}$ & $\phantom{0}177.\phantom{00}$ & \\ + & \textbf{R} & $26.42$ & $17.57$ & $28.97$ & $35.29$ & $44.55$ & $\phantom{0}191.92$ & $\phantom{0}2.84$ \\ +\bottomrule +\end{tabular} + diff --git a/reproducibility/summaries/fb15k_table.tex b/reproducibility/summaries/fb15k_table.tex index a279a6b5b..dda4f721f 100644 --- a/reproducibility/summaries/fb15k_table.tex +++ b/reproducibility/summaries/fb15k_table.tex @@ -1,21 +1,38 @@ -\begin{tabular}{lrrrrrrrrrrrrrrrrrrr} +\begin{tabular}{llrrrrrrr} \toprule -{} & \multicolumn{7}{l}{avg} & \multicolumn{6}{l}{best} & \multicolumn{6}{l}{worst} \\ -{} & AMR (\%) & Hits@1 (\%) & Hits@10 (\%) & Hits@3 (\%) & Hits@5 (\%) & MR & MRR (\%) & Hits@1 (\%) & Hits@10 (\%) & Hits@3 (\%) & Hits@5 (\%) & MR & MRR (\%) & Hits@1 (\%) & Hits@10 (\%) & Hits@3 (\%) & Hits@5 (\%) & MR & MRR (\%) \\ -\textbf{model } & & & & & & & & & & & & & & & & & & & \\ + & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ +\textbf{model} & {} & & & & & & & \\ \midrule -\textbf{ComplEx } & $\phantom{5}$$\phantom{5}$2.46 $\pm$ 0.28 & 10.06 $\pm$ 0.42 & 38.03 $\pm$ 1.02 & 20.82 $\pm$ 0.67 & 27.59 $\pm$ 0.94 & $\phantom{5}$171.16 $\pm$ 16.05 & 19.13 $\pm$ 0.45 & 10.06 $\pm$ 0.42 & 38.03 $\pm$ $\phantom{5}$1.02 & 20.82 $\pm$ $\phantom{5}$0.67 & 27.59 $\pm$ $\phantom{5}$0.94 & $\phantom{5}$171.16 $\pm$ 16.05 & 19.13 $\pm$ 0.45 & 10.06 $\pm$ 0.42 & 38.03 $\pm$ 1.02 & 20.82 $\pm$ 0.67 & 27.59 $\pm$ 0.94 & $\phantom{5}$$\phantom{5}$171.16 $\pm$ 16.05 & 19.13 $\pm$ 0.45 \\ -\textbf{ConvE } & $\phantom{5}$$\phantom{5}$0.73 $\pm$ 0.01 & 48.28 $\pm$ 0.12 & 79.76 $\pm$ 0.07 & 66.99 $\pm$ 0.04 & 73.27 $\pm$ 0.03 & $\phantom{5}$$\phantom{5}$50.76 $\pm$ $\phantom{5}$0.40 & 59.56 $\pm$ 0.06 & 48.28 $\pm$ 0.12 & 79.76 $\pm$ $\phantom{5}$0.07 & 66.99 $\pm$ $\phantom{5}$0.04 & 73.27 $\pm$ $\phantom{5}$0.03 & $\phantom{5}$$\phantom{5}$50.76 $\pm$ $\phantom{5}$0.40 & 59.56 $\pm$ 0.06 & 48.28 $\pm$ 0.12 & 79.76 $\pm$ 0.07 & 66.99 $\pm$ 0.04 & 73.27 $\pm$ 0.03 & $\phantom{5}$$\phantom{5}$$\phantom{5}$50.76 $\pm$ $\phantom{5}$0.40 & 59.56 $\pm$ 0.06 \\ -\textbf{DistMult} & $\phantom{5}$$\phantom{5}$1.86 $\pm$ 0.03 & 16.45 $\pm$ 0.16 & 45.00 $\pm$ 0.25 & 29.10 $\pm$ 0.17 & 35.54 $\pm$ 0.21 & $\phantom{5}$134.02 $\pm$ $\phantom{5}$1.98 & 26.06 $\pm$ 0.17 & 16.45 $\pm$ 0.16 & 45.00 $\pm$ $\phantom{5}$0.25 & 29.10 $\pm$ $\phantom{5}$0.17 & 35.54 $\pm$ $\phantom{5}$0.21 & $\phantom{5}$134.02 $\pm$ $\phantom{5}$1.98 & 26.06 $\pm$ 0.17 & 16.45 $\pm$ 0.16 & 45.00 $\pm$ 0.25 & 29.10 $\pm$ 0.17 & 35.54 $\pm$ 0.21 & $\phantom{5}$$\phantom{5}$134.02 $\pm$ $\phantom{5}$1.98 & 26.06 $\pm$ 0.17 \\ -\textbf{HolE } & $\phantom{5}$$\phantom{5}$2.71 $\pm$ 0.12 & 21.79 $\pm$ 0.19 & 58.84 $\pm$ 0.28 & 39.69 $\pm$ 0.24 & 48.06 $\pm$ 0.30 & $\phantom{5}$193.03 $\pm$ $\phantom{5}$7.61 & 34.15 $\pm$ 0.22 & 21.79 $\pm$ 0.19 & 58.84 $\pm$ $\phantom{5}$0.28 & 39.69 $\pm$ $\phantom{5}$0.24 & 48.06 $\pm$ $\phantom{5}$0.30 & $\phantom{5}$193.03 $\pm$ $\phantom{5}$7.61 & 34.15 $\pm$ 0.22 & 21.79 $\pm$ 0.19 & 58.84 $\pm$ 0.28 & 39.69 $\pm$ 0.24 & 48.06 $\pm$ 0.30 & $\phantom{5}$$\phantom{5}$193.03 $\pm$ $\phantom{5}$7.61 & 34.15 $\pm$ 0.22 \\ -\textbf{KG2E } & $\phantom{5}$78.40 $\pm$ 0.68 & $\phantom{5}$0.11 $\pm$ 0.04 & $\phantom{5}$1.01 $\pm$ 0.14 & $\phantom{5}$0.36 $\pm$ 0.08 & $\phantom{5}$0.56 $\pm$ 0.10 & 5779.07 $\pm$ 51.02 & $\phantom{5}$0.58 $\pm$ 0.07 & $\phantom{5}$0.11 $\pm$ 0.04 & $\phantom{5}$1.01 $\pm$ $\phantom{5}$0.14 & $\phantom{5}$0.36 $\pm$ $\phantom{5}$0.08 & $\phantom{5}$0.56 $\pm$ $\phantom{5}$0.10 & 5779.07 $\pm$ 51.02 & $\phantom{5}$0.58 $\pm$ 0.07 & $\phantom{5}$0.11 $\pm$ 0.04 & $\phantom{5}$1.01 $\pm$ 0.14 & $\phantom{5}$0.36 $\pm$ 0.08 & $\phantom{5}$0.56 $\pm$ 0.10 & $\phantom{5}$5779.07 $\pm$ 51.02 & $\phantom{5}$0.58 $\pm$ 0.07 \\ -\textbf{RotatE } & $\phantom{5}$$\phantom{5}$0.63 $\pm$ 0.00 & 41.53 $\pm$ 0.06 & 78.67 $\pm$ 0.08 & 64.14 $\pm$ 0.07 & 71.23 $\pm$ 0.05 & $\phantom{5}$$\phantom{5}$42.28 $\pm$ $\phantom{5}$0.13 & 55.00 $\pm$ 0.06 & 41.53 $\pm$ 0.06 & 78.67 $\pm$ $\phantom{5}$0.08 & 64.14 $\pm$ $\phantom{5}$0.07 & 71.23 $\pm$ $\phantom{5}$0.05 & $\phantom{5}$$\phantom{5}$42.28 $\pm$ $\phantom{5}$0.13 & 55.00 $\pm$ 0.06 & 41.53 $\pm$ 0.06 & 78.67 $\pm$ 0.08 & 64.14 $\pm$ 0.07 & 71.23 $\pm$ 0.05 & $\phantom{5}$$\phantom{5}$$\phantom{5}$42.28 $\pm$ $\phantom{5}$0.13 & 55.00 $\pm$ 0.06 \\ -\textbf{SimplE } & 100.02 $\pm$ 0.03 & $\phantom{5}$0.01 $\pm$ 0.00 & $\phantom{5}$0.06 $\pm$ 0.01 & $\phantom{5}$0.03 $\pm$ 0.00 & $\phantom{5}$0.04 $\pm$ 0.00 & 7395.75 $\pm$ $\phantom{5}$2.02 & $\phantom{5}$0.04 $\pm$ 0.00 & 11.58 $\pm$ 6.42 & 54.28 $\pm$ 15.80 & 24.16 $\pm$ 10.95 & 34.73 $\pm$ 13.40 & $\phantom{5}$139.34 $\pm$ 49.45 & 23.90 $\pm$ 8.79 & $\phantom{5}$0.01 $\pm$ 0.00 & $\phantom{5}$0.06 $\pm$ 0.01 & $\phantom{5}$0.03 $\pm$ 0.00 & $\phantom{5}$0.04 $\pm$ 0.00 & 14652.16 $\pm$ 45.71 & $\phantom{5}$0.03 $\pm$ 0.00 \\ -\textbf{TransD } & $\phantom{5}$$\phantom{5}$2.29 $\pm$ 0.09 & 21.22 $\pm$ 0.03 & 58.71 $\pm$ 0.14 & 40.48 $\pm$ 0.10 & 48.57 $\pm$ 0.09 & $\phantom{5}$153.37 $\pm$ $\phantom{5}$5.35 & 33.99 $\pm$ 0.03 & 21.22 $\pm$ 0.03 & 58.71 $\pm$ $\phantom{5}$0.14 & 40.48 $\pm$ $\phantom{5}$0.10 & 48.57 $\pm$ $\phantom{5}$0.09 & $\phantom{5}$153.37 $\pm$ $\phantom{5}$5.35 & 33.99 $\pm$ 0.03 & 21.22 $\pm$ 0.03 & 58.71 $\pm$ 0.14 & 40.48 $\pm$ 0.10 & 48.57 $\pm$ 0.09 & $\phantom{5}$$\phantom{5}$153.37 $\pm$ $\phantom{5}$5.35 & 33.99 $\pm$ 0.03 \\ -\textbf{TransE } & $\phantom{5}$$\phantom{5}$1.78 $\pm$ 0.01 & 15.23 $\pm$ 0.16 & 47.34 $\pm$ 0.18 & 29.85 $\pm$ 0.24 & 37.18 $\pm$ 0.24 & $\phantom{5}$127.92 $\pm$ $\phantom{5}$0.86 & 26.01 $\pm$ 0.17 & 15.23 $\pm$ 0.16 & 47.34 $\pm$ $\phantom{5}$0.18 & 29.85 $\pm$ $\phantom{5}$0.24 & 37.18 $\pm$ $\phantom{5}$0.24 & $\phantom{5}$127.92 $\pm$ $\phantom{5}$0.86 & 26.01 $\pm$ 0.17 & 15.23 $\pm$ 0.16 & 47.34 $\pm$ 0.18 & 29.85 $\pm$ 0.24 & 37.18 $\pm$ 0.24 & $\phantom{5}$$\phantom{5}$127.92 $\pm$ $\phantom{5}$0.86 & 26.01 $\pm$ 0.17 \\ -\textbf{TransH } & $\phantom{5}$85.63 $\pm$ 0.40 & $\phantom{5}$1.69 $\pm$ 0.25 & $\phantom{5}$3.74 $\pm$ 0.18 & $\phantom{5}$2.95 $\pm$ 0.20 & $\phantom{5}$3.29 $\pm$ 0.22 & 6320.02 $\pm$ 30.37 & $\phantom{5}$2.54 $\pm$ 0.20 & $\phantom{5}$1.69 $\pm$ 0.25 & $\phantom{5}$3.74 $\pm$ $\phantom{5}$0.18 & $\phantom{5}$2.95 $\pm$ $\phantom{5}$0.20 & $\phantom{5}$3.29 $\pm$ $\phantom{5}$0.22 & 6320.00 $\pm$ 30.37 & $\phantom{5}$2.54 $\pm$ 0.20 & $\phantom{5}$1.69 $\pm$ 0.25 & $\phantom{5}$3.74 $\pm$ 0.18 & $\phantom{5}$2.95 $\pm$ 0.20 & $\phantom{5}$3.29 $\pm$ 0.22 & $\phantom{5}$6320.05 $\pm$ 30.37 & $\phantom{5}$2.54 $\pm$ 0.20 \\ -\textbf{TransR } & $\phantom{5}$91.99 $\pm$ 0.22 & $\phantom{5}$0.37 $\pm$ 0.00 & $\phantom{5}$1.03 $\pm$ 0.07 & $\phantom{5}$0.63 $\pm$ 0.04 & $\phantom{5}$0.78 $\pm$ 0.06 & 6795.95 $\pm$ 16.65 & $\phantom{5}$0.65 $\pm$ 0.02 & $\phantom{5}$0.37 $\pm$ 0.00 & $\phantom{5}$1.03 $\pm$ $\phantom{5}$0.07 & $\phantom{5}$0.63 $\pm$ $\phantom{5}$0.04 & $\phantom{5}$0.78 $\pm$ $\phantom{5}$0.06 & 6795.94 $\pm$ 16.65 & $\phantom{5}$0.65 $\pm$ 0.02 & $\phantom{5}$0.37 $\pm$ 0.00 & $\phantom{5}$1.03 $\pm$ 0.07 & $\phantom{5}$0.63 $\pm$ 0.04 & $\phantom{5}$0.78 $\pm$ 0.06 & $\phantom{5}$6795.95 $\pm$ 16.65 & $\phantom{5}$0.65 $\pm$ 0.02 \\ -\textbf{TuckER } & $\phantom{5}$99.11 $\pm$ 0.39 & $\phantom{5}$0.01 $\pm$ 0.00 & $\phantom{5}$0.15 $\pm$ 0.17 & $\phantom{5}$0.02 $\pm$ 0.00 & $\phantom{5}$0.03 $\pm$ 0.01 & 7327.77 $\pm$ 29.22 & $\phantom{5}$0.07 $\pm$ 0.02 & $\phantom{5}$0.01 $\pm$ 0.00 & $\phantom{5}$0.15 $\pm$ $\phantom{5}$0.17 & $\phantom{5}$0.02 $\pm$ $\phantom{5}$0.00 & $\phantom{5}$0.03 $\pm$ $\phantom{5}$0.01 & 7327.77 $\pm$ 29.22 & $\phantom{5}$0.07 $\pm$ 0.02 & $\phantom{5}$0.01 $\pm$ 0.00 & $\phantom{5}$0.15 $\pm$ 0.17 & $\phantom{5}$0.02 $\pm$ 0.00 & $\phantom{5}$0.03 $\pm$ 0.01 & $\phantom{5}$7327.77 $\pm$ 29.22 & $\phantom{5}$0.07 $\pm$ 0.02 \\ +\textbf{ComplEx} & \textbf{pub} & $69.2\phantom{0 \pm 0.00}$ & $59.9\phantom{0 \pm 0.00}$ & $75.9\phantom{0 \pm 00.00}$ & & $84.\phantom{00 \pm 00.00}$ & & \\ + & \textbf{R} & $19.13 \pm 0.45$ & $10.06 \pm 0.42$ & $20.82 \pm \phantom{0}0.67$ & $27.59 \pm \phantom{0}0.94$ & $38.03 \pm \phantom{0}1.02$ & $\phantom{00}171.16 \pm 16.05$ & $\phantom{00}2.46 \pm 0.28$ \\\midrule +\textbf{ConvE} & \textbf{pub} & $65.7\phantom{0 \pm 0.00}$ & $55.8\phantom{0 \pm 0.00}$ & $72.3\phantom{0 \pm 00.00}$ & & $83.1\phantom{0 \pm 00.00}$ & $\phantom{000}51.\phantom{00 \pm 00.00}$ & \\ + & \textbf{R} & $59.56 \pm 0.06$ & $48.28 \pm 0.12$ & $66.99 \pm \phantom{0}0.04$ & $73.27 \pm \phantom{0}0.03$ & $79.76 \pm \phantom{0}0.07$ & $\phantom{000}50.76 \pm \phantom{0}0.40$ & $\phantom{00}0.73 \pm 0.01$ \\\midrule +\textbf{DistMult} & \textbf{pub} & $35.\phantom{00 \pm 0.00}$ & & & & $57.7\phantom{0 \pm 00.00}$ & & \\ + & \textbf{R} & $26.06 \pm 0.17$ & $16.45 \pm 0.16$ & $29.10 \pm \phantom{0}0.17$ & $35.54 \pm \phantom{0}0.21$ & $45.00 \pm \phantom{0}0.25$ & $\phantom{00}134.02 \pm \phantom{0}1.98$ & $\phantom{00}1.86 \pm 0.03$ \\\midrule +\textbf{HolE} & \textbf{pub} & $52.4\phantom{0 \pm 0.00}$ & $40.2\phantom{0 \pm 0.00}$ & $61.3\phantom{0 \pm 00.00}$ & & $73.9\phantom{0 \pm 00.00}$ & & \\ + & \textbf{R} & $34.15 \pm 0.22$ & $21.79 \pm 0.19$ & $39.69 \pm \phantom{0}0.24$ & $48.06 \pm \phantom{0}0.30$ & $58.84 \pm \phantom{0}0.28$ & $\phantom{00}193.03 \pm \phantom{0}7.61$ & $\phantom{00}2.71 \pm 0.12$ \\\midrule +\textbf{KG2E} & \textbf{pub} & & & & & $71.5\phantom{0 \pm 00.00}$ & $\phantom{000}59.\phantom{00 \pm 00.00}$ & \\ + & \textbf{R} & $\phantom{0}0.58 \pm 0.07$ & $\phantom{0}0.11 \pm 0.04$ & $\phantom{0}0.36 \pm \phantom{0}0.08$ & $\phantom{0}0.56 \pm \phantom{0}0.10$ & $\phantom{0}1.01 \pm \phantom{0}0.14$ & $\phantom{0}5779.07 \pm 51.02$ & $\phantom{0}78.40 \pm 0.68$ \\\midrule +\textbf{RotatE} & \textbf{pub} & $79.7\phantom{0 \pm 0.00}$ & $74.6\phantom{0 \pm 0.00}$ & $83.\phantom{00 \pm 00.00}$ & & $88.4\phantom{0 \pm 00.00}$ & $\phantom{000}40.\phantom{00 \pm 00.00}$ & \\ + & \textbf{R} & $55.00 \pm 0.06$ & $41.53 \pm 0.06$ & $64.14 \pm \phantom{0}0.07$ & $71.23 \pm \phantom{0}0.05$ & $78.67 \pm \phantom{0}0.08$ & $\phantom{000}42.28 \pm \phantom{0}0.13$ & $\phantom{00}0.63 \pm 0.00$ \\\midrule +\textbf{SimplE} & \textbf{pub} & $72.7\phantom{0 \pm 0.00}$ & $66.\phantom{00 \pm 0.00}$ & $77.3\phantom{0 \pm 00.00}$ & & $83.8\phantom{0 \pm 00.00}$ & & \\ + & \textbf{R} & $\phantom{0}0.04 \pm 0.00$ & $\phantom{0}0.01 \pm 0.00$ & $\phantom{0}0.03 \pm \phantom{0}0.00$ & $\phantom{0}0.04 \pm \phantom{0}0.00$ & $\phantom{0}0.06 \pm \phantom{0}0.01$ & $\phantom{0}7395.75 \pm \phantom{0}2.02$ & $100.02 \pm 0.03$ \\ + & \textbf{O} & $23.90 \pm 8.79$ & $11.58 \pm 6.42$ & $24.16 \pm 10.95$ & $34.73 \pm 13.40$ & $54.28 \pm 15.80$ & $\phantom{00}139.34 \pm 49.45$ & \\ + & \textbf{P} & $\phantom{0}0.03 \pm 0.00$ & $\phantom{0}0.01 \pm 0.00$ & $\phantom{0}0.03 \pm \phantom{0}0.00$ & $\phantom{0}0.04 \pm \phantom{0}0.00$ & $\phantom{0}0.06 \pm \phantom{0}0.01$ & $14652.16 \pm 45.71$ & \\\midrule +\textbf{TransD} & \textbf{pub} & & & & & $77.3\phantom{0 \pm 00.00}$ & $\phantom{000}91.\phantom{00 \pm 00.00}$ & \\ + & \textbf{R} & $33.99 \pm 0.03$ & $21.22 \pm 0.03$ & $40.48 \pm \phantom{0}0.10$ & $48.57 \pm \phantom{0}0.09$ & $58.71 \pm \phantom{0}0.14$ & $\phantom{00}153.37 \pm \phantom{0}5.35$ & $\phantom{00}2.29 \pm 0.09$ \\\midrule +\textbf{TransE} & \textbf{pub} & & & & & $47.1\phantom{0 \pm 00.00}$ & $\phantom{00}125.\phantom{00 \pm 00.00}$ & \\ + & \textbf{R} & $26.01 \pm 0.17$ & $15.23 \pm 0.16$ & $29.85 \pm \phantom{0}0.24$ & $37.18 \pm \phantom{0}0.24$ & $47.34 \pm \phantom{0}0.18$ & $\phantom{00}127.92 \pm \phantom{0}0.86$ & $\phantom{00}1.78 \pm 0.01$ \\\midrule +\textbf{TransH} & \textbf{pub} & & & & & $64.4\phantom{0 \pm 00.00}$ & $\phantom{000}87.\phantom{00 \pm 00.00}$ & \\ + & \textbf{R} & $\phantom{0}2.54 \pm 0.20$ & $\phantom{0}1.69 \pm 0.25$ & $\phantom{0}2.95 \pm \phantom{0}0.20$ & $\phantom{0}3.29 \pm \phantom{0}0.22$ & $\phantom{0}3.74 \pm \phantom{0}0.18$ & $\phantom{0}6320.02 \pm 30.37$ & $\phantom{0}85.63 \pm 0.40$ \\ + & \textbf{O} & $\phantom{0}2.54 \pm 0.20$ & $\phantom{0}1.69 \pm 0.25$ & $\phantom{0}2.95 \pm \phantom{0}0.20$ & $\phantom{0}3.29 \pm \phantom{0}0.22$ & $\phantom{0}3.74 \pm \phantom{0}0.18$ & $\phantom{0}6320.00 \pm 30.37$ & \\ + & \textbf{P} & $\phantom{0}2.54 \pm 0.20$ & $\phantom{0}1.69 \pm 0.25$ & $\phantom{0}2.95 \pm \phantom{0}0.20$ & $\phantom{0}3.29 \pm \phantom{0}0.22$ & $\phantom{0}3.74 \pm \phantom{0}0.18$ & $\phantom{0}6320.05 \pm 30.37$ & \\\midrule +\textbf{TransR} & \textbf{pub} & & & & & $68.7\phantom{0 \pm 00.00}$ & $\phantom{000}77.\phantom{00 \pm 00.00}$ & \\ + & \textbf{R} & $\phantom{0}0.65 \pm 0.02$ & $\phantom{0}0.37 \pm 0.00$ & $\phantom{0}0.63 \pm \phantom{0}0.04$ & $\phantom{0}0.78 \pm \phantom{0}0.06$ & $\phantom{0}1.03 \pm \phantom{0}0.07$ & $\phantom{0}6795.95 \pm 16.65$ & $\phantom{0}91.99 \pm 0.22$ \\ + & \textbf{O} & $\phantom{0}0.65 \pm 0.02$ & $\phantom{0}0.37 \pm 0.00$ & $\phantom{0}0.63 \pm \phantom{0}0.04$ & $\phantom{0}0.78 \pm \phantom{0}0.06$ & $\phantom{0}1.03 \pm \phantom{0}0.07$ & $\phantom{0}6795.94 \pm 16.65$ & \\ + & \textbf{P} & $\phantom{0}0.65 \pm 0.02$ & $\phantom{0}0.37 \pm 0.00$ & $\phantom{0}0.63 \pm \phantom{0}0.04$ & $\phantom{0}0.78 \pm \phantom{0}0.06$ & $\phantom{0}1.03 \pm \phantom{0}0.07$ & $\phantom{0}6795.95 \pm 16.65$ & \\\midrule +\textbf{TuckER} & \textbf{pub} & $79.5\phantom{0 \pm 0.00}$ & $74.1\phantom{0 \pm 0.00}$ & $83.3\phantom{0 \pm 00.00}$ & & $89.2\phantom{0 \pm 00.00}$ & & \\ + & \textbf{R} & $\phantom{0}0.07 \pm 0.02$ & $\phantom{0}0.01 \pm 0.00$ & $\phantom{0}0.02 \pm \phantom{0}0.00$ & $\phantom{0}0.03 \pm \phantom{0}0.01$ & $\phantom{0}0.15 \pm \phantom{0}0.17$ & $\phantom{0}7327.77 \pm 29.22$ & $\phantom{0}99.11 \pm 0.39$ \\ \bottomrule \end{tabular} diff --git a/reproducibility/summaries/fb15k_table_without_std.tex b/reproducibility/summaries/fb15k_table_without_std.tex new file mode 100644 index 000000000..da81adbd4 --- /dev/null +++ b/reproducibility/summaries/fb15k_table_without_std.tex @@ -0,0 +1,38 @@ +\begin{tabular}{llrrrrrrr} +\toprule + & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ +\textbf{model} & {} & & & & & & & \\ +\midrule +\textbf{ComplEx} & \textbf{pub} & $69.2\phantom{0}$ & $59.9\phantom{0}$ & $75.9\phantom{0}$ & & $84.\phantom{00}$ & & \\ + & \textbf{R} & $19.13$ & $10.06$ & $20.82$ & $27.59$ & $38.03$ & $\phantom{00}171.16$ & $\phantom{00}2.46$ \\\midrule +\textbf{ConvE} & \textbf{pub} & $65.7\phantom{0}$ & $55.8\phantom{0}$ & $72.3\phantom{0}$ & & $83.1\phantom{0}$ & $\phantom{000}51.\phantom{00}$ & \\ + & \textbf{R} & $59.56$ & $48.28$ & $66.99$ & $73.27$ & $79.76$ & $\phantom{000}50.76$ & $\phantom{00}0.73$ \\\midrule +\textbf{DistMult} & \textbf{pub} & $35.\phantom{00}$ & & & & $57.7\phantom{0}$ & & \\ + & \textbf{R} & $26.06$ & $16.45$ & $29.10$ & $35.54$ & $45.00$ & $\phantom{00}134.02$ & $\phantom{00}1.86$ \\\midrule +\textbf{HolE} & \textbf{pub} & $52.4\phantom{0}$ & $40.2\phantom{0}$ & $61.3\phantom{0}$ & & $73.9\phantom{0}$ & & \\ + & \textbf{R} & $34.15$ & $21.79$ & $39.69$ & $48.06$ & $58.84$ & $\phantom{00}193.03$ & $\phantom{00}2.71$ \\\midrule +\textbf{KG2E} & \textbf{pub} & & & & & $71.5\phantom{0}$ & $\phantom{000}59.\phantom{00}$ & \\ + & \textbf{R} & $\phantom{0}0.58$ & $\phantom{0}0.11$ & $\phantom{0}0.36$ & $\phantom{0}0.56$ & $\phantom{0}1.01$ & $\phantom{0}5779.07$ & $\phantom{0}78.40$ \\\midrule +\textbf{RotatE} & \textbf{pub} & $79.7\phantom{0}$ & $74.6\phantom{0}$ & $83.\phantom{00}$ & & $88.4\phantom{0}$ & $\phantom{000}40.\phantom{00}$ & \\ + & \textbf{R} & $55.00$ & $41.53$ & $64.14$ & $71.23$ & $78.67$ & $\phantom{000}42.28$ & $\phantom{00}0.63$ \\\midrule +\textbf{SimplE} & \textbf{pub} & $72.7\phantom{0}$ & $66.\phantom{00}$ & $77.3\phantom{0}$ & & $83.8\phantom{0}$ & & \\ + & \textbf{R} & $\phantom{0}0.04$ & $\phantom{0}0.01$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $\phantom{0}0.06$ & $\phantom{0}7395.75$ & $100.02$ \\ + & \textbf{O} & $23.90$ & $11.58$ & $24.16$ & $34.73$ & $54.28$ & $\phantom{00}139.34$ & \\ + & \textbf{P} & $\phantom{0}0.03$ & $\phantom{0}0.01$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $\phantom{0}0.06$ & $14652.16$ & \\\midrule +\textbf{TransD} & \textbf{pub} & & & & & $77.3\phantom{0}$ & $\phantom{000}91.\phantom{00}$ & \\ + & \textbf{R} & $33.99$ & $21.22$ & $40.48$ & $48.57$ & $58.71$ & $\phantom{00}153.37$ & $\phantom{00}2.29$ \\\midrule +\textbf{TransE} & \textbf{pub} & & & & & $47.1\phantom{0}$ & $\phantom{00}125.\phantom{00}$ & \\ + & \textbf{R} & $26.01$ & $15.23$ & $29.85$ & $37.18$ & $47.34$ & $\phantom{00}127.92$ & $\phantom{00}1.78$ \\\midrule +\textbf{TransH} & \textbf{pub} & & & & & $64.4\phantom{0}$ & $\phantom{000}87.\phantom{00}$ & \\ + & \textbf{R} & $\phantom{0}2.54$ & $\phantom{0}1.69$ & $\phantom{0}2.95$ & $\phantom{0}3.29$ & $\phantom{0}3.74$ & $\phantom{0}6320.02$ & $\phantom{0}85.63$ \\ + & \textbf{O} & $\phantom{0}2.54$ & $\phantom{0}1.69$ & $\phantom{0}2.95$ & $\phantom{0}3.29$ & $\phantom{0}3.74$ & $\phantom{0}6320.00$ & \\ + & \textbf{P} & $\phantom{0}2.54$ & $\phantom{0}1.69$ & $\phantom{0}2.95$ & $\phantom{0}3.29$ & $\phantom{0}3.74$ & $\phantom{0}6320.05$ & \\\midrule +\textbf{TransR} & \textbf{pub} & & & & & $68.7\phantom{0}$ & $\phantom{000}77.\phantom{00}$ & \\ + & \textbf{R} & $\phantom{0}0.65$ & $\phantom{0}0.37$ & $\phantom{0}0.63$ & $\phantom{0}0.78$ & $\phantom{0}1.03$ & $\phantom{0}6795.95$ & $\phantom{0}91.99$ \\ + & \textbf{O} & $\phantom{0}0.65$ & $\phantom{0}0.37$ & $\phantom{0}0.63$ & $\phantom{0}0.78$ & $\phantom{0}1.03$ & $\phantom{0}6795.94$ & \\ + & \textbf{P} & $\phantom{0}0.65$ & $\phantom{0}0.37$ & $\phantom{0}0.63$ & $\phantom{0}0.78$ & $\phantom{0}1.03$ & $\phantom{0}6795.95$ & \\\midrule +\textbf{TuckER} & \textbf{pub} & $79.5\phantom{0}$ & $74.1\phantom{0}$ & $83.3\phantom{0}$ & & $89.2\phantom{0}$ & & \\ + & \textbf{R} & $\phantom{0}0.07$ & $\phantom{0}0.01$ & $\phantom{0}0.02$ & $\phantom{0}0.03$ & $\phantom{0}0.15$ & $\phantom{0}7327.77$ & $\phantom{0}99.11$ \\ +\bottomrule +\end{tabular} + diff --git a/reproducibility/summaries/results.pdf b/reproducibility/summaries/results.pdf index 44fc2eec0..8bcbcf367 100644 Binary files a/reproducibility/summaries/results.pdf and b/reproducibility/summaries/results.pdf differ diff --git a/reproducibility/summaries/results.tex b/reproducibility/summaries/results.tex index 9470317e5..13ad028e9 100644 --- a/reproducibility/summaries/results.tex +++ b/reproducibility/summaries/results.tex @@ -59,20 +59,20 @@ \subsection{FB15k Average} \centering \begin{tabular}{lrrrrrrr} \toprule -{} & MR & MRR (\%) & AMR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) \\ +{} & MR & MRR (\%) & AMR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) \\ \midrule -\textbf{ComplEx } & $\phantom{5}$171.16 $\pm$ 16.05 & 19.13 $\pm$ 0.45 & $\phantom{5}$$\phantom{5}$2.46 $\pm$ 0.28 & 10.06 $\pm$ 0.42 & 20.82 $\pm$ 0.67 & 27.59 $\pm$ 0.94 & 38.03 $\pm$ 1.02 \\ -\textbf{ConvE } & $\phantom{5}$$\phantom{5}$50.76 $\pm$ $\phantom{5}$0.40 & 59.56 $\pm$ 0.06 & $\phantom{5}$$\phantom{5}$0.73 $\pm$ 0.01 & 48.28 $\pm$ 0.12 & 66.99 $\pm$ 0.04 & 73.27 $\pm$ 0.03 & 79.76 $\pm$ 0.07 \\ -\textbf{DistMult} & $\phantom{5}$134.02 $\pm$ $\phantom{5}$1.98 & 26.06 $\pm$ 0.17 & $\phantom{5}$$\phantom{5}$1.86 $\pm$ 0.03 & 16.45 $\pm$ 0.16 & 29.10 $\pm$ 0.17 & 35.54 $\pm$ 0.21 & 45.00 $\pm$ 0.25 \\ -\textbf{HolE } & $\phantom{5}$193.03 $\pm$ $\phantom{5}$7.61 & 34.15 $\pm$ 0.22 & $\phantom{5}$$\phantom{5}$2.71 $\pm$ 0.12 & 21.79 $\pm$ 0.19 & 39.69 $\pm$ 0.24 & 48.06 $\pm$ 0.30 & 58.84 $\pm$ 0.28 \\ -\textbf{KG2E } & 5779.07 $\pm$ 51.02 & $\phantom{5}$0.58 $\pm$ 0.07 & $\phantom{5}$78.40 $\pm$ 0.68 & $\phantom{5}$0.11 $\pm$ 0.04 & $\phantom{5}$0.36 $\pm$ 0.08 & $\phantom{5}$0.56 $\pm$ 0.10 & $\phantom{5}$1.01 $\pm$ 0.14 \\ -\textbf{RotatE } & $\phantom{5}$$\phantom{5}$42.28 $\pm$ $\phantom{5}$0.13 & 55.00 $\pm$ 0.06 & $\phantom{5}$$\phantom{5}$0.63 $\pm$ 0.00 & 41.53 $\pm$ 0.06 & 64.14 $\pm$ 0.07 & 71.23 $\pm$ 0.05 & 78.67 $\pm$ 0.08 \\ -\textbf{SimplE } & 7395.75 $\pm$ $\phantom{5}$2.02 & $\phantom{5}$0.04 $\pm$ 0.00 & 100.02 $\pm$ 0.03 & $\phantom{5}$0.01 $\pm$ 0.00 & $\phantom{5}$0.03 $\pm$ 0.00 & $\phantom{5}$0.04 $\pm$ 0.00 & $\phantom{5}$0.06 $\pm$ 0.01 \\ -\textbf{TransD } & $\phantom{5}$153.37 $\pm$ $\phantom{5}$5.35 & 33.99 $\pm$ 0.03 & $\phantom{5}$$\phantom{5}$2.29 $\pm$ 0.09 & 21.22 $\pm$ 0.03 & 40.48 $\pm$ 0.10 & 48.57 $\pm$ 0.09 & 58.71 $\pm$ 0.14 \\ -\textbf{TransE } & $\phantom{5}$127.92 $\pm$ $\phantom{5}$0.86 & 26.01 $\pm$ 0.17 & $\phantom{5}$$\phantom{5}$1.78 $\pm$ 0.01 & 15.23 $\pm$ 0.16 & 29.85 $\pm$ 0.24 & 37.18 $\pm$ 0.24 & 47.34 $\pm$ 0.18 \\ -\textbf{TransH } & 6320.02 $\pm$ 30.37 & $\phantom{5}$2.54 $\pm$ 0.20 & $\phantom{5}$85.63 $\pm$ 0.40 & $\phantom{5}$1.69 $\pm$ 0.25 & $\phantom{5}$2.95 $\pm$ 0.20 & $\phantom{5}$3.29 $\pm$ 0.22 & $\phantom{5}$3.74 $\pm$ 0.18 \\ -\textbf{TransR } & 6795.95 $\pm$ 16.65 & $\phantom{5}$0.65 $\pm$ 0.02 & $\phantom{5}$91.99 $\pm$ 0.22 & $\phantom{5}$0.37 $\pm$ 0.00 & $\phantom{5}$0.63 $\pm$ 0.04 & $\phantom{5}$0.78 $\pm$ 0.06 & $\phantom{5}$1.03 $\pm$ 0.07 \\ -\textbf{TuckER } & 7327.77 $\pm$ 29.22 & $\phantom{5}$0.07 $\pm$ 0.02 & $\phantom{5}$99.11 $\pm$ 0.39 & $\phantom{5}$0.01 $\pm$ 0.00 & $\phantom{5}$0.02 $\pm$ 0.00 & $\phantom{5}$0.03 $\pm$ 0.01 & $\phantom{5}$0.15 $\pm$ 0.17 \\ +\textbf{ComplEx } & $\phantom{5}$$\phantom{5}$171.16 $\pm$ 16.05 & 19.13 $\pm$ 0.45 & $\phantom{5}$$\phantom{5}$2.46 $\pm$ 0.28 & 10.06 $\pm$ $\phantom{5}$0.42 & 20.82 $\pm$ $\phantom{5}$0.67 & 27.59 $\pm$ $\phantom{5}$0.94 & 38.03 $\pm$ $\phantom{5}$1.02 \\ +\textbf{ConvE } & $\phantom{5}$$\phantom{5}$$\phantom{5}$50.76 $\pm$ $\phantom{5}$0.40 & 59.56 $\pm$ 0.06 & $\phantom{5}$$\phantom{5}$0.73 $\pm$ 0.01 & 48.28 $\pm$ $\phantom{5}$0.12 & 66.99 $\pm$ $\phantom{5}$0.04 & 73.27 $\pm$ $\phantom{5}$0.03 & 79.76 $\pm$ $\phantom{5}$0.07 \\ +\textbf{DistMult} & $\phantom{5}$$\phantom{5}$134.02 $\pm$ $\phantom{5}$1.98 & 26.06 $\pm$ 0.17 & $\phantom{5}$$\phantom{5}$1.86 $\pm$ 0.03 & 16.45 $\pm$ $\phantom{5}$0.16 & 29.10 $\pm$ $\phantom{5}$0.17 & 35.54 $\pm$ $\phantom{5}$0.21 & 45.00 $\pm$ $\phantom{5}$0.25 \\ +\textbf{HolE } & $\phantom{5}$$\phantom{5}$193.03 $\pm$ $\phantom{5}$7.61 & 34.15 $\pm$ 0.22 & $\phantom{5}$$\phantom{5}$2.71 $\pm$ 0.12 & 21.79 $\pm$ $\phantom{5}$0.19 & 39.69 $\pm$ $\phantom{5}$0.24 & 48.06 $\pm$ $\phantom{5}$0.30 & 58.84 $\pm$ $\phantom{5}$0.28 \\ +\textbf{KG2E } & $\phantom{5}$5779.07 $\pm$ 51.02 & $\phantom{5}$0.58 $\pm$ 0.07 & $\phantom{5}$78.40 $\pm$ 0.68 & $\phantom{5}$0.11 $\pm$ $\phantom{5}$0.04 & $\phantom{5}$0.36 $\pm$ $\phantom{5}$0.08 & $\phantom{5}$0.56 $\pm$ $\phantom{5}$0.10 & $\phantom{5}$1.01 $\pm$ $\phantom{5}$0.14 \\ +\textbf{RotatE } & $\phantom{5}$$\phantom{5}$$\phantom{5}$42.28 $\pm$ $\phantom{5}$0.13 & 55.00 $\pm$ 0.06 & $\phantom{5}$$\phantom{5}$0.63 $\pm$ 0.00 & 41.53 $\pm$ $\phantom{5}$0.06 & 64.14 $\pm$ $\phantom{5}$0.07 & 71.23 $\pm$ $\phantom{5}$0.05 & 78.67 $\pm$ $\phantom{5}$0.08 \\ +\textbf{SimplE } & $\phantom{5}$7395.75 $\pm$ $\phantom{5}$2.02 & $\phantom{5}$0.04 $\pm$ 0.00 & 100.02 $\pm$ 0.03 & $\phantom{5}$0.01 $\pm$ $\phantom{5}$0.00 & $\phantom{5}$0.03 $\pm$ $\phantom{5}$0.00 & $\phantom{5}$0.04 $\pm$ $\phantom{5}$0.00 & $\phantom{5}$0.06 $\pm$ $\phantom{5}$0.01 \\ +\textbf{TransD } & $\phantom{5}$$\phantom{5}$153.37 $\pm$ $\phantom{5}$5.35 & 33.99 $\pm$ 0.03 & $\phantom{5}$$\phantom{5}$2.29 $\pm$ 0.09 & 21.22 $\pm$ $\phantom{5}$0.03 & 40.48 $\pm$ $\phantom{5}$0.10 & 48.57 $\pm$ $\phantom{5}$0.09 & 58.71 $\pm$ $\phantom{5}$0.14 \\ +\textbf{TransE } & $\phantom{5}$$\phantom{5}$127.92 $\pm$ $\phantom{5}$0.86 & 26.01 $\pm$ 0.17 & $\phantom{5}$$\phantom{5}$1.78 $\pm$ 0.01 & 15.23 $\pm$ $\phantom{5}$0.16 & 29.85 $\pm$ $\phantom{5}$0.24 & 37.18 $\pm$ $\phantom{5}$0.24 & 47.34 $\pm$ $\phantom{5}$0.18 \\ +\textbf{TransH } & $\phantom{5}$6320.02 $\pm$ 30.37 & $\phantom{5}$2.54 $\pm$ 0.20 & $\phantom{5}$85.63 $\pm$ 0.40 & $\phantom{5}$1.69 $\pm$ $\phantom{5}$0.25 & $\phantom{5}$2.95 $\pm$ $\phantom{5}$0.20 & $\phantom{5}$3.29 $\pm$ $\phantom{5}$0.22 & $\phantom{5}$3.74 $\pm$ $\phantom{5}$0.18 \\ +\textbf{TransR } & $\phantom{5}$6795.95 $\pm$ 16.65 & $\phantom{5}$0.65 $\pm$ 0.02 & $\phantom{5}$91.99 $\pm$ 0.22 & $\phantom{5}$0.37 $\pm$ $\phantom{5}$0.00 & $\phantom{5}$0.63 $\pm$ $\phantom{5}$0.04 & $\phantom{5}$0.78 $\pm$ $\phantom{5}$0.06 & $\phantom{5}$1.03 $\pm$ $\phantom{5}$0.07 \\ +\textbf{TuckER } & $\phantom{5}$7327.77 $\pm$ 29.22 & $\phantom{5}$0.07 $\pm$ 0.02 & $\phantom{5}$99.11 $\pm$ 0.39 & $\phantom{5}$0.01 $\pm$ $\phantom{5}$0.00 & $\phantom{5}$0.02 $\pm$ $\phantom{5}$0.00 & $\phantom{5}$0.03 $\pm$ $\phantom{5}$0.01 & $\phantom{5}$0.15 $\pm$ $\phantom{5}$0.17 \\ \bottomrule \end{tabular} @@ -84,20 +84,20 @@ \subsection{FB15k Best} \centering \begin{tabular}{lrrrrrr} \toprule -{} & MR & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) \\ +{} & MR & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) \\ \midrule -\textbf{ComplEx } & $\phantom{5}$171.16 $\pm$ 16.05 & 19.13 $\pm$ 0.45 & 10.06 $\pm$ 0.42 & 20.82 $\pm$ $\phantom{5}$0.67 & 27.59 $\pm$ $\phantom{5}$0.94 & 38.03 $\pm$ $\phantom{5}$1.02 \\ -\textbf{ConvE } & $\phantom{5}$$\phantom{5}$50.76 $\pm$ $\phantom{5}$0.40 & 59.56 $\pm$ 0.06 & 48.28 $\pm$ 0.12 & 66.99 $\pm$ $\phantom{5}$0.04 & 73.27 $\pm$ $\phantom{5}$0.03 & 79.76 $\pm$ $\phantom{5}$0.07 \\ -\textbf{DistMult} & $\phantom{5}$134.02 $\pm$ $\phantom{5}$1.98 & 26.06 $\pm$ 0.17 & 16.45 $\pm$ 0.16 & 29.10 $\pm$ $\phantom{5}$0.17 & 35.54 $\pm$ $\phantom{5}$0.21 & 45.00 $\pm$ $\phantom{5}$0.25 \\ -\textbf{HolE } & $\phantom{5}$193.03 $\pm$ $\phantom{5}$7.61 & 34.15 $\pm$ 0.22 & 21.79 $\pm$ 0.19 & 39.69 $\pm$ $\phantom{5}$0.24 & 48.06 $\pm$ $\phantom{5}$0.30 & 58.84 $\pm$ $\phantom{5}$0.28 \\ -\textbf{KG2E } & 5779.07 $\pm$ 51.02 & $\phantom{5}$0.58 $\pm$ 0.07 & $\phantom{5}$0.11 $\pm$ 0.04 & $\phantom{5}$0.36 $\pm$ $\phantom{5}$0.08 & $\phantom{5}$0.56 $\pm$ $\phantom{5}$0.10 & $\phantom{5}$1.01 $\pm$ $\phantom{5}$0.14 \\ -\textbf{RotatE } & $\phantom{5}$$\phantom{5}$42.28 $\pm$ $\phantom{5}$0.13 & 55.00 $\pm$ 0.06 & 41.53 $\pm$ 0.06 & 64.14 $\pm$ $\phantom{5}$0.07 & 71.23 $\pm$ $\phantom{5}$0.05 & 78.67 $\pm$ $\phantom{5}$0.08 \\ -\textbf{SimplE } & $\phantom{5}$139.34 $\pm$ 49.45 & 23.90 $\pm$ 8.79 & 11.58 $\pm$ 6.42 & 24.16 $\pm$ 10.95 & 34.73 $\pm$ 13.40 & 54.28 $\pm$ 15.80 \\ -\textbf{TransD } & $\phantom{5}$153.37 $\pm$ $\phantom{5}$5.35 & 33.99 $\pm$ 0.03 & 21.22 $\pm$ 0.03 & 40.48 $\pm$ $\phantom{5}$0.10 & 48.57 $\pm$ $\phantom{5}$0.09 & 58.71 $\pm$ $\phantom{5}$0.14 \\ -\textbf{TransE } & $\phantom{5}$127.92 $\pm$ $\phantom{5}$0.86 & 26.01 $\pm$ 0.17 & 15.23 $\pm$ 0.16 & 29.85 $\pm$ $\phantom{5}$0.24 & 37.18 $\pm$ $\phantom{5}$0.24 & 47.34 $\pm$ $\phantom{5}$0.18 \\ -\textbf{TransH } & 6320.00 $\pm$ 30.37 & $\phantom{5}$2.54 $\pm$ 0.20 & $\phantom{5}$1.69 $\pm$ 0.25 & $\phantom{5}$2.95 $\pm$ $\phantom{5}$0.20 & $\phantom{5}$3.29 $\pm$ $\phantom{5}$0.22 & $\phantom{5}$3.74 $\pm$ $\phantom{5}$0.18 \\ -\textbf{TransR } & 6795.94 $\pm$ 16.65 & $\phantom{5}$0.65 $\pm$ 0.02 & $\phantom{5}$0.37 $\pm$ 0.00 & $\phantom{5}$0.63 $\pm$ $\phantom{5}$0.04 & $\phantom{5}$0.78 $\pm$ $\phantom{5}$0.06 & $\phantom{5}$1.03 $\pm$ $\phantom{5}$0.07 \\ -\textbf{TuckER } & 7327.77 $\pm$ 29.22 & $\phantom{5}$0.07 $\pm$ 0.02 & $\phantom{5}$0.01 $\pm$ 0.00 & $\phantom{5}$0.02 $\pm$ $\phantom{5}$0.00 & $\phantom{5}$0.03 $\pm$ $\phantom{5}$0.01 & $\phantom{5}$0.15 $\pm$ $\phantom{5}$0.17 \\ +\textbf{ComplEx } & $\phantom{5}$$\phantom{5}$171.16 $\pm$ 16.05 & 19.13 $\pm$ 0.45 & 10.06 $\pm$ $\phantom{5}$0.42 & 20.82 $\pm$ $\phantom{5}$0.67 & 27.59 $\pm$ $\phantom{5}$0.94 & 38.03 $\pm$ $\phantom{5}$1.02 \\ +\textbf{ConvE } & $\phantom{5}$$\phantom{5}$$\phantom{5}$50.76 $\pm$ $\phantom{5}$0.40 & 59.56 $\pm$ 0.06 & 48.28 $\pm$ $\phantom{5}$0.12 & 66.99 $\pm$ $\phantom{5}$0.04 & 73.27 $\pm$ $\phantom{5}$0.03 & 79.76 $\pm$ $\phantom{5}$0.07 \\ +\textbf{DistMult} & $\phantom{5}$$\phantom{5}$134.02 $\pm$ $\phantom{5}$1.98 & 26.06 $\pm$ 0.17 & 16.45 $\pm$ $\phantom{5}$0.16 & 29.10 $\pm$ $\phantom{5}$0.17 & 35.54 $\pm$ $\phantom{5}$0.21 & 45.00 $\pm$ $\phantom{5}$0.25 \\ +\textbf{HolE } & $\phantom{5}$$\phantom{5}$193.03 $\pm$ $\phantom{5}$7.61 & 34.15 $\pm$ 0.22 & 21.79 $\pm$ $\phantom{5}$0.19 & 39.69 $\pm$ $\phantom{5}$0.24 & 48.06 $\pm$ $\phantom{5}$0.30 & 58.84 $\pm$ $\phantom{5}$0.28 \\ +\textbf{KG2E } & $\phantom{5}$5779.07 $\pm$ 51.02 & $\phantom{5}$0.58 $\pm$ 0.07 & $\phantom{5}$0.11 $\pm$ $\phantom{5}$0.04 & $\phantom{5}$0.36 $\pm$ $\phantom{5}$0.08 & $\phantom{5}$0.56 $\pm$ $\phantom{5}$0.10 & $\phantom{5}$1.01 $\pm$ $\phantom{5}$0.14 \\ +\textbf{RotatE } & $\phantom{5}$$\phantom{5}$$\phantom{5}$42.28 $\pm$ $\phantom{5}$0.13 & 55.00 $\pm$ 0.06 & 41.53 $\pm$ $\phantom{5}$0.06 & 64.14 $\pm$ $\phantom{5}$0.07 & 71.23 $\pm$ $\phantom{5}$0.05 & 78.67 $\pm$ $\phantom{5}$0.08 \\ +\textbf{SimplE } & $\phantom{5}$$\phantom{5}$139.34 $\pm$ 49.45 & 23.90 $\pm$ 8.79 & 11.58 $\pm$ $\phantom{5}$6.42 & 24.16 $\pm$ 10.95 & 34.73 $\pm$ 13.40 & 54.28 $\pm$ 15.80 \\ +\textbf{TransD } & $\phantom{5}$$\phantom{5}$153.37 $\pm$ $\phantom{5}$5.35 & 33.99 $\pm$ 0.03 & 21.22 $\pm$ $\phantom{5}$0.03 & 40.48 $\pm$ $\phantom{5}$0.10 & 48.57 $\pm$ $\phantom{5}$0.09 & 58.71 $\pm$ $\phantom{5}$0.14 \\ +\textbf{TransE } & $\phantom{5}$$\phantom{5}$127.92 $\pm$ $\phantom{5}$0.86 & 26.01 $\pm$ 0.17 & 15.23 $\pm$ $\phantom{5}$0.16 & 29.85 $\pm$ $\phantom{5}$0.24 & 37.18 $\pm$ $\phantom{5}$0.24 & 47.34 $\pm$ $\phantom{5}$0.18 \\ +\textbf{TransH } & $\phantom{5}$6320.00 $\pm$ 30.37 & $\phantom{5}$2.54 $\pm$ 0.20 & $\phantom{5}$1.69 $\pm$ $\phantom{5}$0.25 & $\phantom{5}$2.95 $\pm$ $\phantom{5}$0.20 & $\phantom{5}$3.29 $\pm$ $\phantom{5}$0.22 & $\phantom{5}$3.74 $\pm$ $\phantom{5}$0.18 \\ +\textbf{TransR } & $\phantom{5}$6795.94 $\pm$ 16.65 & $\phantom{5}$0.65 $\pm$ 0.02 & $\phantom{5}$0.37 $\pm$ $\phantom{5}$0.00 & $\phantom{5}$0.63 $\pm$ $\phantom{5}$0.04 & $\phantom{5}$0.78 $\pm$ $\phantom{5}$0.06 & $\phantom{5}$1.03 $\pm$ $\phantom{5}$0.07 \\ +\textbf{TuckER } & $\phantom{5}$7327.77 $\pm$ 29.22 & $\phantom{5}$0.07 $\pm$ 0.02 & $\phantom{5}$0.01 $\pm$ $\phantom{5}$0.00 & $\phantom{5}$0.02 $\pm$ $\phantom{5}$0.00 & $\phantom{5}$0.03 $\pm$ $\phantom{5}$0.01 & $\phantom{5}$0.15 $\pm$ $\phantom{5}$0.17 \\ \bottomrule \end{tabular} @@ -109,20 +109,20 @@ \subsection{FB15k Worst} \centering \begin{tabular}{lrrrrrr} \toprule -{} & MR & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) \\ +{} & MR & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) \\ \midrule -\textbf{ComplEx } & $\phantom{5}$171.16 $\pm$ 16.05 & 19.13 $\pm$ 0.45 & 10.06 $\pm$ 0.42 & 20.82 $\pm$ $\phantom{5}$0.67 & 27.59 $\pm$ $\phantom{5}$0.94 & 38.03 $\pm$ $\phantom{5}$1.02 \\ -\textbf{ConvE } & $\phantom{5}$$\phantom{5}$50.76 $\pm$ $\phantom{5}$0.40 & 59.56 $\pm$ 0.06 & 48.28 $\pm$ 0.12 & 66.99 $\pm$ $\phantom{5}$0.04 & 73.27 $\pm$ $\phantom{5}$0.03 & 79.76 $\pm$ $\phantom{5}$0.07 \\ -\textbf{DistMult} & $\phantom{5}$134.02 $\pm$ $\phantom{5}$1.98 & 26.06 $\pm$ 0.17 & 16.45 $\pm$ 0.16 & 29.10 $\pm$ $\phantom{5}$0.17 & 35.54 $\pm$ $\phantom{5}$0.21 & 45.00 $\pm$ $\phantom{5}$0.25 \\ -\textbf{HolE } & $\phantom{5}$193.03 $\pm$ $\phantom{5}$7.61 & 34.15 $\pm$ 0.22 & 21.79 $\pm$ 0.19 & 39.69 $\pm$ $\phantom{5}$0.24 & 48.06 $\pm$ $\phantom{5}$0.30 & 58.84 $\pm$ $\phantom{5}$0.28 \\ -\textbf{KG2E } & 5779.07 $\pm$ 51.02 & $\phantom{5}$0.58 $\pm$ 0.07 & $\phantom{5}$0.11 $\pm$ 0.04 & $\phantom{5}$0.36 $\pm$ $\phantom{5}$0.08 & $\phantom{5}$0.56 $\pm$ $\phantom{5}$0.10 & $\phantom{5}$1.01 $\pm$ $\phantom{5}$0.14 \\ -\textbf{RotatE } & $\phantom{5}$$\phantom{5}$42.28 $\pm$ $\phantom{5}$0.13 & 55.00 $\pm$ 0.06 & 41.53 $\pm$ 0.06 & 64.14 $\pm$ $\phantom{5}$0.07 & 71.23 $\pm$ $\phantom{5}$0.05 & 78.67 $\pm$ $\phantom{5}$0.08 \\ -\textbf{SimplE } & $\phantom{5}$139.34 $\pm$ 49.45 & 23.90 $\pm$ 8.79 & 11.58 $\pm$ 6.42 & 24.16 $\pm$ 10.95 & 34.73 $\pm$ 13.40 & 54.28 $\pm$ 15.80 \\ -\textbf{TransD } & $\phantom{5}$153.37 $\pm$ $\phantom{5}$5.35 & 33.99 $\pm$ 0.03 & 21.22 $\pm$ 0.03 & 40.48 $\pm$ $\phantom{5}$0.10 & 48.57 $\pm$ $\phantom{5}$0.09 & 58.71 $\pm$ $\phantom{5}$0.14 \\ -\textbf{TransE } & $\phantom{5}$127.92 $\pm$ $\phantom{5}$0.86 & 26.01 $\pm$ 0.17 & 15.23 $\pm$ 0.16 & 29.85 $\pm$ $\phantom{5}$0.24 & 37.18 $\pm$ $\phantom{5}$0.24 & 47.34 $\pm$ $\phantom{5}$0.18 \\ -\textbf{TransH } & 6320.00 $\pm$ 30.37 & $\phantom{5}$2.54 $\pm$ 0.20 & $\phantom{5}$1.69 $\pm$ 0.25 & $\phantom{5}$2.95 $\pm$ $\phantom{5}$0.20 & $\phantom{5}$3.29 $\pm$ $\phantom{5}$0.22 & $\phantom{5}$3.74 $\pm$ $\phantom{5}$0.18 \\ -\textbf{TransR } & 6795.94 $\pm$ 16.65 & $\phantom{5}$0.65 $\pm$ 0.02 & $\phantom{5}$0.37 $\pm$ 0.00 & $\phantom{5}$0.63 $\pm$ $\phantom{5}$0.04 & $\phantom{5}$0.78 $\pm$ $\phantom{5}$0.06 & $\phantom{5}$1.03 $\pm$ $\phantom{5}$0.07 \\ -\textbf{TuckER } & 7327.77 $\pm$ 29.22 & $\phantom{5}$0.07 $\pm$ 0.02 & $\phantom{5}$0.01 $\pm$ 0.00 & $\phantom{5}$0.02 $\pm$ $\phantom{5}$0.00 & $\phantom{5}$0.03 $\pm$ $\phantom{5}$0.01 & $\phantom{5}$0.15 $\pm$ $\phantom{5}$0.17 \\ +\textbf{ComplEx } & $\phantom{5}$$\phantom{5}$171.16 $\pm$ 16.05 & 19.13 $\pm$ 0.45 & 10.06 $\pm$ $\phantom{5}$0.42 & 20.82 $\pm$ $\phantom{5}$0.67 & 27.59 $\pm$ $\phantom{5}$0.94 & 38.03 $\pm$ $\phantom{5}$1.02 \\ +\textbf{ConvE } & $\phantom{5}$$\phantom{5}$$\phantom{5}$50.76 $\pm$ $\phantom{5}$0.40 & 59.56 $\pm$ 0.06 & 48.28 $\pm$ $\phantom{5}$0.12 & 66.99 $\pm$ $\phantom{5}$0.04 & 73.27 $\pm$ $\phantom{5}$0.03 & 79.76 $\pm$ $\phantom{5}$0.07 \\ +\textbf{DistMult} & $\phantom{5}$$\phantom{5}$134.02 $\pm$ $\phantom{5}$1.98 & 26.06 $\pm$ 0.17 & 16.45 $\pm$ $\phantom{5}$0.16 & 29.10 $\pm$ $\phantom{5}$0.17 & 35.54 $\pm$ $\phantom{5}$0.21 & 45.00 $\pm$ $\phantom{5}$0.25 \\ +\textbf{HolE } & $\phantom{5}$$\phantom{5}$193.03 $\pm$ $\phantom{5}$7.61 & 34.15 $\pm$ 0.22 & 21.79 $\pm$ $\phantom{5}$0.19 & 39.69 $\pm$ $\phantom{5}$0.24 & 48.06 $\pm$ $\phantom{5}$0.30 & 58.84 $\pm$ $\phantom{5}$0.28 \\ +\textbf{KG2E } & $\phantom{5}$5779.07 $\pm$ 51.02 & $\phantom{5}$0.58 $\pm$ 0.07 & $\phantom{5}$0.11 $\pm$ $\phantom{5}$0.04 & $\phantom{5}$0.36 $\pm$ $\phantom{5}$0.08 & $\phantom{5}$0.56 $\pm$ $\phantom{5}$0.10 & $\phantom{5}$1.01 $\pm$ $\phantom{5}$0.14 \\ +\textbf{RotatE } & $\phantom{5}$$\phantom{5}$$\phantom{5}$42.28 $\pm$ $\phantom{5}$0.13 & 55.00 $\pm$ 0.06 & 41.53 $\pm$ $\phantom{5}$0.06 & 64.14 $\pm$ $\phantom{5}$0.07 & 71.23 $\pm$ $\phantom{5}$0.05 & 78.67 $\pm$ $\phantom{5}$0.08 \\ +\textbf{SimplE } & $\phantom{5}$$\phantom{5}$139.34 $\pm$ 49.45 & 23.90 $\pm$ 8.79 & 11.58 $\pm$ $\phantom{5}$6.42 & 24.16 $\pm$ 10.95 & 34.73 $\pm$ 13.40 & 54.28 $\pm$ 15.80 \\ +\textbf{TransD } & $\phantom{5}$$\phantom{5}$153.37 $\pm$ $\phantom{5}$5.35 & 33.99 $\pm$ 0.03 & 21.22 $\pm$ $\phantom{5}$0.03 & 40.48 $\pm$ $\phantom{5}$0.10 & 48.57 $\pm$ $\phantom{5}$0.09 & 58.71 $\pm$ $\phantom{5}$0.14 \\ +\textbf{TransE } & $\phantom{5}$$\phantom{5}$127.92 $\pm$ $\phantom{5}$0.86 & 26.01 $\pm$ 0.17 & 15.23 $\pm$ $\phantom{5}$0.16 & 29.85 $\pm$ $\phantom{5}$0.24 & 37.18 $\pm$ $\phantom{5}$0.24 & 47.34 $\pm$ $\phantom{5}$0.18 \\ +\textbf{TransH } & $\phantom{5}$6320.00 $\pm$ 30.37 & $\phantom{5}$2.54 $\pm$ 0.20 & $\phantom{5}$1.69 $\pm$ $\phantom{5}$0.25 & $\phantom{5}$2.95 $\pm$ $\phantom{5}$0.20 & $\phantom{5}$3.29 $\pm$ $\phantom{5}$0.22 & $\phantom{5}$3.74 $\pm$ $\phantom{5}$0.18 \\ +\textbf{TransR } & $\phantom{5}$6795.94 $\pm$ 16.65 & $\phantom{5}$0.65 $\pm$ 0.02 & $\phantom{5}$0.37 $\pm$ $\phantom{5}$0.00 & $\phantom{5}$0.63 $\pm$ $\phantom{5}$0.04 & $\phantom{5}$0.78 $\pm$ $\phantom{5}$0.06 & $\phantom{5}$1.03 $\pm$ $\phantom{5}$0.07 \\ +\textbf{TuckER } & $\phantom{5}$7327.77 $\pm$ 29.22 & $\phantom{5}$0.07 $\pm$ 0.02 & $\phantom{5}$0.01 $\pm$ $\phantom{5}$0.00 & $\phantom{5}$0.02 $\pm$ $\phantom{5}$0.00 & $\phantom{5}$0.03 $\pm$ $\phantom{5}$0.01 & $\phantom{5}$0.15 $\pm$ $\phantom{5}$0.17 \\ \bottomrule \end{tabular} diff --git a/reproducibility/summaries/wn18_table.tex b/reproducibility/summaries/wn18_table.tex index 20f42285e..03462c9c0 100644 --- a/reproducibility/summaries/wn18_table.tex +++ b/reproducibility/summaries/wn18_table.tex @@ -1,21 +1,42 @@ -\begin{tabular}{lrrrrrrrrrrrrrrrrrrr} +\begin{tabular}{llrrrrrrr} \toprule -{} & \multicolumn{7}{l}{avg} & \multicolumn{6}{l}{best} & \multicolumn{6}{l}{worst} \\ -{} & AMR (\%) & Hits@1 (\%) & Hits@10 (\%) & Hits@3 (\%) & Hits@5 (\%) & MR & MRR (\%) & Hits@1 (\%) & Hits@10 (\%) & Hits@3 (\%) & Hits@5 (\%) & MR & MRR (\%) & Hits@1 (\%) & Hits@10 (\%) & Hits@3 (\%) & Hits@5 (\%) & MR & MRR (\%) \\ -\textbf{model } & & & & & & & & & & & & & & & & & & & \\ + & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ +\textbf{model} & {} & & & & & & & \\ \midrule -\textbf{ComplEx } & $\phantom{5}$$\phantom{5}$2.21 $\pm$ 0.31 & 12.36 $\pm$ 1.96 & 32.92 $\pm$ 4.40 & 20.66 $\pm$ 2.75 & 25.24 $\pm$ 3.33 & $\phantom{5}$$\phantom{5}$452.67 $\pm$ $\phantom{5}$63.05 & 19.49 $\pm$ 2.55 & 12.36 $\pm$ 1.96 & 32.92 $\pm$ 4.40 & 20.66 $\pm$ 2.75 & 25.24 $\pm$ 3.33 & $\phantom{5}$$\phantom{5}$452.67 $\pm$ $\phantom{5}$63.05 & 19.49 $\pm$ 2.55 & 12.36 $\pm$ 1.96 & 32.92 $\pm$ 4.40 & 20.66 $\pm$ 2.75 & 25.24 $\pm$ 3.33 & $\phantom{5}$$\phantom{5}$452.67 $\pm$ $\phantom{5}$63.05 & 19.49 $\pm$ 2.55 \\ -\textbf{ConvE } & $\phantom{5}$$\phantom{5}$2.17 $\pm$ 0.07 & 85.14 $\pm$ 0.10 & 94.85 $\pm$ 0.06 & 91.76 $\pm$ 0.11 & 93.29 $\pm$ 0.04 & $\phantom{5}$$\phantom{5}$444.40 $\pm$ $\phantom{5}$14.82 & 88.81 $\pm$ 0.09 & 85.14 $\pm$ 0.10 & 94.85 $\pm$ 0.06 & 91.76 $\pm$ 0.11 & 93.29 $\pm$ 0.04 & $\phantom{5}$$\phantom{5}$444.40 $\pm$ $\phantom{5}$14.82 & 88.81 $\pm$ 0.09 & 85.14 $\pm$ 0.10 & 94.85 $\pm$ 0.06 & 91.76 $\pm$ 0.11 & 93.29 $\pm$ 0.04 & $\phantom{5}$$\phantom{5}$444.40 $\pm$ $\phantom{5}$14.82 & 88.81 $\pm$ 0.09 \\ -\textbf{DistMult} & $\phantom{5}$$\phantom{5}$2.24 $\pm$ 0.12 & 67.45 $\pm$ 0.34 & 92.72 $\pm$ 0.18 & 85.94 $\pm$ 0.21 & 89.52 $\pm$ 0.25 & $\phantom{5}$$\phantom{5}$458.64 $\pm$ $\phantom{5}$23.96 & 77.44 $\pm$ 0.22 & 67.45 $\pm$ 0.34 & 92.72 $\pm$ 0.18 & 85.94 $\pm$ 0.21 & 89.52 $\pm$ 0.25 & $\phantom{5}$$\phantom{5}$458.64 $\pm$ $\phantom{5}$23.96 & 77.44 $\pm$ 0.22 & 67.45 $\pm$ 0.34 & 92.72 $\pm$ 0.18 & 85.94 $\pm$ 0.21 & 89.52 $\pm$ 0.25 & $\phantom{5}$$\phantom{5}$458.64 $\pm$ $\phantom{5}$23.96 & 77.44 $\pm$ 0.22 \\ -\textbf{HolE } & $\phantom{5}$$\phantom{5}$3.97 $\pm$ 0.14 & 59.29 $\pm$ 0.53 & 88.61 $\pm$ 0.42 & 79.29 $\pm$ 0.47 & 84.12 $\pm$ 0.36 & $\phantom{5}$$\phantom{5}$812.64 $\pm$ $\phantom{5}$28.33 & 70.44 $\pm$ 0.45 & 59.29 $\pm$ 0.53 & 88.61 $\pm$ 0.42 & 79.29 $\pm$ 0.47 & 84.12 $\pm$ 0.36 & $\phantom{5}$$\phantom{5}$812.63 $\pm$ $\phantom{5}$28.33 & 70.44 $\pm$ 0.45 & 59.29 $\pm$ 0.53 & 88.61 $\pm$ 0.42 & 79.29 $\pm$ 0.47 & 84.12 $\pm$ 0.36 & $\phantom{5}$$\phantom{5}$812.64 $\pm$ $\phantom{5}$28.34 & 70.44 $\pm$ 0.45 \\ -\textbf{KG2E } & $\phantom{5}$13.25 $\pm$ 0.22 & $\phantom{5}$1.35 $\pm$ 0.22 & $\phantom{5}$7.02 $\pm$ 0.43 & $\phantom{5}$3.21 $\pm$ 0.31 & $\phantom{5}$4.57 $\pm$ 0.34 & $\phantom{5}$2708.89 $\pm$ $\phantom{5}$44.57 & $\phantom{5}$3.61 $\pm$ 0.26 & $\phantom{5}$1.35 $\pm$ 0.22 & $\phantom{5}$7.02 $\pm$ 0.43 & $\phantom{5}$3.21 $\pm$ 0.31 & $\phantom{5}$4.57 $\pm$ 0.34 & $\phantom{5}$2708.88 $\pm$ $\phantom{5}$44.57 & $\phantom{5}$3.61 $\pm$ 0.26 & $\phantom{5}$1.35 $\pm$ 0.22 & $\phantom{5}$7.02 $\pm$ 0.43 & $\phantom{5}$3.21 $\pm$ 0.31 & $\phantom{5}$4.57 $\pm$ 0.34 & $\phantom{5}$2708.89 $\pm$ $\phantom{5}$44.57 & $\phantom{5}$3.61 $\pm$ 0.26 \\ -\textbf{RotatE } & $\phantom{5}$$\phantom{5}$0.61 $\pm$ 0.01 & 82.17 $\pm$ 0.20 & 95.28 $\pm$ 0.08 & 91.53 $\pm$ 0.12 & 93.44 $\pm$ 0.07 & $\phantom{5}$$\phantom{5}$123.68 $\pm$ $\phantom{5}$$\phantom{5}$1.71 & 87.29 $\pm$ 0.12 & 82.17 $\pm$ 0.20 & 95.28 $\pm$ 0.08 & 91.53 $\pm$ 0.12 & 93.44 $\pm$ 0.07 & $\phantom{5}$$\phantom{5}$123.68 $\pm$ $\phantom{5}$$\phantom{5}$1.71 & 87.29 $\pm$ 0.12 & 82.17 $\pm$ 0.20 & 95.28 $\pm$ 0.08 & 91.53 $\pm$ 0.12 & 93.44 $\pm$ 0.07 & $\phantom{5}$$\phantom{5}$123.68 $\pm$ $\phantom{5}$$\phantom{5}$1.71 & 87.29 $\pm$ 0.12 \\ -\textbf{SimplE } & $\phantom{5}$99.57 $\pm$ 0.21 & $\phantom{5}$0.01 $\pm$ 0.01 & $\phantom{5}$0.07 $\pm$ 0.03 & $\phantom{5}$0.03 $\pm$ 0.00 & $\phantom{5}$0.04 $\pm$ 0.01 & 20376.43 $\pm$ $\phantom{5}$42.30 & $\phantom{5}$0.04 $\pm$ 0.01 & 33.93 $\pm$ 4.32 & 47.01 $\pm$ 2.66 & 39.59 $\pm$ 4.67 & 42.76 $\pm$ 3.73 & $\phantom{5}$$\phantom{5}$384.53 $\pm$ $\phantom{5}$66.45 & 38.48 $\pm$ 4.00 & $\phantom{5}$0.01 $\pm$ 0.01 & $\phantom{5}$0.07 $\pm$ 0.03 & $\phantom{5}$0.03 $\pm$ 0.00 & $\phantom{5}$0.04 $\pm$ 0.01 & 40368.33 $\pm$ 114.95 & $\phantom{5}$0.03 $\pm$ 0.01 \\ -\textbf{TransD } & $\phantom{5}$$\phantom{5}$2.17 $\pm$ 0.13 & $\phantom{5}$3.94 $\pm$ 0.27 & 87.27 $\pm$ 0.41 & 65.63 $\pm$ 0.55 & 79.64 $\pm$ 0.43 & $\phantom{5}$$\phantom{5}$444.39 $\pm$ $\phantom{5}$25.61 & 36.22 $\pm$ 0.12 & $\phantom{5}$3.94 $\pm$ 0.27 & 87.27 $\pm$ 0.41 & 65.63 $\pm$ 0.55 & 79.64 $\pm$ 0.43 & $\phantom{5}$$\phantom{5}$444.39 $\pm$ $\phantom{5}$25.61 & 36.22 $\pm$ 0.12 & $\phantom{5}$3.94 $\pm$ 0.27 & 87.27 $\pm$ 0.41 & 65.63 $\pm$ 0.55 & 79.64 $\pm$ 0.43 & $\phantom{5}$$\phantom{5}$444.39 $\pm$ $\phantom{5}$25.61 & 36.22 $\pm$ 0.12 \\ -\textbf{TransE } & $\phantom{5}$$\phantom{5}$2.29 $\pm$ 0.07 & $\phantom{5}$9.99 $\pm$ 1.82 & 84.25 $\pm$ 0.33 & 64.74 $\pm$ 0.91 & 75.44 $\pm$ 0.48 & $\phantom{5}$$\phantom{5}$468.24 $\pm$ $\phantom{5}$13.64 & 39.19 $\pm$ 1.21 & $\phantom{5}$9.99 $\pm$ 1.82 & 84.25 $\pm$ 0.33 & 64.74 $\pm$ 0.91 & 75.44 $\pm$ 0.48 & $\phantom{5}$$\phantom{5}$468.24 $\pm$ $\phantom{5}$13.64 & 39.19 $\pm$ 1.21 & $\phantom{5}$9.99 $\pm$ 1.82 & 84.25 $\pm$ 0.33 & 64.74 $\pm$ 0.91 & 75.44 $\pm$ 0.48 & $\phantom{5}$$\phantom{5}$468.24 $\pm$ $\phantom{5}$13.64 & 39.19 $\pm$ 1.21 \\ -\textbf{TransH } & $\phantom{5}$96.16 $\pm$ 0.09 & $\phantom{5}$0.04 $\pm$ 0.02 & $\phantom{5}$0.39 $\pm$ 0.11 & $\phantom{5}$0.19 $\pm$ 0.10 & $\phantom{5}$0.29 $\pm$ 0.14 & 19678.04 $\pm$ $\phantom{5}$18.92 & $\phantom{5}$0.18 $\pm$ 0.04 & $\phantom{5}$0.04 $\pm$ 0.02 & $\phantom{5}$0.39 $\pm$ 0.11 & $\phantom{5}$0.19 $\pm$ 0.10 & $\phantom{5}$0.29 $\pm$ 0.14 & 19678.02 $\pm$ $\phantom{5}$18.92 & $\phantom{5}$0.18 $\pm$ 0.04 & $\phantom{5}$0.04 $\pm$ 0.02 & $\phantom{5}$0.39 $\pm$ 0.11 & $\phantom{5}$0.19 $\pm$ 0.10 & $\phantom{5}$0.29 $\pm$ 0.14 & 19678.06 $\pm$ $\phantom{5}$18.92 & $\phantom{5}$0.18 $\pm$ 0.04 \\ -\textbf{TransR } & $\phantom{5}$96.20 $\pm$ 0.49 & $\phantom{5}$0.00 $\pm$ 0.00 & $\phantom{5}$0.11 $\pm$ 0.06 & $\phantom{5}$0.04 $\pm$ 0.02 & $\phantom{5}$0.05 $\pm$ 0.03 & 19686.49 $\pm$ 100.97 & $\phantom{5}$0.06 $\pm$ 0.02 & $\phantom{5}$0.00 $\pm$ 0.00 & $\phantom{5}$0.11 $\pm$ 0.06 & $\phantom{5}$0.04 $\pm$ 0.02 & $\phantom{5}$0.05 $\pm$ 0.03 & 19686.49 $\pm$ 100.97 & $\phantom{5}$0.06 $\pm$ 0.02 & $\phantom{5}$0.00 $\pm$ 0.00 & $\phantom{5}$0.11 $\pm$ 0.06 & $\phantom{5}$0.04 $\pm$ 0.02 & $\phantom{5}$0.05 $\pm$ 0.03 & 19686.50 $\pm$ 100.97 & $\phantom{5}$0.06 $\pm$ 0.02 \\ -\textbf{TuckER } & 100.78 $\pm$ 0.75 & $\phantom{5}$0.00 $\pm$ 0.00 & $\phantom{5}$0.04 $\pm$ 0.03 & $\phantom{5}$0.02 $\pm$ 0.01 & $\phantom{5}$0.03 $\pm$ 0.03 & 20622.46 $\pm$ 153.52 & $\phantom{5}$0.03 $\pm$ 0.01 & $\phantom{5}$0.00 $\pm$ 0.00 & $\phantom{5}$0.04 $\pm$ 0.03 & $\phantom{5}$0.02 $\pm$ 0.01 & $\phantom{5}$0.03 $\pm$ 0.03 & 20622.46 $\pm$ 153.52 & $\phantom{5}$0.03 $\pm$ 0.01 & $\phantom{5}$0.00 $\pm$ 0.00 & $\phantom{5}$0.04 $\pm$ 0.03 & $\phantom{5}$0.02 $\pm$ 0.01 & $\phantom{5}$0.03 $\pm$ 0.03 & 20622.46 $\pm$ 153.52 & $\phantom{5}$0.03 $\pm$ 0.01 \\ +\textbf{ComplEx} & \textbf{pub} & $94.1\phantom{0 \pm 0.00}$ & $93.6\phantom{0 \pm 0.00}$ & $94.5\phantom{0 \pm 0.00}$ & & $94.7\phantom{0 \pm 0.00}$ & & \\ + & \textbf{R} & $19.49 \pm 2.55$ & $12.36 \pm 1.96$ & $20.66 \pm 2.75$ & $25.24 \pm 3.33$ & $32.92 \pm 4.40$ & $\phantom{00}452.67 \pm \phantom{0}63.05$ & $\phantom{00}2.21 \pm 0.31$ \\\midrule +\textbf{ConvE} & \textbf{pub} & $94.3\phantom{0 \pm 0.00}$ & $93.5\phantom{0 \pm 0.00}$ & $94.6\phantom{0 \pm 0.00}$ & & $95.6\phantom{0 \pm 0.00}$ & $\phantom{00}374.\phantom{00 \pm 000.00}$ & \\ + & \textbf{R} & $88.81 \pm 0.09$ & $85.14 \pm 0.10$ & $91.76 \pm 0.11$ & $93.29 \pm 0.04$ & $94.85 \pm 0.06$ & $\phantom{00}444.40 \pm \phantom{0}14.82$ & $\phantom{00}2.17 \pm 0.07$ \\\midrule +\textbf{DistMult} & \textbf{pub} & $83.\phantom{00 \pm 0.00}$ & & & & $94.2\phantom{0 \pm 0.00}$ & & \\ + & \textbf{R} & $77.44 \pm 0.22$ & $67.45 \pm 0.34$ & $85.94 \pm 0.21$ & $89.52 \pm 0.25$ & $92.72 \pm 0.18$ & $\phantom{00}458.64 \pm \phantom{0}23.96$ & $\phantom{00}2.24 \pm 0.12$ \\\midrule +\textbf{HolE} & \textbf{pub} & $93.8\phantom{0 \pm 0.00}$ & $93.\phantom{00 \pm 0.00}$ & $94.5\phantom{0 \pm 0.00}$ & & $94.9\phantom{0 \pm 0.00}$ & & \\ + & \textbf{R} & $70.44 \pm 0.45$ & $59.29 \pm 0.53$ & $79.29 \pm 0.47$ & $84.12 \pm 0.36$ & $88.61 \pm 0.42$ & $\phantom{00}812.64 \pm \phantom{0}28.33$ & $\phantom{00}3.97 \pm 0.14$ \\ + & \textbf{O} & $70.44 \pm 0.45$ & $59.29 \pm 0.53$ & $79.29 \pm 0.47$ & $84.12 \pm 0.36$ & $88.61 \pm 0.42$ & $\phantom{00}812.63 \pm \phantom{0}28.33$ & \\ + & \textbf{P} & $70.44 \pm 0.45$ & $59.29 \pm 0.53$ & $79.29 \pm 0.47$ & $84.12 \pm 0.36$ & $88.61 \pm 0.42$ & $\phantom{00}812.64 \pm \phantom{0}28.34$ & \\\midrule +\textbf{KG2E} & \textbf{pub} & & & & & $92.8\phantom{0 \pm 0.00}$ & $\phantom{00}331.\phantom{00 \pm 000.00}$ & \\ + & \textbf{R} & $\phantom{0}3.61 \pm 0.26$ & $\phantom{0}1.35 \pm 0.22$ & $\phantom{0}3.21 \pm 0.31$ & $\phantom{0}4.57 \pm 0.34$ & $\phantom{0}7.02 \pm 0.43$ & $\phantom{0}2708.89 \pm \phantom{0}44.57$ & $\phantom{0}13.25 \pm 0.22$ \\ + & \textbf{O} & $\phantom{0}3.61 \pm 0.26$ & $\phantom{0}1.35 \pm 0.22$ & $\phantom{0}3.21 \pm 0.31$ & $\phantom{0}4.57 \pm 0.34$ & $\phantom{0}7.02 \pm 0.43$ & $\phantom{0}2708.88 \pm \phantom{0}44.57$ & \\ + & \textbf{P} & $\phantom{0}3.61 \pm 0.26$ & $\phantom{0}1.35 \pm 0.22$ & $\phantom{0}3.21 \pm 0.31$ & $\phantom{0}4.57 \pm 0.34$ & $\phantom{0}7.02 \pm 0.43$ & $\phantom{0}2708.89 \pm \phantom{0}44.57$ & \\\midrule +\textbf{RotatE} & \textbf{pub} & $94.9\phantom{0 \pm 0.00}$ & $94.4\phantom{0 \pm 0.00}$ & $95.2\phantom{0 \pm 0.00}$ & & $95.9\phantom{0 \pm 0.00}$ & $\phantom{00}309.\phantom{00 \pm 000.00}$ & \\ + & \textbf{R} & $87.29 \pm 0.12$ & $82.17 \pm 0.20$ & $91.53 \pm 0.12$ & $93.44 \pm 0.07$ & $95.28 \pm 0.08$ & $\phantom{00}123.68 \pm \phantom{00}1.71$ & $\phantom{00}0.61 \pm 0.01$ \\\midrule +\textbf{SimplE} & \textbf{pub} & $94.2\phantom{0 \pm 0.00}$ & $93.9\phantom{0 \pm 0.00}$ & $94.4\phantom{0 \pm 0.00}$ & & $94.7\phantom{0 \pm 0.00}$ & & \\ + & \textbf{R} & $\phantom{0}0.04 \pm 0.01$ & $\phantom{0}0.01 \pm 0.01$ & $\phantom{0}0.03 \pm 0.00$ & $\phantom{0}0.04 \pm 0.01$ & $\phantom{0}0.07 \pm 0.03$ & $20376.43 \pm \phantom{0}42.30$ & $\phantom{0}99.57 \pm 0.21$ \\ + & \textbf{O} & $38.48 \pm 4.00$ & $33.93 \pm 4.32$ & $39.59 \pm 4.67$ & $42.76 \pm 3.73$ & $47.01 \pm 2.66$ & $\phantom{00}384.53 \pm \phantom{0}66.45$ & \\ + & \textbf{P} & $\phantom{0}0.03 \pm 0.01$ & $\phantom{0}0.01 \pm 0.01$ & $\phantom{0}0.03 \pm 0.00$ & $\phantom{0}0.04 \pm 0.01$ & $\phantom{0}0.07 \pm 0.03$ & $40368.33 \pm 114.95$ & \\\midrule +\textbf{TransD} & \textbf{pub} & & & & & $92.2\phantom{0 \pm 0.00}$ & $\phantom{00}212.\phantom{00 \pm 000.00}$ & \\ + & \textbf{R} & $36.22 \pm 0.12$ & $\phantom{0}3.94 \pm 0.27$ & $65.63 \pm 0.55$ & $79.64 \pm 0.43$ & $87.27 \pm 0.41$ & $\phantom{00}444.39 \pm \phantom{0}25.61$ & $\phantom{00}2.17 \pm 0.13$ \\\midrule +\textbf{TransE} & \textbf{pub} & & & & & $89.2\phantom{0 \pm 0.00}$ & $\phantom{00}251.\phantom{00 \pm 000.00}$ & \\ + & \textbf{R} & $39.19 \pm 1.21$ & $\phantom{0}9.99 \pm 1.82$ & $64.74 \pm 0.91$ & $75.44 \pm 0.48$ & $84.25 \pm 0.33$ & $\phantom{00}468.24 \pm \phantom{0}13.64$ & $\phantom{00}2.29 \pm 0.07$ \\\midrule +\textbf{TransH} & \textbf{pub} & & & & & $82.3\phantom{0 \pm 0.00}$ & $\phantom{00}388.\phantom{00 \pm 000.00}$ & \\ + & \textbf{R} & $\phantom{0}0.18 \pm 0.04$ & $\phantom{0}0.04 \pm 0.02$ & $\phantom{0}0.19 \pm 0.10$ & $\phantom{0}0.29 \pm 0.14$ & $\phantom{0}0.39 \pm 0.11$ & $19678.04 \pm \phantom{0}18.92$ & $\phantom{0}96.16 \pm 0.09$ \\ + & \textbf{O} & $\phantom{0}0.18 \pm 0.04$ & $\phantom{0}0.04 \pm 0.02$ & $\phantom{0}0.19 \pm 0.10$ & $\phantom{0}0.29 \pm 0.14$ & $\phantom{0}0.39 \pm 0.11$ & $19678.02 \pm \phantom{0}18.92$ & \\ + & \textbf{P} & $\phantom{0}0.18 \pm 0.04$ & $\phantom{0}0.04 \pm 0.02$ & $\phantom{0}0.19 \pm 0.10$ & $\phantom{0}0.29 \pm 0.14$ & $\phantom{0}0.39 \pm 0.11$ & $19678.06 \pm \phantom{0}18.92$ & \\\midrule +\textbf{TransR} & \textbf{pub} & & & & & $92.\phantom{00 \pm 0.00}$ & $\phantom{00}225.\phantom{00 \pm 000.00}$ & \\ + & \textbf{R} & $\phantom{0}0.06 \pm 0.02$ & $\phantom{0}0.00 \pm 0.00$ & $\phantom{0}0.04 \pm 0.02$ & $\phantom{0}0.05 \pm 0.03$ & $\phantom{0}0.11 \pm 0.06$ & $19686.49 \pm 100.97$ & $\phantom{0}96.20 \pm 0.49$ \\ + & \textbf{O} & $\phantom{0}0.06 \pm 0.02$ & $\phantom{0}0.00 \pm 0.00$ & $\phantom{0}0.04 \pm 0.02$ & $\phantom{0}0.05 \pm 0.03$ & $\phantom{0}0.11 \pm 0.06$ & $19686.49 \pm 100.97$ & \\ + & \textbf{P} & $\phantom{0}0.06 \pm 0.02$ & $\phantom{0}0.00 \pm 0.00$ & $\phantom{0}0.04 \pm 0.02$ & $\phantom{0}0.05 \pm 0.03$ & $\phantom{0}0.11 \pm 0.06$ & $19686.50 \pm 100.97$ & \\\midrule +\textbf{TuckER} & \textbf{pub} & $95.3\phantom{0 \pm 0.00}$ & $94.9\phantom{0 \pm 0.00}$ & $95.5\phantom{0 \pm 0.00}$ & & $95.8\phantom{0 \pm 0.00}$ & & \\ + & \textbf{R} & $\phantom{0}0.03 \pm 0.01$ & $\phantom{0}0.00 \pm 0.00$ & $\phantom{0}0.02 \pm 0.01$ & $\phantom{0}0.03 \pm 0.03$ & $\phantom{0}0.04 \pm 0.03$ & $20622.46 \pm 153.52$ & $100.78 \pm 0.75$ \\ \bottomrule \end{tabular} diff --git a/reproducibility/summaries/wn18_table_without_std.tex b/reproducibility/summaries/wn18_table_without_std.tex new file mode 100644 index 000000000..6ca0f1383 --- /dev/null +++ b/reproducibility/summaries/wn18_table_without_std.tex @@ -0,0 +1,42 @@ +\begin{tabular}{llrrrrrrr} +\toprule + & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ +\textbf{model} & {} & & & & & & & \\ +\midrule +\textbf{ComplEx} & \textbf{pub} & $94.1\phantom{0}$ & $93.6\phantom{0}$ & $94.5\phantom{0}$ & & $94.7\phantom{0}$ & & \\ + & \textbf{R} & $19.49$ & $12.36$ & $20.66$ & $25.24$ & $32.92$ & $\phantom{00}452.67$ & $\phantom{00}2.21$ \\\midrule +\textbf{ConvE} & \textbf{pub} & $94.3\phantom{0}$ & $93.5\phantom{0}$ & $94.6\phantom{0}$ & & $95.6\phantom{0}$ & $\phantom{00}374.\phantom{00}$ & \\ + & \textbf{R} & $88.81$ & $85.14$ & $91.76$ & $93.29$ & $94.85$ & $\phantom{00}444.40$ & $\phantom{00}2.17$ \\\midrule +\textbf{DistMult} & \textbf{pub} & $83.\phantom{00}$ & & & & $94.2\phantom{0}$ & & \\ + & \textbf{R} & $77.44$ & $67.45$ & $85.94$ & $89.52$ & $92.72$ & $\phantom{00}458.64$ & $\phantom{00}2.24$ \\\midrule +\textbf{HolE} & \textbf{pub} & $93.8\phantom{0}$ & $93.\phantom{00}$ & $94.5\phantom{0}$ & & $94.9\phantom{0}$ & & \\ + & \textbf{R} & $70.44$ & $59.29$ & $79.29$ & $84.12$ & $88.61$ & $\phantom{00}812.64$ & $\phantom{00}3.97$ \\ + & \textbf{O} & $70.44$ & $59.29$ & $79.29$ & $84.12$ & $88.61$ & $\phantom{00}812.63$ & \\ + & \textbf{P} & $70.44$ & $59.29$ & $79.29$ & $84.12$ & $88.61$ & $\phantom{00}812.64$ & \\\midrule +\textbf{KG2E} & \textbf{pub} & & & & & $92.8\phantom{0}$ & $\phantom{00}331.\phantom{00}$ & \\ + & \textbf{R} & $\phantom{0}3.61$ & $\phantom{0}1.35$ & $\phantom{0}3.21$ & $\phantom{0}4.57$ & $\phantom{0}7.02$ & $\phantom{0}2708.89$ & $\phantom{0}13.25$ \\ + & \textbf{O} & $\phantom{0}3.61$ & $\phantom{0}1.35$ & $\phantom{0}3.21$ & $\phantom{0}4.57$ & $\phantom{0}7.02$ & $\phantom{0}2708.88$ & \\ + & \textbf{P} & $\phantom{0}3.61$ & $\phantom{0}1.35$ & $\phantom{0}3.21$ & $\phantom{0}4.57$ & $\phantom{0}7.02$ & $\phantom{0}2708.89$ & \\\midrule +\textbf{RotatE} & \textbf{pub} & $94.9\phantom{0}$ & $94.4\phantom{0}$ & $95.2\phantom{0}$ & & $95.9\phantom{0}$ & $\phantom{00}309.\phantom{00}$ & \\ + & \textbf{R} & $87.29$ & $82.17$ & $91.53$ & $93.44$ & $95.28$ & $\phantom{00}123.68$ & $\phantom{00}0.61$ \\\midrule +\textbf{SimplE} & \textbf{pub} & $94.2\phantom{0}$ & $93.9\phantom{0}$ & $94.4\phantom{0}$ & & $94.7\phantom{0}$ & & \\ + & \textbf{R} & $\phantom{0}0.04$ & $\phantom{0}0.01$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $\phantom{0}0.07$ & $20376.43$ & $\phantom{0}99.57$ \\ + & \textbf{O} & $38.48$ & $33.93$ & $39.59$ & $42.76$ & $47.01$ & $\phantom{00}384.53$ & \\ + & \textbf{P} & $\phantom{0}0.03$ & $\phantom{0}0.01$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $\phantom{0}0.07$ & $40368.33$ & \\\midrule +\textbf{TransD} & \textbf{pub} & & & & & $92.2\phantom{0}$ & $\phantom{00}212.\phantom{00}$ & \\ + & \textbf{R} & $36.22$ & $\phantom{0}3.94$ & $65.63$ & $79.64$ & $87.27$ & $\phantom{00}444.39$ & $\phantom{00}2.17$ \\\midrule +\textbf{TransE} & \textbf{pub} & & & & & $89.2\phantom{0}$ & $\phantom{00}251.\phantom{00}$ & \\ + & \textbf{R} & $39.19$ & $\phantom{0}9.99$ & $64.74$ & $75.44$ & $84.25$ & $\phantom{00}468.24$ & $\phantom{00}2.29$ \\\midrule +\textbf{TransH} & \textbf{pub} & & & & & $82.3\phantom{0}$ & $\phantom{00}388.\phantom{00}$ & \\ + & \textbf{R} & $\phantom{0}0.18$ & $\phantom{0}0.04$ & $\phantom{0}0.19$ & $\phantom{0}0.29$ & $\phantom{0}0.39$ & $19678.04$ & $\phantom{0}96.16$ \\ + & \textbf{O} & $\phantom{0}0.18$ & $\phantom{0}0.04$ & $\phantom{0}0.19$ & $\phantom{0}0.29$ & $\phantom{0}0.39$ & $19678.02$ & \\ + & \textbf{P} & $\phantom{0}0.18$ & $\phantom{0}0.04$ & $\phantom{0}0.19$ & $\phantom{0}0.29$ & $\phantom{0}0.39$ & $19678.06$ & \\\midrule +\textbf{TransR} & \textbf{pub} & & & & & $92.\phantom{00}$ & $\phantom{00}225.\phantom{00}$ & \\ + & \textbf{R} & $\phantom{0}0.06$ & $\phantom{0}0.00$ & $\phantom{0}0.04$ & $\phantom{0}0.05$ & $\phantom{0}0.11$ & $19686.49$ & $\phantom{0}96.20$ \\ + & \textbf{O} & $\phantom{0}0.06$ & $\phantom{0}0.00$ & $\phantom{0}0.04$ & $\phantom{0}0.05$ & $\phantom{0}0.11$ & $19686.49$ & \\ + & \textbf{P} & $\phantom{0}0.06$ & $\phantom{0}0.00$ & $\phantom{0}0.04$ & $\phantom{0}0.05$ & $\phantom{0}0.11$ & $19686.50$ & \\\midrule +\textbf{TuckER} & \textbf{pub} & $95.3\phantom{0}$ & $94.9\phantom{0}$ & $95.5\phantom{0}$ & & $95.8\phantom{0}$ & & \\ + & \textbf{R} & $\phantom{0}0.03$ & $\phantom{0}0.00$ & $\phantom{0}0.02$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $20622.46$ & $100.78$ \\ +\bottomrule +\end{tabular} + diff --git a/reproducibility/summaries/wn18rr_table.tex b/reproducibility/summaries/wn18rr_table.tex index e968b913d..d206e1fc6 100644 --- a/reproducibility/summaries/wn18rr_table.tex +++ b/reproducibility/summaries/wn18rr_table.tex @@ -1,12 +1,16 @@ -\begin{tabular}{lrrrrrrrrrrrrrrrrrrr} +\begin{tabular}{llrrrrrrr} \toprule -{} & \multicolumn{7}{l}{avg} & \multicolumn{6}{l}{best} & \multicolumn{6}{l}{worst} \\ -{} & AMR (\%) & Hits@1 (\%) & Hits@10 (\%) & Hits@3 (\%) & Hits@5 (\%) & MR & MRR (\%) & Hits@1 (\%) & Hits@10 (\%) & Hits@3 (\%) & Hits@5 (\%) & MR & MRR (\%) & Hits@1 (\%) & Hits@10 (\%) & Hits@3 (\%) & Hits@5 (\%) & MR & MRR (\%) \\ -\textbf{model } & & & & & & & & & & & & & & & & & & & \\ + & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ +\textbf{model} & {} & & & & & & & \\ \midrule -\textbf{ConvE } & 26.49 $\pm$ 0.25 & 40.98 $\pm$ 0.22 & 51.76 $\pm$ 0.13 & 46.49 $\pm$ 0.14 & 48.92 $\pm$ 0.23 & $\phantom{5}$5369.49 $\pm$ $\phantom{5}$50.92 & 44.69 $\pm$ 0.21 & 40.98 $\pm$ 0.22 & 51.76 $\pm$ 0.13 & 46.49 $\pm$ 0.14 & 48.92 $\pm$ 0.23 & $\phantom{5}$5369.49 $\pm$ $\phantom{5}$50.92 & 44.69 $\pm$ 0.21 & 40.98 $\pm$ 0.22 & 51.76 $\pm$ 0.13 & 46.49 $\pm$ 0.14 & 48.92 $\pm$ 0.23 & $\phantom{5}$5369.49 $\pm$ $\phantom{5}$50.92 & 44.69 $\pm$ 0.21 \\ -\textbf{ConvKB} & 67.27 $\pm$ 3.52 & $\phantom{5}$0.09 $\pm$ 0.03 & $\phantom{5}$0.57 $\pm$ 0.16 & $\phantom{5}$0.21 $\pm$ 0.07 & $\phantom{5}$0.32 $\pm$ 0.10 & 13634.66 $\pm$ 714.24 & $\phantom{5}$0.30 $\pm$ 0.07 & $\phantom{5}$0.09 $\pm$ 0.03 & $\phantom{5}$0.57 $\pm$ 0.16 & $\phantom{5}$0.21 $\pm$ 0.07 & $\phantom{5}$0.32 $\pm$ 0.10 & 13634.65 $\pm$ 714.24 & $\phantom{5}$0.30 $\pm$ 0.07 & $\phantom{5}$0.09 $\pm$ 0.03 & $\phantom{5}$0.57 $\pm$ 0.16 & $\phantom{5}$0.21 $\pm$ 0.07 & $\phantom{5}$0.32 $\pm$ 0.10 & 13634.66 $\pm$ 714.24 & $\phantom{5}$0.30 $\pm$ 0.07 \\ -\textbf{RotatE} & 21.03 $\pm$ 0.45 & 44.02 $\pm$ 0.15 & 56.51 $\pm$ 0.26 & 50.55 $\pm$ 0.12 & 52.98 $\pm$ 0.11 & $\phantom{5}$4263.32 $\pm$ $\phantom{5}$90.33 & 48.40 $\pm$ 0.09 & 44.02 $\pm$ 0.15 & 56.51 $\pm$ 0.26 & 50.55 $\pm$ 0.12 & 52.98 $\pm$ 0.11 & $\phantom{5}$4263.32 $\pm$ $\phantom{5}$90.33 & 48.40 $\pm$ 0.09 & 44.02 $\pm$ 0.15 & 56.51 $\pm$ 0.26 & 50.55 $\pm$ 0.12 & 52.98 $\pm$ 0.11 & $\phantom{5}$4263.32 $\pm$ $\phantom{5}$90.33 & 48.40 $\pm$ 0.09 \\ +\textbf{ConvE} & \textbf{pub} & $43.\phantom{00 \pm 0.00}$ & $\phantom{0}4.\phantom{00 \pm 0.00}$ & $44.\phantom{00 \pm 0.00}$ & & $52.\phantom{00 \pm 0.00}$ & $\phantom{0}4187.\phantom{00 \pm 000.00}$ & \\ + & \textbf{R} & $44.69 \pm 0.21$ & $40.98 \pm 0.22$ & $46.49 \pm 0.14$ & $48.92 \pm 0.23$ & $51.76 \pm 0.13$ & $\phantom{0}5369.49 \pm \phantom{0}50.92$ & $26.49 \pm 0.25$ \\\midrule +\textbf{ConvKB} & \textbf{pub} & $24.8\phantom{0 \pm 0.00}$ & & & & $52.5\phantom{0 \pm 0.00}$ & $\phantom{0}2554.\phantom{00 \pm 000.00}$ & \\ + & \textbf{R} & $\phantom{0}0.30 \pm 0.07$ & $\phantom{0}0.09 \pm 0.03$ & $\phantom{0}0.21 \pm 0.07$ & $\phantom{0}0.32 \pm 0.10$ & $\phantom{0}0.57 \pm 0.16$ & $13634.66 \pm 714.24$ & $67.27 \pm 3.52$ \\ + & \textbf{O} & $\phantom{0}0.30 \pm 0.07$ & $\phantom{0}0.09 \pm 0.03$ & $\phantom{0}0.21 \pm 0.07$ & $\phantom{0}0.32 \pm 0.10$ & $\phantom{0}0.57 \pm 0.16$ & $13634.65 \pm 714.24$ & \\ + & \textbf{P} & $\phantom{0}0.30 \pm 0.07$ & $\phantom{0}0.09 \pm 0.03$ & $\phantom{0}0.21 \pm 0.07$ & $\phantom{0}0.32 \pm 0.10$ & $\phantom{0}0.57 \pm 0.16$ & $13634.66 \pm 714.24$ & \\\midrule +\textbf{RotatE} & \textbf{pub} & $47.6\phantom{0 \pm 0.00}$ & $42.8\phantom{0 \pm 0.00}$ & $49.2\phantom{0 \pm 0.00}$ & & $57.1\phantom{0 \pm 0.00}$ & $\phantom{0}3340.\phantom{00 \pm 000.00}$ & \\ + & \textbf{R} & $48.40 \pm 0.09$ & $44.02 \pm 0.15$ & $50.55 \pm 0.12$ & $52.98 \pm 0.11$ & $56.51 \pm 0.26$ & $\phantom{0}4263.32 \pm \phantom{0}90.33$ & $21.03 \pm 0.45$ \\ \bottomrule \end{tabular} diff --git a/reproducibility/summaries/wn18rr_table_without_std.tex b/reproducibility/summaries/wn18rr_table_without_std.tex new file mode 100644 index 000000000..5b2d483a1 --- /dev/null +++ b/reproducibility/summaries/wn18rr_table_without_std.tex @@ -0,0 +1,16 @@ +\begin{tabular}{llrrrrrrr} +\toprule + & & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\ +\textbf{model} & {} & & & & & & & \\ +\midrule +\textbf{ConvE} & \textbf{pub} & $43.\phantom{00}$ & $\phantom{0}4.\phantom{00}$ & $44.\phantom{00}$ & & $52.\phantom{00}$ & $\phantom{0}4187.\phantom{00}$ & \\ + & \textbf{R} & $44.69$ & $40.98$ & $46.49$ & $48.92$ & $51.76$ & $\phantom{0}5369.49$ & $26.49$ \\\midrule +\textbf{ConvKB} & \textbf{pub} & $24.8\phantom{0}$ & & & & $52.5\phantom{0}$ & $\phantom{0}2554.\phantom{00}$ & \\ + & \textbf{R} & $\phantom{0}0.30$ & $\phantom{0}0.09$ & $\phantom{0}0.21$ & $\phantom{0}0.32$ & $\phantom{0}0.57$ & $13634.66$ & $67.27$ \\ + & \textbf{O} & $\phantom{0}0.30$ & $\phantom{0}0.09$ & $\phantom{0}0.21$ & $\phantom{0}0.32$ & $\phantom{0}0.57$ & $13634.65$ & \\ + & \textbf{P} & $\phantom{0}0.30$ & $\phantom{0}0.09$ & $\phantom{0}0.21$ & $\phantom{0}0.32$ & $\phantom{0}0.57$ & $13634.66$ & \\\midrule +\textbf{RotatE} & \textbf{pub} & $47.6\phantom{0}$ & $42.8\phantom{0}$ & $49.2\phantom{0}$ & & $57.1\phantom{0}$ & $\phantom{0}3340.\phantom{00}$ & \\ + & \textbf{R} & $48.40$ & $44.02$ & $50.55$ & $52.98$ & $56.51$ & $\phantom{0}4263.32$ & $21.03$ \\ +\bottomrule +\end{tabular} +