Update result table with published results

reproducibility/configs/complex/trouillon2016_complex_fb15k.json

@@ -1,7 +1,7 @@
 {
   "metadata": {
     "title": "Learn FB15K Data Set with ComplEx as described by Trouillon et al., 2016",
-    "comments": "In the paper it is not mentioned that they compute the mean of the vector norms as done in the published code base."
+    "comments": "In the paper it is not mentioned that they compute the mean of the vector norms as done in the published code base. The rank type can be found at https://github.com/ttrouill/complex/blob/67fef2324d5a7695f7b932b5606d2e4b6de09a41/efe/evaluation.py#L295."
   },
   "pipeline": {
     "dataset": "fb15k",
@@ -36,5 +36,17 @@
     "evaluator_kwargs": {
       "filtered": true
     }
+  },
+  "results": {
+    "mean_reciprocal_rank": {
+      "best": 0.692
+    },
+    "hits_at_k": {
+      "best": {
+        "1": 0.599,
+        "3": 0.759,
+        "10": 0.840
+      }
+    }
   }
 }

reproducibility/configs/complex/trouillon2016_complex_wn18.json

@@ -1,7 +1,7 @@
 {
   "metadata": {
     "title": "Learn WN18 Data Set with ComplEx as described by Trouillon et al., 2016",
-    "comments": "In the paper it is not mentioned that they compute the mean of the vector norms as done in the published code base."
+    "comments": "In the paper it is not mentioned that they compute the mean of the vector norms as done in the published code base.  The rank type can be found at https://github.com/ttrouill/complex/blob/67fef2324d5a7695f7b932b5606d2e4b6de09a41/efe/evaluation.py#L295."
   },
   "pipeline": {
     "dataset": "wn18",
@@ -36,5 +36,17 @@
     "evaluator_kwargs": {
       "filtered": true
     }
+  },
+  "results": {
+    "mean_reciprocal_rank": {
+      "best": 0.941
+    },
+    "hits_at_k": {
+      "best": {
+        "1": 0.936,
+        "3": 0.945,
+        "10": 0.947
+      }
+    }
   }
-}
+}

reproducibility/configs/conve/dettmers2018_conve_fb15k.json

@@ -1,6 +1,7 @@
 {
   "metadata": {
-    "title": "Learn FB15K Data Set with ConvE as described by Dettmers et al., 2018"
+    "title": "Learn FB15K Data Set with ConvE as described by Dettmers et al., 2018",
+    "comment": "For evaluation, the non-deterministic rank definition is used, cf. https://github.com/TimDettmers/ConvE/blob/5feb358eb7dbd1f534978cdc4c20ee0bf919148a/evaluation.py#L67."
   },
   "pipeline": {
     "dataset": "fb15k",
@@ -18,7 +19,8 @@
       "kernel_width": 3,
       "input_dropout": 0.2,
       "feature_map_dropout": 0.2,
-      "output_dropout": 0.3
+      "output_dropout": 0.3,
+      "apply_batch_normalization": true
     },
     "optimizer": "Adam",
     "optimizer_kwargs": {
@@ -36,9 +38,21 @@
     },
     "evaluator_kwargs": {
       "filtered": true
+    }
+  },
+  "results": {
+    "mean_rank": {
+      "nondeterministic": 51
+    },
+    "hits_at_k": {
+      "nondeterministic": {
+        "1": 0.558,
+        "3": 0.723,
+        "10": 0.831
+      }
     },
-    "evaluation_kwargs": {
-      "batch_size": 16
+    "mean_reciprocal_rank": {
+      "nondeterministic": 0.657
     }
   }
 }

reproducibility/configs/conve/dettmers2018_conve_fb15k237.json

@@ -1,6 +1,7 @@
 {
   "metadata": {
-    "title": "Learn FB15K-237 Data Set with ConvE as described by Dettmers et al., 2018"
+    "title": "Learn FB15K-237 Data Set with ConvE as described by Dettmers et al., 2018",
+    "comment": "For evaluation, the non-deterministic rank definition is used, cf. https://github.com/TimDettmers/ConvE/blob/5feb358eb7dbd1f534978cdc4c20ee0bf919148a/evaluation.py#L67."
   },
   "pipeline": {
     "dataset": "fb15k237",
@@ -18,7 +19,8 @@
       "kernel_width": 3,
       "input_dropout": 0.2,
       "feature_map_dropout": 0.2,
-      "output_dropout": 0.3
+      "output_dropout": 0.3,
+      "apply_batch_normalization": true
     },
     "optimizer": "Adam",
     "optimizer_kwargs": {
@@ -36,9 +38,21 @@
     },
     "evaluator_kwargs": {
       "filtered": true
+    }
+  },
+  "results": {
+    "mean_rank": {
+      "nondeterministic": 244
+    },
+    "hits_at_k": {
+      "nondeterministic": {
+        "1": 0.237,
+        "3": 0.356,
+        "10": 0.501
+      }
     },
-    "evaluation_kwargs": {
-      "batch_size": 16
+    "mean_reciprocal_rank": {
+      "nondeterministic": 0.325
     }
   }
 }

reproducibility/configs/conve/dettmers2018_conve_wn18.json

@@ -1,6 +1,7 @@
 {
   "metadata": {
-    "title": "Learn WN18 Data Set with ConvE as described by Dettmers et al., 2018"
+    "title": "Learn WN18 Data Set with ConvE as described by Dettmers et al., 2018",
+    "comment": "For evaluation, the non-deterministic rank definition is used, cf. https://github.com/TimDettmers/ConvE/blob/5feb358eb7dbd1f534978cdc4c20ee0bf919148a/evaluation.py#L67."
   },
   "pipeline": {
     "dataset": "wn18",
@@ -18,7 +19,8 @@
       "kernel_width": 3,
       "input_dropout": 0.2,
       "feature_map_dropout": 0.2,
-      "output_dropout": 0.3
+      "output_dropout": 0.3,
+      "apply_batch_normalization": true
     },
     "optimizer": "Adam",
     "optimizer_kwargs": {
@@ -36,9 +38,21 @@
     },
     "evaluator_kwargs": {
       "filtered": true
+    }
+  },
+  "results": {
+    "mean_rank": {
+      "nondeterministic": 374
+    },
+    "hits_at_k": {
+      "nondeterministic": {
+        "1": 0.935,
+        "3": 0.946,
+        "10": 0.956
+      }
     },
-    "evaluation_kwargs": {
-      "batch_size": 16
+    "mean_reciprocal_rank": {
+      "nondeterministic": 0.943
     }
   }
 }

reproducibility/configs/conve/dettmers2018_conve_wn18rr.json

@@ -1,13 +1,14 @@
 {
   "metadata": {
-    "title": "Learn WN18RR Data Set with ConvE as described by Dettmers et al., 2018"
+    "title": "Learn WN18RR Data Set with ConvE as described by Dettmers et al., 2018",
+    "comment": "For evaluation, the non-deterministic rank definition is used, cf. https://github.com/TimDettmers/ConvE/blob/5feb358eb7dbd1f534978cdc4c20ee0bf919148a/evaluation.py#L67."
   },
   "pipeline": {
     "dataset": "wn18rr",
+    "model": "ConvE",
     "dataset_kwargs": {
       "create_inverse_triples": true
     },
-    "model": "ConvE",
     "model_kwargs": {
       "embedding_dim": 200,
       "input_channels": 1,
@@ -18,7 +19,8 @@
       "kernel_width": 3,
       "input_dropout": 0.2,
       "feature_map_dropout": 0.2,
-      "output_dropout": 0.3
+      "output_dropout": 0.3,
+      "apply_batch_normalization": true
     },
     "optimizer": "Adam",
     "optimizer_kwargs": {
@@ -36,9 +38,21 @@
     },
     "evaluator_kwargs": {
       "filtered": true
+    }
+  },
+  "results": {
+    "mean_rank": {
+      "nondeterministic": 4187
+    },
+    "hits_at_k": {
+      "nondeterministic": {
+        "1": 0.40,
+        "3": 0.44,
+        "10": 0.52
+      }
     },
-    "evaluation_kwargs": {
-      "batch_size": 16
+    "mean_reciprocal_rank": {
+      "nondeterministic": 0.43
     }
   }
 }

reproducibility/configs/convkb/nguyen2018_convkb_fb15k237.json

@@ -1,7 +1,7 @@
 {
   "metadata": {
     "title": "Learn FB15k-237 Data Set with ConvKB as described by Nguyen et al., 2018",
-    "comments": "regularization weight is set to 0.0005, because in the paper the regularization term is multiplied with (regularization weight)/2"
+    "comments": "regularization weight is set to 0.0005, because in the paper the regularization term is multiplied with (regularization weight)/2. The evaluation is done with the optimistic rank, cf. https://github.com/daiquocnguyen/ConvKB/blob/ba02c0665a80751676289a8d5570dc420465a9ff/eval.py#L207-L236."
   },
   "pipeline": {
     "dataset": "fb15k237",
@@ -37,9 +37,19 @@
     },
     "evaluator_kwargs": {
       "filtered": true
+    }
+  },
+  "results": {
+    "mean_rank": {
+      "best": 257
+    },
+    "hits_at_k": {
+      "best": {
+        "10": 0.517
+      }
     },
-    "evaluation_kwargs": {
-      "batch_size": 16
+    "mean_reciprocal_rank": {
+      "best": 0.396
     }
   }
 }

reproducibility/configs/convkb/nguyen2018_convkb_wn18rr.json

@@ -1,7 +1,7 @@
 {
   "metadata": {
     "title": "Learn WN18RR Data Set with ConvKB as described by Nguyen et al., 2018",
-    "comments": "regularization weight is set to 0.0005, because in the paper the regularization term is multiplied with (regularization weight)/2"
+    "comments": "regularization weight is set to 0.0005, because in the paper the regularization term is multiplied with (regularization weight)/2. The evaluation is done with the optimistic rank, cf. https://github.com/daiquocnguyen/ConvKB/blob/ba02c0665a80751676289a8d5570dc420465a9ff/eval.py#L207-L236."
   },
   "pipeline": {
     "dataset": "wn18rr",
@@ -37,9 +37,19 @@
     },
     "evaluator_kwargs": {
       "filtered": true
+    }
+  },
+  "results": {
+    "mean_rank": {
+      "best": 2554
+    },
+    "hits_at_k": {
+      "best": {
+        "10": 0.525
+      }
     },
-    "evaluation_kwargs": {
-      "batch_size": 16
+    "mean_reciprocal_rank": {
+      "best": 0.248
     }
   }
 }

reproducibility/configs/convkb/nguyen2018_transe_fb15k237.json

@@ -30,9 +30,6 @@
     },
     "evaluator_kwargs": {
       "filtered": true
-    },
-    "evaluation_kwargs": {
-      "batch_size": 16
     }
   }
 }

reproducibility/configs/convkb/nguyen2018_transe_wn18rr.json

@@ -20,7 +20,7 @@
       "margin": 5
     },
     "training_loop": "OWA",
-    "negative_sampler": "bernoulli",
+    "negative_sampler_cls": "bernoulli",
     "negative_sampler_kwargs": {
       "num_negs_per_pos": 1
     },
@@ -30,9 +30,6 @@
     },
     "evaluator_kwargs": {
       "filtered": true
-    },
-    "evaluation_kwargs": {
-      "batch_size": 16
     }
   }
 }

reproducibility/configs/distmult/yang2014_distmult_fb15k.json

@@ -37,5 +37,15 @@
     "evaluator_kwargs": {
       "filtered": true
     }
+  },
+  "results": {
+    "hits_at_k": {
+      "unknown": {
+        "10": 0.577
+      }
+    },
+    "mean_reciprocal_rank": {
+      "unknown": 0.35
+    }
   }
 }

reproducibility/configs/distmult/yang2014_distmult_wn18.json

@@ -37,5 +37,15 @@
     "evaluator_kwargs": {
       "filtered": true
     }
+  },
+  "results": {
+    "hits_at_k": {
+      "unknown": {
+        "10": 0.942
+      }
+    },
+    "mean_reciprocal_rank": {
+      "unknown": 0.83
+    }
   }
 }