integrated SlackForwarder in ExtractionRecorder

dbpedia · Feb 21, 2017 · 6c9ef60 · 6c9ef60
1 parent 0474735
commit 6c9ef60
Show file tree

Hide file tree

Showing 15 changed files with 326 additions and 390 deletions.
diff --git a/core/pom.xml b/core/pom.xml
@@ -176,6 +176,23 @@
             <version>1.8.3</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.scalaj</groupId>
+            <artifactId>scalaj-http_2.11</artifactId>
+            <version>2.2.1</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.jena</groupId>
+            <artifactId>jena-core</artifactId>
+            <version>2.13.0</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.jena</groupId>
+            <artifactId>jena-querybuilder</artifactId>
+            <version>2.13.0</version>
+        </dependency>
     </dependencies>
 
 </project>

diff --git a/core/src/main/resources/datasetdefinitions.json b/core/src/main/resources/datasetdefinitions.json
@@ -377,13 +377,13 @@
       "defaultgraph": "dataset"
     },
     "nif_page_structure": {
-      "name": "Template Parameters",
+      "name": "NIF page structure",
       "traits":"LinkedData, Published",
       "desc": "This dataset contains the page structure of a Wikipedia page, divided in sections and paragraphs.",
       "defaultgraph":"nif"
     },
     "nif_text_links": {
-      "name": "Template Parameters",
+      "name": "NIF Text Links",
       "traits":"LinkedData, Published",
       "desc": "This dataset contains all text links of a Wikipedia page, detailed in the NIF format.",
       "defaultgraph":"nif"
@@ -395,13 +395,13 @@
       "defaultgraph":"nif"
     },
     "raw_tables": {
-      "name": "raw html tables",
+      "name": "Raw html tables",
       "traits":"LinkedData, Temporary",
       "desc": "These are the raw html tables collected during the NIF extraction; to be further processed later.",
       "defaultgraph": "dataset"
     },
     "equations": {
-      "name": "equations in MathML",
+      "name": "Equations in MathML",
       "traits":"LinkedData",
       "desc": "This are all equations collected during the NIF extraction, transformed into MathML XML syntax.",
       "defaultgraph": "dataset"

diff --git a/core/src/main/resources/nifextractionconfig.json b/core/src/main/resources/nifextractionconfig.json
@@ -283,6 +283,75 @@
 
     ]
   },
+  "ru":{
+    "nif-find-pageend":[
+      "span[id*=.D0.9F.D1.80.D0.B8.D0.BC.D0.B5.D1.87.D0.B0.D0.BD.D0.B8.D1.8F]",
+      "span[id*=.D0.9B.D0.B8.D1.82.D0.B5.D1.80.D0.B0.D1.82.D1.83.D1.80.D0.B0]",
+      "span[id*=.D0.A1.D1.81.D1.8B.D0.BB.D0.BA.D0.B8]",
+      "span[id*=Ang_mga_gi_basihan_niini]"
+    ],
+    "nif-find-next-title":[
+
+    ],
+    "nif-find-toc":[
+
+    ],
+    "nif-remove-elements":[
+
+    ],
+    "nif-replace-elements":[
+
+    ],
+    "nif-note-elements":[
+
+    ]
+  },
+  "war":{
+    "nif-find-pageend":[
+      "span[id*=Mga_kasarigan]",
+      "span[id*=Mga_Sumpay_ha_Gawas]"
+    ],
+    "nif-find-next-title":[
+
+    ],
+    "nif-find-toc":[
+
+    ],
+    "nif-remove-elements":[
+
+    ],
+    "nif-replace-elements":[
+
+    ],
+    "nif-note-elements":[
+
+    ]
+  },
+  "vi":{
+    "nif-find-pageend":[
+      "span[id*=Ch.C3.BA_th.C3.ADch]",
+      "span[id*=Tham_kh.E1.BA.A3o]",
+      "span[id*=.C4.90.E1.BB.8Dc_th.C3.AAm]",
+      "span[id*=Li.C3.AAn_k.E1.BA.BFt_ngo.C3.A0i]",
+      "span[id*=Ghi_ch.C3.BA]",
+      "span[id*=Xem_th.C3.AAm]"
+    ],
+    "nif-find-next-title":[
+
+    ],
+    "nif-find-toc":[
+
+    ],
+    "nif-remove-elements":[
+
+    ],
+    "nif-replace-elements":[
+
+    ],
+    "nif-note-elements":[
+
+    ]
+  },
   "en":{
     "nif-find-pageend":[
 

diff --git a/core/src/main/resources/universal.properties b/core/src/main/resources/universal.properties
@@ -7,7 +7,13 @@ base-dir=/data/extraction-data/2016-10
 dbpedia-version=2016-10
 
 # The log file directory - used to store all log files created in the course of all extractions
-#log-dir=/data/extraction-data/2016-10
+# log-dir=/data/extraction-data/2016-10
+
+# to forward extraction summaries and warnings via the slack API, use this option
+# slack-webhook=https://hooks.slack.com/services/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
+# slack-username=username
+# slack-summary-threshold=1000000
+# slack-exception-threshold=10
 
 # wiki suffix: should be 'wiki'
 wiki-name=wiki