Renamed passage evaluation components, added keyterm evaluation

components and reporter. Addes two new tables to sqlite file. Fixed
#4 #9

src/main/resources/helloqa/collection/decorated-collection-reader.yaml

@@ -1,7 +1,8 @@
 inherit: fs.fs-collection-reader
 
 decorators: |
-  - inherit: helloqa.collection.gs-decorator
+  - inherit: helloqa.collection.keyterm-gs-decorator
+  - inherit: helloqa.collection.passage-gs-decorator
 
 persistence-provider: |
   inherit: helloqa.db.local-persistence-provider

src/main/resources/helloqa/collection/keyterm-gs-decorator.yaml

@@ -0,0 +1,4 @@
+class: edu.cmu.lti.oaqa.framework.eval.KeytermGoldStandardDecorator
+
+persistence-provider: |
+  inherit: helloqa.collection.keyterm-gs-persistence-provider

src/main/resources/helloqa/collection/keyterm-gs-persistence-provider.yaml

@@ -0,0 +1,4 @@
+class: edu.cmu.lti.oaqa.framework.eval.gs.KeytermGoldStandardFilePersistenceProvider
+DataSet: GUTENBERG
+PathPattern: classpath:/gs/small.keyterm
+LineSyntax: (\d+)\|(\d+) (\d+)\|(.*)

src/main/resources/helloqa/collection/gs-decorator.yaml → src/main/resources/helloqa/collection/passage-gs-decorator.yaml

@@ -1,4 +1,4 @@
 class: edu.cmu.lti.oaqa.framework.eval.PassageDatasetDecorator
 
 persistence-provider: |
-  inherit: helloqa.collection.gs-persistence-provider
+  inherit: helloqa.collection.passage-gs-persistence-provider

src/main/resources/helloqa/eval/keyterm-aggregator-consumer.yaml

@@ -0,0 +1,5 @@
+class: edu.cmu.lti.oaqa.framework.eval.keyterm.KeytermEvalConsumer
+name: keyterm-evaluator  
+
+aggregators: |
+  - inherit: helloqa.eval.keyterm-aggregator

src/main/resources/helloqa/eval/keyterm-aggregator.yaml

@@ -0,0 +1,3 @@
+class: edu.cmu.lti.oaqa.framework.eval.retrieval.RetrievalEvalAggregator
+persistence-provider: |
+  inherit: helloqa.eval.keyterm-evaluation-persistence-provider

src/main/resources/helloqa/eval/keyterm-evaluation-persistence-provider.yaml

@@ -0,0 +1,20 @@
+class: edu.cmu.lti.oaqa.framework.eval.retrieval.impl.JdbcRetrievalEvalPersistenceProvider
+select-passage-aggregates-query: |
+  SELECT experimentId, traceId, relevant_retrieved, retrieved,
+   relevant, avep, count, binary_relevant, stage
+  FROM keyterm_aggregates WHERE experimentId = ? AND stage = ?
+insert-f-measure-eval-query: |
+  INSERT INTO keyterm_eval
+   (experimentId, traceId, evaluator, prec, recall, f1, 
+   map, binary_recall, count, stage, traceHash) 
+  VALUES (?,?,?,?,?,?,?,?,?,?,?)
+delete-f-measure-eval-query: |
+  DELETE FROM keyterm_eval WHERE experimentId = ? AND stage = ?
+insert-passage-aggregates-query: |
+  INSERT INTO keyterm_aggregates (experimentId, traceId, 
+   aggregator, relevant_retrieved, retrieved, relevant, 
+   avep, count, binary_relevant, sequenceId, stage, traceHash) 
+  VALUES (?,?,?,?,?,?,?,?,?,?,?,?)
+delete-passage-aggr-eval-query: |
+  DELETE FROM keyterm_aggregates 
+  WHERE experimentId = ? AND traceHash = ? AND aggregator = ? AND sequenceId = ?  

src/main/resources/helloqa/eval/keyterm-evaluator-consumer.yaml

@@ -0,0 +1,4 @@
+class: edu.cmu.lti.oaqa.framework.eval.TraceConsumer
+name: retrieval-measures-evaluator 
+experiment-listeners: |
+  - inherit: helloqa.eval.keyterm-evaluator

src/main/resources/helloqa/eval/keyterm-evaluator.yaml

@@ -0,0 +1,3 @@
+class: edu.cmu.lti.oaqa.framework.eval.retrieval.RetrievalMeasuresEvaluator
+persistence-provider: |
+  inherit: helloqa.eval.keyterm-evaluation-persistence-provider

src/main/resources/helloqa/helloqa.yaml

@@ -13,33 +13,40 @@ collection-reader:
 #  dataset: GUTENBERG
 #  suffix: txt
 #  dir: /input
-  
+
 pipeline:
   - inherit: jdbc.sqlite.cse.phase  
     name: keyterm-extractor
     options: |
       - inherit: helloqa.keyterm.simple
-  
+
   - inherit: jdbc.sqlite.cse.phase 
     name: retrieval-strategist
     options: |
       - inherit: helloqa.retrieval.simple-solr-strategist
-      
+          
   - inherit: jdbc.sqlite.cse.phase  
     name: passage-extractor
     options: |
       - inherit: helloqa.passage.simple
+      
+  - inherit: helloqa.eval.keyterm-aggregator-consumer
 
   - inherit: jdbc.eval.retrieval-aggregator-consumer
-
-  - inherit: helloqa.eval.passage-map-aggregator-consumer
-
+      
+  - inherit: helloqa.eval.passage-map-aggregator-consumer      
+    
 post-process: 
+  - inherit: helloqa.eval.keyterm-evaluator-consumer
+  - inherit: report.csv-report-generator
+    builders: |
+      - inherit: helloqa.report.keyterm-report-component
+
   - inherit: jdbc.eval.retrieval-evaluator-consumer
   - inherit: report.csv-report-generator
     builders: |
       - inherit: jdbc.report.f-measure-report-component
-      
+          
   - inherit: helloqa.eval.passage-map-evaluator-consumer
   - inherit: report.csv-report-generator
     builders: |

src/main/resources/helloqa/report/keyterm-report-component.yaml

@@ -0,0 +1,9 @@
+class: edu.cmu.lti.oaqa.framework.report.DatabaseReportComponentBuilder
+keys: evaluator,traceId
+headers: Evaluator,Configuration,Precision,Recall,F-1,MAP,Binary Recall,Count
+fields: evaluator,traceId,prec,recall,f1,map,binary_recall,count
+formats: "%s,%s,%.4f,%.4f,%.4f,%.4f,%.4f,%d"
+query: |
+  SELECT evaluator,traceId,prec,recall,f1,map,binary_recall,count FROM keyterm_eval 
+  WHERE experimentId = ? AND stage = ?
+