From f02fdeecee216c4c7617f26657ecf1620de88dc5 Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Wed, 29 May 2013 17:12:22 +0700 Subject: [PATCH 01/59] Issue #235 - fix for updating surveyInstanceSummary --- .../mapping/analytics/dao/SurveyInstanceSummaryDao.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/GAE/src/org/waterforpeople/mapping/analytics/dao/SurveyInstanceSummaryDao.java b/GAE/src/org/waterforpeople/mapping/analytics/dao/SurveyInstanceSummaryDao.java index a1f98d7b66..5d2069621d 100644 --- a/GAE/src/org/waterforpeople/mapping/analytics/dao/SurveyInstanceSummaryDao.java +++ b/GAE/src/org/waterforpeople/mapping/analytics/dao/SurveyInstanceSummaryDao.java @@ -62,7 +62,8 @@ public static synchronized void incrementCount(String community, .declareParameters("String countryCodeParam, String communityCodeParam, Date collectionDateParam"); // have to import the date class before we can use it query.declareImports("import java.util.Date"); - List results = (List) query.execute(community, country, colDate); + List results = (List) query.execute(country,community, colDate); + SurveyInstanceSummary summary = null; if (results == null || results.size() == 0) { summary = new SurveyInstanceSummary(); From 569c228e2e1048d7099e458f8be27d9ccc525e8f Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Thu, 30 May 2013 08:46:58 +0700 Subject: [PATCH 02/59] Issue #237 - Add row number to inspect data rows --- .../app/js/lib/controllers/data-controllers.js | 1 + .../lib/views/data/inspect-data-table-views.js | 17 +++++++++++++++++ .../templates/navData/inspect-data.handlebars | 2 ++ 3 files changed, 20 insertions(+) diff --git a/Dashboard/app/js/lib/controllers/data-controllers.js b/Dashboard/app/js/lib/controllers/data-controllers.js index 38f5ce9335..f712382799 100644 --- a/Dashboard/app/js/lib/controllers/data-controllers.js +++ b/Dashboard/app/js/lib/controllers/data-controllers.js @@ -40,6 +40,7 @@ FLOW.surveyInstanceControl = Ember.ArrayController.create({ selectedSurvey: null, content: null, sinceArray: [], + pagenumber:0, populate: function() { this.get('sinceArray').pushObject(FLOW.metaControl.get('since')); diff --git a/Dashboard/app/js/lib/views/data/inspect-data-table-views.js b/Dashboard/app/js/lib/views/data/inspect-data-table-views.js index 61a9fef81c..502b994622 100644 --- a/Dashboard/app/js/lib/views/data/inspect-data-table-views.js +++ b/Dashboard/app/js/lib/views/data/inspect-data-table-views.js @@ -17,6 +17,9 @@ FLOW.inspectDataTableView = FLOW.View.extend({ FLOW.selectedControl.set('selectedSurvey',null); FLOW.dateControl.set('toDate',null); FLOW.dateControl.set('fromDate',null); + FLOW.locationControl.set('selectedLevel1',null); + FLOW.locationControl.set('selectedLevel2',null); + FLOW.surveyInstanceControl.set('pageNumber',0); }, // do a new query @@ -54,12 +57,14 @@ FLOW.inspectDataTableView = FLOW.View.extend({ doNextPage: function() { FLOW.surveyInstanceControl.get('sinceArray').pushObject(FLOW.metaControl.get('since')); this.doInstanceQuery(); + FLOW.surveyInstanceControl.set('pageNumber',FLOW.surveyInstanceControl.get('pageNumber') + 1); }, doPrevPage: function() { FLOW.surveyInstanceControl.get('sinceArray').popObject(); FLOW.metaControl.set('since', FLOW.surveyInstanceControl.get('sinceArray')[FLOW.surveyInstanceControl.get('sinceArray').length - 1]); this.doInstanceQuery(); + FLOW.surveyInstanceControl.set('pageNumber',FLOW.surveyInstanceControl.get('pageNumber') - 1); }, // If the number of items in the previous call was 20 (a full page) we assume that there are more. @@ -186,3 +191,15 @@ FLOW.DataItemView = FLOW.View.extend({ } } }); + +FLOW.DataNumView = FLOW.View.extend({ + tagName: 'span', + content:null, + rownum:null, + + init: function(){ + var index; + index = FLOW.surveyInstanceControl.content.indexOf(this.get('content')); + this.set('rownum',index + 1 + 20 * FLOW.surveyInstanceControl.get('pageNumber')); + } +}); diff --git a/Dashboard/app/js/templates/navData/inspect-data.handlebars b/Dashboard/app/js/templates/navData/inspect-data.handlebars index 3984f6ea88..c21e5d2cdf 100644 --- a/Dashboard/app/js/templates/navData/inspect-data.handlebars +++ b/Dashboard/app/js/templates/navData/inspect-data.handlebars @@ -60,6 +60,7 @@ + @@ -72,6 +73,7 @@ {{#each SI in FLOW.surveyInstanceControl}} {{#unless SI.isDeleted}} + From fc4302ec917e0f637c48a9bbd0d7acd17f28bbf0 Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Thu, 30 May 2013 08:49:01 +0700 Subject: [PATCH 03/59] Issue #237 - Edit fixtures of surveyInstances to have different collection dates --- Dashboard/app/js/lib/models/fixtures.js | 129 +++++------------------- 1 file changed, 26 insertions(+), 103 deletions(-) diff --git a/Dashboard/app/js/lib/models/fixtures.js b/Dashboard/app/js/lib/models/fixtures.js index 19fa6fba13..21bb827306 100644 --- a/Dashboard/app/js/lib/models/fixtures.js +++ b/Dashboard/app/js/lib/models/fixtures.js @@ -913,263 +913,186 @@ FLOW.QuestionAnswer.FIXTURES = [{ FLOW.SurveyInstance.FIXTURES = [{ submitterName: "Community (CBO)", - collectionDate: 1334938302000, + collectionDate: 1334938302001, surveyCode: "Mars / Initial question du planteur", deviceIdentifier: "1. WP Ownership", placemarkId: 732033, id: 734238 }, { submitterName: "Functional ( in use)", - collectionDate: 1334938302000, + collectionDate: 1334938302002, surveyCode: "Mars / Initial question du planteur", deviceIdentifier: "1.Functional status", placemarkId: 732033, id: 734234 }, { submitterName: "Unsafe", - collectionDate: 1334938302000, + collectionDate: 1334938302003, surveyCode: "Mars / Initial question du planteur", deviceIdentifier: "2a. Quantitative in-field assessment", placemarkId: 732033, id: 735246 }, { submitterName: "Coloured (whitish- brownish)", - collectionDate: 1334938302000, + collectionDate: 1334938302004, surveyCode: "Mars / Initial question du planteur", deviceIdentifier: "2b.Qualitative in-field assessment", placemarkId: 732033, id: 735245 }, { submitterName: "Good- practically always", - collectionDate: 1334938302000, + collectionDate: 1334938302005, surveyCode: "Mars / Initial question du planteur", deviceIdentifier: "3. Reliability", placemarkId: 732033, id: 734235 }, { submitterName: "Yes", - collectionDate: 1334938302000, + collectionDate: 1334938302006, surveyCode: "Mars / Initial question du planteur", deviceIdentifier: "B.Sufficient for HHs", placemarkId: 732033, id: 732228 }, { submitterName: "Yes", - collectionDate: 1334938302000, + collectionDate: 1334938302006, surveyCode: "Mars / Initial question du planteur", deviceIdentifier: "C. Sufficient for livestock", placemarkId: 732033, id: 735242 }, { submitterName: "ahero youth", - collectionDate: 1334938302000, + collectionDate: 1334938302007, surveyCode: "Mars / Initial question du planteur", deviceIdentifier: "CBO, specify", placemarkId: 732033, id: 732222 }, { submitterName: "Unknown", - collectionDate: 1334938302000, + collectionDate: 1334938302008, surveyCode: "Mars / Initial question du planteur", deviceIdentifier: "Committee in place:", placemarkId: 732033, id: 735249 }, { submitterName: "all", - collectionDate: 1334938302000, + collectionDate: 1334938302009, surveyCode: "Mars / Initial question du planteur", deviceIdentifier: "Community, specify", placemarkId: 732033, id: 732224 }, { submitterName: "Name one", - collectionDate: 1334938302000, + collectionDate: 1334938302010, surveyCode: "Mars / Initial question du planteur", deviceIdentifier: "Contact one", placemarkId: 732033, id: 735244 }, { submitterName: "40", - collectionDate: 1334938302000, + collectionDate: 1334938302011, surveyCode: "Mars / Initial question du planteur", deviceIdentifier: "D. HHs # served/day", placemarkId: 732033, id: 735241 }, { submitterName: "20/04/12", - collectionDate: 1334938302000, + collectionDate: 1334938302012, surveyCode: "Mars / Initial question du planteur", deviceIdentifier: "Date Record", placemarkId: 732033, id: 728181 }, { submitterName: "No", - collectionDate: 1334938302000, + collectionDate: 1334938302013, surveyCode: "Mars / Initial question du planteur", deviceIdentifier: "Do you have an SPA?", placemarkId: 732033, id: 734236 }, { submitterName: "community", - collectionDate: 1334938302000, + collectionDate: 1334938302014, surveyCode: "Mars / Initial question du planteur", deviceIdentifier: "Funded by", placemarkId: 732033, id: 728179 }, { submitterName: "-0.16252854|35.07743752|1136.800048828125|7gs8a46", - collectionDate: 1334938302000, + collectionDate: 1334938302015, surveyCode: "Mars / Initial question du planteur", deviceIdentifier: "GPS reading", placemarkId: 732033, id: 732229 }, { submitterName: "alex", - collectionDate: 1334938302000, + collectionDate: 1334938302016, surveyCode: "Mars / Initial question du planteur", deviceIdentifier: "Interviewee", placemarkId: 732033, id: 728178 }, { submitterName: "amara", - collectionDate: 1334938302000, + collectionDate: 1334938302017, surveyCode: "Mars / Initial question du planteur", deviceIdentifier: "Interviewer", placemarkId: 732033, id: 728180 }, { submitterName: "ahero pan", - collectionDate: 1334938302000, + collectionDate: 1334938302018, surveyCode: "Mars / Initial question du planteur", deviceIdentifier: "Name of source/water point", placemarkId: 732033, id: 728182 }, { submitterName: "onuonga", - collectionDate: 1334938302000, + collectionDate: 1334938302019, surveyCode: "Mars / Initial question du planteur", deviceIdentifier: "Names one,specify", placemarkId: 732033, id: 735247 }, { submitterName: "No", - collectionDate: 1334938302000, + collectionDate: 1334938302020, surveyCode: "Mars / Initial question du planteur", deviceIdentifier: "O&M cost recovery", placemarkId: 732033, id: 732225 }, { submitterName: "LVNWSB", - collectionDate: 1334938302000, + collectionDate: 1334938302021, surveyCode: "Mars / Initial question du planteur", deviceIdentifier: "Organisation", placemarkId: 732033, id: 728183 }, { submitterName: "No", - collectionDate: 1334938302000, + collectionDate: 1334938302022, surveyCode: "Mars / Initial question du planteur", deviceIdentifier: "Part of the piped scheme", placemarkId: 732033, id: 734229 }, { submitterName: "/mnt/sdcard/fieldsurvey/surveyal/8/7/9/8/5/wfpPhoto18652367987985.jpg", - collectionDate: 1334938302000, + collectionDate: 1334938302023, surveyCode: "Mars / Initial question du planteur", deviceIdentifier: "Photo", placemarkId: 732033, id: 732230 }, { submitterName: "No", - collectionDate: 1334938302000, + collectionDate: 1334938302024, surveyCode: "Mars / Initial question du planteur", deviceIdentifier: "Specify none", placemarkId: 732033, id: 732223 }, { submitterName: "tura", - collectionDate: 1334938302000, + collectionDate: 1334938302025, surveyCode: "Mars / Initial question du planteur", deviceIdentifier: "Sub-location", placemarkId: 732033, id: 734231 -}, { - submitterName: "< 1 hour", - collectionDate: 1334938302000, - surveyCode: "Mars / Initial question du planteur", - deviceIdentifier: "Time", - placemarkId: 732033, - id: 732227 -}, { - submitterName: "Dam/Pan(runoff harvesting)", - collectionDate: 1334938302000, - surveyCode: "Mars / Initial question du planteur", - deviceIdentifier: "Unimproved", - placemarkId: 732033, - id: 734233 -}, { - submitterName: "ksm/040", - collectionDate: 1334938302000, - surveyCode: "Mars / Initial question du planteur", - deviceIdentifier: "WP ID", - placemarkId: 732033, - id: 728177 -}, { - submitterName: "Community (technician) Name/NO", - collectionDate: 1334938302000, - surveyCode: "Mars / Initial question du planteur", - deviceIdentifier: "WP Maintenance", - placemarkId: 732033, - id: 735248 -}, { - submitterName: "Directly managed by the CBO", - collectionDate: 1334938302000, - surveyCode: "Mars / Initial question du planteur", - deviceIdentifier: "WP Management", - placemarkId: 732033, - id: 734237 -}, { - submitterName: "Year round", - collectionDate: 1334938302000, - surveyCode: "Mars / Initial question du planteur", - deviceIdentifier: "Water Availability", - placemarkId: 732033, - id: 732226 -}, { - submitterName: "None", - collectionDate: 1334938302000, - surveyCode: "Mars / Initial question du planteur", - deviceIdentifier: "Water Payment", - placemarkId: 732033, - id: 735250 -}, { - submitterName: "30", - collectionDate: 1334938302000, - surveyCode: "Mars / Initial question du planteur", - deviceIdentifier: "Water consumption per ( in dry season)", - placemarkId: 732033, - id: 735243 -}, { - submitterName: "Unimproved", - collectionDate: 1334938302000, - surveyCode: "Mars / Initial question du planteur", - deviceIdentifier: "Water source type", - placemarkId: 732033, - id: 734232 -}, { - submitterName: "No", - collectionDate: 1334938302000, - surveyCode: "Mars / Initial question du planteur", - deviceIdentifier: "Within WSP", - placemarkId: 732033, - id: 734230 -}, { - submitterName: "2004", - collectionDate: 1334938302000, - surveyCode: "Mars / Initial question du planteur", - deviceIdentifier: "Year Constructed", - placemarkId: 732033, - id: 732231 }]; FLOW.SurveyQuestionSummary.FIXTURES = [{ From d348d1e9a897f7bc78e2d393ead2e6a87d6816fa Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Thu, 23 May 2013 12:34:18 +0700 Subject: [PATCH 04/59] Issue #182 and #188- turn on photo size reminder and photo resize by default --- .../gallatinsystems/survey/device/dao/SurveyDbAdapter.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/dao/SurveyDbAdapter.java b/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/dao/SurveyDbAdapter.java index 97e789833d..b56567b030 100644 --- a/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/dao/SurveyDbAdapter.java +++ b/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/dao/SurveyDbAdapter.java @@ -129,7 +129,8 @@ public class SurveyDbAdapter { // "insert into survey values(943186,'Community Water Point', 1.0,'Survey','res','cw943186','en','N','N')", // "insert into survey values(1007024,'Household Interview', 1.0,'Survey','res','hh1007024','en','N','N')", // "insert into survey values(971189,'Public Institution', 1.0,'Survey','res','pi971189','en','N','N')", - + + // insert default values "insert into preferences values('survey.language','0')", "insert into preferences values('user.storelast','false')", "insert into preferences values('data.cellular.upload','0')", @@ -144,7 +145,9 @@ public class SurveyDbAdapter { "insert into preferences values('precache.points.limit','200')", "insert into preferences values('survey.textsize','LARGE')", "insert into preferences values('survey.checkforupdates','0')", - "insert into preferences values('remoteexception.upload','0')" }; + "insert into preferences values('remoteexception.upload','0')", + "insert into preferences values('survey.media.photo.shrink','true')", + "insert into preferences values('survey.media.photo.sizereminder','true')" }; private static final String DATABASE_NAME = "surveydata"; private static final String SURVEY_TABLE = "survey"; From 2b0c897eca5fad4af17ad823811dbac626591acc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Perdomo?= Date: Tue, 4 Jun 2013 11:56:57 +0200 Subject: [PATCH 05/59] Issue #245 - Set the connection and read timeout to 5min --- .../org/waterforpeople/mapping/app/web/TaskServlet.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/GAE/src/org/waterforpeople/mapping/app/web/TaskServlet.java b/GAE/src/org/waterforpeople/mapping/app/web/TaskServlet.java index fda50f12b9..72ffc5491d 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/TaskServlet.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/TaskServlet.java @@ -21,6 +21,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.URL; +import java.net.URLConnection; import java.security.GeneralSecurityException; import java.security.MessageDigest; import java.text.DateFormat; @@ -86,6 +87,7 @@ public class TaskServlet extends AbstractRestApiServlet { private SurveyInstanceDAO siDao; private final static String EMAIL_FROM_ADDRESS_KEY = "emailFromAddress"; private TreeMap recepientList = null; + private static final int CONNECTION_TIMEOUT = 5 * 60 * 1000; // 5min public TaskServlet() { DEVICE_FILE_PATH = com.gallatinsystems.common.util.PropertyUtil @@ -116,8 +118,13 @@ private ArrayList processFile( try { DeviceFilesDao dfDao = new DeviceFilesDao(); + URL url = new URL(DEVICE_FILE_PATH + fileName); - BufferedInputStream bis = new BufferedInputStream(url.openStream()); + URLConnection conn = url.openConnection(); + conn.setConnectTimeout(CONNECTION_TIMEOUT); + conn.setReadTimeout(CONNECTION_TIMEOUT); + + BufferedInputStream bis = new BufferedInputStream(conn.getInputStream()); ZipInputStream zis = new ZipInputStream(bis); List dfList = null; DeviceFiles deviceFile = null; From 311483a0c22d38be2542c8795ee23232ee902786 Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Wed, 5 Jun 2013 14:35:53 +0200 Subject: [PATCH 06/59] Issue #242 - Add extra fixtures for surveyGroups --- Dashboard/app/js/lib/models/fixtures.js | 98 ++++++++++++++++++++++++- 1 file changed, 97 insertions(+), 1 deletion(-) diff --git a/Dashboard/app/js/lib/models/fixtures.js b/Dashboard/app/js/lib/models/fixtures.js index 21bb827306..fb1e00ac56 100644 --- a/Dashboard/app/js/lib/models/fixtures.js +++ b/Dashboard/app/js/lib/models/fixtures.js @@ -17,7 +17,103 @@ FLOW.SurveyGroup.FIXTURES = [{ }, { id: 5, keyId: 5, - code: 'Akvo test surveys' + code: '1 Akvo test surveys' +}, { + id: 6, + keyId: 6, + code: '2 Akvo test surveys' +}, { + id: 7, + keyId: 7, + code: '3 Akvo test surveys' +}, { + id: 8, + keyId: 8, + code: '4 Akvo test surveys' +}, { + id: 9, + keyId: 9, + code: '5 Akvo test surveys' +}, { + id: 10, + keyId:11, + code: '6 Akvo test surveys' +}, { + id: 12, + keyId: 12, + code: '7 Akvo test surveys' +}, { + id: 13, + keyId: 14, + code: '8 Akvo test surveys' +}, { + id: 15, + keyId: 15, + code: '9 Akvo test surveys' +}, { + id: 16, + keyId: 16, + code: '10 Akvo test surveys' +}, { + id: 17, + keyId: 17, + code: '11 Akvo test surveys' +}, { + id: 18, + keyId: 18, + code: '12 Akvo test surveys' +}, { + id: 19, + keyId: 19, + code: '13 Akvo test surveys' +}, { + id: 21, + keyId: 21, + code: '14 Akvo test surveys' +}, { + id: 22, + keyId:22, + code: '15 Akvo test surveys' +}, { + id: 23, + keyId: 23, + code: '16 Akvo test surveys' +}, { + id: 24, + keyId: 24, + code: '17 Akvo test surveys' +}, { + id: 25, + keyId: 25, + code: '18 Akvo test surveys' +}, { + id: 26, + keyId: 26, + code: '19 Akvo test surveys' +}, { + id: 27, + keyId: 27, + code: '20 Akvo test surveys' +}, { + id: 28, + keyId: 28, + code: '21 Akvo test surveys' +}, { + id: 29, + keyId: 29, + code: '22 Akvo test surveys' +}, { + id: 30, + keyId: 30, + code: '23 Akvo test surveys' +}, { + id: 31, + keyId: 31, + code: '24 Akvo test surveys' +}, { + id: 32, + keyId: 32, + code: '25 Akvo test surveys' }]; From 89ff08a10378262f2fe6fe9ecae4e942aec9c326 Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Wed, 5 Jun 2013 15:21:27 +0200 Subject: [PATCH 07/59] Issue #242 - remove bug in height cutoff computation --- Dashboard/app/css/main.css | 2 +- Dashboard/app/js/lib/views/surveys/survey-group-views.js | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/Dashboard/app/css/main.css b/Dashboard/app/css/main.css index 4d1955d0e6..5d38db9779 100644 --- a/Dashboard/app/css/main.css +++ b/Dashboard/app/css/main.css @@ -1163,7 +1163,7 @@ th.action { width: 7%; } .no-textshadow #groupBar h2, .no-textshadow #newSurveyInfo h2 { color: rgb(125, 125, 125); } nav.menuGroupWrap { overflow-y: hidden; - height: 361px; + height: 361px; background: rgb(58,58,58); } nav.menuGroupWrap ul.menuGroup { } diff --git a/Dashboard/app/js/lib/views/surveys/survey-group-views.js b/Dashboard/app/js/lib/views/surveys/survey-group-views.js index 3805ec32ba..0c1d1df301 100644 --- a/Dashboard/app/js/lib/views/surveys/survey-group-views.js +++ b/Dashboard/app/js/lib/views/surveys/survey-group-views.js @@ -199,9 +199,6 @@ FLOW.JavascriptSurveyGroupListView = FLOW.View.extend({ $('.menuGroupWrap').animate({ 'scrollTop': scroll + 72 }, 155); - if(scroll > menuHeight) { - $('.scrollDown').addClass("FadeIt"); - } }); }, From ea0c4a1fb7bba1f355a535c20e351a74233396de Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Wed, 5 Jun 2013 17:46:12 +0200 Subject: [PATCH 08/59] revert change in 569c228e2e, remove reference to locationControl --- Dashboard/app/js/lib/views/data/inspect-data-table-views.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/Dashboard/app/js/lib/views/data/inspect-data-table-views.js b/Dashboard/app/js/lib/views/data/inspect-data-table-views.js index 502b994622..3b7454802b 100644 --- a/Dashboard/app/js/lib/views/data/inspect-data-table-views.js +++ b/Dashboard/app/js/lib/views/data/inspect-data-table-views.js @@ -17,8 +17,6 @@ FLOW.inspectDataTableView = FLOW.View.extend({ FLOW.selectedControl.set('selectedSurvey',null); FLOW.dateControl.set('toDate',null); FLOW.dateControl.set('fromDate',null); - FLOW.locationControl.set('selectedLevel1',null); - FLOW.locationControl.set('selectedLevel2',null); FLOW.surveyInstanceControl.set('pageNumber',0); }, From 4c1c1472c5720ab66cdba5f41336fd665ee7a2a1 Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Wed, 5 Jun 2013 17:01:28 +0200 Subject: [PATCH 09/59] Issue #236 - fix date handling in preview data --- .../app/js/lib/views/data/question-answer-view.js | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Dashboard/app/js/lib/views/data/question-answer-view.js b/Dashboard/app/js/lib/views/data/question-answer-view.js index deec414f48..2d357039c6 100644 --- a/Dashboard/app/js/lib/views/data/question-answer-view.js +++ b/Dashboard/app/js/lib/views/data/question-answer-view.js @@ -104,8 +104,18 @@ doInit: function() { }, doSave: function() { + var tempDate = null; if(this.get('isDateType')){ - this.content.set('value',Date.parse(this.get('date'))); + if (Ember.empty(this.get('date'))){ + this.content.set('value',null); + } else { + tempDate = Date.parse(this.get('date')); + if (!isNaN(tempDate)){ + this.content.set('value',tempDate); + } else { + this.content.set('value',null); + } + } } else if (this.get('isOptionType')){ this.content.set('value',this.optionChoice.get('value')); } else if (this.get('isNumberType')){ From 1c0c03f4934b89c9a0cc1bcfafa5a071c4058bd2 Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Thu, 6 Jun 2013 13:23:15 +0200 Subject: [PATCH 10/59] add access to fixtures for development --- Dashboard/app/js/lib/main.js | 1 + Dashboard/app/js/lib/models/store_def-common.js | 1 + 2 files changed, 2 insertions(+) diff --git a/Dashboard/app/js/lib/main.js b/Dashboard/app/js/lib/main.js index 31f8143ba7..6fc769a15f 100755 --- a/Dashboard/app/js/lib/main.js +++ b/Dashboard/app/js/lib/main.js @@ -1,6 +1,7 @@ require('akvo-flow/all_locales'); require('akvo-flow/models/FLOWrest-adapter-v2-common'); require('akvo-flow/models/models'); +//require('akvo-flow/models/fixtures'); require('akvo-flow/flowenv'); require('akvo-flow/controllers/controllers'); require('akvo-flow/views/views'); diff --git a/Dashboard/app/js/lib/models/store_def-common.js b/Dashboard/app/js/lib/models/store_def-common.js index cd608dc0d3..1199265dd0 100644 --- a/Dashboard/app/js/lib/models/store_def-common.js +++ b/Dashboard/app/js/lib/models/store_def-common.js @@ -2,6 +2,7 @@ var host = "http://" + window.location.host; FLOW.store = DS.Store.create({ revision: 10, adapter:DS.FLOWRESTAdapter.create({bulkCommit:false, namespace:"rest", url:host}) + //adapter: DS.FixtureAdapter }); DS.JSONTransforms.array = { From 4f7b455c75901286958ada88d6cd09cc6e2861e6 Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Thu, 6 Jun 2013 14:42:22 +0200 Subject: [PATCH 11/59] Issue #247 - display (clickable) string of spaces when value is empty --- Dashboard/app/js/lib/views/views.js | 6 ++++++ Dashboard/app/js/templates/navData/inspect-data.handlebars | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Dashboard/app/js/lib/views/views.js b/Dashboard/app/js/lib/views/views.js index d17fe33b8c..c746120d4b 100644 --- a/Dashboard/app/js/lib/views/views.js +++ b/Dashboard/app/js/lib/views/views.js @@ -80,6 +80,12 @@ Ember.Handlebars.registerHelper('t', function(i18nKey, options) { }); +Ember.Handlebars.registerHelper('if_blank', function(item) { + var text; + text = Ember.get(this,item); + return (text && text.replace(/\s/g,"").length) ? new Handlebars.SafeString('') : new Handlebars.SafeString('          '); +}); + Ember.Handlebars.registerHelper('tooltip', function(i18nKey) { var tooltip; try { diff --git a/Dashboard/app/js/templates/navData/inspect-data.handlebars b/Dashboard/app/js/templates/navData/inspect-data.handlebars index c21e5d2cdf..70ac902c79 100644 --- a/Dashboard/app/js/templates/navData/inspect-data.handlebars +++ b/Dashboard/app/js/templates/navData/inspect-data.handlebars @@ -181,7 +181,7 @@ {{#if view.isDateType}} {{#with QA}}{{date3 value}}{{/with}} {{else}} - {{QA.value}} + {{QA.value}} {{#with QA}}{{if_blank value}}{{/with}} {{/if}} {{/if}} {{/if}} From a9108fb87c819d08f70acfac2c30b439026fc7b6 Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Fri, 31 May 2013 11:59:34 +0700 Subject: [PATCH 12/59] Issue #237 - correct row number display --- Dashboard/app/js/lib/controllers/data-controllers.js | 2 +- Dashboard/app/js/lib/views/data/inspect-data-table-views.js | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Dashboard/app/js/lib/controllers/data-controllers.js b/Dashboard/app/js/lib/controllers/data-controllers.js index f712382799..2329c818cc 100644 --- a/Dashboard/app/js/lib/controllers/data-controllers.js +++ b/Dashboard/app/js/lib/controllers/data-controllers.js @@ -40,7 +40,7 @@ FLOW.surveyInstanceControl = Ember.ArrayController.create({ selectedSurvey: null, content: null, sinceArray: [], - pagenumber:0, + pageNumber:0, populate: function() { this.get('sinceArray').pushObject(FLOW.metaControl.get('since')); diff --git a/Dashboard/app/js/lib/views/data/inspect-data-table-views.js b/Dashboard/app/js/lib/views/data/inspect-data-table-views.js index 3b7454802b..7b5e11acce 100644 --- a/Dashboard/app/js/lib/views/data/inspect-data-table-views.js +++ b/Dashboard/app/js/lib/views/data/inspect-data-table-views.js @@ -23,6 +23,7 @@ FLOW.inspectDataTableView = FLOW.View.extend({ // do a new query doFindSurveyInstances: function() { FLOW.surveyInstanceControl.get('sinceArray').clear(); + FLOW.surveyInstanceControl.set('pageNumber',-1); FLOW.metaControl.set('since', null); this.doNextPage(); }, @@ -155,10 +156,6 @@ FLOW.inspectDataTableView = FLOW.View.extend({ } }, - // doSaveSI: function(event) { - // this.set('showEditSurveyInstanceWindowBool', false); - // }, - doShowDeleteSIDialog: function(event) { FLOW.dialogControl.set('activeAction', 'delSI'); FLOW.dialogControl.set('showCANCEL', true); From 297116c57b8b981f6c70d8f7403993d5f36ba5b2 Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Fri, 7 Jun 2013 13:37:59 +0200 Subject: [PATCH 13/59] Issue #218 - fix deletion of surveyedLocales to work with HR datastore --- .../mapping/dao/SurveyInstanceDAO.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java b/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java index 4bf5291978..2738f18395 100644 --- a/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java +++ b/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java @@ -476,19 +476,20 @@ public void deleteSurveyInstance(SurveyInstance item) { } // delete surveyalValue items - List vals = localeDao + List valsForInstance = localeDao .listSurveyalValuesByInstance(surveyInstanceId); - if (vals != null && vals.size() > 0) { - Long localeId = vals.get(0).getSurveyedLocaleId(); - localeDao.delete(vals); - // now see if there are any other values for the same locale - List otherVals = localeDao + if (valsForInstance != null && valsForInstance.size() > 0) { + Long localeId = valsForInstance.get(0).getSurveyedLocaleId(); + List valsForLocale = localeDao .listValuesByLocale(localeId); - if (otherVals == null || otherVals.size() == 0) { + + // now see if there are any other values for the same locale + if (valsForLocale != null && valsForLocale.size() <= valsForInstance.size()) { // if there are no other values, delete the locale SurveyedLocale l = localeDao.getByKey(localeId); localeDao.delete(l); } + localeDao.delete(valsForInstance); } } From f856ed37576932673f459898591979c9b1f14432 Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Mon, 10 Jun 2013 21:47:34 +0200 Subject: [PATCH 14/59] Issue #251 - only store cursor for surveyInstance REST results --- Dashboard/app/js/lib/models/FLOWrest-adapter-v2-common.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dashboard/app/js/lib/models/FLOWrest-adapter-v2-common.js b/Dashboard/app/js/lib/models/FLOWrest-adapter-v2-common.js index 201a361df6..71344df48f 100644 --- a/Dashboard/app/js/lib/models/FLOWrest-adapter-v2-common.js +++ b/Dashboard/app/js/lib/models/FLOWrest-adapter-v2-common.js @@ -25,10 +25,10 @@ DS.FLOWRESTAdapter = DS.RESTAdapter.extend({ this._super(store, type, json, root); // only change metaControl info if there is actual meta info in the server response if (Object.keys(this.extractMeta(json)).length !== 0) { - FLOW.metaControl.set('since', this.extractMeta(json).since); - FLOW.metaControl.set('num', this.extractMeta(json).num); if (type == 'FLOW.SurveyInstance') { FLOW.metaControl.set('numSILoaded', this.extractMeta(json).num); + FLOW.metaControl.set('since', this.extractMeta(json).since); + FLOW.metaControl.set('num', this.extractMeta(json).num); } msg = this.extractMeta(json).message; status = this.extractMeta(json).status; From 4af2f13385eec4a1a28312d44854636f24757394 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Perdomo?= Date: Tue, 11 Jun 2013 11:07:40 +0200 Subject: [PATCH 15/59] Issue #252 - Include Piwik tracking code --- Dashboard/app/dashboard.html | 17 +++++++++++++++++ Dashboard/app/public.html | 16 ++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/Dashboard/app/dashboard.html b/Dashboard/app/dashboard.html index 1abdbb885d..4880fdc5a7 100644 --- a/Dashboard/app/dashboard.html +++ b/Dashboard/app/dashboard.html @@ -48,5 +48,22 @@ loader.require('akvo-flow/main'); + + diff --git a/Dashboard/app/public.html b/Dashboard/app/public.html index 2037ec6e34..a4e24e6e36 100644 --- a/Dashboard/app/public.html +++ b/Dashboard/app/public.html @@ -45,5 +45,21 @@ loader.require('akvo-flow/main-public'); + From ada35501f44a865be1e258d777dafd5b0dbe980b Mon Sep 17 00:00:00 2001 From: Loic Date: Tue, 11 Jun 2013 17:30:24 +0200 Subject: [PATCH 16/59] add war/index.html to .gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b13c0e6988..e02080fe8e 100644 --- a/.gitignore +++ b/.gitignore @@ -64,4 +64,5 @@ reports/target/* # Vagrant -.vagrant \ No newline at end of file +.vagrant +GAE/war/index.html From 0c6f0bf9b3cfb6353fa76d036252b663ce95642f Mon Sep 17 00:00:00 2001 From: Stellan Lagerstrom Date: Thu, 13 Jun 2013 17:22:15 +0200 Subject: [PATCH 17/59] Make instance configuration depend only on survey.properties. R.arrays.servers now contains only server choices for debugging --- .../device/survey/res/values/arrays.xml | 8 +-- .../device/survey/res/values/strings.xml | 2 +- .../device/activity/PreferencesActivity.java | 56 +++++++++++++++++-- .../survey/device/dao/SurveyDbAdapter.java | 2 +- .../survey/device/util/ConstantUtil.java | 2 +- 5 files changed, 58 insertions(+), 12 deletions(-) diff --git a/WFPMapping/device/survey/res/values/arrays.xml b/WFPMapping/device/survey/res/values/arrays.xml index b6dfd4af65..eb3d36b89e 100644 --- a/WFPMapping/device/survey/res/values/arrays.xml +++ b/WFPMapping/device/survey/res/values/arrays.xml @@ -42,10 +42,10 @@ Never - - - http://@@@@.appspot.com - + + http://192.168.1.100 + http://127.0.0.1 + millimeters centimeters diff --git a/WFPMapping/device/survey/res/values/strings.xml b/WFPMapping/device/survey/res/values/strings.xml index df64708d80..57d9ff2615 100644 --- a/WFPMapping/device/survey/res/values/strings.xml +++ b/WFPMapping/device/survey/res/values/strings.xml @@ -1,6 +1,6 @@ - 1.10.5 + 1.10.6 Field Survey No Storage Space Low Storage Space diff --git a/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/activity/PreferencesActivity.java b/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/activity/PreferencesActivity.java index 346ffa661b..2f05cb55ce 100644 --- a/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/activity/PreferencesActivity.java +++ b/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/activity/PreferencesActivity.java @@ -41,6 +41,7 @@ import com.gallatinsystems.survey.device.util.ArrayPreferenceData; import com.gallatinsystems.survey.device.util.ArrayPreferenceUtil; import com.gallatinsystems.survey.device.util.ConstantUtil; +import com.gallatinsystems.survey.device.util.PropertyUtil; import com.gallatinsystems.survey.device.util.ViewUtil; /** @@ -76,6 +77,7 @@ public class PreferencesActivity extends Activity implements OnClickListener, private String[] uploadArray; private String[] precacheHelpArray; private String[] serverArray; + private PropertyUtil props; public void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); @@ -100,6 +102,8 @@ public void onCreate(Bundle savedInstanceState) { radiusTextView = (TextView) findViewById(R.id.radiusvalue); Resources res = getResources(); + props = new PropertyUtil(res); + uploadArray = res.getStringArray(R.array.celluploadoptions); precacheHelpArray = res.getStringArray(R.array.precachehelpoptions); @@ -178,10 +182,12 @@ private void populateFields() { } val = settings.get(ConstantUtil.SERVER_SETTING_KEY); - if (val != null) { + if (val != null && val.trim().length() > 0) { serverTextView.setText(serverArray[Integer.parseInt(val)]); + } else { + serverTextView.setText(props.getProperty(ConstantUtil.SERVER_BASE)); } - + val = settings.get(ConstantUtil.DEVICE_IDENT_KEY); if (val != null) { identTextView.setText(val); @@ -334,10 +340,10 @@ public void onAuthenticated() { new ViewUtil.AdminAuthDialogListener() { @Override public void onAuthenticated() { - showPreferenceDialog(R.string.serverlabel, - R.array.servers, + showPreferenceDialogBase(R.string.serverlabel, + props.getProperty(ConstantUtil.SERVER_BASE), ConstantUtil.SERVER_SETTING_KEY, - serverArray, serverTextView, null, null); + serverArray, serverTextView); } }); @@ -431,6 +437,46 @@ public void onClick(DialogInterface dialog, int which) { }); builder.show(); } + /** + * displays a dialog that allows the user to choose a setting from a string + * array + * + * @param titleId + * - resource id of dialog title + * @param baseValue + * - Value resulting in empty setting value + * @param settingKey + * - key of setting to edit + * @param valueArray + * - string array containing values + * @param currentValView + * - view to update with value selected + */ + private void showPreferenceDialogBase(int titleId, String baseValue, + final String settingKey, final String[] valueArray, + final TextView currentValView) { + AlertDialog.Builder builder = new AlertDialog.Builder(this); + final String[] extraValueArray = new String[valueArray.length + 1]; + extraValueArray[0] = baseValue; + for (int i = 0; i < valueArray.length; i++ ){ + extraValueArray[i+1] = valueArray[i]; + } + builder.setTitle(titleId).setItems(extraValueArray, + new DialogInterface.OnClickListener() { + @Override + public void onClick(DialogInterface dialog, int which) { + if (which == 0) + database.savePreference(settingKey, ""); + else + database.savePreference(settingKey, (which - 1) + ""); + currentValView.setText(extraValueArray[which]); + if(dialog != null){ + dialog.dismiss(); + } + } + }); + builder.show(); + } /** * saves the value of the checkbox to the database diff --git a/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/dao/SurveyDbAdapter.java b/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/dao/SurveyDbAdapter.java index b56567b030..4be5e4331e 100644 --- a/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/dao/SurveyDbAdapter.java +++ b/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/dao/SurveyDbAdapter.java @@ -139,7 +139,7 @@ public class SurveyDbAdapter { "insert into preferences values('user.lastuser.id','')", "insert into preferences values('location.sendbeacon','true')", "insert into preferences values('survey.precachehelp','1')", - "insert into preferences values('upload.server','0')", + "insert into preferences values('backend.server','')", "insert into preferences values('screen.keepon','true')", "insert into preferences values('precache.points.countries','2')", "insert into preferences values('precache.points.limit','200')", diff --git a/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/util/ConstantUtil.java b/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/util/ConstantUtil.java index eba477b52a..ad9c675e8b 100644 --- a/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/util/ConstantUtil.java +++ b/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/util/ConstantUtil.java @@ -206,7 +206,7 @@ public class ConstantUtil { public static final String LAST_USER_SETTING_KEY = "user.lastuser.id"; public static final String LOCATION_BEACON_SETTING_KEY = "location.sendbeacon"; public static final String PRECACHE_SETTING_KEY = "survey.precachehelp"; - public static final String SERVER_SETTING_KEY = "upload.server"; + public static final String SERVER_SETTING_KEY = "backend.server"; public static final String SCREEN_ON_KEY = "screen.keepon"; public static final String PRECACHE_POINT_COUNTRY_KEY = "precache.points.countries"; public static final String PRECACHE_POINT_LIMIT_KEY = "precache.points.limit"; From c01fdce40722cc9752d4ced86c5251dedd565dc1 Mon Sep 17 00:00:00 2001 From: Stellan Lagerstrom Date: Fri, 14 Jun 2013 12:23:41 +0200 Subject: [PATCH 18/59] Add summary of changes for released versions --- WFPMapping/device/survey/README | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/WFPMapping/device/survey/README b/WFPMapping/device/survey/README index c423a37201..5ff654a160 100644 --- a/WFPMapping/device/survey/README +++ b/WFPMapping/device/survey/README @@ -1,6 +1,6 @@ This is the root directory of the Flow Field Survey application. -The project depends on Android version +The project depends on Android version 2.0.0 Before the project will build, the survey.properties file must be placed in the following location: res/raw/ @@ -10,3 +10,15 @@ This file is slated to be replaced via a central configuration service in an upc + +Summary of new features and changes for released versions +--------------------------------------------------------- + +1.10.6 Always include serverBase from survey.properties in server selection. Makes instance builds independent of res/values/arrays.xml + +1.10.5 Prevent display rotation from forgetting a taken photo. Fix survey status partially-successful icon bug. + +1.10.3 Report OS version to server in beacon call. + +1.10.2 Shrink photos, warn about large media files, report IMEI to server. + From 845684f22b87ba848ecfb6434ec33512dd10e3c7 Mon Sep 17 00:00:00 2001 From: Stellan Lagerstrom Date: Mon, 17 Jun 2013 14:50:52 +0200 Subject: [PATCH 19/59] Sanitize zipfile data in answer, deviceId, displayName and email columns more fully, to prevent broken uploads. Clean up and optimize buffer append code. --- .../device/service/DataSyncService.java | 90 +++++++++---------- 1 file changed, 44 insertions(+), 46 deletions(-) diff --git a/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/service/DataSyncService.java b/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/service/DataSyncService.java index 80dc66be13..5b44fa446d 100644 --- a/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/service/DataSyncService.java +++ b/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/service/DataSyncService.java @@ -76,6 +76,7 @@ public class DataSyncService extends Service { private static final String TAG = "DATA_SYNC_SERVICE"; private static final String NOTHING = "NADA"; private static final String DELIMITER = "\t"; + private static final String SPACE = "\u0020"; //safe from source whitespace reformatting private static final String SIGNING_KEY_PROP = "signingKey"; private static final String SIGNING_ALGORITHM = "HmacSHA1"; @@ -618,62 +619,48 @@ private void processSurveyData(StringBuilder buf, } else { deviceIdentifier = cleanVal(deviceIdentifier); } + //evaluate indices once, outside the loop + int survey_fk_col = data.getColumnIndexOrThrow(SurveyDbAdapter.SURVEY_FK_COL); + int pk_id_col = data.getColumnIndexOrThrow(SurveyDbAdapter.PK_ID_COL); + int question_fk_col = data.getColumnIndexOrThrow(SurveyDbAdapter.QUESTION_FK_COL); + int answer_type_col = data.getColumnIndexOrThrow(SurveyDbAdapter.ANSWER_TYPE_COL); + int answer_col = data.getColumnIndexOrThrow(SurveyDbAdapter.ANSWER_COL); + int disp_name_col = data.getColumnIndexOrThrow(SurveyDbAdapter.DISP_NAME_COL); + int email_col = data.getColumnIndexOrThrow(SurveyDbAdapter.EMAIL_COL); + int submitted_date_col = data.getColumnIndexOrThrow(SurveyDbAdapter.SUBMITTED_DATE_COL); + int scored_val_col = data.getColumnIndexOrThrow(SurveyDbAdapter.SCORED_VAL_COL); + int strength_col = data.getColumnIndexOrThrow(SurveyDbAdapter.STRENGTH_COL); + int uuid_col = data.getColumnIndexOrThrow(SurveyDbAdapter.UUID_COL); + do { - - String value = data.getString(data - .getColumnIndexOrThrow(SurveyDbAdapter.ANSWER_COL)); + // Sanitize answer value. No newlines or tabs! + String value = data.getString(answer_col); if (value != null) { + value = value.replace("\n", SPACE); + value = value.replace(DELIMITER, SPACE); value = value.trim(); - value = value.replaceAll("\\n", " "); } - - if (value == null || value.trim().length() == 0) { + // never send empty answers + if (value == null || value.length() == 0) { continue; } - buf.append( - data.getString(data - .getColumnIndexOrThrow(SurveyDbAdapter.SURVEY_FK_COL))) - .append(DELIMITER); - - String type = data - .getString(data - .getColumnIndexOrThrow(SurveyDbAdapter.ANSWER_TYPE_COL)); - buf.append(data.getString(data - .getColumnIndexOrThrow(SurveyDbAdapter.PK_ID_COL))); - buf.append(DELIMITER) - .append(data.getString(data - .getColumnIndexOrThrow(SurveyDbAdapter.QUESTION_FK_COL))); + buf.append(data.getString(survey_fk_col)); + String respId = data.getString(pk_id_col); + buf.append(DELIMITER).append(respId); + buf.append(DELIMITER).append(data.getString(question_fk_col)); + String type = data.getString(answer_type_col); buf.append(DELIMITER).append(type); buf.append(DELIMITER).append(value); - - buf.append(DELIMITER) - .append(cleanVal(data.getString(data - .getColumnIndexOrThrow(SurveyDbAdapter.DISP_NAME_COL)))); - buf.append(DELIMITER) - .append(cleanVal(data.getString(data - .getColumnIndexOrThrow(SurveyDbAdapter.EMAIL_COL)))); - buf.append(DELIMITER) - .append(data.getString(data - .getColumnIndexOrThrow(SurveyDbAdapter.SUBMITTED_DATE_COL))); + buf.append(DELIMITER).append(cleanVal(data.getString(disp_name_col))); + buf.append(DELIMITER).append(cleanVal(data.getString(email_col))); + buf.append(DELIMITER).append(data.getString(submitted_date_col)); buf.append(DELIMITER).append(deviceIdentifier); - String scoredVal = data - .getString(data - .getColumnIndexOrThrow(SurveyDbAdapter.SCORED_VAL_COL)); - buf.append(DELIMITER).append( - scoredVal != null ? scoredVal : ""); - String strength = data - .getString(data - .getColumnIndexOrThrow(SurveyDbAdapter.STRENGTH_COL)); - buf.append(DELIMITER).append( - strength != null ? strength : ""); - buf.append(DELIMITER) - .append(data.getString(data - .getColumnIndexOrThrow(SurveyDbAdapter.UUID_COL))); + buf.append(DELIMITER).append(neverNull(data.getString(scored_val_col))); + buf.append(DELIMITER).append(neverNull(data.getString(strength_col))); + buf.append(DELIMITER).append(data.getString(uuid_col)); buf.append("\n"); - String respId = data.getString(data - .getColumnIndexOrThrow(SurveyDbAdapter.PK_ID_COL)); if (ConstantUtil.IMAGE_RESPONSE_TYPE.equals(type) || ConstantUtil.VIDEO_RESPONSE_TYPE.equals(type)) { ArrayList paths = imagePaths.get(respId); @@ -696,18 +683,29 @@ private void processSurveyData(StringBuilder buf, } } + //replace troublesome chars in user-provided values + //replaceAll() compiles a Pattern, and so is inefficient inside a loop private String cleanVal(String val) { if (val != null) { if (val.contains(DELIMITER)) { - val = val.replaceAll(DELIMITER, " "); + val = val.replace(DELIMITER, SPACE); } if (val.contains(",")) { - val.replaceAll(",", " "); + val = val.replace(",", SPACE); + } + if (val.contains("\n")) { + val = val.replace("\n", SPACE); } } return val; } + private String neverNull(String val) { + if (val != null) { + return val; + } else return ""; + } + /** * sends the zip file containing data/images to the server via an http * upload From 5b1dce8e80b8c794e4e4f944a76dc7082341e47e Mon Sep 17 00:00:00 2001 From: Stellan Lagerstrom Date: Tue, 18 Jun 2013 00:10:40 +0200 Subject: [PATCH 20/59] Prevent control characters, such as Tab, in device IDs. --- .../survey/device/activity/PreferencesActivity.java | 11 +++++++---- .../survey/device/util/StringUtil.java | 12 ++++++++++++ .../gallatinsystems/survey/device/util/ViewUtil.java | 1 + 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/activity/PreferencesActivity.java b/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/activity/PreferencesActivity.java index 2f05cb55ce..41dabbf734 100644 --- a/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/activity/PreferencesActivity.java +++ b/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/activity/PreferencesActivity.java @@ -42,6 +42,7 @@ import com.gallatinsystems.survey.device.util.ArrayPreferenceUtil; import com.gallatinsystems.survey.device.util.ConstantUtil; import com.gallatinsystems.survey.device.util.PropertyUtil; +import com.gallatinsystems.survey.device.util.StringUtil; import com.gallatinsystems.survey.device.util.ViewUtil; /** @@ -372,6 +373,8 @@ public void onClick(DialogInterface dialog, int clicked) { public void onAuthenticated() { final EditText inputView = new EditText( PreferencesActivity.this); + //one line only + inputView.setSingleLine(); ViewUtil.ShowTextInputDialog( PreferencesActivity.this, R.string.identlabel, @@ -381,12 +384,12 @@ public void onAuthenticated() { public void onClick( DialogInterface dialog, int which) { - identTextView.setText(inputView - .getText()); + String s = StringUtil.ControlToSPace(inputView.getText().toString()); + //drop any control chars, especially tabs + identTextView.setText(s); database.savePreference( ConstantUtil.DEVICE_IDENT_KEY, - inputView.getText() - .toString()); + s); } }); } diff --git a/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/util/StringUtil.java b/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/util/StringUtil.java index 7e880b01f3..32f5c09941 100644 --- a/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/util/StringUtil.java +++ b/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/util/StringUtil.java @@ -37,4 +37,16 @@ public static boolean isNullOrEmpty(String s) { return false; } } + + public static String ControlToSPace(String val) { + String result = ""; + for (int i= 0; i < val.length(); i++) { + if (val.charAt(i) < '\u0020') + result = result + '\u0020'; + else + result = result + val.charAt(i); + } + + return result; + } } diff --git a/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/util/ViewUtil.java b/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/util/ViewUtil.java index cd4c307f2f..7a7e526862 100644 --- a/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/util/ViewUtil.java +++ b/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/util/ViewUtil.java @@ -379,6 +379,7 @@ public void onClick( public static void showAdminAuthDialog(final Context parentContext, final AdminAuthDialogListener listener) { final EditText input = new EditText(parentContext); + input.setSingleLine(); ShowTextInputDialog(parentContext, R.string.authtitle, R.string.authtext, input, new DialogInterface.OnClickListener() { From 957c5abc541a638f48676738a277c0afe9dcd22c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Perdomo?= Date: Tue, 18 Jun 2013 19:34:30 +0200 Subject: [PATCH 21/59] Issue #256 - Enable REST security based on RestAuthFilter --- .../framework/servlet/RestAuthFilter.java | 21 +---- .../app/web/DataProcessorRestServlet.java | 2 +- .../mapping/app/web/TestHarnessServlet.java | 2 +- .../GraphicalSurveySummaryExporter.java | 19 +++-- .../mapping/dataexport/OfflineExport.java | 2 +- .../mapping/dataexport/RawDataExporter.java | 13 +-- .../RawDataSpreadsheetImporter.java | 2 +- .../dataexport/SurveyFormExporter.java | 10 +-- .../dataexport/SurveyReplicationImporter.java | 28 +++--- .../dataexport/SurveySpreadsheetImporter.java | 5 +- .../dataexport/SurveySummaryExporter.java | 40 +++++---- .../service/BulkDataServiceClient.java | 85 +++++++++++-------- 12 files changed, 118 insertions(+), 111 deletions(-) diff --git a/GAE/src/com/gallatinsystems/framework/servlet/RestAuthFilter.java b/GAE/src/com/gallatinsystems/framework/servlet/RestAuthFilter.java index efff0e3576..0bf7c5eea6 100644 --- a/GAE/src/com/gallatinsystems/framework/servlet/RestAuthFilter.java +++ b/GAE/src/com/gallatinsystems/framework/servlet/RestAuthFilter.java @@ -51,8 +51,6 @@ public class RestAuthFilter implements Filter { private static final long MAX_TIME = 60000; - private static final String[] RESTRICTED_ACTIONS = { "delete", "save", - "update", "create", "purge", "reset" }; private static final Logger log = Logger.getLogger(RestAuthFilter.class .getName()); private static final String ENABLED_PROP = "enableRestSecurity"; @@ -93,10 +91,7 @@ private boolean isAuthorized(ServletRequest req) throws Exception { String incomingHash = null; long incomingTimestamp = 0; List names = new ArrayList(); - if (paramMap != null - && isRestrictedAction((String[]) paramMap - .get(RestRequest.ACTION_PARAM))) { - + if (paramMap != null) { names.addAll(paramMap.keySet()); Collections.sort(names); StringBuilder builder = new StringBuilder(); @@ -124,6 +119,7 @@ && isRestrictedAction((String[]) paramMap } } else { incomingHash = ((String[]) paramMap.get(name))[0]; + incomingHash = incomingHash.replaceAll(" ", "+"); } } @@ -134,6 +130,7 @@ && isRestrictedAction((String[]) paramMap // Do something but for now return false; return false; } + if (ourHash.equals(incomingHash)) { return isTimestampValid(incomingTimestamp); } else { @@ -143,18 +140,6 @@ && isRestrictedAction((String[]) paramMap return false; } } - return true; - } - - private boolean isRestrictedAction(String[] action) { - if (action != null && action.length > 0) { - String actionVal = action[0].trim().toLowerCase(); - for (int i = 0; i < RESTRICTED_ACTIONS.length; i++) { - if (actionVal.contains(RESTRICTED_ACTIONS[i])) { - return true; - } - } - } return false; } diff --git a/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java b/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java index 8455d8b7bb..5a3503c80c 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java @@ -90,7 +90,7 @@ protected RestResponse handleRequest(RestRequest req) throws Exception { } else if (DataProcessorRequest.IMPORT_REMOTE_SURVEY_ACTION .equalsIgnoreCase(dpReq.getAction())) { SurveyReplicationImporter sri = new SurveyReplicationImporter(); - sri.executeImport(dpReq.getSource(), dpReq.getSurveyId()); + sri.executeImport(dpReq.getSource(), dpReq.getSurveyId(), null); //FIXME } else if (DataProcessorRequest.RESCORE_AP_ACTION .equalsIgnoreCase(dpReq.getAction())) { rescoreAp(dpReq.getCountry()); diff --git a/GAE/src/org/waterforpeople/mapping/app/web/TestHarnessServlet.java b/GAE/src/org/waterforpeople/mapping/app/web/TestHarnessServlet.java index 896af69de6..c4c129bca4 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/TestHarnessServlet.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/TestHarnessServlet.java @@ -1366,7 +1366,7 @@ else if ("saveSurveyGroupRefactor".equals(action)) { } else if ("importallsurveys".equals(action)) { // Only run in dev hence hardcoding SurveyReplicationImporter sri = new SurveyReplicationImporter(); - sri.executeImport("http://watermapmonitordev.appspot.com", null); + sri.executeImport("http://watermapmonitordev.appspot.com", null, null); // sri.executeImport("http://localhost:8888", // "http://localhost:8888"); diff --git a/GAE/src/org/waterforpeople/mapping/dataexport/GraphicalSurveySummaryExporter.java b/GAE/src/org/waterforpeople/mapping/dataexport/GraphicalSurveySummaryExporter.java index d2e2f3f35f..6d8f0c6fdb 100644 --- a/GAE/src/org/waterforpeople/mapping/dataexport/GraphicalSurveySummaryExporter.java +++ b/GAE/src/org/waterforpeople/mapping/dataexport/GraphicalSurveySummaryExporter.java @@ -292,7 +292,7 @@ public void export(Map criteria, File fileName, LOADING_QUESTIONS.get(locale))); Map> questionMap = loadAllQuestions( criteria.get(SurveyRestRequest.SURVEY_ID_PARAM), - performGeoRollup, serverBase); + performGeoRollup, serverBase, criteria.get("apiKey")); if (questionMap != null) { for (List qList : questionMap.values()) { for (QuestionDto q : qList) { @@ -305,7 +305,7 @@ public void export(Map criteria, File fileName, // translations SwingUtilities.invokeLater(new StatusUpdater(currentStep++, LOADING_DETAILS.get(locale))); - loadFullQuestions(questionMap); + loadFullQuestions(questionMap, criteria.get("apiKey")); } else { currentStep++; } @@ -325,7 +325,7 @@ public void export(Map criteria, File fileName, SummaryModel model = fetchAndWriteRawData( criteria.get(SurveyRestRequest.SURVEY_ID_PARAM), - serverBase, questionMap, wb, isFullReport, fileName); + serverBase, questionMap, wb, isFullReport, fileName, criteria.get("apiKey")); if (isFullReport) { SwingUtilities.invokeLater(new StatusUpdater(currentStep++, WRITING_SUMMARY.get(locale))); @@ -377,8 +377,9 @@ public int compare(String o1, String o2) { protected SummaryModel fetchAndWriteRawData(String surveyId, final String serverBase, Map> questionMap, Workbook wb, - final boolean generateSummary, File outputFile) throws Exception { + final boolean generateSummary, File outputFile, String apiKey) throws Exception { final SummaryModel model = new SummaryModel(); + final String key = apiKey; final Sheet sheet = wb.createSheet(RAW_DATA_LABEL.get(locale)); int curRow = 1; @@ -404,7 +405,7 @@ protected SummaryModel fetchAndWriteRawData(String surveyId, SwingUtilities.invokeLater(new StatusUpdater(currentStep++, LOADING_INSTANCES.get(locale))); Map instanceMap = BulkDataServiceClient - .fetchInstanceIds(surveyId, serverBase); + .fetchInstanceIds(surveyId, serverBase, key); SwingUtilities.invokeLater(new StatusUpdater(currentStep++, LOADING_INSTANCE_DETAILS.get(locale))); @@ -424,12 +425,12 @@ public void run() { Map responseMap = BulkDataServiceClient .fetchQuestionResponses(instanceId, - serverBase); + serverBase, key); SurveyInstanceDto dto = BulkDataServiceClient .findSurveyInstance( Long.parseLong(instanceId.trim()), - serverBase); + serverBase, key); if (dto != null) { done = true; } @@ -999,13 +1000,13 @@ protected void processOptions(Map options) { * @param questionMap */ private void loadFullQuestions( - Map> questionMap) { + Map> questionMap, String apiKey) { for (List questionList : questionMap.values()) { for (int i = 0; i < questionList.size(); i++) { try { QuestionDto newQ = BulkDataServiceClient .loadQuestionDetails(serverBase, questionList - .get(i).getKeyId()); + .get(i).getKeyId(), apiKey); if (newQ != null) { questionList.set(i, newQ); } diff --git a/GAE/src/org/waterforpeople/mapping/dataexport/OfflineExport.java b/GAE/src/org/waterforpeople/mapping/dataexport/OfflineExport.java index 1ffa194460..4e09c461cd 100644 --- a/GAE/src/org/waterforpeople/mapping/dataexport/OfflineExport.java +++ b/GAE/src/org/waterforpeople/mapping/dataexport/OfflineExport.java @@ -75,7 +75,7 @@ public void export(Map criteria, File outputFile, if (inputFiles != null && inputFiles.size() > 0) { Map> questionMap = loadAllQuestions( criteria.get(SurveyRestRequest.SURVEY_ID_PARAM), false, - serverBase); + serverBase, criteria.get("apiKey")); Workbook wb = new XSSFWorkbook(); diff --git a/GAE/src/org/waterforpeople/mapping/dataexport/RawDataExporter.java b/GAE/src/org/waterforpeople/mapping/dataexport/RawDataExporter.java index 39871ca407..24f8daf3a9 100644 --- a/GAE/src/org/waterforpeople/mapping/dataexport/RawDataExporter.java +++ b/GAE/src/org/waterforpeople/mapping/dataexport/RawDataExporter.java @@ -48,6 +48,7 @@ public class RawDataExporter extends AbstractDataExporter { private String serverBase; private String surveyId; private String imgPrefix = null; + private String apiKey; public static final String SURVEY_ID = "surveyId"; private Map questionMap; @@ -60,12 +61,14 @@ public void export(Map criteria, File fileName, this.serverBase = serverBase; surveyId = criteria.get(SURVEY_ID); imgPrefix = options.get("imagePrefix"); + apiKey = criteria.get("apiKey"); Writer pw = null; System.out.println("In CSV exporter"); + final String apiKey = criteria.get("apiKey"); try { Object[] results = BulkDataServiceClient.loadQuestions(surveyId, - serverBase); + serverBase, apiKey); if (results != null) { keyList = (List) results[0]; questionMap = (Map) results[1]; @@ -95,7 +98,7 @@ public void export(String serverBase, Long surveyIdentifier, Writer pw) { this.surveyId = surveyIdentifier.toString(); this.serverBase = serverBase; Object[] results = BulkDataServiceClient.loadQuestions(surveyId, - serverBase); + serverBase, apiKey); if (results != null) { keyList = (List) results[0]; questionMap = (Map) results[1]; @@ -135,7 +138,7 @@ private void writeHeader(Writer pw, Map questions) private void exportInstances(Writer pw, List idList) throws Exception { Map instances = BulkDataServiceClient.fetchInstanceIds( - surveyId, serverBase); + surveyId, serverBase, apiKey); if (instances != null) { String imagePrefix = imgPrefix != null ? imgPrefix : IMAGE_PREFIX; @@ -150,7 +153,7 @@ private void exportInstances(Writer pw, List idList) if (instanceId != null && instanceId.trim().length() > 0) { try { Map responses = BulkDataServiceClient - .fetchQuestionResponses(instanceId, serverBase); + .fetchQuestionResponses(instanceId, serverBase, apiKey); if (responses != null && responses.size() > 0) { pw.write(instanceId); pw.write("\t"); @@ -159,7 +162,7 @@ private void exportInstances(Writer pw, List idList) SurveyInstanceDto dto = BulkDataServiceClient .findSurveyInstance( Long.parseLong(instanceId.trim()), - serverBase); + serverBase, apiKey); if (dto != null) { String name = dto.getSubmitterName(); if (name != null) { diff --git a/GAE/src/org/waterforpeople/mapping/dataexport/RawDataSpreadsheetImporter.java b/GAE/src/org/waterforpeople/mapping/dataexport/RawDataSpreadsheetImporter.java index 6ad0351ff6..5bb3783b8c 100644 --- a/GAE/src/org/waterforpeople/mapping/dataexport/RawDataSpreadsheetImporter.java +++ b/GAE/src/org/waterforpeople/mapping/dataexport/RawDataSpreadsheetImporter.java @@ -145,7 +145,7 @@ public void executeImport(File file, String serverBase, } HashMap questionIDColMap = new HashMap(); Object[] results = BulkDataServiceClient.loadQuestions( - getSurveyId().toString(), serverBase); + getSurveyId().toString(), serverBase, criteria.get("apiKey")); Map questionMap = null; if (results != null) { diff --git a/GAE/src/org/waterforpeople/mapping/dataexport/SurveyFormExporter.java b/GAE/src/org/waterforpeople/mapping/dataexport/SurveyFormExporter.java index 75ceaaf03f..6bdf350fad 100644 --- a/GAE/src/org/waterforpeople/mapping/dataexport/SurveyFormExporter.java +++ b/GAE/src/org/waterforpeople/mapping/dataexport/SurveyFormExporter.java @@ -66,7 +66,7 @@ public class SurveyFormExporter implements DataExporter { public void export(Map criteria, File fileName, String serverBase, Map options) { try { - populateQuestionMap(criteria.get(SURVEY_ID_KEY), serverBase); + populateQuestionMap(criteria.get(SURVEY_ID_KEY), serverBase, criteria.get("apiKey")); writeSurvey(surveyTitle, fileName, groupList, questionMap); } catch (Exception e) { System.out.println("Could not write survey"); @@ -81,22 +81,22 @@ public void export(Map criteria, File fileName, * will populate a number of member variables to store the results. * */ - private void populateQuestionMap(String surveyId, String serverBase) + private void populateQuestionMap(String surveyId, String serverBase, String apiKey) throws Exception { groupList = BulkDataServiceClient.fetchQuestionGroups(serverBase, - surveyId); + surveyId, apiKey); questionMap = new HashMap>(); idToNumberMap = new HashMap(); if (groupList != null) { Long count = 1L; for (QuestionGroupDto group : groupList) { List questions = BulkDataServiceClient - .fetchQuestions(serverBase, group.getKeyId()); + .fetchQuestions(serverBase, group.getKeyId(), apiKey); if (questions != null) { List fullQuestions = new ArrayList(); for (QuestionDto q : questions) { QuestionDto fullQ = BulkDataServiceClient - .loadQuestionDetails(serverBase, q.getKeyId()); + .loadQuestionDetails(serverBase, q.getKeyId(), apiKey); if (fullQ != null) { fullQuestions.add(fullQ); idToNumberMap.put(fullQ.getKeyId(), count++); diff --git a/GAE/src/org/waterforpeople/mapping/dataexport/SurveyReplicationImporter.java b/GAE/src/org/waterforpeople/mapping/dataexport/SurveyReplicationImporter.java index d6984f5bf4..3874dee9f0 100644 --- a/GAE/src/org/waterforpeople/mapping/dataexport/SurveyReplicationImporter.java +++ b/GAE/src/org/waterforpeople/mapping/dataexport/SurveyReplicationImporter.java @@ -41,20 +41,20 @@ public class SurveyReplicationImporter { - public void executeImport(String sourceBase, Long surveyId) { + public void executeImport(String sourceBase, Long surveyId, String apiKey) { SurveyGroupDAO sgDao = new SurveyGroupDAO(); SurveyDAO sDao = new SurveyDAO(); QuestionGroupDao qgDao = new QuestionGroupDao(); QuestionDao qDao = new QuestionDao(); boolean hasFoundSurvey = false; try { - for (SurveyGroup sg : fetchSurveyGroups(sourceBase)) { + for (SurveyGroup sg : fetchSurveyGroups(sourceBase, apiKey)) { System.out.println("surveygroup: " + sg.getName() + ":" + sg.getCode()); if (surveyId == null) { sgDao.save(sg); } - for (Survey s : fetchSurveys(sg.getKey().getId(), sourceBase)) { + for (Survey s : fetchSurveys(sg.getKey().getId(), sourceBase, apiKey)) { System.out.println(" survey:" + s.getCode()); if (surveyId != null && surveyId.equals(s.getKey().getId())) { sgDao.save(sg); @@ -69,11 +69,11 @@ public void executeImport(String sourceBase, Long surveyId) { } for (QuestionGroup qg : fetchQuestionGroups(s.getKey() - .getId(), sourceBase)) { + .getId(), sourceBase, apiKey)) { System.out.println(" qg:" + qg.getCode()); qgDao.save(qg); for (Question q : fetchQuestions(qg.getKey().getId(), - sourceBase)) { + sourceBase, apiKey)) { System.out.println(" q" + q.getText()); qDao.save(q, qg.getKey().getId()); } @@ -91,34 +91,34 @@ public void executeImport(String sourceBase, Long surveyId) { } - public List fetchSurveyGroups(String serverBase) + public List fetchSurveyGroups(String serverBase, String apiKey) throws Exception { List sgDtoList = BulkDataServiceClient - .fetchSurveyGroups(serverBase); + .fetchSurveyGroups(serverBase, apiKey); List sgList = new ArrayList(); return copyAndCreateList(sgList, sgDtoList, SurveyGroup.class); } - public List fetchSurveys(Long surveyGroupId, String serverBase) + public List fetchSurveys(Long surveyGroupId, String serverBase, String apiKey) throws Exception { List surveyDtoList = BulkDataServiceClient.fetchSurveys( - surveyGroupId, serverBase); + surveyGroupId, serverBase, apiKey); List surveyList = new ArrayList(); return copyAndCreateList(surveyList, surveyDtoList, Survey.class); } public List fetchQuestionGroups(Long surveyId, - String serverBase) throws Exception { + String serverBase, String apiKey) throws Exception { List qgDtoList = BulkDataServiceClient - .fetchQuestionGroups(serverBase, surveyId.toString()); + .fetchQuestionGroups(serverBase, surveyId.toString(), apiKey); List qgList = new ArrayList(); return copyAndCreateList(qgList, qgDtoList, QuestionGroup.class); } - public List fetchQuestions(Long questionGroupId, String serverBase) + public List fetchQuestions(Long questionGroupId, String serverBase, String apiKey) throws Exception { List qgDtoList = BulkDataServiceClient.fetchQuestions( - serverBase, questionGroupId); + serverBase, questionGroupId, apiKey); List qList = new ArrayList(); SurveyServiceImpl ssi = new SurveyServiceImpl(); @@ -127,7 +127,7 @@ public List fetchQuestions(Long questionGroupId, String serverBase) for (int i = 0; i < 3; i++) { try { dtoDetail = (QuestionDto) BulkDataServiceClient - .loadQuestionDetails(serverBase, dto.getKeyId()); + .loadQuestionDetails(serverBase, dto.getKeyId(), apiKey); break; } catch (IOException iex) { System.out.print("Retrying because of timeout."); diff --git a/GAE/src/org/waterforpeople/mapping/dataexport/SurveySpreadsheetImporter.java b/GAE/src/org/waterforpeople/mapping/dataexport/SurveySpreadsheetImporter.java index 204eb51aa8..9c5364a2d4 100644 --- a/GAE/src/org/waterforpeople/mapping/dataexport/SurveySpreadsheetImporter.java +++ b/GAE/src/org/waterforpeople/mapping/dataexport/SurveySpreadsheetImporter.java @@ -79,18 +79,19 @@ public void executeImport(File file, String serverBase, inp = new FileInputStream(file); HSSFWorkbook wb = new HSSFWorkbook(new POIFSFileSystem(inp)); sheet1 = wb.getSheetAt(0); + String apiKey = criteria.get("apiKey"); if (!isWholeSurvey) { // even though there is a header row, we want lastRowNum since // rows are 0 indexed int questionCount = sheet1.getLastRowNum(); // figure out the starting order QuestionDto startingQuestion = BulkDataServiceClient - .loadQuestionDetails(serverBase, beforeQuestionId); + .loadQuestionDetails(serverBase, beforeQuestionId, apiKey); startRow = startingQuestion.getOrder(); // now get all the questions List questionsInGroup = BulkDataServiceClient .fetchQuestions(serverBase, - startingQuestion.getQuestionGroupId()); + startingQuestion.getQuestionGroupId(), apiKey); if (questionsInGroup != null) { // we only need to reorder the group into which we're diff --git a/GAE/src/org/waterforpeople/mapping/dataexport/SurveySummaryExporter.java b/GAE/src/org/waterforpeople/mapping/dataexport/SurveySummaryExporter.java index bd398bbda6..21bd2d8da0 100644 --- a/GAE/src/org/waterforpeople/mapping/dataexport/SurveySummaryExporter.java +++ b/GAE/src/org/waterforpeople/mapping/dataexport/SurveySummaryExporter.java @@ -46,6 +46,7 @@ import org.waterforpeople.mapping.dataexport.service.BulkDataServiceClient; import com.gallatinsystems.framework.dataexport.applet.AbstractDataExporter; +import com.google.api.server.spi.config.Api; /** * @@ -59,7 +60,7 @@ public class SurveySummaryExporter extends AbstractDataExporter { public static final String RESPONSE_KEY = "dtoList"; - private static final String SERVLET_URL = "/surveyrestapi?action="; + private static final String SERVLET_URL = "/surveyrestapi"; private static final NumberFormat PCT_FMT = new DecimalFormat("0.00"); protected static final String[] ROLLUP_QUESTIONS = { "Sector/Cell", "Department", "Province", "Municipality", "Region", "District", @@ -106,11 +107,11 @@ public void export(Map criteria, File fileName, writeHeader(pw, dia.getDoRollup()); Map> questionMap = loadAllQuestions( criteria.get(SurveyRestRequest.SURVEY_ID_PARAM), true, - serverBase); + serverBase, criteria.get("apiKey")); if (questionMap.size() > 0) { SummaryModel model = buildDataModel( criteria.get(SurveyRestRequest.SURVEY_ID_PARAM), - serverBase); + serverBase, criteria.get("apiKey")); for (QuestionGroupDto group : orderedGroupList) { for (QuestionDto question : questionMap.get(group)) { pw.print(model.outputQuestion(group.getDisplayName() @@ -131,14 +132,14 @@ public void export(Map criteria, File fileName, } } - protected SummaryModel buildDataModel(String surveyId, String serverBase) + protected SummaryModel buildDataModel(String surveyId, String serverBase, String apiKey) throws Exception { SummaryModel model = new SummaryModel(); Map instanceMap = BulkDataServiceClient - .fetchInstanceIds(surveyId, serverBase); + .fetchInstanceIds(surveyId, serverBase, apiKey); for (String instanceId : instanceMap.keySet()) { Map responseMap = BulkDataServiceClient - .fetchQuestionResponses(instanceId, serverBase); + .fetchQuestionResponses(instanceId, serverBase, apiKey); Set rollups = null; if (rollupOrder != null && rollupOrder.size() > 0) { rollups = formRollupStrings(responseMap); @@ -181,14 +182,14 @@ protected Set formRollupStrings(Map responseMap) { } protected Map> loadAllQuestions( - String surveyId, boolean performRollups, String serverBase) + String surveyId, boolean performRollups, String serverBase, String apiKey) throws Exception { Map> questionMap = new HashMap>(); - orderedGroupList = fetchQuestionGroups(serverBase, surveyId); + orderedGroupList = fetchQuestionGroups(serverBase, surveyId, apiKey); rollupOrder = new ArrayList(); for (QuestionGroupDto group : orderedGroupList) { List questions = fetchQuestions(serverBase, - group.getKeyId()); + group.getKeyId(), apiKey); if (performRollups && questions != null) { for (QuestionDto q : questions) { @@ -212,18 +213,23 @@ private void writeHeader(PrintWriter pw, boolean isRolledUp) { } } - protected List fetchQuestions(String serverBase, Long groupId) + protected List fetchQuestions(String serverBase, Long groupId, String apiKey) throws Exception { - return parseQuestions(fetchDataFromServer(serverBase + SERVLET_URL - + SurveyRestRequest.LIST_QUESTION_ACTION + "&" - + SurveyRestRequest.QUESTION_GROUP_ID_PARAM + "=" + groupId)); + + return parseQuestions(BulkDataServiceClient.fetchDataFromServer( + serverBase + SERVLET_URL, "?action=" + + SurveyRestRequest.LIST_QUESTION_ACTION + "&" + + SurveyRestRequest.QUESTION_GROUP_ID_PARAM + "=" + + groupId, true, apiKey)); } protected List fetchQuestionGroups(String serverBase, - String surveyId) throws Exception { - return parseQuestionGroups(fetchDataFromServer(serverBase + SERVLET_URL - + SurveyRestRequest.LIST_GROUP_ACTION + "&" - + SurveyRestRequest.SURVEY_ID_PARAM + "=" + surveyId)); + String surveyId, String apiKey) throws Exception { + return parseQuestionGroups(BulkDataServiceClient.fetchDataFromServer( + serverBase + SERVLET_URL, "?action=" + + SurveyRestRequest.LIST_GROUP_ACTION + "&" + + SurveyRestRequest.SURVEY_ID_PARAM + "=" + surveyId, + true, apiKey)); } protected List parseQuestionGroups(String response) diff --git a/GAE/src/org/waterforpeople/mapping/dataexport/service/BulkDataServiceClient.java b/GAE/src/org/waterforpeople/mapping/dataexport/service/BulkDataServiceClient.java index dd9d4e104d..f7e2c41b62 100644 --- a/GAE/src/org/waterforpeople/mapping/dataexport/service/BulkDataServiceClient.java +++ b/GAE/src/org/waterforpeople/mapping/dataexport/service/BulkDataServiceClient.java @@ -68,9 +68,9 @@ * */ public class BulkDataServiceClient { - private static final String DATA_SERVLET_PATH = "/databackout?action="; + private static final String DATA_SERVLET_PATH = "/databackout"; public static final String RESPONSE_KEY = "dtoList"; - private static final String SURVEY_SERVLET_PATH = "/surveyrestapi?action="; + private static final String SURVEY_SERVLET_PATH = "/surveyrestapi"; private static final String DEVICE_FILES_SERVLET_PATH = "/devicefilesrestapi?action="; private static final String ACCESS_POINT_SERVLET_PATH = "/accesspoint?action=search"; @@ -84,12 +84,12 @@ public class BulkDataServiceClient { * @throws Exception */ public static Map fetchQuestionResponses(String instanceId, - String serverBase) throws Exception { + String serverBase, String apiKey) throws Exception { String instanceValues = fetchDataFromServer(serverBase - + DATA_SERVLET_PATH + + DATA_SERVLET_PATH, "?action=" + DataBackoutRequest.LIST_INSTANCE_RESPONSE_ACTION + "&" + DataBackoutRequest.SURVEY_INSTANCE_ID_PARAM + "=" - + instanceId); + + instanceId, true, apiKey); return parseInstanceValues(instanceValues); } @@ -293,12 +293,16 @@ private static List parsePlacemarks(String response) * @throws Exception */ public static Map fetchInstanceIds(String surveyId, - String serverBase) throws Exception { + String serverBase, String apiKey) throws Exception { Map values = new HashMap(); - String instanceString = fetchDataFromServer(serverBase - + DATA_SERVLET_PATH + DataBackoutRequest.LIST_INSTANCE_ACTION - + "&" + DataBackoutRequest.SURVEY_ID_PARAM + "=" + surveyId - + "&" + DataBackoutRequest.INCLUDE_DATE_PARAM + "=true"); + + String instanceString = fetchDataFromServer(serverBase + DATA_SERVLET_PATH, + "?action=" + DataBackoutRequest.LIST_INSTANCE_ACTION + + "&" + DataBackoutRequest.SURVEY_ID_PARAM + + "=" + surveyId + "&" + + DataBackoutRequest.INCLUDE_DATE_PARAM + + "=true", true, apiKey); + if (instanceString != null && instanceString.trim().length() != 0) { StringTokenizer strTok = new StringTokenizer(instanceString, ","); while (strTok.hasMoreTokens()) { @@ -319,7 +323,7 @@ public static Map fetchInstanceIds(String surveyId, public static void main(String[] args) { try { Map results = BulkDataServiceClient - .fetchInstanceIds(args[1], args[0]); + .fetchInstanceIds(args[1], args[0], args[2]); if (results != null) { for (Entry entry : results.entrySet()) { System.out @@ -384,14 +388,15 @@ private static Map parseInstanceValues(String data) { * @return */ public static QuestionDto loadQuestionDetails(String serverBase, - Long questionId) throws Exception { + Long questionId, String apiKey) throws Exception { List dtoList = null; dtoList = parseQuestions(fetchDataFromServer(serverBase - + SURVEY_SERVLET_PATH + + SURVEY_SERVLET_PATH, "?action=" + SurveyRestRequest.GET_QUESTION_DETAILS_ACTION + "&" - + SurveyRestRequest.QUESTION_ID_PARAM + "=" + questionId)); + + SurveyRestRequest.QUESTION_ID_PARAM + "=" + questionId, true, + apiKey)); if (dtoList != null && dtoList.size() > 0) { return dtoList.get(0); @@ -411,17 +416,17 @@ public static QuestionDto loadQuestionDetails(String serverBase, * @return * @throws Exception */ - public static Object[] loadQuestions(String surveyId, String serverBase) + public static Object[] loadQuestions(String surveyId, String serverBase, String apiKey) throws Exception { Object[] results = new Object[2]; Map questions = new HashMap(); List groups = fetchQuestionGroups(serverBase, - surveyId); + surveyId, apiKey); List keyList = new ArrayList(); if (groups != null) { for (QuestionGroupDto group : groups) { List questionDtos = fetchQuestions(serverBase, - group.getKeyId()); + group.getKeyId(), apiKey); if (questionDtos != null) { for (QuestionDto question : questionDtos) { keyList.add(question.getKeyId().toString()); @@ -445,11 +450,12 @@ public static Object[] loadQuestions(String surveyId, String serverBase) * @throws Exception */ public static List fetchQuestions(String serverBase, - Long groupId) throws Exception { + Long groupId, String apiKey) throws Exception { return parseQuestions(fetchDataFromServer(serverBase - + SURVEY_SERVLET_PATH + SurveyRestRequest.LIST_QUESTION_ACTION - + "&" + SurveyRestRequest.QUESTION_GROUP_ID_PARAM + "=" - + groupId)); + + SURVEY_SERVLET_PATH, "?action=" + + SurveyRestRequest.LIST_QUESTION_ACTION + "&" + + SurveyRestRequest.QUESTION_GROUP_ID_PARAM + "=" + groupId, + true, apiKey)); } /** @@ -461,11 +467,12 @@ public static List fetchQuestions(String serverBase, * @throws Exception */ public static SurveyInstanceDto findSurveyInstance(Long id, - String serverBase) throws Exception { + String serverBase, String apiKey) throws Exception { return parseSurveyInstance(fetchDataFromServer(serverBase - + SURVEY_SERVLET_PATH + + SURVEY_SERVLET_PATH, "?action=" + SurveyRestRequest.GET_SURVEY_INSTANCE_ACTION + "&" - + SurveyRestRequest.INSTANCE_PARAM + "=" + id)); + + SurveyRestRequest.INSTANCE_PARAM + "=" + id, true, + apiKey)); } /** @@ -477,10 +484,12 @@ public static SurveyInstanceDto findSurveyInstance(Long id, * @throws Exception */ public static List fetchQuestionGroups(String serverBase, - String surveyId) throws Exception { + String surveyId, String apiKey) throws Exception { return parseQuestionGroups(fetchDataFromServer(serverBase - + SURVEY_SERVLET_PATH + SurveyRestRequest.LIST_GROUP_ACTION - + "&" + SurveyRestRequest.SURVEY_ID_PARAM + "=" + surveyId)); + + SURVEY_SERVLET_PATH, "?action=" + + SurveyRestRequest.LIST_GROUP_ACTION + "&" + + SurveyRestRequest.SURVEY_ID_PARAM + "=" + surveyId, true, + apiKey)); } /** @@ -491,11 +500,12 @@ public static List fetchQuestionGroups(String serverBase, * @return * @throws Exception */ - public static List fetchSurveyGroups(String serverBase) + public static List fetchSurveyGroups(String serverBase, String apiKey) throws Exception { return parseSurveyGroups(fetchDataFromServer(serverBase - + SURVEY_SERVLET_PATH - + SurveyRestRequest.LIST_SURVEY_GROUPS_ACTION)); + + SURVEY_SERVLET_PATH, "?action=" + + SurveyRestRequest.LIST_SURVEY_GROUPS_ACTION, true, + apiKey)); } /** @@ -507,11 +517,12 @@ public static List fetchSurveyGroups(String serverBase) * @throws Exception */ public static List fetchSurveys(Long surveyGroupId, - String serverBase) throws Exception { + String serverBase, String apiKey) throws Exception { return parseSurveys(fetchDataFromServer(serverBase - + SURVEY_SERVLET_PATH + SurveyRestRequest.LIST_SURVEYS_ACTION - + "&" + SurveyRestRequest.SURVEY_GROUP_ID_PARAM + "=" - + surveyGroupId)); + + SURVEY_SERVLET_PATH, + "?action=" + SurveyRestRequest.LIST_SURVEYS_ACTION + "&" + + SurveyRestRequest.SURVEY_GROUP_ID_PARAM + "=" + + surveyGroupId, true, apiKey)); } /** @@ -959,9 +970,9 @@ private static TreeMap parseTranslations( * @throws Exception */ public static String fetchDataFromServer(String baseUrl, - String queryString, boolean shouldSign, String key) + String queryString, boolean shouldSign, String apiKey) throws Exception { - if (shouldSign && key != null) { + if (shouldSign && apiKey != null) { if (queryString == null) { queryString = new String(); } else { @@ -976,7 +987,7 @@ public static String fetchDataFromServer(String baseUrl, + URLEncoder.encode(df.format(new Date()), "UTF-8"); queryString = sortQueryString(queryString); queryString += "&" + RestRequest.HASH_PARAM + "=" - + MD5Util.generateHMAC(queryString, key); + + MD5Util.generateHMAC(queryString, apiKey); } return fetchDataFromServer(baseUrl + ((queryString != null && queryString.trim().length() > 0) ? "?" From 3e20c3f2df10a212eac891f4eeb3754dc1eddd38 Mon Sep 17 00:00:00 2001 From: Stellan Lagerstrom Date: Wed, 19 Jun 2013 03:07:56 +0200 Subject: [PATCH 22/59] Issue #255, prevent control chars in user names and email addresses --- .../device/activity/UserEditActivity.java | 19 ++++++------------- .../survey/device/util/StringUtil.java | 18 +++++++++++++++++- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/activity/UserEditActivity.java b/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/activity/UserEditActivity.java index 6a30cb060f..01974906e1 100644 --- a/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/activity/UserEditActivity.java +++ b/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/activity/UserEditActivity.java @@ -16,6 +16,7 @@ package com.gallatinsystems.survey.device.activity; +import android.annotation.SuppressLint; import android.app.Activity; import android.database.Cursor; import android.os.Bundle; @@ -27,6 +28,7 @@ import com.gallatinsystems.survey.device.R; import com.gallatinsystems.survey.device.dao.SurveyDbAdapter; import com.gallatinsystems.survey.device.util.ConstantUtil; +import com.gallatinsystems.survey.device.util.StringUtil; /** * this activity is used to edit a user's profile information and persist it to @@ -41,6 +43,7 @@ public class UserEditActivity extends Activity { private Long userId; private SurveyDbAdapter databaseAdaptor; + @SuppressLint("UseValueOf") @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); @@ -120,22 +123,12 @@ protected void onDestroy() { } /** - * save the name and email address to the db + * save the sanitized name and email address to the db */ private void saveState() { - String name = displayName.getText().toString(); - String email = emailAddr.getText().toString(); - name = cleanupString(name); - email = cleanupString(email); + String name = StringUtil.ControlCommaToSPace(displayName.getText().toString()).trim(); + String email = StringUtil.ControlCommaToSPace(emailAddr.getText().toString()).trim(); databaseAdaptor.createOrUpdateUser(userId, name, email); } - private String cleanupString(String input) { - if (input != null) { - input = input.trim(); - input = input.replaceAll("\n", " "); - input = input.replaceAll(",", " "); - } - return input; - } } diff --git a/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/util/StringUtil.java b/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/util/StringUtil.java index 32f5c09941..26167b57cd 100644 --- a/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/util/StringUtil.java +++ b/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/util/StringUtil.java @@ -37,7 +37,9 @@ public static boolean isNullOrEmpty(String s) { return false; } } - + + //copy a string transforming all control chars + //(like newline and tab) into spaces public static String ControlToSPace(String val) { String result = ""; for (int i= 0; i < val.length(); i++) { @@ -49,4 +51,18 @@ public static String ControlToSPace(String val) { return result; } + + //copy a string transforming all control chars + //(like newline and tab) and comma into spaces + public static String ControlCommaToSPace(String val) { + String result = ""; + for (int i= 0; i < val.length(); i++) { + if (val.charAt(i) < '\u0020' || val.charAt(i) == ',') + result = result + '\u0020'; + else + result = result + val.charAt(i); + } + + return result; + } } From c0626f901d84708de172189712f2ca8a42aa565c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Perdomo?= Date: Wed, 19 Jun 2013 10:09:04 +0200 Subject: [PATCH 23/59] Issue #256 - Pass the uploadUrl as parameter * Pass the `uploadUrl` as option for identifying the instance * Remove the `?` from the querystring --- Dashboard/app/js/lib/views/reports/export-reports-views.js | 2 ++ .../mapping/dataexport/SurveySummaryExporter.java | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Dashboard/app/js/lib/views/reports/export-reports-views.js b/Dashboard/app/js/lib/views/reports/export-reports-views.js index 0c2ba259b1..7b50b375e6 100644 --- a/Dashboard/app/js/lib/views/reports/export-reports-views.js +++ b/Dashboard/app/js/lib/views/reports/export-reports-views.js @@ -44,7 +44,9 @@ FLOW.ReportLoader = Ember.Object.create({ criteria = Ember.copy(this.get('payloads')[exportType]); criteria.surveyId = '' + surveyId; criteria.baseURL = location.protocol + '//' + location.host; + criteria.opts.imgPrefix = FLOW.Env.photo_url_root; + criteria.opts.uploadUrl = FLOW.Env.surveyuploadurl; if (opts) { Ember.keys(opts).forEach(function (k) { diff --git a/GAE/src/org/waterforpeople/mapping/dataexport/SurveySummaryExporter.java b/GAE/src/org/waterforpeople/mapping/dataexport/SurveySummaryExporter.java index 21bd2d8da0..cfc566f84b 100644 --- a/GAE/src/org/waterforpeople/mapping/dataexport/SurveySummaryExporter.java +++ b/GAE/src/org/waterforpeople/mapping/dataexport/SurveySummaryExporter.java @@ -217,7 +217,7 @@ protected List fetchQuestions(String serverBase, Long groupId, Stri throws Exception { return parseQuestions(BulkDataServiceClient.fetchDataFromServer( - serverBase + SERVLET_URL, "?action=" + serverBase + SERVLET_URL, "action=" + SurveyRestRequest.LIST_QUESTION_ACTION + "&" + SurveyRestRequest.QUESTION_GROUP_ID_PARAM + "=" + groupId, true, apiKey)); @@ -226,7 +226,7 @@ protected List fetchQuestions(String serverBase, Long groupId, Stri protected List fetchQuestionGroups(String serverBase, String surveyId, String apiKey) throws Exception { return parseQuestionGroups(BulkDataServiceClient.fetchDataFromServer( - serverBase + SERVLET_URL, "?action=" + serverBase + SERVLET_URL, "action=" + SurveyRestRequest.LIST_GROUP_ACTION + "&" + SurveyRestRequest.SURVEY_ID_PARAM + "=" + surveyId, true, apiKey)); From 957f15dafd4d67fca08b22b273264217fedc08a3 Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Wed, 19 Jun 2013 11:52:53 +0200 Subject: [PATCH 24/59] Issue #185 - action to fix faulty questionAnswerStore type fields in datastore --- .../app/web/DataProcessorRestServlet.java | 52 ++++++++++++++++++- .../app/web/dto/DataProcessorRequest.java | 1 + .../app/web/rest/ActionRestService.java | 15 ++++++ .../mapping/dao/SurveyInstanceDAO.java | 14 +++++ 4 files changed, 81 insertions(+), 1 deletion(-) diff --git a/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java b/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java index 5a3503c80c..09bd5b9b00 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java @@ -24,10 +24,13 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.logging.Level; import java.util.zip.ZipInputStream; +import java.util.logging.Logger; import javax.servlet.http.HttpServletRequest; + import org.waterforpeople.mapping.analytics.dao.SurveyQuestionSummaryDao; import org.waterforpeople.mapping.analytics.domain.SurveyQuestionSummary; import org.waterforpeople.mapping.app.web.dto.DataProcessorRequest; @@ -64,7 +67,7 @@ * */ public class DataProcessorRestServlet extends AbstractRestApiServlet { - + private static final Logger log = Logger.getLogger("DataProcessorRestServlet"); private static final long serialVersionUID = -7902002525342262821L; private static final String REBUILD_Q_SUM_STATUS_KEY = "rebuildQuestionSummary"; private static final String VALUE_TYPE = "VALUE"; @@ -103,6 +106,8 @@ protected RestResponse handleRequest(RestRequest req) throws Exception { } else if (DataProcessorRequest.TRIM_OPTIONS.equalsIgnoreCase(dpReq .getAction())) { trimOptions(); + } else if (DataProcessorRequest.FIX_OPTIONS2VALUES_ACTION.equalsIgnoreCase(dpReq.getAction())){ + fixOptions2Values(dpReq.getCursor()); } return new RestResponse(); } @@ -509,4 +514,49 @@ public static void sendProjectUpdateTask(String country, String cursor) { } + /** + * fixes wrong Types in questionAnswerStore objects. When cleaned data is + * uploaded using an excel file, the type of the answer is set according + * to the type of the question, while the device sets the type according + * to a different convention. The action handles 500 items in one call, + * and invokes new tasks as necessary if there are more items. + * + * @param cursor + * @author M.T. Westra + */ + public static void fixOptions2Values(String cursorString) { + SurveyInstanceDAO siDao = new SurveyInstanceDAO(); + QuestionAnswerStoreDao qasDao = new QuestionAnswerStoreDao(); + List qasList = siDao.listqaOPTION_FREETEXT_NUMBER_SCAN_PHOTO(cursorString,500); + List qasChangedList = new ArrayList(); + log.log(Level.INFO, "Running fixOptions2Values, cursor at " + cursorString); + if (qasList != null) { + String cursor = SurveyInstanceDAO.getCursor(qasList); + for (QuestionAnswerStore qas : qasList) { + + if (Question.Type.OPTION.toString().equals(qas.getType()) + || Question.Type.NUMBER.toString().equals(qas.getType()) + || Question.Type.FREE_TEXT.toString().equals(qas.getType()) + || Question.Type.SCAN.toString().equals(qas.getType())){ + qas.setType("VALUE"); + qasChangedList.add(qas); + } else if (Question.Type.PHOTO.toString().equals(qas.getType())){ + qas.setType("IMAGE"); + qasChangedList.add(qas); + } + } + qasDao.save(qasChangedList); + // if there are more, invoke another task + if (qasList.size() == 500) { + log.log(Level.INFO, "invoking another fixOptions task"); + Queue queue = QueueFactory.getDefaultQueue(); + TaskOptions options = TaskOptions.Builder + .withUrl("/app_worker/dataprocessor") + .param(DataProcessorRequest.ACTION_PARAM, + DataProcessorRequest.FIX_OPTIONS2VALUES_ACTION) + .param(DataProcessorRequest.CURSOR_PARAM, cursor); + queue.add(options); + } + } + } } diff --git a/GAE/src/org/waterforpeople/mapping/app/web/dto/DataProcessorRequest.java b/GAE/src/org/waterforpeople/mapping/app/web/dto/DataProcessorRequest.java index ea876f301b..0f8647614a 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/dto/DataProcessorRequest.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/dto/DataProcessorRequest.java @@ -32,6 +32,7 @@ public class DataProcessorRequest extends RestRequest { public static final String PROJECT_FLAG_UPDATE_ACTION = "projectFlagUpdate"; public static final String REBUILD_QUESTION_SUMMARY_ACTION = "rebuildQuestionSummary"; public static final String IMPORT_REMOTE_SURVEY_ACTION = "importRemoteSurvey"; + public static final String FIX_OPTIONS2VALUES_ACTION = "fixOptions2Values"; public static final String FIX_NULL_SUBMITTER_ACTION = "fixNullSubmitter"; public static final String FIX_DUPLICATE_OTHER_TEXT_ACTION = "fixDuplicateOtherText"; public static final String TRIM_OPTIONS = "trimOptions"; diff --git a/GAE/src/org/waterforpeople/mapping/app/web/rest/ActionRestService.java b/GAE/src/org/waterforpeople/mapping/app/web/rest/ActionRestService.java index 4fc1830597..453c07c6b0 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/rest/ActionRestService.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/rest/ActionRestService.java @@ -29,6 +29,7 @@ import org.waterforpeople.mapping.analytics.dao.SurveyInstanceSummaryDao; import org.waterforpeople.mapping.analytics.domain.SurveyInstanceSummary; import org.waterforpeople.mapping.app.web.dto.BootstrapGeneratorRequest; +import org.waterforpeople.mapping.app.web.dto.DataProcessorRequest; import org.waterforpeople.mapping.app.web.rest.dto.RestStatusDto; import org.waterforpeople.mapping.dao.SurveyInstanceDAO; import org.waterforpeople.mapping.app.gwt.server.survey.SurveyServiceImpl; @@ -78,6 +79,8 @@ public Map doAction( statusDto.setMessage(message); } else if ("removeZeroValues".equals(action)) { status = removeZeroMinMaxValues(); + } else if ("fixOptions2Values".equals(action)){ + status = fixOptions2Values(); } statusDto.setStatus(status); @@ -150,6 +153,18 @@ private String publishSurvey(Long surveyId) { return "publishing requested"; } + private String fixOptions2Values() { + Queue queue = QueueFactory.getDefaultQueue(); + TaskOptions options = TaskOptions.Builder + .withUrl("/app_worker/dataprocessor") + .param(DataProcessorRequest.ACTION_PARAM, + DataProcessorRequest.FIX_OPTIONS2VALUES_ACTION); + queue.add(options); + + return "fixing opions to values in surveyInstances requested"; + } + + private String generateBootstrapFile(Long[] surveyIdList, String dbInstructions, String notificationEmail) { diff --git a/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java b/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java index 2738f18395..d045ec89dc 100644 --- a/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java +++ b/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java @@ -605,6 +605,20 @@ public List listInstanceBySubmitter(String submitter) { } } + + /** lists surveyInstances of particular types + * + * + * + */ + public List listqaOPTION_FREETEXT_NUMBER_SCAN_PHOTO(String cursorString, Integer pageSize){ + PersistenceManager pm = PersistenceFilter.getManager(); + javax.jdo.Query q = pm.newQuery(QuestionAnswerStore.class); + q.setFilter("type == 'OPTION' || type == 'NUMBER' || type == 'FREE_TEXT' || type == 'SCAN' || type == 'PHOTO'"); + prepareCursor(cursorString, pageSize, q); + return (List) q.execute(); + } + /** * finds a single survey instance by uuid. This method will NOT load all * QuestionAnswerStore objects. From 0962905b014d10415a3ddad7168b4eea74234735 Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Wed, 19 Jun 2013 12:26:17 +0200 Subject: [PATCH 25/59] Issue #185 - rawDataSpreadsheetImporter uses same convention as device for answer type --- .../dataexport/RawDataSpreadsheetImporter.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/GAE/src/org/waterforpeople/mapping/dataexport/RawDataSpreadsheetImporter.java b/GAE/src/org/waterforpeople/mapping/dataexport/RawDataSpreadsheetImporter.java index 5bb3783b8c..74f6231c9e 100644 --- a/GAE/src/org/waterforpeople/mapping/dataexport/RawDataSpreadsheetImporter.java +++ b/GAE/src/org/waterforpeople/mapping/dataexport/RawDataSpreadsheetImporter.java @@ -219,14 +219,18 @@ public void executeImport(File file, String serverBase, QuestionDto question = questionMap.get(questionIDColMap .get(cell.getColumnIndex())); QuestionType type = null; + // VALUE is default, it is valid for NUMBER, FREE_TEXT, SCAN, OPTION String typeString = "VALUE"; if (question != null) { type = question.getType(); - typeString = type.toString(); - if (QuestionType.GEO == type - || QuestionType.PHOTO == type - || QuestionType.VIDEO == type) { - typeString = type.toString(); + if (QuestionType.GEO == type){ + typeString = "GEO"; + } else if (QuestionType.PHOTO == type) { + typeString = "IMAGE"; + } else if (QuestionType.VIDEO == type) { + typeString = "VIDEO"; + } else if (QuestionType.DATE == type) { + typeString = "DATE"; } } else if (questionIDColMap.get(cell.getColumnIndex()) .startsWith("--")) { From ac9242dae5f138baf68a3b061e7f93411aa64c80 Mon Sep 17 00:00:00 2001 From: Stellan Lagerstrom Date: Wed, 19 Jun 2013 14:06:53 +0200 Subject: [PATCH 26/59] Clean up last traces of the APK's arrays.xml instance dependency --- GAE/build.xml | 1 - WFPMapping/device/survey/res/values/arrays.xml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/GAE/build.xml b/GAE/build.xml index 800eb2816a..08f9e3906e 100644 --- a/GAE/build.xml +++ b/GAE/build.xml @@ -103,7 +103,6 @@ - diff --git a/WFPMapping/device/survey/res/values/arrays.xml b/WFPMapping/device/survey/res/values/arrays.xml index eb3d36b89e..24bdc7ff64 100644 --- a/WFPMapping/device/survey/res/values/arrays.xml +++ b/WFPMapping/device/survey/res/values/arrays.xml @@ -45,7 +45,7 @@ http://192.168.1.100 http://127.0.0.1 - + millimeters centimeters From 2258868de2638cf93ef33fe434605e723388c852 Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Wed, 19 Jun 2013 17:19:41 +0200 Subject: [PATCH 27/59] Issue #258 - fix validation for min-max on number question --- Dashboard/app/js/lib/views/surveys/question-view.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dashboard/app/js/lib/views/surveys/question-view.js b/Dashboard/app/js/lib/views/surveys/question-view.js index 3a6973f3c7..5129fe6c52 100644 --- a/Dashboard/app/js/lib/views/surveys/question-view.js +++ b/Dashboard/app/js/lib/views/surveys/question-view.js @@ -158,7 +158,7 @@ FLOW.QuestionView = FLOW.View.extend({ // validation if (this.type.get('value') == 'NUMBER'){ if (!Ember.empty(this.get('minVal')) && !Ember.empty(this.get('maxVal')) ){ - if (this.get('minVal') >= this.get('maxVal')){ + if (parseFloat(this.get('minVal')) >= parseFloat(this.get('maxVal'))){ FLOW.dialogControl.set('activeAction', 'ignore'); FLOW.dialogControl.set('header', Ember.String.loc('_min_max_not_correct')); FLOW.dialogControl.set('message', Ember.String.loc('_min_larger_than_max_or_equal')); From 01cf19ec05143ef280128bc1f4410cfb110dc9d5 Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Thu, 20 Jun 2013 10:33:20 +0200 Subject: [PATCH 28/59] Issue #185 - more subtle implementation of questionAnswer filter --- .../mapping/app/web/DataProcessorRestServlet.java | 2 +- .../mapping/dao/SurveyInstanceDAO.java | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java b/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java index 09bd5b9b00..493d749083 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java @@ -527,7 +527,7 @@ public static void sendProjectUpdateTask(String country, String cursor) { public static void fixOptions2Values(String cursorString) { SurveyInstanceDAO siDao = new SurveyInstanceDAO(); QuestionAnswerStoreDao qasDao = new QuestionAnswerStoreDao(); - List qasList = siDao.listqaOPTION_FREETEXT_NUMBER_SCAN_PHOTO(cursorString,500); + List qasList = siDao.listQAOptions(cursorString,500,"OPTION","FREE_TEXT","NUMBER","SCAN","PHOTO"); List qasChangedList = new ArrayList(); log.log(Level.INFO, "Running fixOptions2Values, cursor at " + cursorString); if (qasList != null) { diff --git a/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java b/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java index d045ec89dc..cf295b6917 100644 --- a/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java +++ b/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java @@ -606,15 +606,16 @@ public List listInstanceBySubmitter(String submitter) { } - /** lists surveyInstances of particular types - * - * - * + /** lists surveyInstances of particular types passed in */ - public List listqaOPTION_FREETEXT_NUMBER_SCAN_PHOTO(String cursorString, Integer pageSize){ + public List listQAOptions(String cursorString, Integer pageSize, String... options){ PersistenceManager pm = PersistenceFilter.getManager(); javax.jdo.Query q = pm.newQuery(QuestionAnswerStore.class); - q.setFilter("type == 'OPTION' || type == 'NUMBER' || type == 'FREE_TEXT' || type == 'SCAN' || type == 'PHOTO'"); + StringBuffer filter = new StringBuffer(); + for(String op :options) { + filter.append("type == '").append(op).append("' ||"); + } + q.setFilter(filter.substring(0,filter.length()-3).toString()); prepareCursor(cursorString, pageSize, q); return (List) q.execute(); } From 6be4e7009d50aff927813ea653d6e8d736bbec41 Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Thu, 20 Jun 2013 13:49:04 +0200 Subject: [PATCH 29/59] Issue #185 - add comment to surveyInstanceSummarizer --- .../mapping/analytics/SurveyInstanceSummarizer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/GAE/src/org/waterforpeople/mapping/analytics/SurveyInstanceSummarizer.java b/GAE/src/org/waterforpeople/mapping/analytics/SurveyInstanceSummarizer.java index 22e2a92558..d82e97737f 100644 --- a/GAE/src/org/waterforpeople/mapping/analytics/SurveyInstanceSummarizer.java +++ b/GAE/src/org/waterforpeople/mapping/analytics/SurveyInstanceSummarizer.java @@ -53,6 +53,7 @@ public SurveyInstanceSummarizer() { /** * looks up a survey instance then finds it's corresponding country and (if * possible) sublevel1 using the GIS serviceF + * A second version of this function is present in dataProcessorRestService. */ @Override public boolean performSummarization(String key, String type, String value, From 459e28732aa326a4cf8b6962a4d6b90540568e62 Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Thu, 20 Jun 2013 13:49:33 +0200 Subject: [PATCH 30/59] Issue #185 - add surveyInstanceSummarizer code to DataProcessorRestServlet --- .../app/web/DataProcessorRestServlet.java | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java b/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java index 493d749083..2b3f3c2780 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java @@ -31,6 +31,8 @@ import javax.servlet.http.HttpServletRequest; +import org.waterforpeople.mapping.analytics.SurveyInstanceSummarizer; +import org.waterforpeople.mapping.analytics.dao.SurveyInstanceSummaryDao; import org.waterforpeople.mapping.analytics.dao.SurveyQuestionSummaryDao; import org.waterforpeople.mapping.analytics.domain.SurveyQuestionSummary; import org.waterforpeople.mapping.app.web.dto.DataProcessorRequest; @@ -40,6 +42,7 @@ import org.waterforpeople.mapping.dao.SurveyInstanceDAO; import org.waterforpeople.mapping.dataexport.SurveyReplicationImporter; import org.waterforpeople.mapping.domain.AccessPoint; +import org.waterforpeople.mapping.domain.GeoCoordinates; import org.waterforpeople.mapping.domain.QuestionAnswerStore; import org.waterforpeople.mapping.domain.SurveyInstance; @@ -48,6 +51,9 @@ import com.gallatinsystems.framework.rest.AbstractRestApiServlet; import com.gallatinsystems.framework.rest.RestRequest; import com.gallatinsystems.framework.rest.RestResponse; +import com.gallatinsystems.gis.location.GeoLocationService; +import com.gallatinsystems.gis.location.GeoLocationServiceGeonamesImpl; +import com.gallatinsystems.gis.location.GeoPlace; import com.gallatinsystems.operations.dao.ProcessingStatusDao; import com.gallatinsystems.operations.domain.ProcessingStatus; import com.gallatinsystems.survey.dao.QuestionDao; @@ -108,6 +114,8 @@ protected RestResponse handleRequest(RestRequest req) throws Exception { trimOptions(); } else if (DataProcessorRequest.FIX_OPTIONS2VALUES_ACTION.equalsIgnoreCase(dpReq.getAction())){ fixOptions2Values(dpReq.getCursor()); + } else if (DataProcessorRequest.SURVEY_INSTANCE_SUMMARIZER.equalsIgnoreCase(dpReq.getAction())){ + surveyInstanceSummarizer(dpReq.getSurveyInstanceId(),dpReq.getQasId()); } return new RestResponse(); } @@ -559,4 +567,34 @@ public static void fixOptions2Values(String cursorString) { } } } + + public static void surveyInstanceSummarizer(Long surveyInstanceId, Long qasId) { + SurveyInstanceDAO siDao = new SurveyInstanceDAO(); + QuestionAnswerStoreDao qasDao = new QuestionAnswerStoreDao(); + boolean success = false; + if (surveyInstanceId != null) { + SurveyInstance si = siDao.getByKey(surveyInstanceId); + if (si != null && qasId != null) { + QuestionAnswerStore qas = qasDao.getByKey(qasId); + if (qas != null){ + GeoCoordinates geoC = null; + if (qas.getValue() != null && qas.getValue().trim().length() > 0) { + geoC = GeoCoordinates.extractGeoCoordinate(qas.getValue()); + } + if (geoC != null) { + GeoLocationService gisService = new GeoLocationServiceGeonamesImpl(); + GeoPlace gp = gisService.findDetailedGeoPlace(geoC.getLatitude().toString(), geoC.getLongitude().toString()); + if (gp != null) { + SurveyInstanceSummaryDao.incrementCount(gp.getSub1(), gp.getCountryCode(), qas.getCollectionDate()); + success = true; + } + } + } + } + } + if (!success) { + log.log(Level.SEVERE,"Couldnt find geoplace for instance. Instance id: " + surveyInstanceId); + } + } + } From 653d6429e88cab5cfa659594eae2b59aeeb250e9 Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Thu, 20 Jun 2013 13:50:05 +0200 Subject: [PATCH 31/59] Issue #185 - add logging to RawDataRestServlet --- .../waterforpeople/mapping/app/web/RawDataRestServlet.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/GAE/src/org/waterforpeople/mapping/app/web/RawDataRestServlet.java b/GAE/src/org/waterforpeople/mapping/app/web/RawDataRestServlet.java index 5f1264ad96..9d184672f4 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/RawDataRestServlet.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/RawDataRestServlet.java @@ -20,6 +20,8 @@ import java.util.Date; import java.util.List; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import javax.servlet.http.HttpServletRequest; @@ -51,7 +53,7 @@ import com.google.appengine.api.taskqueue.TaskOptions; public class RawDataRestServlet extends AbstractRestApiServlet { - + private static final Logger log = Logger.getLogger("RawDataRestServlet"); private static final long serialVersionUID = 2409014651721639814L; private SurveyInstanceDAO instanceDao; @@ -187,6 +189,7 @@ protected RestResponse handleRequest(RestRequest req) throws Exception { } else if (RawDataImportRequest.UPDATE_SUMMARIES_ACTION .equalsIgnoreCase(importReq.getAction())) { // first rebuild the summaries + log.log(Level.INFO, "Rebuilding summaries for surveyId " + importReq.getSurveyId().toString()); TaskOptions options = TaskOptions.Builder.withUrl( "/app_worker/dataprocessor").param( DataProcessorRequest.ACTION_PARAM, From def1d8104e13288a3a925ea21f41ec1d1ef5043a Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Thu, 20 Jun 2013 13:50:53 +0200 Subject: [PATCH 32/59] Issue #185 - disable surveyInstance summarisers in TaskServlet --- .../org/waterforpeople/mapping/app/web/TaskServlet.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/GAE/src/org/waterforpeople/mapping/app/web/TaskServlet.java b/GAE/src/org/waterforpeople/mapping/app/web/TaskServlet.java index 72ffc5491d..51fa82b1a1 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/TaskServlet.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/TaskServlet.java @@ -42,6 +42,7 @@ import javax.crypto.spec.SecretKeySpec; import javax.servlet.http.HttpServletRequest; +import org.waterforpeople.mapping.app.web.dto.DataProcessorRequest; import org.waterforpeople.mapping.app.web.dto.TaskRequest; import org.waterforpeople.mapping.dao.DeviceFilesDao; import org.waterforpeople.mapping.dao.SurveyInstanceDAO; @@ -480,10 +481,10 @@ private void ingestFile(TaskRequest req) { log.info("Received Task Queue calls for surveyInstanceKey: " + instance.getKey().getId() + ""); aph.processSurveyInstance(instance.getKey().getId() + ""); - summQueue.add(TaskOptions.Builder.withUrl("/app_worker/datasummarization").param( - "objectKey", instance.getKey().getId() + "").param( - "type", "SurveyInstance")); - // process the "new" domain structure +// summQueue.add(TaskOptions.Builder.withUrl("/app_worker/datasummarization").param( +// "objectKey", instance.getKey().getId() + "").param( +// "type", "SurveyInstance")); +// // process the "new" domain structure defaultQueue.add(TaskOptions.Builder.withUrl("/app_worker/surveyalservlet").param( SurveyalRestRequest.ACTION_PARAM, From 6ee785985f992a9d906de30a5210615f04f9d704 Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Thu, 20 Jun 2013 13:51:28 +0200 Subject: [PATCH 33/59] Issue #185 - add surveyInstanceId and qasId fields to DataProcessorRequest class --- .../app/web/dto/DataProcessorRequest.java | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/GAE/src/org/waterforpeople/mapping/app/web/dto/DataProcessorRequest.java b/GAE/src/org/waterforpeople/mapping/app/web/dto/DataProcessorRequest.java index 0f8647614a..43763c1e8d 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/dto/DataProcessorRequest.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/dto/DataProcessorRequest.java @@ -35,15 +35,20 @@ public class DataProcessorRequest extends RestRequest { public static final String FIX_OPTIONS2VALUES_ACTION = "fixOptions2Values"; public static final String FIX_NULL_SUBMITTER_ACTION = "fixNullSubmitter"; public static final String FIX_DUPLICATE_OTHER_TEXT_ACTION = "fixDuplicateOtherText"; + public static final String SURVEY_INSTANCE_SUMMARIZER = "surveyInstanceSummarizer"; public static final String TRIM_OPTIONS = "trimOptions"; public static final String RESCORE_AP_ACTION = "rescoreAp"; public static final String SOURCE_PARAM = "source"; public static final String COUNTRY_PARAM = "country"; public static final String SURVEY_ID_PARAM = "surveyId"; + public static final String SURVEY_INSTANCE_PARAM = "surveyInstanceId"; + public static final String QAS_ID_PARAM = "qasId"; private String country; private String source; private Long surveyId; + private Long surveyInstanceId; + private Long qasId; @Override protected void populateFields(HttpServletRequest req) throws Exception { @@ -58,6 +63,24 @@ protected void populateFields(HttpServletRequest req) throws Exception { + " must be an integer")); } } + if (req.getParameter(SURVEY_INSTANCE_PARAM) != null) { + try { + setSurveyInstanceId(new Long(req.getParameter(SURVEY_INSTANCE_PARAM).trim())); + } catch (Exception e) { + addError(new RestError(RestError.BAD_DATATYPE_CODE, + RestError.BAD_DATATYPE_MESSAGE, SURVEY_INSTANCE_PARAM + + " must be an integer")); + } + } + if (req.getParameter(QAS_ID_PARAM) != null) { + try { + setQasId(new Long(req.getParameter(QAS_ID_PARAM).trim())); + } catch (Exception e) { + addError(new RestError(RestError.BAD_DATATYPE_CODE, + RestError.BAD_DATATYPE_MESSAGE, QAS_ID_PARAM + + " must be an integer")); + } + } } @@ -91,4 +114,20 @@ public Long getSurveyId() { return surveyId; } + public Long getSurveyInstanceId() { + return surveyInstanceId; + } + + public void setSurveyInstanceId(Long surveyInstanceId) { + this.surveyInstanceId = surveyInstanceId; + } + + public Long getQasId() { + return qasId; + } + + public void setQasId(Long qasId) { + this.qasId = qasId; + } + } From 73d180de6ccc8a20bfedef190abd2d67bc43132a Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Thu, 20 Jun 2013 13:51:53 +0200 Subject: [PATCH 34/59] Issue #185 - add summariser code to surveyInstanceDao --- .../mapping/dao/SurveyInstanceDAO.java | 59 ++++++++++++++++++- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java b/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java index cf295b6917..5eb2142565 100644 --- a/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java +++ b/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java @@ -28,6 +28,7 @@ import javax.jdo.Query; import org.waterforpeople.mapping.analytics.dao.SurveyQuestionSummaryDao; +import org.waterforpeople.mapping.app.web.dto.DataProcessorRequest; import org.waterforpeople.mapping.domain.QuestionAnswerStore; import org.waterforpeople.mapping.domain.Status.StatusCode; import org.waterforpeople.mapping.domain.SurveyInstance; @@ -46,12 +47,16 @@ import com.google.appengine.api.datastore.Entity; import com.google.appengine.api.datastore.PreparedQuery; import com.google.appengine.api.datastore.Query.FilterOperator; +import com.google.appengine.api.taskqueue.Queue; +import com.google.appengine.api.taskqueue.QueueFactory; +import com.google.appengine.api.taskqueue.TaskOptions; public class SurveyInstanceDAO extends BaseDAO { private static final Logger logger = Logger .getLogger(SurveyInstanceDAO.class.getName()); - + // the set of unparsedLines we have here represent values from one surveyInstance + // as they are split up in the TaskServlet task. public SurveyInstance save(Date collectionDate, DeviceFiles deviceFile, Long userID, List unparsedLines) { @@ -60,10 +65,13 @@ public SurveyInstance save(Date collectionDate, DeviceFiles deviceFile, si.setDeviceFile(deviceFile); si.setUserID(userID); String delimiter = "\t"; + Boolean surveyInstanceIsNew = true; + Long geoQasId = null; final QuestionAnswerStoreDao qasDao = new QuestionAnswerStoreDao(); ArrayList qasList = new ArrayList(); + for (String line : unparsedLines) { String[] parts = line.split(delimiter); @@ -123,7 +131,7 @@ public SurveyInstance save(Date collectionDate, DeviceFiles deviceFile, si.setDeviceIdentifier(parts[8].trim()); } } - + // if this is the first time round, save the surveyInstance or use an existing one if (si.getSurveyId() == null) { try { if (collDate != null) { @@ -144,6 +152,7 @@ public SurveyInstance save(Date collectionDate, DeviceFiles deviceFile, } } si = save(si); + } catch (NumberFormatException e) { logger.log(Level.SEVERE, "Could not parse survey id: " + parts[0], e); @@ -196,9 +205,55 @@ public SurveyInstance save(Date collectionDate, DeviceFiles deviceFile, StatusCode.PROCESSED_WITH_ERRORS); } si.setQuestionAnswersStore(qasList); + + QuestionDao questionDao = new QuestionDao(); + List qOptionList = questionDao.listQuestionByType(si.getSurveyId(), Question.Type.OPTION); + + for (QuestionAnswerStore qas : qasList){ + if (Question.Type.GEO.toString().equals(qas.getType())){ + geoQasId = qas.getKey().getId(); + } + // update count of questionAnswerSummary objects + if (isSummarizable(qas, qOptionList)) { + SurveyQuestionSummaryDao.incrementCount(qas,1); + } + } + + // invoke a task to update corresponding surveyInstanceSummary objects + if (surveyInstanceIsNew && geoQasId != null){ + Queue summQueue = QueueFactory.getQueue("dataSummarization"); + summQueue.add(TaskOptions.Builder.withUrl("/app_worker/dataprocessor").param( + DataProcessorRequest.ACTION_PARAM, DataProcessorRequest.SURVEY_INSTANCE_SUMMARIZER) + .param("surveyInstanceId", si.getKey().getId() + "") + .param("qasId", geoQasId + "")); + } return si; } + /** + * returns true if the question type for the answer object is an OPTION type + * @param answer + * @param questions + * @return + */ + private boolean isSummarizable(QuestionAnswerStore answer, + List questions) { + if (questions != null && answer != null) { + long id = Long.parseLong(answer.getQuestionID()); + for (Question q : questions) { + if (q.getKey().getId() == id) { + return true; + } + } + return false; + } else { + return false; + } + } + + + + public SurveyInstanceDAO() { super(SurveyInstance.class); } From ed61bfb37ca42899cd927227988a8ad56f1898d8 Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Fri, 21 Jun 2013 11:53:32 +0200 Subject: [PATCH 35/59] Issue #185 - make geocode parsing more robust --- .../mapping/domain/GeoCoordinates.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/GAE/src/org/waterforpeople/mapping/domain/GeoCoordinates.java b/GAE/src/org/waterforpeople/mapping/domain/GeoCoordinates.java index 7dbc0abbab..81928070ad 100644 --- a/GAE/src/org/waterforpeople/mapping/domain/GeoCoordinates.java +++ b/GAE/src/org/waterforpeople/mapping/domain/GeoCoordinates.java @@ -61,15 +61,25 @@ public static GeoCoordinates extractGeoCoordinate(String line) { gc = new GeoCoordinates(); String[] coordinates = line.split("\\|"); if (coordinates.length > 1) { - gc.setLatitude(new Double(coordinates[0])); - gc.setLongitude(new Double(coordinates[1])); + try { + gc.setLatitude(Double.parseDouble(coordinates[0])); + gc.setLongitude(Double.parseDouble(coordinates[1])); + } catch (NumberFormatException nfe) { + // if we can't parse the lat/lon, the whole operation should fail + return null; + } } else { return null; } if (coordinates.length > 2) { if (coordinates[2] != null && coordinates[2].trim().length() > 0) { - gc.setAltitude(new Double(coordinates[2])); + try { + gc.setAltitude(Double.parseDouble(coordinates[2])); + } catch (NumberFormatException nfe) { + // the altitude cannot be parsed as double, so set it to null + gc.setAltitude(null); + } } } if (coordinates.length > 3) { From 174e61316ea8394ad5a68543173c307eb25324e1 Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Fri, 21 Jun 2013 13:04:12 +0200 Subject: [PATCH 36/59] Issue #185 - fix rebuild question summaries to handle multiple answers --- .../app/web/DataProcessorRestServlet.java | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java b/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java index 2b3f3c2780..6cf83ae3b7 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java @@ -407,13 +407,24 @@ private Map> summarizeQuestionAnswerStore( countMap = new HashMap(); summaryMap.put(qas.getQuestionID(), countMap); } - Long count = countMap.get(val); - if (count == null) { - count = 1L; + + // split up multiple answers + String[] answers; + if (val != null && val.contains("|")) { + answers = val.split("\\|"); } else { - count = count + 1; + answers = new String[] { val }; + } + // perform count + for (int i = 0; i < answers.length; i++) { + Long count = countMap.get(answers[i]); + if (count == null) { + count = 1L; + } else { + count = count + 1; + } + countMap.put(answers[i], count); } - countMap.put(val, count); } } } From 3e81a1f712bb4194e14a6ef0a91f17f326c55c5e Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Fri, 21 Jun 2013 13:56:04 +0200 Subject: [PATCH 37/59] Issue #251 - fix comparison of type --- Dashboard/app/js/lib/models/FLOWrest-adapter-v2-common.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dashboard/app/js/lib/models/FLOWrest-adapter-v2-common.js b/Dashboard/app/js/lib/models/FLOWrest-adapter-v2-common.js index 71344df48f..43a3196de3 100644 --- a/Dashboard/app/js/lib/models/FLOWrest-adapter-v2-common.js +++ b/Dashboard/app/js/lib/models/FLOWrest-adapter-v2-common.js @@ -25,7 +25,7 @@ DS.FLOWRESTAdapter = DS.RESTAdapter.extend({ this._super(store, type, json, root); // only change metaControl info if there is actual meta info in the server response if (Object.keys(this.extractMeta(json)).length !== 0) { - if (type == 'FLOW.SurveyInstance') { + if (type == FLOW.SurveyInstance) { FLOW.metaControl.set('numSILoaded', this.extractMeta(json).num); FLOW.metaControl.set('since', this.extractMeta(json).since); FLOW.metaControl.set('num', this.extractMeta(json).num); From 570443439443e5eda667896106f050729f9259b6 Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Fri, 21 Jun 2013 13:56:32 +0200 Subject: [PATCH 38/59] Issue #258 - add validation to see if min/max are numbers --- Dashboard/app/js/lib/views/surveys/question-view.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Dashboard/app/js/lib/views/surveys/question-view.js b/Dashboard/app/js/lib/views/surveys/question-view.js index 5129fe6c52..2b8ae4879f 100644 --- a/Dashboard/app/js/lib/views/surveys/question-view.js +++ b/Dashboard/app/js/lib/views/surveys/question-view.js @@ -158,6 +158,16 @@ FLOW.QuestionView = FLOW.View.extend({ // validation if (this.type.get('value') == 'NUMBER'){ if (!Ember.empty(this.get('minVal')) && !Ember.empty(this.get('maxVal')) ){ + + if (isNaN(this.get('minVal')) || isNaN(this.get('maxVal'))){ + FLOW.dialogControl.set('activeAction', 'ignore'); + FLOW.dialogControl.set('header', Ember.String.loc('_min_max_not_number')); + FLOW.dialogControl.set('message', Ember.String.loc('_min_max_not_number_message')); + FLOW.dialogControl.set('showCANCEL', false); + FLOW.dialogControl.set('showDialog', true); + return; + } + if (parseFloat(this.get('minVal')) >= parseFloat(this.get('maxVal'))){ FLOW.dialogControl.set('activeAction', 'ignore'); FLOW.dialogControl.set('header', Ember.String.loc('_min_max_not_correct')); From 9b2cf603354a118ce8ef0521b56eb168469cbf84 Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Fri, 21 Jun 2013 14:01:20 +0200 Subject: [PATCH 39/59] Issue #258 - add error message texts to locale --- GAE/src/locale/en.properties | 2 ++ GAE/src/locale/ui-strings.properties | 2 ++ 2 files changed, 4 insertions(+) diff --git a/GAE/src/locale/en.properties b/GAE/src/locale/en.properties index 8719292226..a1febefe12 100644 --- a/GAE/src/locale/en.properties +++ b/GAE/src/locale/en.properties @@ -180,6 +180,7 @@ Number\ of\ question\ groups = Number of question groups Number\ of\ questions = Number of questions OK = OK Omit\ Charts = Omit Charts +One\ of\ the\ values\ you\ put\ in\ the\ minimum\ or\ maximum\ field\ is\ not\ a\ number.\ Please\ fix\ this. = One of the values you put in the minimum or maximum field is not a number. Please fix this. Option = Option Option\ details = Option details Optional = Optional @@ -286,6 +287,7 @@ User = User User\ name = User name Username = Username Users = Users +Value\ is\ not\ a\ number = Value is not a number Value\ type = Value type Version = Version Version\ Number\ helps\ you\ keep\ track\ of\ the\ number\ of\ times\ the\ survey\ has\ been\ revised\ and\ published.\ So,\ if\ the\ survey\ has\ been\ revised\ and\ published\ five\ times,\ it\ will\ be\ Version\ 5.0 = Version Number helps you keep track of the number of times the survey has been revised and published. So, if the survey has been revised and published five times, it will be Version 5.0 diff --git a/GAE/src/locale/ui-strings.properties b/GAE/src/locale/ui-strings.properties index e4e03bdc18..c3d73b843e 100644 --- a/GAE/src/locale/ui-strings.properties +++ b/GAE/src/locale/ui-strings.properties @@ -155,6 +155,8 @@ _message = Message _messages = Messages _min_larger_than_max_or_equal = The minimum value is larger then or equal to the maximum value. This won't work. _min_max_not_correct = The minimum and maximum value do not make sense +_min_max_not_number = Value is not a number +_min_max_not_number_message = One of the values you put in the minimum or maximum field is not a number. Please fix this. _min_val = Minimum value _missing_db_instructions = You must provide some DB instructions _modified = Modified From 054ec0744d9beafa63641b51b21eb99fd54a358a Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Fri, 21 Jun 2013 20:06:32 +0200 Subject: [PATCH 40/59] Issue #185 - questionAnswerSummary computation also uses OTHER values --- .../app/web/DataProcessorRestServlet.java | 164 +++++++++++------- 1 file changed, 98 insertions(+), 66 deletions(-) diff --git a/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java b/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java index 6cf83ae3b7..9663960313 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java @@ -30,7 +30,6 @@ import javax.servlet.http.HttpServletRequest; - import org.waterforpeople.mapping.analytics.SurveyInstanceSummarizer; import org.waterforpeople.mapping.analytics.dao.SurveyInstanceSummaryDao; import org.waterforpeople.mapping.analytics.dao.SurveyQuestionSummaryDao; @@ -73,10 +72,12 @@ * */ public class DataProcessorRestServlet extends AbstractRestApiServlet { - private static final Logger log = Logger.getLogger("DataProcessorRestServlet"); + private static final Logger log = Logger + .getLogger("DataProcessorRestServlet"); private static final long serialVersionUID = -7902002525342262821L; private static final String REBUILD_Q_SUM_STATUS_KEY = "rebuildQuestionSummary"; private static final String VALUE_TYPE = "VALUE"; + private static final String OTHER_TYPE = "OTHER"; private static final Integer QAS_PAGE_SIZE = 300; @Override @@ -99,7 +100,7 @@ protected RestResponse handleRequest(RestRequest req) throws Exception { } else if (DataProcessorRequest.IMPORT_REMOTE_SURVEY_ACTION .equalsIgnoreCase(dpReq.getAction())) { SurveyReplicationImporter sri = new SurveyReplicationImporter(); - sri.executeImport(dpReq.getSource(), dpReq.getSurveyId(), null); //FIXME + sri.executeImport(dpReq.getSource(), dpReq.getSurveyId(), null); // FIXME } else if (DataProcessorRequest.RESCORE_AP_ACTION .equalsIgnoreCase(dpReq.getAction())) { rescoreAp(dpReq.getCountry()); @@ -112,10 +113,13 @@ protected RestResponse handleRequest(RestRequest req) throws Exception { } else if (DataProcessorRequest.TRIM_OPTIONS.equalsIgnoreCase(dpReq .getAction())) { trimOptions(); - } else if (DataProcessorRequest.FIX_OPTIONS2VALUES_ACTION.equalsIgnoreCase(dpReq.getAction())){ + } else if (DataProcessorRequest.FIX_OPTIONS2VALUES_ACTION + .equalsIgnoreCase(dpReq.getAction())) { fixOptions2Values(dpReq.getCursor()); - } else if (DataProcessorRequest.SURVEY_INSTANCE_SUMMARIZER.equalsIgnoreCase(dpReq.getAction())){ - surveyInstanceSummarizer(dpReq.getSurveyInstanceId(),dpReq.getQasId()); + } else if (DataProcessorRequest.SURVEY_INSTANCE_SUMMARIZER + .equalsIgnoreCase(dpReq.getAction())) { + surveyInstanceSummarizer(dpReq.getSurveyInstanceId(), + dpReq.getQasId()); } return new RestResponse(); } @@ -387,48 +391,63 @@ private Map> summarizeQuestionAnswerStore( String cursor = null; Map> summaryMap = new HashMap>(); List qasList = null; - do { - qasList = qasDao.listByTypeAndDate(VALUE_TYPE, surveyId, sinceDate, - cursor, QAS_PAGE_SIZE); - if (qasList != null && qasList.size() > 0) { - cursor = QuestionAnswerStoreDao.getCursor(qasList); - for (QuestionAnswerStore qas : qasList) { - if (isSummarizable(qas, qList)) { - String val = qas.getValue(); - // skip images since the summary is meaningless in those - // cases - if (val == null - || val.toLowerCase().trim().endsWith(".jpg")) { - continue; - } - Map countMap = summaryMap.get(qas - .getQuestionID()); - if (countMap == null) { - countMap = new HashMap(); - summaryMap.put(qas.getQuestionID(), countMap); - } - - // split up multiple answers - String[] answers; - if (val != null && val.contains("|")) { - answers = val.split("\\|"); - } else { - answers = new String[] { val }; - } - // perform count - for (int i = 0; i < answers.length; i++) { - Long count = countMap.get(answers[i]); - if (count == null) { - count = 1L; + + String responseType = VALUE_TYPE; + // first do VALUE_TYPE, then do OTHER_TYPE. We need to do it this way + // because the datastore does not support contains() queries with a cursor + for (int run = 1; run <= 2; run++) { + do { + // We need both VALUE and OTHER QAS because both can originate + // from an option question + qasList = qasDao.listByTypeAndDate(responseType, surveyId, + sinceDate, cursor, QAS_PAGE_SIZE); + + if (qasList != null && qasList.size() > 0) { + cursor = QuestionAnswerStoreDao.getCursor(qasList); + + for (QuestionAnswerStore qas : qasList) { + if (isSummarizable(qas, qList)) { + String val = qas.getValue(); + // skip images since the summary is meaningless in + // those + // cases + if (val == null + || val.toLowerCase().trim() + .endsWith(".jpg")) { + continue; + } + Map countMap = summaryMap.get(qas + .getQuestionID()); + if (countMap == null) { + countMap = new HashMap(); + summaryMap.put(qas.getQuestionID(), countMap); + } + + // split up multiple answers + String[] answers; + if (val != null && val.contains("|")) { + answers = val.split("\\|"); } else { - count = count + 1; + answers = new String[] { val }; + } + // perform count + for (int i = 0; i < answers.length; i++) { + Long count = countMap.get(answers[i]); + if (count == null) { + count = 1L; + } else { + count = count + 1; + } + countMap.put(answers[i], count); } - countMap.put(answers[i], count); } } } - } - } while (qasList != null && qasList.size() > 0); + } while (qasList != null && qasList.size() > 0); + responseType = OTHER_TYPE; + cursor = null; + } + return summaryMap; } @@ -534,35 +553,39 @@ public static void sendProjectUpdateTask(String country, String cursor) { } /** - * fixes wrong Types in questionAnswerStore objects. When cleaned data is - * uploaded using an excel file, the type of the answer is set according - * to the type of the question, while the device sets the type according - * to a different convention. The action handles 500 items in one call, - * and invokes new tasks as necessary if there are more items. + * fixes wrong Types in questionAnswerStore objects. When cleaned data is + * uploaded using an excel file, the type of the answer is set according to + * the type of the question, while the device sets the type according to a + * different convention. The action handles 500 items in one call, and + * invokes new tasks as necessary if there are more items. * * @param cursor * @author M.T. Westra */ public static void fixOptions2Values(String cursorString) { SurveyInstanceDAO siDao = new SurveyInstanceDAO(); - QuestionAnswerStoreDao qasDao = new QuestionAnswerStoreDao(); - List qasList = siDao.listQAOptions(cursorString,500,"OPTION","FREE_TEXT","NUMBER","SCAN","PHOTO"); + QuestionAnswerStoreDao qasDao = new QuestionAnswerStoreDao(); + List qasList = siDao.listQAOptions(cursorString, + 500, "OPTION", "FREE_TEXT", "NUMBER", "SCAN", "PHOTO"); List qasChangedList = new ArrayList(); - log.log(Level.INFO, "Running fixOptions2Values, cursor at " + cursorString); + log.log(Level.INFO, "Running fixOptions2Values, cursor at " + + cursorString); if (qasList != null) { String cursor = SurveyInstanceDAO.getCursor(qasList); for (QuestionAnswerStore qas : qasList) { if (Question.Type.OPTION.toString().equals(qas.getType()) - || Question.Type.NUMBER.toString().equals(qas.getType()) - || Question.Type.FREE_TEXT.toString().equals(qas.getType()) - || Question.Type.SCAN.toString().equals(qas.getType())){ + || Question.Type.NUMBER.toString() + .equals(qas.getType()) + || Question.Type.FREE_TEXT.toString().equals( + qas.getType()) + || Question.Type.SCAN.toString().equals(qas.getType())) { qas.setType("VALUE"); qasChangedList.add(qas); - } else if (Question.Type.PHOTO.toString().equals(qas.getType())){ + } else if (Question.Type.PHOTO.toString().equals(qas.getType())) { qas.setType("IMAGE"); qasChangedList.add(qas); - } + } } qasDao.save(qasChangedList); // if there are more, invoke another task @@ -576,10 +599,11 @@ public static void fixOptions2Values(String cursorString) { .param(DataProcessorRequest.CURSOR_PARAM, cursor); queue.add(options); } - } + } } - - public static void surveyInstanceSummarizer(Long surveyInstanceId, Long qasId) { + + public static void surveyInstanceSummarizer(Long surveyInstanceId, + Long qasId) { SurveyInstanceDAO siDao = new SurveyInstanceDAO(); QuestionAnswerStoreDao qasDao = new QuestionAnswerStoreDao(); boolean success = false; @@ -587,16 +611,22 @@ public static void surveyInstanceSummarizer(Long surveyInstanceId, Long qasId) { SurveyInstance si = siDao.getByKey(surveyInstanceId); if (si != null && qasId != null) { QuestionAnswerStore qas = qasDao.getByKey(qasId); - if (qas != null){ + if (qas != null) { GeoCoordinates geoC = null; - if (qas.getValue() != null && qas.getValue().trim().length() > 0) { - geoC = GeoCoordinates.extractGeoCoordinate(qas.getValue()); + if (qas.getValue() != null + && qas.getValue().trim().length() > 0) { + geoC = GeoCoordinates.extractGeoCoordinate(qas + .getValue()); } if (geoC != null) { GeoLocationService gisService = new GeoLocationServiceGeonamesImpl(); - GeoPlace gp = gisService.findDetailedGeoPlace(geoC.getLatitude().toString(), geoC.getLongitude().toString()); + GeoPlace gp = gisService.findDetailedGeoPlace(geoC + .getLatitude().toString(), geoC.getLongitude() + .toString()); if (gp != null) { - SurveyInstanceSummaryDao.incrementCount(gp.getSub1(), gp.getCountryCode(), qas.getCollectionDate()); + SurveyInstanceSummaryDao.incrementCount( + gp.getSub1(), gp.getCountryCode(), + qas.getCollectionDate()); success = true; } } @@ -604,8 +634,10 @@ public static void surveyInstanceSummarizer(Long surveyInstanceId, Long qasId) { } } if (!success) { - log.log(Level.SEVERE,"Couldnt find geoplace for instance. Instance id: " + surveyInstanceId); + log.log(Level.SEVERE, + "Couldnt find geoplace for instance. Instance id: " + + surveyInstanceId); } } - + } From 5fa91a121bd54061c1709a5e0d6c935782eb2c53 Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Fri, 21 Jun 2013 20:07:07 +0200 Subject: [PATCH 41/59] Issue #185 - code cleanup --- GAE/src/org/waterforpeople/mapping/app/web/TaskServlet.java | 2 -- .../org/waterforpeople/mapping/dao/SurveyInstanceDAO.java | 5 +++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/GAE/src/org/waterforpeople/mapping/app/web/TaskServlet.java b/GAE/src/org/waterforpeople/mapping/app/web/TaskServlet.java index 51fa82b1a1..8f92798812 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/TaskServlet.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/TaskServlet.java @@ -42,7 +42,6 @@ import javax.crypto.spec.SecretKeySpec; import javax.servlet.http.HttpServletRequest; -import org.waterforpeople.mapping.app.web.dto.DataProcessorRequest; import org.waterforpeople.mapping.app.web.dto.TaskRequest; import org.waterforpeople.mapping.dao.DeviceFilesDao; import org.waterforpeople.mapping.dao.SurveyInstanceDAO; @@ -457,7 +456,6 @@ private void ingestFile(TaskRequest req) { req.getOffset()); Map surveyMap = new HashMap(); SurveyDAO surveyDao = new SurveyDAO(); - Queue summQueue = QueueFactory.getQueue("dataSummarization"); Queue defaultQueue = QueueFactory.getDefaultQueue(); for (SurveyInstance instance : surveyInstances) { Survey s = surveyMap.get(instance.getSurveyId()); diff --git a/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java b/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java index 5eb2142565..689dbba530 100644 --- a/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java +++ b/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java @@ -17,6 +17,7 @@ package org.waterforpeople.mapping.dao; import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -661,13 +662,13 @@ public List listInstanceBySubmitter(String submitter) { } - /** lists surveyInstances of particular types passed in + /** lists questionAnswerStore objects of particular types passed in */ public List listQAOptions(String cursorString, Integer pageSize, String... options){ PersistenceManager pm = PersistenceFilter.getManager(); javax.jdo.Query q = pm.newQuery(QuestionAnswerStore.class); StringBuffer filter = new StringBuffer(); - for(String op :options) { + for (String op : options) { filter.append("type == '").append(op).append("' ||"); } q.setFilter(filter.substring(0,filter.length()-3).toString()); From a64ba10c9a46ed85f9890248e611c36c73a2e4fb Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Mon, 24 Jun 2013 12:08:37 +0200 Subject: [PATCH 42/59] "Return is a statement, not a function call" - cleanup --- .../js/lib/controllers/device-controllers.js | 4 ++-- .../js/lib/controllers/survey-controllers.js | 20 +++++++++---------- .../app/js/lib/views/reports/report-views.js | 8 ++++---- .../app/js/lib/views/surveys/preview-view.js | 2 +- .../app/js/lib/views/surveys/question-view.js | 12 +++++------ .../lib/views/surveys/survey-details-views.js | 12 +++++------ Dashboard/app/js/lib/views/views-public.js | 2 +- Dashboard/app/js/lib/views/views.js | 12 +++++------ 8 files changed, 36 insertions(+), 36 deletions(-) diff --git a/Dashboard/app/js/lib/controllers/device-controllers.js b/Dashboard/app/js/lib/controllers/device-controllers.js index 067e44e69e..2e9a2efce8 100644 --- a/Dashboard/app/js/lib/controllers/device-controllers.js +++ b/Dashboard/app/js/lib/controllers/device-controllers.js @@ -74,11 +74,11 @@ FLOW.devicesInGroupControl = Ember.ArrayController.create({ // 1 means all unassigned devices if(deviceGroupId == 1) { this.set('content', FLOW.store.filter(FLOW.Device, function(item) { - return(Ember.empty(item.get('deviceGroup'))); + return Ember.empty(item.get('deviceGroup')); })); } else { this.set('content', FLOW.store.filter(FLOW.Device, function(item) { - return(parseInt(item.get('deviceGroup'), 10) == deviceGroupId); + return parseInt(item.get('deviceGroup'), 10) == deviceGroupId; })); } } diff --git a/Dashboard/app/js/lib/controllers/survey-controllers.js b/Dashboard/app/js/lib/controllers/survey-controllers.js index a89777f6ed..3b61cea413 100644 --- a/Dashboard/app/js/lib/controllers/survey-controllers.js +++ b/Dashboard/app/js/lib/controllers/survey-controllers.js @@ -142,7 +142,7 @@ FLOW.surveyGroupControl = Ember.ArrayController.create({ return true; } }); - return(surveys.get('content').length > 0); + return surveys.get('content').length > 0; }, deleteSurveyGroup: function(keyId){ @@ -166,7 +166,7 @@ FLOW.surveyControl = Ember.ArrayController.create({ if(FLOW.selectedControl.get('selectedSurveyGroup') && FLOW.selectedControl.selectedSurveyGroup.get('keyId') > 0) { sgId = FLOW.selectedControl.selectedSurveyGroup.get('keyId'); this.set('content', FLOW.store.filter(FLOW.Survey, function(item) { - return(item.get('surveyGroupId') == sgId); + return item.get('surveyGroupId') == sgId; })); } else { this.set('content', null); @@ -178,7 +178,7 @@ FLOW.surveyControl = Ember.ArrayController.create({ if(FLOW.selectedControl.get('selectedSurveyGroup') && FLOW.selectedControl.selectedSurveyGroup.get('keyId') > 0) { sgId = FLOW.selectedControl.selectedSurveyGroup.get('keyId'); this.set('publishedContent', FLOW.store.filter(FLOW.Survey, function(item) { - return (item.get('surveyGroupId') == sgId && item.get('status') == 'PUBLISHED'); + return item.get('surveyGroupId') == sgId && item.get('status') == 'PUBLISHED'; })); } else { this.set('publishedContent', null); @@ -226,7 +226,7 @@ FLOW.questionGroupControl = Ember.ArrayController.create({ if(!Ember.empty(FLOW.selectedControl.selectedSurvey.get('keyId'))) { sId = FLOW.selectedControl.selectedSurvey.get('keyId'); this.set('content', FLOW.store.filter(FLOW.QuestionGroup, function(item) { - return(item.get('surveyId') == sId); + return item.get('surveyId') == sId; })); } else { // this happens when we have created a new survey, which has no id yet @@ -299,7 +299,7 @@ FLOW.questionControl = Ember.ArrayController.create({ // restore order questionsInGroup = FLOW.store.filter(FLOW.Question, function(item) { - return(item.get('questionGroupId') == qgId); + return item.get('questionGroupId') == qgId; }); questionsInGroup.forEach(function(item) { @@ -315,7 +315,7 @@ FLOW.questionControl = Ember.ArrayController.create({ if(FLOW.selectedControl.get('selectedSurvey') && FLOW.selectedControl.selectedSurvey.get('keyId') > 0) { sId = FLOW.selectedControl.selectedSurvey.get('keyId'); this.set('filterContent', FLOW.store.filter(FLOW.Question, function(item) { - return(item.get('surveyId') == sId); + return item.get('surveyId') == sId; })); } else { this.set('filterContent',null); @@ -327,7 +327,7 @@ FLOW.questionControl = Ember.ArrayController.create({ if(FLOW.selectedControl.get('selectedQuestionGroup') && FLOW.selectedControl.selectedSurvey.get('keyId') > 0) { var qId = FLOW.selectedControl.selectedQuestionGroup.get('keyId'); this.set('content', FLOW.store.filter(FLOW.Question, function(item) { - return(item.get('questionGroupId') == qId); + return item.get('questionGroupId') == qId; })); } }.observes('FLOW.selectedControl.selectedQuestionGroup'), @@ -337,7 +337,7 @@ FLOW.questionControl = Ember.ArrayController.create({ if(FLOW.selectedControl.get('selectedSurvey')) { sId = FLOW.selectedControl.selectedSurvey.get('keyId'); this.set('OPTIONcontent', FLOW.store.filter(FLOW.Question, function(item) { - return(item.get('type') == 'OPTION' && item.get('surveyId') == sId); + return item.get('type') == 'OPTION' && item.get('surveyId') == sId; })); } else { this.set('OPTIONcontent', null); @@ -359,7 +359,7 @@ FLOW.questionControl = Ember.ArrayController.create({ if(qgOrder > questionGroupOrder) {return false;} if(qgOrder < questionGroupOrder) {return true;} // when we arrive there qgOrder = questionGroupOrder, so we have to check question order - return (item.get('order') < questionOrder); + return item.get('order') < questionOrder; }); this.set('earlierOptionQuestions', optionQuestionList); @@ -416,7 +416,7 @@ FLOW.notificationControl = Ember.ArrayController.create({ if(FLOW.selectedControl.get('selectedSurvey') && FLOW.selectedControl.selectedSurvey.get('keyId') > 0) { sId = FLOW.selectedControl.selectedSurvey.get('keyId'); this.set('content', FLOW.store.filter(FLOW.NotificationSubscription, function(item) { - return(item.get('entityId') == sId); + return item.get('entityId') == sId; })); } }.observes('FLOW.selectedControl.selectedSurvey') diff --git a/Dashboard/app/js/lib/views/reports/report-views.js b/Dashboard/app/js/lib/views/reports/report-views.js index 0aeb168a71..d1f40cc754 100644 --- a/Dashboard/app/js/lib/views/reports/report-views.js +++ b/Dashboard/app/js/lib/views/reports/report-views.js @@ -6,7 +6,7 @@ FLOW.chartView = FLOW.View.extend({ compactSmaller: true, isDoughnut: function() { - return(this.chartType.get('value') == 'doughnut'); + return this.chartType.get('value') == 'doughnut'; }.property('this.chartType'), init: function() { @@ -59,7 +59,7 @@ FLOW.chartView = FLOW.View.extend({ // sort smallest first chartData.sort(function(a, b) { - return(a.percentage >= b.percentage); + return a.percentage >= b.percentage; }); @@ -106,7 +106,7 @@ FLOW.chartView = FLOW.View.extend({ // sort smallest first chartData.sort(function(a, b) { - return(a.percentage <= b.percentage); + return a.percentage <= b.percentage; }); FLOW.chartDataControl.set('chartData', chartData); FLOW.chartDataControl.set('maxPer', maxPer); @@ -125,7 +125,7 @@ FLOW.chartView = FLOW.View.extend({ // sort smallest first chartData.sort(function(a, b) { - return(a.percentage <= b.percentage); + return a.percentage <= b.percentage; }); FLOW.chartDataControl.set('chartData', chartData); FLOW.chartDataControl.set('maxPer', maxPer); diff --git a/Dashboard/app/js/lib/views/surveys/preview-view.js b/Dashboard/app/js/lib/views/surveys/preview-view.js index 801d5e80a8..d7c5fd45ca 100644 --- a/Dashboard/app/js/lib/views/surveys/preview-view.js +++ b/Dashboard/app/js/lib/views/surveys/preview-view.js @@ -15,7 +15,7 @@ FLOW.PreviewQuestionGroupView = FLOW.View.extend({ this._super(); qgId = this.content.get('keyId'); this.set('QGcontent', FLOW.store.filter(FLOW.Question, function(item) { - return(item.get('questionGroupId') == qgId); + return item.get('questionGroupId') == qgId; })); } }); diff --git a/Dashboard/app/js/lib/views/surveys/question-view.js b/Dashboard/app/js/lib/views/surveys/question-view.js index 2b8ae4879f..a75d199ae0 100644 --- a/Dashboard/app/js/lib/views/surveys/question-view.js +++ b/Dashboard/app/js/lib/views/surveys/question-view.js @@ -50,7 +50,7 @@ FLOW.QuestionView = FLOW.View.extend({ amOptionType: function() { if(this.type) { - return(this.type.get('value') == 'OPTION'); + return this.type.get('value') == 'OPTION'; } else { return false; } @@ -58,7 +58,7 @@ FLOW.QuestionView = FLOW.View.extend({ amNumberType: function() { if(this.type) { - return(this.type.get('value') == 'NUMBER'); + return this.type.get('value') == 'NUMBER'; } else { return false; } @@ -68,7 +68,7 @@ FLOW.QuestionView = FLOW.View.extend({ var val; if(!Ember.none(this.type)) { val = this.type.get('value'); - return(val == 'GEO' || val == 'FREE_TEXT' || val == 'PHOTO' || val == 'VIDEO' || val == 'BARCODE'); + return val == 'GEO' || val == 'FREE_TEXT' || val == 'PHOTO' || val == 'VIDEO' || val == 'BARCODE'; } }.property('this.type').cacheable(), @@ -277,7 +277,7 @@ FLOW.QuestionView = FLOW.View.extend({ // restore order qgId = FLOW.selectedControl.selectedQuestionGroup.get('keyId'); questionsInGroup = FLOW.store.filter(FLOW.Question, function(item) { - return(item.get('questionGroupId') == qgId); + return item.get('questionGroupId') == qgId; }); origOrder = FLOW.selectedControl.selectedForMoveQuestion.get('order'); @@ -329,7 +329,7 @@ FLOW.QuestionView = FLOW.View.extend({ // restore order qgId = FLOW.selectedControl.selectedQuestionGroup.get('keyId'); questionsInGroup = FLOW.store.filter(FLOW.Question, function(item) { - return(item.get('questionGroupId') == qgId); + return item.get('questionGroupId') == qgId; }); // move items up to make space @@ -378,7 +378,7 @@ FLOW.QuestionView = FLOW.View.extend({ // restore order qgId = FLOW.selectedControl.selectedQuestionGroup.get('keyId'); questionsInGroup = FLOW.store.filter(FLOW.Question, function(item) { - return(item.get('questionGroupId') == qgId); + return item.get('questionGroupId') == qgId; }); // move items up to make space diff --git a/Dashboard/app/js/lib/views/surveys/survey-details-views.js b/Dashboard/app/js/lib/views/surveys/survey-details-views.js index d19e706a48..851a131814 100644 --- a/Dashboard/app/js/lib/views/surveys/survey-details-views.js +++ b/Dashboard/app/js/lib/views/surveys/survey-details-views.js @@ -64,7 +64,7 @@ FLOW.SurveySidebarView = FLOW.View.extend({ }, isPublished: function() { - return(FLOW.selectedControl.selectedSurvey.get('status') == 'PUBLISHED'); + return FLOW.selectedControl.selectedSurvey.get('status') == 'PUBLISHED'; }.property('FLOW.selectedControl.selectedSurvey.status'), numberQuestions: function() { @@ -243,7 +243,7 @@ FLOW.QuestionGroupItemView = FLOW.View.extend({ questionGroup = FLOW.store.find(FLOW.QuestionGroup, qgDeleteId); qgOrder = questionGroup.get('order'); questionsInGroup = FLOW.store.filter(FLOW.Question,function (item){ - return(item.get('questionGroupId') == qgDeleteId); + return item.get('questionGroupId') == qgDeleteId; }); if (questionsInGroup.get('content').length > 0){ @@ -259,7 +259,7 @@ FLOW.QuestionGroupItemView = FLOW.View.extend({ questionGroup.deleteRecord(); // restore order questionGroupsInSurvey = FLOW.store.filter(FLOW.QuestionGroup, function(item) { - return(item.get('surveyId') == sId); + return item.get('surveyId') == sId; }); questionGroupsInSurvey.forEach(function(item) { @@ -288,7 +288,7 @@ FLOW.QuestionGroupItemView = FLOW.View.extend({ // restore order sId = FLOW.selectedControl.selectedSurvey.get('keyId'); questionGroupsInSurvey = FLOW.store.filter(FLOW.QuestionGroup, function(item) { - return(item.get('surveyId') == sId); + return item.get('surveyId') == sId; }); // move items up to make space @@ -363,7 +363,7 @@ FLOW.QuestionGroupItemView = FLOW.View.extend({ // restore order sId = FLOW.selectedControl.selectedSurvey.get('keyId'); questionGroupsInSurvey = FLOW.store.filter(FLOW.QuestionGroup, function(item) { - return(item.get('surveyId') == sId); + return item.get('surveyId') == sId; }); origOrder = FLOW.selectedControl.selectedForMoveQuestionGroup.get('order'); @@ -413,7 +413,7 @@ FLOW.QuestionGroupItemView = FLOW.View.extend({ sId = FLOW.selectedControl.selectedSurvey.get('keyId'); questionGroupsInSurvey = FLOW.store.filter(FLOW.QuestionGroup, function(item) { - return(item.get('surveyId') == sId); + return item.get('surveyId') == sId; }); // restore order - move items up to make space diff --git a/Dashboard/app/js/lib/views/views-public.js b/Dashboard/app/js/lib/views/views-public.js index 95da3d08e1..fa45b4b46a 100644 --- a/Dashboard/app/js/lib/views/views-public.js +++ b/Dashboard/app/js/lib/views/views-public.js @@ -171,7 +171,7 @@ FLOW.registerViewHelper('date2', Ember.View.extend({ minString = curr_min.toString(); } - return(curr_year + "-" + monthString + "-" + dateString + " " + hourString + ":" + minString); + return curr_year + "-" + monthString + "-" + dateString + " " + hourString + ":" + minString; }).property('content') })); diff --git a/Dashboard/app/js/lib/views/views.js b/Dashboard/app/js/lib/views/views.js index c746120d4b..65bbbc6ac0 100644 --- a/Dashboard/app/js/lib/views/views.js +++ b/Dashboard/app/js/lib/views/views.js @@ -170,7 +170,7 @@ Ember.Handlebars.registerHelper("date", function(property) { var curr_date = d.getDate(); var curr_month = d.getMonth(); var curr_year = d.getFullYear(); - return(curr_date + " " + m_names[curr_month] + " " + curr_year); + return curr_date + " " + m_names[curr_month] + " " + curr_year; }); // format used in devices table @@ -208,7 +208,7 @@ Ember.Handlebars.registerHelper("date1", function(property) { minString = curr_min.toString(); } - return(curr_year + "-" + monthString + "-" + dateString + " " + hourString + ":" + minString); + return curr_year + "-" + monthString + "-" + dateString + " " + hourString + ":" + minString; } else { return ""; } @@ -235,7 +235,7 @@ Ember.Handlebars.registerHelper("date3", function(property) { dateString = curr_date.toString(); } - return(curr_year + "-" + monthString + "-" + dateString); + return curr_year + "-" + monthString + "-" + dateString; } else { return ""; } @@ -302,7 +302,7 @@ FLOW.registerViewHelper('date2', Ember.View.extend({ minString = curr_min.toString(); } - return(curr_year + "-" + monthString + "-" + dateString + " " + hourString + ":" + minString); + return curr_year + "-" + monthString + "-" + dateString + " " + hourString + ":" + minString; }).property('content') })); @@ -613,11 +613,11 @@ FLOW.ColumnView = Ember.View.extend({ classNameBindings: ['isActiveAsc:sorting_asc', 'isActiveDesc:sorting_desc'], isActiveAsc: function() { - return(this.get('item') === FLOW.tableColumnControl.get('selected')) && (FLOW.tableColumnControl.get('sortAscending') === true); + return this.get('item') === FLOW.tableColumnControl.get('selected')) && (FLOW.tableColumnControl.get('sortAscending') === true; }.property('item', 'FLOW.tableColumnControl.selected', 'FLOW.tableColumnControl.sortAscending').cacheable(), isActiveDesc: function() { - return(this.get('item') === FLOW.tableColumnControl.get('selected')) && (FLOW.tableColumnControl.get('sortAscending') === false); + return this.get('item') === FLOW.tableColumnControl.get('selected')) && (FLOW.tableColumnControl.get('sortAscending') === false; }.property('item', 'FLOW.tableColumnControl.selected', 'FLOW.tableColumnControl.sortAscending').cacheable(), sort: function() { From fda9a14a5bcd1c9026aa267bf9a83a16508ca5eb Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Mon, 24 Jun 2013 23:16:10 +0200 Subject: [PATCH 43/59] Issue #185 - counts of surveyInstances can decrease as well --- .../analytics/SurveyInstanceSummarizer.java | 2 +- .../dao/SurveyInstanceSummaryDao.java | 18 +++++++++++++---- .../app/web/DataProcessorRestServlet.java | 6 +++--- .../app/web/dto/DataProcessorRequest.java | 20 +++++++++++++++++++ 4 files changed, 38 insertions(+), 8 deletions(-) diff --git a/GAE/src/org/waterforpeople/mapping/analytics/SurveyInstanceSummarizer.java b/GAE/src/org/waterforpeople/mapping/analytics/SurveyInstanceSummarizer.java index d82e97737f..734994c4c2 100644 --- a/GAE/src/org/waterforpeople/mapping/analytics/SurveyInstanceSummarizer.java +++ b/GAE/src/org/waterforpeople/mapping/analytics/SurveyInstanceSummarizer.java @@ -83,7 +83,7 @@ public boolean performSummarization(String key, String type, String value, if (gp != null) { SurveyInstanceSummaryDao.incrementCount(gp.getSub1(), gp.getCountryCode(), - instance.getCollectionDate()); + instance.getCollectionDate(), 1); success = true; } } else { diff --git a/GAE/src/org/waterforpeople/mapping/analytics/dao/SurveyInstanceSummaryDao.java b/GAE/src/org/waterforpeople/mapping/analytics/dao/SurveyInstanceSummaryDao.java index 5d2069621d..4ee3132592 100644 --- a/GAE/src/org/waterforpeople/mapping/analytics/dao/SurveyInstanceSummaryDao.java +++ b/GAE/src/org/waterforpeople/mapping/analytics/dao/SurveyInstanceSummaryDao.java @@ -20,6 +20,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import javax.jdo.PersistenceManager; @@ -38,6 +40,8 @@ */ public class SurveyInstanceSummaryDao extends BaseDAO { + private static final Logger log = Logger + .getLogger("SurveyInstanceSummaryDao"); public SurveyInstanceSummaryDao() { super(SurveyInstanceSummary.class); } @@ -52,7 +56,7 @@ public SurveyInstanceSummaryDao() { */ @SuppressWarnings("rawtypes") public static synchronized void incrementCount(String community, - String country, Date collectionDate) { + String country, Date collectionDate, int delta) { PersistenceManager pm = PersistenceFilter.getManager(); javax.jdo.Query query = pm.newQuery(SurveyInstanceSummary.class); Date colDate = DateUtil.getDateNoTime(collectionDate); @@ -65,18 +69,24 @@ public static synchronized void incrementCount(String community, List results = (List) query.execute(country,community, colDate); SurveyInstanceSummary summary = null; + SurveyInstanceSummaryDao thisDao = new SurveyInstanceSummaryDao(); if (results == null || results.size() == 0) { summary = new SurveyInstanceSummary(); summary.setCount(1L); summary.setCommunityCode(community); summary.setCountryCode(country); summary.setCollectionDate(colDate); + thisDao.save(summary); } else { summary = (SurveyInstanceSummary) results.get(0); - summary.setCount(summary.getCount() + 1); + summary.setCount(summary.getCount() + delta); + // if the count is zero, delete it + if (summary.getCount() == 0){ + thisDao.delete(summary); + } else { + thisDao.save(summary); + } } - SurveyInstanceSummaryDao thisDao = new SurveyInstanceSummaryDao(); - thisDao.save(summary); } /** diff --git a/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java b/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java index 9663960313..3afc974ffc 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java @@ -119,7 +119,7 @@ protected RestResponse handleRequest(RestRequest req) throws Exception { } else if (DataProcessorRequest.SURVEY_INSTANCE_SUMMARIZER .equalsIgnoreCase(dpReq.getAction())) { surveyInstanceSummarizer(dpReq.getSurveyInstanceId(), - dpReq.getQasId()); + dpReq.getQasId(), dpReq.getDelta()); } return new RestResponse(); } @@ -603,7 +603,7 @@ public static void fixOptions2Values(String cursorString) { } public static void surveyInstanceSummarizer(Long surveyInstanceId, - Long qasId) { + Long qasId, Integer delta) { SurveyInstanceDAO siDao = new SurveyInstanceDAO(); QuestionAnswerStoreDao qasDao = new QuestionAnswerStoreDao(); boolean success = false; @@ -626,7 +626,7 @@ public static void surveyInstanceSummarizer(Long surveyInstanceId, if (gp != null) { SurveyInstanceSummaryDao.incrementCount( gp.getSub1(), gp.getCountryCode(), - qas.getCollectionDate()); + qas.getCollectionDate(), delta.intValue()); success = true; } } diff --git a/GAE/src/org/waterforpeople/mapping/app/web/dto/DataProcessorRequest.java b/GAE/src/org/waterforpeople/mapping/app/web/dto/DataProcessorRequest.java index 43763c1e8d..5312974ae4 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/dto/DataProcessorRequest.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/dto/DataProcessorRequest.java @@ -43,12 +43,14 @@ public class DataProcessorRequest extends RestRequest { public static final String SURVEY_ID_PARAM = "surveyId"; public static final String SURVEY_INSTANCE_PARAM = "surveyInstanceId"; public static final String QAS_ID_PARAM = "qasId"; + public static final String DELTA_PARAM = "delta"; private String country; private String source; private Long surveyId; private Long surveyInstanceId; private Long qasId; + private Integer delta; @Override protected void populateFields(HttpServletRequest req) throws Exception { @@ -81,6 +83,16 @@ protected void populateFields(HttpServletRequest req) throws Exception { + " must be an integer")); } } + + if (req.getParameter(DELTA_PARAM) != null) { + try { + setDelta(new Integer(req.getParameter(DELTA_PARAM).trim())); + } catch (Exception e) { + addError(new RestError(RestError.BAD_DATATYPE_CODE, + RestError.BAD_DATATYPE_MESSAGE, DELTA_PARAM + + " must be an integer")); + } + } } @@ -130,4 +142,12 @@ public void setQasId(Long qasId) { this.qasId = qasId; } + public Integer getDelta() { + return delta; + } + + public void setDelta(Integer delta) { + this.delta = delta; + } + } From eff09f84d9c3cdad50eacb06f3bc57f6c601fed7 Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Mon, 24 Jun 2013 23:16:39 +0200 Subject: [PATCH 44/59] Issue #185 - add comment to surveyInstanceServiceImpl --- .../gwt/server/surveyinstance/SurveyInstanceServiceImpl.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/GAE/src/org/waterforpeople/mapping/app/gwt/server/surveyinstance/SurveyInstanceServiceImpl.java b/GAE/src/org/waterforpeople/mapping/app/gwt/server/surveyinstance/SurveyInstanceServiceImpl.java index 1afd54805e..7e80b3b43b 100644 --- a/GAE/src/org/waterforpeople/mapping/app/gwt/server/surveyinstance/SurveyInstanceServiceImpl.java +++ b/GAE/src/org/waterforpeople/mapping/app/gwt/server/surveyinstance/SurveyInstanceServiceImpl.java @@ -174,8 +174,10 @@ public List updateQuestions( domainList.add(answer); } SurveyInstanceDAO dao = new SurveyInstanceDAO(); - dao.save(domainList); + + // this is not active - questionAnswerSummary objects are updated in bulk + // after the import in RawDataSpreadsheetImporter if (isApproved && processSummaries) { SurveyAttributeMappingDao mappingDao = new SurveyAttributeMappingDao(); // now send a change message for each item From d8b1ef9a21b4322d6e6009ed3b6feace62cc1b3e Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Mon, 24 Jun 2013 23:17:38 +0200 Subject: [PATCH 45/59] Issue #185 - update surveyInstance count on spreadsheet upload and deletion of surveyInstance --- .../mapping/app/web/RawDataRestServlet.java | 18 ++++++++++++++++++ .../mapping/dao/SurveyInstanceDAO.java | 19 ++++++++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/GAE/src/org/waterforpeople/mapping/app/web/RawDataRestServlet.java b/GAE/src/org/waterforpeople/mapping/app/web/RawDataRestServlet.java index 9d184672f4..836c15a219 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/RawDataRestServlet.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/RawDataRestServlet.java @@ -50,6 +50,8 @@ import com.gallatinsystems.survey.domain.Survey; import com.google.appengine.api.backends.BackendServiceFactory; import com.google.appengine.api.datastore.KeyFactory; +import com.google.appengine.api.taskqueue.Queue; +import com.google.appengine.api.taskqueue.QueueFactory; import com.google.appengine.api.taskqueue.TaskOptions; public class RawDataRestServlet extends AbstractRestApiServlet { @@ -77,10 +79,12 @@ protected RestResponse handleRequest(RestRequest req) throws Exception { if (RawDataImportRequest.SAVE_SURVEY_INSTANCE_ACTION.equals(importReq .getAction())) { List dtoList = new ArrayList(); + boolean isNew = false; if (importReq.getSurveyInstanceId() == null && importReq.getSurveyId() != null) { // if the instanceID is null, we need to create one createInstance(importReq); + isNew = true; } for (Map.Entry item : importReq .getQuestionAnswerMap().entrySet()) { @@ -93,7 +97,21 @@ protected RestResponse handleRequest(RestRequest req) throws Exception { qasDto.setCollectionDate(importReq.getCollectionDate()); dtoList.add(qasDto); } + sisi.updateQuestions(dtoList, true, false); + + if (isNew) { + SurveyInstanceDAO siDao = new SurveyInstanceDAO(); + List qasList = siDao.listQuestionAnswerStoreByType(new Long(importReq.getSurveyInstanceId()), "GEO"); + if (qasList != null && qasList.size() > 0) { + Queue summQueue = QueueFactory.getQueue("dataSummarization"); + summQueue.add(TaskOptions.Builder.withUrl("/app_worker/dataprocessor").param( + DataProcessorRequest.ACTION_PARAM, DataProcessorRequest.SURVEY_INSTANCE_SUMMARIZER) + .param("surveyInstanceId", importReq.getSurveyInstanceId() + "") + .param("qasId", qasList.get(0).getKey().getId() + "") + .param("delta",1 + "")); + } + } } else if (RawDataImportRequest.RESET_SURVEY_INSTANCE_ACTION .equals(importReq.getAction())) { SurveyInstance instance = instanceDao.getByKey(importReq diff --git a/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java b/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java index 689dbba530..09f5daeba6 100644 --- a/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java +++ b/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java @@ -29,6 +29,7 @@ import javax.jdo.Query; import org.waterforpeople.mapping.analytics.dao.SurveyQuestionSummaryDao; +import org.waterforpeople.mapping.app.web.DataProcessorRestServlet; import org.waterforpeople.mapping.app.web.dto.DataProcessorRequest; import org.waterforpeople.mapping.domain.QuestionAnswerStore; import org.waterforpeople.mapping.domain.Status.StatusCode; @@ -38,6 +39,7 @@ import com.gallatinsystems.framework.dao.BaseDAO; import com.gallatinsystems.framework.servlet.PersistenceFilter; import com.gallatinsystems.survey.dao.QuestionDao; +import com.gallatinsystems.survey.dao.SurveyUtils; import com.gallatinsystems.survey.domain.Question; import com.gallatinsystems.surveyal.dao.SurveyedLocaleDao; import com.gallatinsystems.surveyal.domain.SurveyalValue; @@ -226,7 +228,8 @@ public SurveyInstance save(Date collectionDate, DeviceFiles deviceFile, summQueue.add(TaskOptions.Builder.withUrl("/app_worker/dataprocessor").param( DataProcessorRequest.ACTION_PARAM, DataProcessorRequest.SURVEY_INSTANCE_SUMMARIZER) .param("surveyInstanceId", si.getKey().getId() + "") - .param("qasId", geoQasId + "")); + .param("qasId", geoQasId + "") + .param("delta",1 + "")); } return si; } @@ -509,9 +512,19 @@ public void deleteSurveyInstance(SurveyInstance item) { List qasList = siDao.listQuestionAnswerStore( surveyInstanceId, null); + // to account for the slim change if we have two geo questions in one surveyInstance + Boolean SISCount_updated = false; if (qasList != null && qasList.size() > 0) { // update the questionAnswerSummary counts for (QuestionAnswerStore qasItem : qasList) { + + // if the questionAnswerStore item is the GEO type, try to update + // the surveyInstanceSummary + if (Question.Type.GEO.toString().equals(qasItem.getType()) && !SISCount_updated){ + DataProcessorRestServlet.surveyInstanceSummarizer(surveyInstanceId, qasItem.getKey().getId(), new Integer(-1)); + SISCount_updated = true; + } + // if the questionAnswerStore item belongs to an OPTION type, // update the count Question q = qDao.getByKey(Long.parseLong(qasItem.getQuestionID())); @@ -528,6 +541,10 @@ public void deleteSurveyInstance(SurveyInstance item) { // delete the surveyInstance SurveyInstance instance = siDao.getByKey(item.getKey()); if (instance != null) { + // send notification + List ids = new ArrayList(); + ids.add(instance.getSurveyId()); + SurveyUtils.notifyReportService(ids, "invalidate"); siDao.delete(instance); } From 0bc2c166a64a7949a23c1a994ae363f6079cb7dc Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Tue, 25 Jun 2013 10:21:19 +0200 Subject: [PATCH 46/59] Fix bracket error in views.js --- Dashboard/app/js/lib/views/views.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dashboard/app/js/lib/views/views.js b/Dashboard/app/js/lib/views/views.js index 65bbbc6ac0..ef927d3ee5 100644 --- a/Dashboard/app/js/lib/views/views.js +++ b/Dashboard/app/js/lib/views/views.js @@ -613,11 +613,11 @@ FLOW.ColumnView = Ember.View.extend({ classNameBindings: ['isActiveAsc:sorting_asc', 'isActiveDesc:sorting_desc'], isActiveAsc: function() { - return this.get('item') === FLOW.tableColumnControl.get('selected')) && (FLOW.tableColumnControl.get('sortAscending') === true; + return this.get('item') === FLOW.tableColumnControl.get('selected') && FLOW.tableColumnControl.get('sortAscending') === true; }.property('item', 'FLOW.tableColumnControl.selected', 'FLOW.tableColumnControl.sortAscending').cacheable(), isActiveDesc: function() { - return this.get('item') === FLOW.tableColumnControl.get('selected')) && (FLOW.tableColumnControl.get('sortAscending') === false; + return this.get('item') === FLOW.tableColumnControl.get('selected') && FLOW.tableColumnControl.get('sortAscending') === false; }.property('item', 'FLOW.tableColumnControl.selected', 'FLOW.tableColumnControl.sortAscending').cacheable(), sort: function() { From d4303a3204ac2cae8306a86872974024d98fa918 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Perdomo?= Date: Fri, 28 Jun 2013 17:46:15 +0200 Subject: [PATCH 47/59] Issue #256 - Fixes issue with RestAuthFilter * Some request made when importing a spreadsheet use the same query parameter (e.g. questionId), the code was only building the querystring for calculating the hash * In the applet code, we "url encode" all parameters except ths `ts` to match the version in the RestAuthFilter * The RawDataImportRequest needs to be able to handle url encoded dates --- .../framework/servlet/RestAuthFilter.java | 19 ++++++++++++------- .../app/web/dto/RawDataImportRequest.java | 8 ++++++-- .../RawDataSpreadsheetImporter.java | 2 +- .../service/BulkDataServiceClient.java | 10 ++++++++-- 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/GAE/src/com/gallatinsystems/framework/servlet/RestAuthFilter.java b/GAE/src/com/gallatinsystems/framework/servlet/RestAuthFilter.java index 0bf7c5eea6..e8794a9062 100644 --- a/GAE/src/com/gallatinsystems/framework/servlet/RestAuthFilter.java +++ b/GAE/src/com/gallatinsystems/framework/servlet/RestAuthFilter.java @@ -100,23 +100,28 @@ private boolean isAuthorized(ServletRequest req) throws Exception { if (builder.length() > 0) { builder.append("&"); } - builder.append(name) - .append("=") - .append(URLEncoder.encode( - ((String[]) paramMap.get(name))[0], "UTF-8")); + if (RestRequest.TIMESTAMP_PARAM.equals(name)) { + String timestamp = ((String[]) paramMap.get(name))[0]; try { DateFormat df = new SimpleDateFormat( "yyyy/MM/dd HH:mm:ss"); df.setTimeZone(TimeZone.getTimeZone("GMT")); - incomingTimestamp = df.parse( - ((String[]) paramMap.get(name))[0]) - .getTime(); + incomingTimestamp = df.parse(timestamp).getTime(); } catch (Exception e) { log.warning("Recived rest api request with invalid timestamp"); return false; } } + String [] vals = ((String[]) paramMap.get(name)); + int count = 0; + for (String v : vals) { + if (count > 0) { + builder.append("&"); + } + builder.append(name).append("=").append(URLEncoder.encode(v, "UTF-8")); + count++; + } } else { incomingHash = ((String[]) paramMap.get(name))[0]; incomingHash = incomingHash.replaceAll(" ", "+"); diff --git a/GAE/src/org/waterforpeople/mapping/app/web/dto/RawDataImportRequest.java b/GAE/src/org/waterforpeople/mapping/app/web/dto/RawDataImportRequest.java index e00fe68457..1cfc1588f7 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/dto/RawDataImportRequest.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/dto/RawDataImportRequest.java @@ -16,6 +16,7 @@ package org.waterforpeople.mapping.app.web.dto; +import java.net.URLDecoder; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -199,8 +200,11 @@ protected void populateFields(HttpServletRequest req) throws Exception { } if (req.getParameter(COLLECTION_DATE_PARAM) != null && req.getParameter(COLLECTION_DATE_PARAM).trim().length() > 0 ) { - collectionDate = IN_FMT.get().parse(req.getParameter( - COLLECTION_DATE_PARAM).trim()); + String colDate = req.getParameter(COLLECTION_DATE_PARAM).trim(); + if (colDate.contains("%") || colDate.contains("+")) { + colDate = URLDecoder.decode(colDate, "UTF-8"); + } + collectionDate = IN_FMT.get().parse(colDate); } if (req.getParameter(SUBMITTER_PARAM) != null) { setSubmitter(req.getParameter(SUBMITTER_PARAM)); diff --git a/GAE/src/org/waterforpeople/mapping/dataexport/RawDataSpreadsheetImporter.java b/GAE/src/org/waterforpeople/mapping/dataexport/RawDataSpreadsheetImporter.java index 74f6231c9e..8e1f5bf320 100644 --- a/GAE/src/org/waterforpeople/mapping/dataexport/RawDataSpreadsheetImporter.java +++ b/GAE/src/org/waterforpeople/mapping/dataexport/RawDataSpreadsheetImporter.java @@ -57,7 +57,7 @@ public class RawDataSpreadsheetImporter implements DataImporter { private static final String SERVLET_URL = "/rawdatarestapi"; private static final String DEFAULT_LOCALE = "en"; public static final String SURVEY_CONFIG_KEY = "surveyId"; - protected static final String KEY_PARAM = "k"; + protected static final String KEY_PARAM = "apiKey"; private static final Map SAVING_DATA; private static final Map COMPLETE; private Long surveyId; diff --git a/GAE/src/org/waterforpeople/mapping/dataexport/service/BulkDataServiceClient.java b/GAE/src/org/waterforpeople/mapping/dataexport/service/BulkDataServiceClient.java index f7e2c41b62..44c06f7160 100644 --- a/GAE/src/org/waterforpeople/mapping/dataexport/service/BulkDataServiceClient.java +++ b/GAE/src/org/waterforpeople/mapping/dataexport/service/BulkDataServiceClient.java @@ -20,6 +20,7 @@ import java.io.DataOutputStream; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.UnsupportedEncodingException; import java.net.HttpURLConnection; import java.net.URL; import java.net.URLEncoder; @@ -1123,7 +1124,7 @@ private static String fetchDataFromServerGET(String fullUrl) return result; } - private static String sortQueryString(String queryString) { + private static String sortQueryString(String queryString) throws UnsupportedEncodingException { String[] parts = queryString.split("&"); List pairs = new ArrayList(); for (int i = 0; i < parts.length; i++) { @@ -1153,7 +1154,12 @@ private static String sortQueryString(String queryString) { if (result.length() > 0) { result.append("&"); } - result.append(nvp.getName()).append("=").append(nvp.getValue()); + result.append(nvp.getName()).append("="); + if(nvp.getName().equals(RestRequest.TIMESTAMP_PARAM)) { + result.append(nvp.getValue()); + } else { + result.append(URLEncoder.encode(nvp.getValue(), "UTF-8")); + } } return result.toString(); } From a84b3d60b0463c867663cca1f8e1a0480f0b869a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Perdomo?= Date: Mon, 1 Jul 2013 09:22:10 +0200 Subject: [PATCH 48/59] Issue #256 - Request on data is URLEncoded * We need to decode the data before spliting it --- .../mapping/app/web/dto/RawDataImportRequest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/GAE/src/org/waterforpeople/mapping/app/web/dto/RawDataImportRequest.java b/GAE/src/org/waterforpeople/mapping/app/web/dto/RawDataImportRequest.java index 1cfc1588f7..b6f7bd3287 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/dto/RawDataImportRequest.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/dto/RawDataImportRequest.java @@ -147,7 +147,7 @@ protected void populateFields(HttpServletRequest req) throws Exception { } if (req.getParameter(FIXED_FIELD_VALUE_PARAM) != null) { fixedFieldValues = new ArrayList(); - String[] vals = req.getParameter(FIXED_FIELD_VALUE_PARAM).split( + String[] vals = URLDecoder.decode(req.getParameter(FIXED_FIELD_VALUE_PARAM), "UTF-8").split( FIELD_VAL_DELIMITER); for (int i = 0; i < vals.length; i++) { fixedFieldValues.add(vals[i]); @@ -157,7 +157,7 @@ protected void populateFields(HttpServletRequest req) throws Exception { String[] answers = req.getParameterValues(QUESTION_ID_PARAM); if (answers != null) { for (int i = 0; i < answers.length; i++) { - String[] parts = answers[i].split("\\|"); + String[] parts = URLDecoder.decode(answers[i], "UTF-8").split("\\|"); String qId = null; String val = null; String type = null; From 1b2e8319f12a8050a63de7f6fc703713929d61a9 Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Mon, 1 Jul 2013 16:53:27 +0200 Subject: [PATCH 49/59] Issue #185 - use QAS_PAGE_SIZE instead of '500' --- .../mapping/app/web/DataProcessorRestServlet.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java b/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java index 3afc974ffc..0f66c3def2 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java @@ -556,7 +556,7 @@ public static void sendProjectUpdateTask(String country, String cursor) { * fixes wrong Types in questionAnswerStore objects. When cleaned data is * uploaded using an excel file, the type of the answer is set according to * the type of the question, while the device sets the type according to a - * different convention. The action handles 500 items in one call, and + * different convention. The action handles QAS_PAGE_SIZE items in one call, and * invokes new tasks as necessary if there are more items. * * @param cursor @@ -566,7 +566,7 @@ public static void fixOptions2Values(String cursorString) { SurveyInstanceDAO siDao = new SurveyInstanceDAO(); QuestionAnswerStoreDao qasDao = new QuestionAnswerStoreDao(); List qasList = siDao.listQAOptions(cursorString, - 500, "OPTION", "FREE_TEXT", "NUMBER", "SCAN", "PHOTO"); + QAS_PAGE_SIZE, "OPTION", "FREE_TEXT", "NUMBER", "SCAN", "PHOTO"); List qasChangedList = new ArrayList(); log.log(Level.INFO, "Running fixOptions2Values, cursor at " + cursorString); @@ -589,7 +589,7 @@ public static void fixOptions2Values(String cursorString) { } qasDao.save(qasChangedList); // if there are more, invoke another task - if (qasList.size() == 500) { + if (qasList.size() == QAS_PAGE_SIZE) { log.log(Level.INFO, "invoking another fixOptions task"); Queue queue = QueueFactory.getDefaultQueue(); TaskOptions options = TaskOptions.Builder From 90084fd182b4fdd15c4d9155b0472370a318bdda Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Mon, 1 Jul 2013 17:53:20 +0200 Subject: [PATCH 50/59] Issue #250 and #185 - don't use cursor for lookup, it doesn't work because we use OR in the query --- .../mapping/app/web/DataProcessorRestServlet.java | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java b/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java index 0f66c3def2..b8ec43acd4 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/DataProcessorRestServlet.java @@ -115,7 +115,7 @@ protected RestResponse handleRequest(RestRequest req) throws Exception { trimOptions(); } else if (DataProcessorRequest.FIX_OPTIONS2VALUES_ACTION .equalsIgnoreCase(dpReq.getAction())) { - fixOptions2Values(dpReq.getCursor()); + fixOptions2Values(); } else if (DataProcessorRequest.SURVEY_INSTANCE_SUMMARIZER .equalsIgnoreCase(dpReq.getAction())) { surveyInstanceSummarizer(dpReq.getSurveyInstanceId(), @@ -562,18 +562,15 @@ public static void sendProjectUpdateTask(String country, String cursor) { * @param cursor * @author M.T. Westra */ - public static void fixOptions2Values(String cursorString) { + public static void fixOptions2Values() { SurveyInstanceDAO siDao = new SurveyInstanceDAO(); QuestionAnswerStoreDao qasDao = new QuestionAnswerStoreDao(); - List qasList = siDao.listQAOptions(cursorString, + List qasList = siDao.listQAOptions(null, QAS_PAGE_SIZE, "OPTION", "FREE_TEXT", "NUMBER", "SCAN", "PHOTO"); List qasChangedList = new ArrayList(); - log.log(Level.INFO, "Running fixOptions2Values, cursor at " - + cursorString); + log.log(Level.INFO, "Running fixOptions2Values"); if (qasList != null) { - String cursor = SurveyInstanceDAO.getCursor(qasList); for (QuestionAnswerStore qas : qasList) { - if (Question.Type.OPTION.toString().equals(qas.getType()) || Question.Type.NUMBER.toString() .equals(qas.getType()) @@ -595,8 +592,7 @@ public static void fixOptions2Values(String cursorString) { TaskOptions options = TaskOptions.Builder .withUrl("/app_worker/dataprocessor") .param(DataProcessorRequest.ACTION_PARAM, - DataProcessorRequest.FIX_OPTIONS2VALUES_ACTION) - .param(DataProcessorRequest.CURSOR_PARAM, cursor); + DataProcessorRequest.FIX_OPTIONS2VALUES_ACTION); queue.add(options); } } From 1753aae2e6e8d80dc37b6b9499051245238e5a4f Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Tue, 2 Jul 2013 10:12:38 +0200 Subject: [PATCH 51/59] Issue #218 - improve implementation of deleting surveyInstances --- .../mapping/dao/SurveyInstanceDAO.java | 35 +++++++++++-------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java b/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java index 09f5daeba6..9ddceff35d 100644 --- a/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java +++ b/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java @@ -505,6 +505,7 @@ public List listQuestionAnswerStoreForQuestion( // TODO update lastSurveyalInstanceId in surveydLocale objects public void deleteSurveyInstance(SurveyInstance item) { SurveyInstanceDAO siDao = new SurveyInstanceDAO(); + QuestionAnswerStoreDao qasDao = new QuestionAnswerStoreDao(); SurveyedLocaleDao localeDao = new SurveyedLocaleDao(); QuestionDao qDao = new QuestionDao(); Long surveyInstanceId = item.getKey().getId(); @@ -535,9 +536,12 @@ public void deleteSurveyInstance(SurveyInstance item) { // delete the questionAnswerStore objects in a single datastore // operation - siDao.delete(qasList); + qasDao.delete(qasList); } + // get the instances that have contributed to the Locale, for later use + List instancesForLocale = siDao.listByProperty("surveyedLocaleId",item.getSurveyedLocaleId(),"Long"); + // delete the surveyInstance SurveyInstance instance = siDao.getByKey(item.getKey()); if (instance != null) { @@ -545,24 +549,27 @@ public void deleteSurveyInstance(SurveyInstance item) { List ids = new ArrayList(); ids.add(instance.getSurveyId()); SurveyUtils.notifyReportService(ids, "invalidate"); + + Long localeId = instance.getSurveyedLocaleId(); + + // now delete the surveyInstance siDao.delete(instance); - } - // delete surveyalValue items - List valsForInstance = localeDao - .listSurveyalValuesByInstance(surveyInstanceId); - if (valsForInstance != null && valsForInstance.size() > 0) { - Long localeId = valsForInstance.get(0).getSurveyedLocaleId(); - List valsForLocale = localeDao - .listValuesByLocale(localeId); - - // now see if there are any other values for the same locale - if (valsForLocale != null && valsForLocale.size() <= valsForInstance.size()) { - // if there are no other values, delete the locale + // delete the corresponding surveyalValues + // SurveyalValues don't have a Dao of their own, therefore we use the SurveyedLocaleDao + List valsForInstance = localeDao + .listSurveyalValuesByInstance(instance.getKey().getId()); + if (valsForInstance != null && valsForInstance.size() > 0) { + localeDao.delete(valsForInstance); + } + + // if there is only one surveyInstance that has contributed to this Locale, + // we can delete the SurveyedLocale. The values should already have been deleted + // in the previous step. + if (instancesForLocale != null && instancesForLocale.size() == 1) { SurveyedLocale l = localeDao.getByKey(localeId); localeDao.delete(l); } - localeDao.delete(valsForInstance); } } From 1a5123d49fe97dd26f33b69a214e706101eee08f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Perdomo?= Date: Tue, 2 Jul 2013 09:21:41 +0200 Subject: [PATCH 52/59] Smoke tests for REST apiKey functionality * Code originally developed by Neha Chriss --- tests/clojure/akvo-tests/README.md | 31 +++++++++++++ tests/clojure/akvo-tests/project.clj | 8 ++++ tests/clojure/akvo-tests/resources/config.clj | 1 + .../akvo-tests/src/akvo_tests/api_key.clj | 44 +++++++++++++++++++ .../akvo-tests/test/akvo_tests/core_test.clj | 7 +++ 5 files changed, 91 insertions(+) create mode 100644 tests/clojure/akvo-tests/README.md create mode 100644 tests/clojure/akvo-tests/project.clj create mode 100644 tests/clojure/akvo-tests/resources/config.clj create mode 100644 tests/clojure/akvo-tests/src/akvo_tests/api_key.clj create mode 100644 tests/clojure/akvo-tests/test/akvo_tests/core_test.clj diff --git a/tests/clojure/akvo-tests/README.md b/tests/clojure/akvo-tests/README.md new file mode 100644 index 0000000000..006a82b150 --- /dev/null +++ b/tests/clojure/akvo-tests/README.md @@ -0,0 +1,31 @@ +# akvo-tests + +Set of smoke tests for [Akvo FLOW](https://github.com/akvo/akvo-flow) + +## Usage + +* Install [Leiningen](http://leiningen.org/) +* Run the tests: + + lein deps + lein test + + +## License + +Copyright (C) 2013 Stichting Akvo (Akvo Foundation) + +This file is part of Akvo FLOW. + +Akvo FLOW is free software: you can redistribute it and modify +it under the terms of the GNU Affero General Public License (AGPL) +as published by the Free Software Foundation, either version 3 of the +License or any later version. + +Akvo FLOW is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License included below for more details. + +The license can also be seen at . + diff --git a/tests/clojure/akvo-tests/project.clj b/tests/clojure/akvo-tests/project.clj new file mode 100644 index 0000000000..42aa19d6b7 --- /dev/null +++ b/tests/clojure/akvo-tests/project.clj @@ -0,0 +1,8 @@ +(defproject akvo-tests "0.1.0" + :description "Akvo FLOW smoke tests" + :url "https://github.com/akvo/akvo-flow" + :license {:name "GNU Affero General Public License" + :url "https://www.gnu.org/licenses/agpl"} + :dependencies [[org.clojure/clojure "1.5.1"] + [clj-http "0.7.3"] + [org.springframework.security.oauth/spring-security-oauth "1.0.4.RELEASE"]]) diff --git a/tests/clojure/akvo-tests/resources/config.clj b/tests/clojure/akvo-tests/resources/config.clj new file mode 100644 index 0000000000..0f94f2fd8b --- /dev/null +++ b/tests/clojure/akvo-tests/resources/config.clj @@ -0,0 +1 @@ +{:apiKey {}} diff --git a/tests/clojure/akvo-tests/src/akvo_tests/api_key.clj b/tests/clojure/akvo-tests/src/akvo_tests/api_key.clj new file mode 100644 index 0000000000..878bc6ac00 --- /dev/null +++ b/tests/clojure/akvo-tests/src/akvo_tests/api_key.clj @@ -0,0 +1,44 @@ +(ns akvo-tests.api-key + "ApiKey Smoke Test" + (:require [clj-http.client :as client]) + (:import java.util.Date + java.text.SimpleDateFormat + java.net.URLEncoder + java.util.TimeZone + javax.crypto.Mac + javax.crypto.spec.SecretKeySpec + org.springframework.security.oauth.common.signature.HMAC_SHA1SignatureMethod)) + +(def date (Date.)) + +(defn tsgen + "Generates ts parameter for flowservices request" + [] + (doto (SimpleDateFormat. "yyyy/MM/dd HH:mm:ss") + (.setTimeZone + (TimeZone/getTimeZone "GMT")))) + +(defn tsenc + "URL Encodes ts parameter for flowservices request" + [] + (URLEncoder/encode (.format (tsgen) date) "UTF-8")) + +(defn genapikey [secret sign] + (.sign (HMAC_SHA1SignatureMethod. (SecretKeySpec. (.getBytes "foo") "HMACSHA1")) "bar")) + +(def query + {"action" "listInstance" "includeDate" "true" "surveyId" "394002" "ts" (tsenc) "h" (genapikey "foo" "bar")}) + +(def querynone + {"action" "listInstance" "includeDate" "true" "surveyId" "394002"}) + +(defn querystring [params] + (loop [ks (sort (keys params)) + result "" ] + (if-not ks + (subs result 0 (- (count result) 1)) + (recur (next ks) + (str result (first ks) "=" (params (first ks)) "&" ))))) + + +(client/get (str "http://flowaglimmerofhope.appspot.com/databackout?" (querystring querynone))) diff --git a/tests/clojure/akvo-tests/test/akvo_tests/core_test.clj b/tests/clojure/akvo-tests/test/akvo_tests/core_test.clj new file mode 100644 index 0000000000..6773abf172 --- /dev/null +++ b/tests/clojure/akvo-tests/test/akvo_tests/core_test.clj @@ -0,0 +1,7 @@ +(ns akvo-tests.core-test + (:require [clojure.test :refer :all] + [akvo-tests.api-key :refer :all])) + +(deftest a-test + (testing "Dummy" + (is (= 0 0)))) From 2565c8e5f0bbfb59996b1c6f68c766c030bd50be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Perdomo?= Date: Tue, 2 Jul 2013 09:24:38 +0200 Subject: [PATCH 53/59] Add Clojure generated folders to .gitignore --- .gitignore | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index e02080fe8e..f11e9c0ce2 100644 --- a/.gitignore +++ b/.gitignore @@ -55,12 +55,9 @@ WFPMapping/device/survey/gen/com/gallatinsystems/survey/device/R.java deploy gwt-unitcache -# Reports service -reports/.externalToolBuilders/* -reports/.lein-failures -reports/.lein-repl-history -reports/bin/* -reports/target/* +# Clojure smoke tests +tests/clojure/akvo-tests/.lein-failures +tests/clojure/akvo-tests/target/* # Vagrant From bd4824b1aaabd4f95c8bf935554643fd591856dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Perdomo?= Date: Tue, 2 Jul 2013 11:41:03 +0200 Subject: [PATCH 54/59] Issue #256 - Refactors initial code and adds test for REST api * Refactor the initial code * 2nd implementation on how to make authenticated requests * TODO: The secret should be based on the config repository Please enter the commit message for your changes. Lines starting --- tests/clojure/akvo-tests/resources/config.clj | 11 ++++- .../akvo-tests/src/akvo_tests/api_key.clj | 47 +++++++++---------- .../akvo-tests/test/akvo_tests/core_test.clj | 23 +++++++-- 3 files changed, 51 insertions(+), 30 deletions(-) diff --git a/tests/clojure/akvo-tests/resources/config.clj b/tests/clojure/akvo-tests/resources/config.clj index 0f94f2fd8b..62d98c6cd1 100644 --- a/tests/clojure/akvo-tests/resources/config.clj +++ b/tests/clojure/akvo-tests/resources/config.clj @@ -1 +1,10 @@ -{:apiKey {}} +{:test-api-key [{"url" "http://akvoflowsandbox.appspot.com/surveyrestapi" + "action" "listGroups" + "surveyId" "1591009"} + {"url" "http://akvoflowsandbox.appspot.com/surveyrestapi" + "action" "listQuestions" + "questionGroupId" "1617008"} + {"url" "http://akvoflowsandbox.appspot.com/databackout" + "action" "listInstance" + "includeDate" "true" + "surveyId" "1591009"}]} diff --git a/tests/clojure/akvo-tests/src/akvo_tests/api_key.clj b/tests/clojure/akvo-tests/src/akvo_tests/api_key.clj index 878bc6ac00..a21dddcea5 100644 --- a/tests/clojure/akvo-tests/src/akvo_tests/api_key.clj +++ b/tests/clojure/akvo-tests/src/akvo_tests/api_key.clj @@ -5,40 +5,37 @@ java.text.SimpleDateFormat java.net.URLEncoder java.util.TimeZone - javax.crypto.Mac javax.crypto.spec.SecretKeySpec org.springframework.security.oauth.common.signature.HMAC_SHA1SignatureMethod)) -(def date (Date.)) - -(defn tsgen - "Generates ts parameter for flowservices request" - [] +(defn- dformat [] (doto (SimpleDateFormat. "yyyy/MM/dd HH:mm:ss") (.setTimeZone (TimeZone/getTimeZone "GMT")))) -(defn tsenc - "URL Encodes ts parameter for flowservices request" +(defn generate-timestamp + "Returns the current date using the format yyyy/MM/dd HH:mm:ss" [] - (URLEncoder/encode (.format (tsgen) date) "UTF-8")) + (.format (dformat) (Date.))) -(defn genapikey [secret sign] - (.sign (HMAC_SHA1SignatureMethod. (SecretKeySpec. (.getBytes "foo") "HMACSHA1")) "bar")) +(defn generate-apikey + "Generates a HMAC-SHA1 signature based on a secret" + [secret sign] + (.sign (HMAC_SHA1SignatureMethod. (SecretKeySpec. (.getBytes secret) "HMAC-SHA1")) sign)) -(def query - {"action" "listInstance" "includeDate" "true" "surveyId" "394002" "ts" (tsenc) "h" (genapikey "foo" "bar")}) - -(def querynone - {"action" "listInstance" "includeDate" "true" "surveyId" "394002"}) +(defn generate-query-string + "Returns a string suitable for a GET request based on a Map of parameters + The parameters are sorted by key, Note: The values are url encoded e.g. -(defn querystring [params] - (loop [ks (sort (keys params)) - result "" ] - (if-not ks - (subs result 0 (- (count result) 1)) - (recur (next ks) - (str result (first ks) "=" (params (first ks)) "&" ))))) + {\"includeDate\" \"true\"} + \"action\" \"listInstance\" + \"ts\" \"2013/07/02 08:09:55\"} - -(client/get (str "http://flowaglimmerofhope.appspot.com/databackout?" (querystring querynone))) + Returns: action=listInstance&includeDate=true&ts=2013%2F07%2F02+08%3A09%3A55" + [params] + (loop [ks (sort (keys params)) + result ""] + (if-not ks + (subs result 0 (- (count result) 1)) ;; removing the last & + (recur (next ks) + (str result (first ks) "=" (URLEncoder/encode (params (first ks)) "UTF-8") "&" ))))) \ No newline at end of file diff --git a/tests/clojure/akvo-tests/test/akvo_tests/core_test.clj b/tests/clojure/akvo-tests/test/akvo_tests/core_test.clj index 6773abf172..5f653c36bf 100644 --- a/tests/clojure/akvo-tests/test/akvo_tests/core_test.clj +++ b/tests/clojure/akvo-tests/test/akvo_tests/core_test.clj @@ -1,7 +1,22 @@ (ns akvo-tests.core-test (:require [clojure.test :refer :all] - [akvo-tests.api-key :refer :all])) + [clojure.java.io :as io :only (resource)] + [akvo-tests.api-key :as api :refer :all] + [clj-http.client :as client :only (get)])) -(deftest a-test - (testing "Dummy" - (is (= 0 0)))) +(def cfg (read-string (slurp (io/resource "config.clj")))) + +(def secret "") ;; FIXME + +(deftest api-key-test + (testing "Testing authenticated requests" + (doseq [data (:test-api-key cfg)] + (let [url (data "url") + base-params (dissoc data "url") + params (assoc base-params "ts" (api/generate-timestamp)) + base-qs (api/generate-query-string params) + req-url (str url "?" base-qs "&h=" (api/generate-apikey secret base-qs))] + (try + (is (= 200 (:status (client/get req-url)))) + (catch Exception e + (prn e))))))) From 6eb9bd01ca122a964a147b19d7a699c534b02380 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Perdomo?= Date: Tue, 2 Jul 2013 12:15:58 +0200 Subject: [PATCH 55/59] Adds eclipse folders and repl files to .gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f11e9c0ce2..bc326f8b27 100644 --- a/.gitignore +++ b/.gitignore @@ -57,8 +57,9 @@ gwt-unitcache # Clojure smoke tests tests/clojure/akvo-tests/.lein-failures +tests/clojure/akvo-tests/.lein-repl-history tests/clojure/akvo-tests/target/* - +tests/clojure/akvo-tests/bin/* # Vagrant .vagrant From d76280aad6a8ebb28904bc6060a58c70c238059b Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Tue, 2 Jul 2013 13:54:52 +0200 Subject: [PATCH 56/59] Issue #218 - improvements to implementation of surveyInstanceDao --- .../mapping/dao/SurveyInstanceDAO.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java b/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java index 9ddceff35d..de3f755f48 100644 --- a/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java +++ b/GAE/src/org/waterforpeople/mapping/dao/SurveyInstanceDAO.java @@ -508,22 +508,23 @@ public void deleteSurveyInstance(SurveyInstance item) { QuestionAnswerStoreDao qasDao = new QuestionAnswerStoreDao(); SurveyedLocaleDao localeDao = new SurveyedLocaleDao(); QuestionDao qDao = new QuestionDao(); + BaseDAO svDao = new BaseDAO(SurveyalValue.class); Long surveyInstanceId = item.getKey().getId(); List qasList = siDao.listQuestionAnswerStore( surveyInstanceId, null); // to account for the slim change if we have two geo questions in one surveyInstance - Boolean SISCount_updated = false; + boolean sisCountUpdated = false; if (qasList != null && qasList.size() > 0) { // update the questionAnswerSummary counts for (QuestionAnswerStore qasItem : qasList) { // if the questionAnswerStore item is the GEO type, try to update // the surveyInstanceSummary - if (Question.Type.GEO.toString().equals(qasItem.getType()) && !SISCount_updated){ - DataProcessorRestServlet.surveyInstanceSummarizer(surveyInstanceId, qasItem.getKey().getId(), new Integer(-1)); - SISCount_updated = true; + if (Question.Type.GEO.toString().equals(qasItem.getType()) && !sisCountUpdated){ + DataProcessorRestServlet.surveyInstanceSummarizer(surveyInstanceId, qasItem.getKey().getId(), -1); + sisCountUpdated = true; } // if the questionAnswerStore item belongs to an OPTION type, @@ -555,12 +556,10 @@ public void deleteSurveyInstance(SurveyInstance item) { // now delete the surveyInstance siDao.delete(instance); - // delete the corresponding surveyalValues - // SurveyalValues don't have a Dao of their own, therefore we use the SurveyedLocaleDao List valsForInstance = localeDao .listSurveyalValuesByInstance(instance.getKey().getId()); if (valsForInstance != null && valsForInstance.size() > 0) { - localeDao.delete(valsForInstance); + svDao.delete(valsForInstance); } // if there is only one surveyInstance that has contributed to this Locale, From e438dc78e37905e7f803ecec6fa9eba06ccd959b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Perdomo?= Date: Tue, 2 Jul 2013 16:21:24 +0200 Subject: [PATCH 57/59] Issue #255 - User Long.valueOf and remove SupressLint annotation --- .../survey/device/activity/UserEditActivity.java | 4 +--- .../gallatinsystems/survey/device/dao/SurveyDbAdapter.java | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/activity/UserEditActivity.java b/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/activity/UserEditActivity.java index 01974906e1..98696fa29e 100644 --- a/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/activity/UserEditActivity.java +++ b/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/activity/UserEditActivity.java @@ -16,7 +16,6 @@ package com.gallatinsystems.survey.device.activity; -import android.annotation.SuppressLint; import android.app.Activity; import android.database.Cursor; import android.os.Bundle; @@ -43,7 +42,6 @@ public class UserEditActivity extends Activity { private Long userId; private SurveyDbAdapter databaseAdaptor; - @SuppressLint("UseValueOf") @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); @@ -62,7 +60,7 @@ protected void onCreate(Bundle savedInstanceState) { .getLong(ConstantUtil.ID_KEY) : null; if (userId == null || userId == 0L) { Bundle extras = getIntent().getExtras(); - userId = extras != null ? new Long(extras + userId = extras != null ? Long.valueOf(extras .getString(ConstantUtil.ID_KEY)) : null; } diff --git a/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/dao/SurveyDbAdapter.java b/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/dao/SurveyDbAdapter.java index 4be5e4331e..517f3f2d9f 100644 --- a/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/dao/SurveyDbAdapter.java +++ b/WFPMapping/device/survey/src/com/gallatinsystems/survey/device/dao/SurveyDbAdapter.java @@ -22,7 +22,6 @@ import java.util.HashSet; import java.util.UUID; -import android.annotation.SuppressLint; import android.content.ContentValues; import android.content.Context; import android.database.Cursor; @@ -179,8 +178,7 @@ public class SurveyDbAdapter { static class DatabaseHelper extends SQLiteOpenHelper { private static SQLiteDatabase database; - @SuppressLint("UseValueOf") - private static volatile Long LOCK_OBJ = new Long(1); + private static volatile Long LOCK_OBJ = 1L; private volatile static int instanceCount = 0; private Context context; From 56242ba7b3452a2298f20db7bb2f45a6e34e7b2f Mon Sep 17 00:00:00 2001 From: Mark Tiele Westra Date: Wed, 3 Jul 2013 09:56:26 +0200 Subject: [PATCH 58/59] Issue #185 - remove commented out code dataSummarization * The surveyInstance summarization now happens in the surveyInstanceDao, so we avoid double counting when a surveyInstance already exists --- GAE/src/org/waterforpeople/mapping/app/web/TaskServlet.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/GAE/src/org/waterforpeople/mapping/app/web/TaskServlet.java b/GAE/src/org/waterforpeople/mapping/app/web/TaskServlet.java index 8f92798812..14c9ebdbc2 100644 --- a/GAE/src/org/waterforpeople/mapping/app/web/TaskServlet.java +++ b/GAE/src/org/waterforpeople/mapping/app/web/TaskServlet.java @@ -479,10 +479,6 @@ private void ingestFile(TaskRequest req) { log.info("Received Task Queue calls for surveyInstanceKey: " + instance.getKey().getId() + ""); aph.processSurveyInstance(instance.getKey().getId() + ""); -// summQueue.add(TaskOptions.Builder.withUrl("/app_worker/datasummarization").param( -// "objectKey", instance.getKey().getId() + "").param( -// "type", "SurveyInstance")); -// // process the "new" domain structure defaultQueue.add(TaskOptions.Builder.withUrl("/app_worker/surveyalservlet").param( SurveyalRestRequest.ACTION_PARAM, From 48fe5433aa2fb966a3485f0f01eedc439eef8089 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Perdomo?= Date: Wed, 3 Jul 2013 11:36:01 +0200 Subject: [PATCH 59/59] Issue #272 - Verify and use the parameter * The InstanceConfigurator was not using the key $restApiKey in appengine-web.vm * We enable rest security by default * We verify that the key is not 'test' nor empty string --- .../instancecreator/app/InstanceConfigurator.java | 3 ++- GAE/war/appengine-web.vm | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/GAE/src/com/gallatinsystems/instancecreator/app/InstanceConfigurator.java b/GAE/src/com/gallatinsystems/instancecreator/app/InstanceConfigurator.java index ac94f734bf..6c3f5c7623 100644 --- a/GAE/src/com/gallatinsystems/instancecreator/app/InstanceConfigurator.java +++ b/GAE/src/com/gallatinsystems/instancecreator/app/InstanceConfigurator.java @@ -25,6 +25,7 @@ import java.util.HashMap; import java.util.Map.Entry; import java.util.TreeMap; +import java.util.UUID; import java.util.logging.Level; import java.util.logging.Logger; @@ -113,7 +114,7 @@ public static void main(String[] args) { String localLocation = args[12]; ic.addAttribute("keystore", args[13]); ic.addAttribute("mapsApiKey", args[14]); - ic.addAttribute("restApiKey",args[15]); + ic.addAttribute("restApiKey", args[15].equals("test") || args[15].equals("") ? UUID.randomUUID().toString() : args[15]); localLocation = ic.createLocalDeployDir(localLocation, args[2]); diff --git a/GAE/war/appengine-web.vm b/GAE/war/appengine-web.vm index 788b623cb2..16869932c5 100644 --- a/GAE/war/appengine-web.vm +++ b/GAE/war/appengine-web.vm @@ -64,8 +64,8 @@ - - + +
{{t _id}} {{t _survey}} {{t _submitter}}
{{#view FLOW.DataNumView contentBinding="SI"}}{{view.rownum}}{{/view}} {{SI.keyId}} {{SI.surveyCode}} {{SI.submitterName}}