Merge branch 'development' of https://github.com/loklak/loklak_server.git into development

Author: Orbiter

bin/upgrade.sh

@@ -1,9 +1,7 @@
 #!/usr/bin/env sh
 cd `dirname $0`/..
 echo "loading latest code changes"
-git pull origin master
-echo "clean up"
-./gradlew clean
-echo "building loklak"
-./gradlew build
+git pull -r
+echo "assembling loklak"
+./gradlew assemble
 bin/restart.sh

src/org/json/JSONObject.java

@@ -297,7 +297,7 @@ public JSONObject(JSONTokener x) throws JSONException {
      */
     public JSONObject(Map<?, ?> m) {
         if (m == null) {
-            this.map = new HashMap<String, Object>();
+            this.map = new LinkedHashMap<String, Object>();
         } else {
             this.map = new HashMap<String, Object>(m.size());
         	for (final Entry<?, ?> e : m.entrySet()) {

src/org/loklak/Caretaker.java

@@ -107,7 +107,7 @@ public void run() {
             if (SuggestServlet.cache.size() > 100) SuggestServlet.cache.clear();
 
             // sleep a bit to prevent that the DoS limit fires at backend server
-            try {Thread.sleep(busy ? 500 : 5000);} catch (InterruptedException e) {}
+            try {Thread.sleep(busy ? 500 : 10000);} catch (InterruptedException e) {}
             if (!this.shallRun) break beat;
             busy = false;
 
@@ -165,32 +165,36 @@ public void run() {
                     try {
                         pendingQueries = LoklakServer.harvester.get_harvest_queries();
                     } catch (IOException e) {
+                        DAO.severe("FAIL SUGGESTIONS cannot get queries from backend: " + e.getMessage(), e);
+                        break hloop;
                     }
 
-                    // in case we have queries
-                    if (pendingQueries > 0) {
-                        Thread[] rts = new Thread[Math.min(pendingQueries, retrieval_forbackend_concurrency)];
-                        final AtomicInteger acccount = new AtomicInteger(0);
-                        for (int j = 0; j < rts.length; j++) {
-                            rts[j] = new Thread() {
-                                public void run() {
-                                    TwitterTimeline tl = LoklakServer.harvester.harvest_timeline();
-                                    if (tl != null && tl.getQuery() != null) {
-                                        /* Thread t = */ LoklakServer.harvester.push_timeline_to_backend(tl);
-                                    }
-                                    int count = tl == null ? 0 : tl.size();
-                                    acccount.addAndGet(count);
+                    // without queries we cannot do harvesting
+                    if (pendingQueries == 0) break hloop;
+                    
+                    // start harvesting
+                    Thread[] rts = new Thread[Math.min(pendingQueries, retrieval_forbackend_concurrency)];
+                    final AtomicInteger acccount = new AtomicInteger(0);
+                    for (int j = 0; j < rts.length; j++) {
+                        rts[j] = new Thread() {
+                            public void run() {
+                                TwitterTimeline tl = LoklakServer.harvester.harvest_timeline();
+                                if (tl != null && tl.getQuery() != null) {
+                                    /* Thread t = */ LoklakServer.harvester.push_timeline_to_backend(tl);
                                 }
-                            };
-                            rts[j].start();
-                            try {Thread.sleep(retrieval_forbackend_sleep_base + random.nextInt(retrieval_forbackend_sleep_randomoffset));} catch (InterruptedException e) {}
-                        }
-                        for (Thread t: rts) t.join();
-                        if (acccount.get() < 0) break hloop;
+                                int count = tl == null ? 0 : tl.size();
+                                acccount.addAndGet(count);
+                            }
+                        };
+                        rts[j].start();
                         try {Thread.sleep(retrieval_forbackend_sleep_base + random.nextInt(retrieval_forbackend_sleep_randomoffset));} catch (InterruptedException e) {}
                     }
+                    for (Thread t: rts) t.join();
+                    if (acccount.get() < 0) break hloop;
+                    try {Thread.sleep(retrieval_forbackend_sleep_base + random.nextInt(retrieval_forbackend_sleep_randomoffset));} catch (InterruptedException e) {}
+                
+                    busy = true;
                 }
-                busy = true;
             }
 
             // run some crawl steps

src/org/loklak/LoklakServer.java

@@ -104,7 +104,6 @@
 import org.loklak.api.p2p.HelloService;
 import org.loklak.api.p2p.PeersServlet;
 import org.loklak.api.p2p.PushServlet;
-import org.loklak.api.search.ConsoleService;
 import org.loklak.api.search.EventBriteCrawlerService;
 import org.loklak.api.search.GenericScraper;
 import org.loklak.api.search.GithubProfileScraper;
@@ -607,7 +606,6 @@ private static void setServerHandler(File dataFile){
                 HelloService.class,
 
                 // search
-                ConsoleService.class,
                 EventBriteCrawlerService.class,
                 MeetupsCrawlerService.class,
                 RSSReaderService.class,

src/org/loklak/api/search/ConsoleService.java

@@ -1302,7 +1302,11 @@ public static ResultList<QueryEntry> SearchLocalQueries(final String q, final in
         ResultList<Map<String, Object>> result = elasticsearch_client.fuzzyquery(IndexName.queries.name(), "query", q, resultCount, sort_field, default_sort_type, sort_order, since, until, range_field);
         queries.setHits(result.getHits());
         for (Map<String, Object> map: result) {
-            queries.add(new QueryEntry(new JSONObject(map)));
+            QueryEntry qe = new QueryEntry(new JSONObject(map));
+            // check a flag value for queries that probably never get new messages
+            if (qe.getMessagePeriod() != QueryEntry.DAY_MILLIS) {
+                queries.add(qe);
+            }
         }
         return queries;
     }
@@ -1415,7 +1419,7 @@ public static TwitterTimeline scrapeTwitter(
         	DAO.severe(e);
         }
 
-        if (recordQuery && Caretaker.acceptQuery4Retrieval(q)) {
+        if (recordQuery && Caretaker.acceptQuery4Retrieval(q) && tl.size() > 0) {
             if (qe == null) {
                 // a new query occurred
                 qe = new QueryEntry(q, timezoneOffset, tl.period(), SourceType.TWITTER, byUserQuery);
@@ -1550,6 +1554,7 @@ public static ArrayList<String> getFrontPeers() {
             for (String peer: remote) testpeers.add(peer);
             return testpeers;
         }
+        /*
         if (frontPeerCache.size() == 0) {
             // add dynamically all peers that contacted myself
             for (Map<String, RemoteAccess> hmap: RemoteAccess.history.values()) {
@@ -1559,6 +1564,7 @@ public static ArrayList<String> getFrontPeers() {
             }
         }
         testpeers.addAll(frontPeerCache);
+        */
         return getBestPeers(testpeers);
     }
 

src/org/loklak/data/DAO.java

@@ -120,6 +120,7 @@ public TwitterTimeline harvest_timeline() {
 
         if (tl != null && tl.size() > 0) {
             // find content query strings and store them in the context cache
+            tl.setQuery(q);
             checkContext(tl, true);
         }
 

src/org/loklak/harvester/TwitterHarvester.java

@@ -218,7 +218,7 @@ private static TwitterTimeline[] search(
 
         // parse
         Elements items = doc.getElementsByClass("stream-item");
-        for (int itemc = 0; itemc < items.size(); itemc++) {
+        itemloop: for (int itemc = 0; itemc < items.size(); itemc++) {
             Element item = items.get(itemc);
             if (debuglog) System.out.println(item.toString());
 
@@ -258,10 +258,12 @@ private static TwitterTimeline[] search(
             }
 
             String tweettimes = timestamp.attr("data-time-ms");
+            if (tweettimes.length() == 0) continue itemloop; // sometimes tweets are not available any more
             long tweettime = Long.parseLong(tweettimes);
-            long snowflaketime = snowflake2millis(Long.parseLong(tweetID));
-            assert tweettime / 1000 == snowflaketime / 1000;
-            
+            // this assertion holds in most, but unfortunately not in all cases, so it is commented out
+            // long snowflaketime = snowflake2millis(Long.parseLong(tweetID));
+            // assert tweettime / 1000L == snowflaketime / 1000L : "tweettime = " + tweettime + ", snowflaketime = " + snowflaketime;
+
             Elements reply = item.getElementsByClass("ProfileTweet-action--reply").get(0).children();
             Elements retweet = item.getElementsByClass("ProfileTweet-action--retweet").get(0).children();
             Elements favourite = item.getElementsByClass("ProfileTweet-action--favorite").get(0).children();
@@ -747,7 +749,7 @@ public Post toJSON(final UserEntry user, final boolean calculatedData, final int
             // the tweet; the cleanup is a helper function which cleans mistakes from the past in scraping
             MessageEntry.TextLinkMap tlm = this.moreData.getText(iflinkexceedslength, urlstub, this.text, this.getLinks(), this.getPostId());
             this.put("text", tlm);
-            if (this.status_id_url != null) this.put("link", this.status_id_url.toExternalForm());
+            if (this.status_id_url != null) this.put("link", this.status_id_url.toExternalForm()); // this is the primary key for retrieval in elasticsearch
             this.put("id_str", this.postId);
             this.put("conversation_id", this.conversationID);
             this.put("conversation_user", this.conversationUserIDs);

src/org/loklak/harvester/TwitterScraper.java

@@ -421,24 +421,21 @@ public static void download(String source_url, File target_file) {
     }
 
     public static byte[] download(String source_url) throws IOException {
+        ClientConnection connection = new ClientConnection(source_url, "");
+        if (connection.inputStream == null) return null;
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        int count;
+        byte[] buffer = new byte[4096];
+        IOException ee = null;
         try {
-            ClientConnection connection = new ClientConnection(source_url, "");
-            if (connection.inputStream == null) return null;
-            ByteArrayOutputStream baos = new ByteArrayOutputStream();
-            int count;
-            byte[] buffer = new byte[4096];
-            try {
-                while ((count = connection.inputStream.read(buffer)) > 0) baos.write(buffer, 0, count);
-            } catch (IOException e) {
-            	DAO.severe(e);
-            } finally {
-                connection.close();
-            }
-            return baos.toByteArray();
+            while ((count = connection.inputStream.read(buffer)) > 0) baos.write(buffer, 0, count);
         } catch (IOException e) {
-        	DAO.severe(e);
-            return null;
+        	ee = e;
+        } finally {
+            connection.close();
         }
+        if (ee != null) throw ee;
+        return baos.toByteArray();
     }
 
     public int getStatusCode() {

src/org/loklak/http/ClientConnection.java

@@ -35,8 +35,8 @@
 
 public abstract class AbstractObjectEntry extends Post implements ObjectEntry {
 
-    public  final static String TIMESTAMP_FIELDNAME = "timestamp";
-    public  final static String CREATED_AT_FIELDNAME = "created_at";
+    public  final static String TIMESTAMP_FIELDNAME = "timestamp";   // the harvesting time, NOT used for identification
+    public  final static String CREATED_AT_FIELDNAME = "created_at"; // the tweet time as embedded in the tweet, not used for identification either
 
     public AbstractObjectEntry() {
     }

src/org/loklak/objects/AbstractObjectEntry.java

@@ -64,7 +64,7 @@
  */
 public class QueryEntry extends AbstractObjectEntry implements ObjectEntry {
 
-    private final static long DAY_MILLIS = 1000L * 60L * 60L * 24L;
+    public  final static long DAY_MILLIS = 1000L * 60L * 60L * 24L;
     private final static int RETRIEVAL_CONSTANT = 20; // the number of messages that we get with each retrieval at maximum
 
     protected String query;           // the query in the exact way as the user typed it in
@@ -144,7 +144,7 @@ public void update(final long message_period, final boolean byUserQuery) {
             this.query_last = this.retrieval_last;
         }
         long new_message_period = message_period; // can be Long.MAX_VALUE if less than 2 messages are in timeline!
-        int new_messages_per_day = (int) (DAY_MILLIS / new_message_period); // this is an interpolation based on the last tweet list, can be 0!
+        int new_messages_per_day = (int) (DAY_MILLIS / (new_message_period + 1)); // this is an interpolation based on the last tweet list, can be 0!
         if (new_message_period == Long.MAX_VALUE || new_messages_per_day == 0) {
             this.message_period = DAY_MILLIS;
         } else {
@@ -217,6 +217,10 @@ public int getQueryCount() {
     public int getRetrievalCount() {
         return this.retrieval_count;
     }
+    
+    public long getMessagePeriod() {
+        return this.message_period;
+    }
 
     public int getMessagesPerDay() {
         return this.messages_per_day;

src/org/loklak/objects/QueryEntry.java

@@ -131,10 +131,6 @@ public JSONObject toJSON(boolean withEnrichedData, String metadata_field_name, S
         return json;
     }
 
-    public SusiThought toSusi(boolean withEnrichedData) throws JSONException {
-        return toSusi(withEnrichedData, new SusiThought());
-    }
-    
     private SusiThought toSusi(boolean withEnrichedData, SusiThought json) throws JSONException {
         json
             .setQuery(this.query)

src/org/loklak/objects/TwitterTimeline.java

@@ -20,9 +20,6 @@
 
 package org.loklak.susi;
 
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
 import org.json.JSONArray;
 import org.json.JSONObject;
 
@@ -44,37 +41,6 @@ public SusiThought() {
         this.data_name = "data";
     }
 
-    /**
-     * create a clone of a json object as a SusiThought object
-     * @param json the 'other' thought, probably an exported and re-imported thought
-     */
-    public SusiThought(JSONObject json) {
-        this();
-        if (json.has(this.metadata_name)) this.put(this.metadata_name, json.getJSONObject(this.metadata_name));
-        if (json.has(this.data_name)) this.setData(json.getJSONArray(this.data_name));
-        if (json.has("actions")) this.put("actions", json.getJSONArray("actions"));
-    }
-    
-    /**
-     * Create an initial thought using the matcher on an expression.
-     * Such an expression is like the input from a text source which contains keywords
-     * that are essential for the thought. The matcher extracts such information.
-     * Matching informations are named using the order of the appearance of the information pieces.
-     * The first information is named '1', the second '2' and so on. The whole information which contained
-     * the matching information is named '0'.
-     * @param matcher
-     */
-    public SusiThought(Matcher matcher) {
-        this();
-        this.setOffset(0).setHits(1);
-        JSONObject row = new JSONObject();
-        row.put("0", matcher.group(0));
-        for (int i = 0; i < matcher.groupCount(); i++) {
-            row.put(Integer.toString(i + 1), matcher.group(i + 1));
-        }
-        this.setData(new JSONArray().put(row));
-    }
-    
     @Deprecated
     public SusiThought(String metadata_name, String data_name) {
         super(true);
@@ -202,67 +168,5 @@ public JSONArray getData() {
         this.put(data_name, a);
         return a;
     }
-    
-    /**
-     * Merging of data is required during an mind-meld.
-     * To meld two thoughts, we combine their data arrays into one.
-     * The resulting table has the maximum length of the source tables
-     * @param table the information to be melted into our existing table.
-     * @return the thought
-     */
-    public SusiThought mergeData(JSONArray table1) {
-        JSONArray table0 = this.getData();
-        while (table0.length() < table1.length()) table0.put(new JSONObject());
-        for (int i = 0; i < table1.length(); i++) {
-            table0.getJSONObject(i).putAll(table1.getJSONObject(i));
-        }
-        setData(table0);
-        return this;
-    }
-    
-    /**
-     * If during thinking we observe something that we want to memorize, we can memorize this here
-     * @param featureName the object key
-     * @param observation the object value
-     * @return the thought
-     */
-    public SusiThought addObservation(String featureName, String observation) {
-        JSONArray data = getData();
-        for (int i = 0; i < data.length(); i++) {
-            JSONObject spark = data.getJSONObject(i);
-            if (!spark.has(featureName)) {
-                spark.put(featureName, observation);
-                return this;
-            }
-        }
-        data.put(new JSONObject().put(featureName, observation));
-        return this;
-    }
-    
-    public static final Pattern variable_pattern = Pattern.compile("\\$.*?\\$");
-    
-    /**
-     * Unification applies a piece of memory within the current argument to a statement
-     * which creates an instantiated statement
-     * @param statement
-     * @return the instantiated statement with elements of the argument applied as much as possible
-     */
-    public String unify(String statement) {
-        JSONArray table = this.getData();
-        if (table != null && table.length() > 0) {
-            JSONObject row = table.getJSONObject(0);
-            for (String key: row.keySet()) {
-                int i = statement.indexOf("$" + key + "$");
-                if (i >= 0) {
-                    statement = statement.substring(0, i) + row.get(key).toString() + statement.substring(i + key.length() + 2);
-                }
-            }
-        }
-        return statement;
-    }
-    
-    public static void main(String[] args) {
-        SusiThought t = new SusiThought().addObservation("a", "letter-a");
-        System.out.println(t.unify("the letter $a$"));
-    }
+
 }