Date search

Makefile

@@ -14,3 +14,6 @@ run-frontend:
 # Run this command with "make run -j2", that way backend and frontend run simultaneously
 run: run-backend run-frontend
 
+# Only runs backend tests (currently there are no frontend tests anyway)
+test:
+	cd backend && $(MAKE) test

backend/Makefile

@@ -3,5 +3,9 @@ install:
 	env/bin/pip install -e .[dev]
 
 run:
-	FLASK_APP=backend.app ELOGY_CONFIG_FILE=../config.py env/bin/flask run -p 8888
+	FLASK_APP=backend.app ELOGY_CONFIG_FILE=../config.py env/bin/flask run -p 8888 --reload
 
+# Skip performance tests because they are slow.
+.PHONY: test
+test:
+	env/bin/pytest test -k 'not performance'

backend/backend/api/entries.py

@@ -1,9 +1,11 @@
+from datetime import datetime
 import logging
 
 from flask import request, send_file
 from flask_restful import Resource, marshal, marshal_with, abort
+import pytz
 from webargs.fields import (Integer, Str, Boolean, Dict, List,
-                            Nested, Email, LocalDateTime)
+                            Nested, Email, LocalDateTime, Date)
 from webargs.flaskparser import use_args
 
 from ..db import Entry, Logbook, EntryLock
@@ -131,13 +133,24 @@ def put(self, args, entry_id, logbook_id=None):
         return entry
 
 
+class NaiveDate(Date):
+    """Takes a date, and interprets it as midnight in the local timezone."""
+    def _deserialize(self, *args, **kwargs):
+        date = super()._deserialize(*args, **kwargs)
+        tz = datetime.now().astimezone().tzinfo
+        dt = datetime.combine(date, datetime.min.time())
+        return dt.replace(tzinfo=tz).astimezone(pytz.utc)
+
+
 entries_args = {
     "title": Str(),
     "content": Str(),
     "authors": Str(),
     "attachments": Str(),
     "attribute": List(Str(validate=lambda s: len(s.split(":")) == 2)),
     "metadata": List(Str(validate=lambda s: len(s.split(":")) == 2)),
+    "from": NaiveDate(),
+    "until": NaiveDate(),
     "archived": Boolean(),
     "ignore_children": Boolean(),
     "n": Integer(missing=50),
@@ -169,6 +182,8 @@ def get(self, args, logbook_id=None):
                                attachment_filter=args.get("attachments"),
                                attribute_filter=attributes,
                                metadata_filter=metadata,
+                               from_timestamp=args.get("from"),
+                               until_timestamp=args.get("until"),
                                n=args["n"], offset=args.get("offset"),
                                sort_by_timestamp=args.get("sort_by_timestamp"))
             entries = logbook.get_entries(**search_args)
@@ -182,6 +197,8 @@ def get(self, args, logbook_id=None):
                                attachment_filter=args.get("attachments"),
                                attribute_filter=attributes,
                                metadata_filter=metadata,
+                               from_timestamp=args.get("from"),
+                               until_timestamp=args.get("until"),
                                n=args["n"], offset=args.get("offset"),
                                sort_by_timestamp=args.get("sort_by_timestamp"))
             entries = Entry.search(**search_args)

backend/backend/app.py

@@ -24,7 +24,7 @@
             static_url_path="/static")
 app.config.from_envvar('ELOGY_CONFIG_FILE')
 
-app.secret_key = app.config["SECRET"]
+app.secret_key = app.config.get("SECRET", "not-very-secret")
 
 # add some hooks for debugging purposes
 @app.before_request

backend/backend/db.py

@@ -571,6 +571,7 @@ def search(cls, logbook=None, followups=False,
                attribute_filter=None, content_filter=None,
                title_filter=None, author_filter=None,
                attachment_filter=None, metadata_filter=None,
+               from_timestamp=None, until_timestamp=None,
                sort_by_timestamp=True):
 
         # Note: this is all pretty messy. The reason we're building
@@ -740,11 +741,20 @@ def search(cls, logbook=None, followups=False,
         # Check if we're searching, in that case we want to show all entries.
         if followups or any([title_filter, content_filter, author_filter,
                              metadata_filter, attribute_filter, attachment_filter]):
-            query += " GROUP BY thread"
+            query += " GROUP BY thread HAVING 1"
         else:
             # We're not searching. In this case we'll only show
             query += " GROUP BY thread HAVING entry.follows_id IS NULL"
 
+        # Since we're using timestamp, which is an aggregate, it must be filtered
+        # here instead of in the WHERE clause.
+        if from_timestamp:
+            query += " AND (entry.created_at >= datetime(?) OR timestamp >= datetime(?))\n "
+            variables.extend([from_timestamp, from_timestamp])
+        if until_timestamp:
+            query += " AND (entry.created_at <= ? OR timestamp <= ?)\n"
+            variables.extend([until_timestamp, until_timestamp])
+
         # sort newest first, taking into account the last edit if any
         # TODO: does this make sense? Should we only consider creation date?
         order_by = sort_by_timestamp and "timestamp" or "entry.created_at"
@@ -753,7 +763,7 @@ def search(cls, logbook=None, followups=False,
             query += " LIMIT {}".format(n)
             if offset:
                 query += " OFFSET {}".format(offset)
-        logging.debug("query=%r, variables=%r" % (query, variables))
+        logging.warning("query=%r, variables=%r" % (query, variables))
         return Entry.raw(query, *variables)
 
     @classmethod
@@ -804,7 +814,7 @@ def search_(cls, logbook=None, followups=False,
                 # We're using the SQLite JSON1 extension to pick the
                 # attribute value out of the JSON encoded field.
                 # TODO: regexp?
-                attr = Entry.attributes.extract(name)
+                attr = Entry.attributes[name]
                 # Note: The reason we're just using 'contains' here
                 # (it's a substring match) is to support "multioption"
                 # attributes. They are represented as a JSON array and
@@ -815,7 +825,7 @@ def search_(cls, logbook=None, followups=False,
 
         if metadata_filter:
             for name, value in metadata_filter:
-                field = Entry.metadata.extract(name)
+                field = Entry.metadata[name]
                 result = result.where(field.contains(value))
 
         # TODO: how to group the results properly? If searching, we

backend/test/fixtures.py

@@ -19,7 +19,7 @@ def elogy(request):
     os.environ["ELOGY_CONFIG_FILE"] = "../test/config.py"
 
     def run_elogy():
-        from elogy.app import app
+        from backend.app import app
         app.run()
 
     proc = Process(target=run_elogy)
@@ -33,7 +33,7 @@ def run_elogy():
 @pytest.fixture(scope="function")
 def db(request):
 
-    from elogy.db import db, setup_database
+    from backend.db import db, setup_database
 
     setup_database(":memory:", close=False)
     return db
@@ -45,7 +45,7 @@ def db(request):
 @pytest.fixture(scope="module")
 def elogy_client(request):
     os.environ["ELOGY_CONFIG_FILE"] = "../test/config.py"
-    from elogy.app import app
+    from backend.app import app
     with app.test_client() as c:
         yield c
     try:

backend/test/test_api.py

@@ -135,7 +135,6 @@ def test_update_entry(elogy_client):
         elogy_client.put("/api/logbooks/{logbook[id]}/entries/{entry[id]}/"
                          .format(logbook=logbook, entry=entry),
                          data=new_in_entry))["entry"]
-    print(out_entry)
     assert out_entry["title"] == new_in_entry["title"]
     assert out_entry["content"] == new_in_entry["content"]
     assert out_entry["id"] == entry["id"]
@@ -159,7 +158,7 @@ def test_update_entry(elogy_client):
         elogy_client.get(
             "/api/logbooks/{logbook[id]}/entries/{entry[id]}/revisions/"
             .format(logbook=logbook, entry=entry)))["entry_changes"]
-
+    assert len(revisions) == 1
 
 def test_move_entry(elogy_client):
     in_logbook1, logbook1 = make_logbook(elogy_client)
@@ -188,14 +187,14 @@ def test_move_entry(elogy_client):
         elogy_client.get(
             "/api/entries/{entry[id]}/revisions/0"
             .format(entry=entry)))["entry"]
-    print(old_entry_version)
     assert old_entry_version["logbook"]["id"] == logbook1["id"]
     assert old_entry_version["revision_n"] == 0
 
     revisions = decode_response(
         elogy_client.get(
             "/api/logbooks/{logbook[id]}/entries/{entry[id]}/revisions/"
             .format(logbook=logbook2, entry=entry)))["entry_changes"]
+    assert len(revisions) == 1
 
 
 def test_create_entry_followup(elogy_client):
@@ -463,7 +462,7 @@ def test_create_attachment_with_single_quotes(elogy_client):
 
 def test_entry_search(elogy_client):
 
-    # TODO: expand to cover all ways to search
+    # TODO: expand to cover all ways to search, in various permutations
 
     # create a bunch of logbooks and entries
     in_logbook1, logbook1 = make_logbook(elogy_client)

backend/test/test_db.py

@@ -1,8 +1,9 @@
+from datetime import datetime
 from operator import attrgetter
 
 from .fixtures import db
-from elogy.db import Entry
-from elogy.db import Logbook, LogbookRevision
+from backend.db import Entry
+from backend.db import Logbook, LogbookRevision
 
 
 # Logbook
@@ -309,6 +310,67 @@ def test_entry_title_search(db):
     assert result.title == "Third entry"
 
 
+def test_entry_date_search(db):
+    lb = Logbook.create(name="Logbook1")
+
+    entries = [
+        {
+            "logbook": lb,
+            "title": "Z",
+            "content": "This content is great!",
+            "created_at": datetime(2019, 1, 14, 12, 0, 0)
+        },
+        {
+            "logbook": lb,
+            "title": "A",
+            "content": "This content is great!",
+            "created_at": datetime(2019, 1, 15, 12, 0, 0)
+        },
+        {
+            "logbook": lb,
+            "title": "B",
+            "content": "Some very neat content.",
+            "created_at": datetime(2019, 1, 17, 12, 0, 0)
+        },
+        {
+            "logbook": lb,
+            "title": "C",
+            "content": "Not so bad content either.",
+            "created_at": datetime(2019, 1, 18, 12, 0, 0)
+        },
+        {
+            "logbook": lb,
+            "title": "C",
+            "content": "Not so bad content either.",
+            "created_at": datetime(2019, 1, 19, 12, 0, 0),
+            "last_changed_at": datetime(2019, 2, 6, 12, 0, 0)
+        }
+    ]
+
+    # create entries
+    for entry in entries:
+        entry = Entry.create(**entry)
+        entry.save()
+
+    # include the from date
+    results = list(Entry.search(logbook=lb, from_timestamp=datetime(2019, 1, 17, 0, 0, 0)))
+    assert {r.title for r in results} == {"B", "C"}
+
+    # include the until_date
+    results = list(Entry.search(logbook=lb, until_timestamp=datetime(2019, 1, 17, 23, 59, 59)))
+    assert {r.title for r in results} == {"Z", "A", "B"}
+
+    # date interval
+    results = list(Entry.search(logbook=lb,
+                                from_timestamp=datetime(2019, 1, 15, 0, 0, 0),
+                                until_timestamp=datetime(2019, 1, 17, 23, 59, 59)))
+    assert {r.title for r in results} == {"A", "B"}
+
+    # also looks at change timestamp
+    results = list(Entry.search(logbook=lb, from_timestamp=datetime(2019, 2, 1)))
+    assert {r.title for r in results} == {"C"}
+
+
 def test_entry_search_followups(db):
     lb = Logbook.create(name="Logbook1")
 

frontend/src/app.js

@@ -55,7 +55,6 @@ class NoEntry extends React.Component {
 
     render() {
         const logbookId = parseInt(this.props.match.params.logbookId);
-        console.log(this.props.match.location);
         return (
             <div className="empty">
                 <i className="fa fa-arrow-left" /> Select an entry to read it

frontend/src/logbook.js

@@ -194,7 +194,7 @@ class Logbook extends React.Component {
                 ? parseInt(this.props.match.params.entryId, 10)
                 : null,
             query = parseQuery(this.props.location.search),
-            filter = ["title", "content", "authors", "attachments"]
+            searchFilters = ["title", "content", "authors", "attachments", "from", "until"]
                 .filter(key => query[key])
                 .map(key => (
                     <span key={key} className="filter">
@@ -280,7 +280,7 @@ class Logbook extends React.Component {
                             New logbook
                         </Link>
                     </div>
-                    <div className="filters"> {filter} </div>
+                    <div className="filters"> {searchFilters} </div>
                     <div className="attributes">{attributes}</div>
                     { this.state.entries.length === 0 ? null : 
                         <div className="date-sorting">

frontend/src/search.js

@@ -1,6 +1,6 @@
 import React from "react";
 import { Route } from "react-router-dom";
-import { parseQuery } from "./util.js";
+import { parseQuery, formatISODateString, getLocalTimezoneOffset } from "./util.js";
 import "./search.css";
 
 class QuickSearch extends React.Component {
@@ -11,6 +11,8 @@ class QuickSearch extends React.Component {
             title: null,
             authors: null,
             attachments: null,
+            from: null,
+            until: null,
             ignore_children: false
         };
     }
@@ -54,6 +56,8 @@ class QuickSearch extends React.Component {
                 title: null,
                 authors: null,
                 attachments: null,
+                from: null,
+                until: null,
                 ignore_children: false
             },
             this.onSubmit.bind(this, history, event)
@@ -115,6 +119,36 @@ class QuickSearch extends React.Component {
                             placeholder="Attachments"
                             onChange={this.onChange.bind(this)}
                         />
+                        <table>
+                            <tbody>
+                                <tr>
+                                    <td>From:</td>
+                                    <td>
+                                        <input type="date"
+                                            name="from"
+                                            value={ this.state.from || "" }
+                                            max={ this.state.until }
+                                            onChange={this.onChange.bind(this)}
+                                            title="Only entries created/changed on or after this date."
+                                        />
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td>
+                                        Until:
+                                    </td>
+                                    <td>
+                                        <input type="date"
+                                            name="until"
+                                            value={ this.state.until || "" }
+                                            min={ this.state.from }
+                                            onChange={this.onChange.bind(this)}
+                                            title="Only entries created/changed before this date."
+                                        />
+                                    </td>                                    
+                                </tr>
+                            </tbody>
+                        </table>
                         <label title="Whether to include entries in logbooks contained in the selected logbook.">
                             <input
                                 type="checkbox"