Implemented events and posts

swarm_to_sqlite/utils.py

@@ -17,6 +17,17 @@ def save_checkin(checkin, db):
         checkin["venue"] = venue["id"]
     else:
         checkin["venue"] = None
+    if "event" in checkin:
+        event = checkin.pop("event")
+        categories = event.pop("categories")
+        e = db["events"].upsert(event, pk="id", alter=True)
+        for category in categories:
+            cleanup_category(category)
+            e.m2m("categories", category, pk="id")
+        checkin["event"] = event["id"]
+    else:
+        checkin["event"] = None
+
     checkin["createdAt"] = datetime.datetime.fromtimestamp(
         checkin["createdAt"]
     ).isoformat()
@@ -27,11 +38,13 @@ def save_checkin(checkin, db):
         users_likes.extend(group["items"])
     del checkin["likes"]
     photos = checkin.pop("photos")["items"]
+    posts = (checkin.pop("posts") or {}).get("items") or []
     if checkin.get("createdBy"):
         created_by_user = checkin.pop("createdBy")
         cleanup_user(created_by_user)
         db["users"].upsert(created_by_user, pk="id")
         checkin["createdBy"] = created_by_user["id"]
+    checkin["comments_count"] = checkin.pop("comments")["count"]
     # Actually save the checkin
     checkins_table = db["checkins"].upsert(
         checkin,
@@ -58,13 +71,31 @@ def save_checkin(checkin, db):
         db["users"].upsert(user, pk="id")
         photo["user"] = user["id"]
         photos_table.upsert(photo)
+    # Handle posts
+    posts_table = db.table("posts", pk="id")
+    for post in posts:
+        post["createdAt"] = datetime.datetime.fromtimestamp(
+            post["createdAt"]
+        ).isoformat()
+        post["post_source"] = (
+            db["post_sources"].upsert(post.pop("source"), pk="id").last_pk
+        )
+        post["checkin"] = checkin["id"]
+        posts_table.upsert(post, foreign_keys=("post_source", "checkin"))
+
     # Add checkin.createdBy => users.id foreign key last, provided the
     # users table exists
     if "users" in db.table_names() and "createdBy" in db["checkins"].columns_dict:
         try:
             db["checkins"].add_foreign_key("createdBy", "users", "id")
         except AlterError:
             pass
+    # Same for checkin.event
+    if "events" in db.table_names() and "event" in db["checkins"].columns_dict:
+        try:
+            db["checkins"].add_foreign_key("event", "events", "id")
+        except AlterError:
+            pass
 
 
 def cleanup_user(user):

tests/test_save_checkin.py

@@ -29,6 +29,10 @@ def test_tables(converted):
         "sources",
         "checkins",
         "photos",
+        "categories_events",
+        "events",
+        "posts",
+        "post_sources",
     } == set(converted.table_names())
 
 
@@ -67,6 +71,27 @@ def test_venue(converted):
     ] == categories
 
 
+def test_event(converted):
+    event = list(converted["events"].rows)[0]
+    assert {"id": "5bf8e4fb646e38002c472397", "name": "A movie"} == event
+    categories = list(
+        converted["categories"].rows_where(
+            "id in (select categories_id from categories_events where events_id = '5bf8e4fb646e38002c472397')"
+        )
+    )
+    assert [
+        {
+            "id": "4dfb90c6bd413dd705e8f897",
+            "name": "Movie",
+            "pluralName": "Movies",
+            "shortName": "Movie",
+            "primary": 1,
+            "icon_prefix": "https://ss3.4sqi.net/img/categories_v2/arts_entertainment/movietheater_",
+            "icon_suffix": ".png",
+        }
+    ] == categories
+
+
 def test_likes(converted):
     likes = list(converted["likes"].rows)
     assert [
@@ -179,11 +204,38 @@ def test_photos(converted):
     ] == photos
 
 
+def test_posts(converted):
+    assert [
+        ForeignKey(
+            table="posts", column="checkin", other_table="checkins", other_column="id"
+        ),
+        ForeignKey(
+            table="posts",
+            column="post_source",
+            other_table="post_sources",
+            other_column="id",
+        ),
+    ] == converted["posts"].foreign_keys
+    posts = list(converted["posts"].rows)
+    assert [
+        {
+            "id": "58994045e386e304939156e0",
+            "createdAt": "2017-02-07T04:34:29",
+            "text": "The samosa chaat appetizer (easily enough for two or even four people) was a revelation - I've never tasted anything quite like it before, absolutely delicious. Chicken tika masala was amazing too.",
+            "url": "https://foursquare.com/item/58994045668af77dae50b376",
+            "contentId": "58994045668af77dae50b376",
+            "post_source": "UJXJTUHR42CKGO54KXQWGUZJL3OJKMKMVHGJ1SWIOC5TRKAC",
+            "checkin": "592b2cfe09e28339ac543fde",
+        }
+    ] == posts
+
+
 def test_checkin_with_no_event():
     checkin = load_checkin()
-    # If no event in checkin, event column should not be there
+    # If no event in checkin, event column should be None
     del checkin["event"]
     db = sqlite_utils.Database(":memory:")
     utils.save_checkin(checkin, db)
     assert 1 == db["checkins"].count
-    assert "event" not in db["checkins"].columns_dict
+    row = list(db["checkins"].rows)[0]
+    assert row["event"] is None