trending words endpoint

backend/alembic/versions/4b97d64fa93d_add_db_for_trending_words.py

@@ -0,0 +1,39 @@
+"""Add db for trending words
+
+Revision ID: 4b97d64fa93d
+Revises: 4e16c97d06e7
+Create Date: 2021-03-08 06:17:07.485897
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "4b97d64fa93d"
+down_revision = "4e16c97d06e7"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "trending_words",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("created_at", sa.DateTime(), nullable=True),
+        sa.Column("updated_at", sa.DateTime(), nullable=True),
+        sa.Column("data", sa.JSON(), nullable=True),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_trending_words_id"), "trending_words", ["id"], unique=False
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f("ix_trending_words_id"), table_name="trending_words")
+    op.drop_table("trending_words")
+    # ### end Alembic commands ###

backend/requirements.txt

@@ -14,3 +14,4 @@ pandas
 jenkspy
 sigfig
 GitPython
+nltk

backend/router/stories.py

@@ -216,3 +216,9 @@ def explore(
     return crud.get_story_feed(
         db, current_story.id, current_story.latitude, current_story.longitude
     )
+
+
+@router.get("/trending")
+def trending(db: Session = Depends(get_db)):
+    db_trending = crud.get_trending_words(db)
+    return db_trending

backend/stories/crud.py

@@ -1,13 +1,22 @@
 from typing import List
 import random
+import datetime
 
 from sqlalchemy.orm import Session, joinedload
 from sqlalchemy.sql.expression import func, and_
+import nltk
+from nltk.corpus import stopwords
+from nltk.probability import FreqDist
+import string
+import asyncio
 
 from database import Base
 from users.models import User
 from . import models, schemas
 
+nltk.download("stopwords")
+sampling_trending_words = False
+
 
 def update(model_id: int, dto: schemas.BaseModel, model: Base, db: Session):
     item_as_dict = dict(dto)
@@ -198,3 +207,58 @@ def rand_per_story(arr: [models.MyStory]):
             output.append(ms)
 
     return output
+
+
+def get_trending_words(db: Session):
+    db_trending = db.query(models.Trending).first()
+    now = datetime.datetime.now()
+
+    if not db_trending or (now - db_trending.updated_at).days >= 7:
+        run_sample_task(db, db_trending)
+
+    return db_trending
+
+
+def run_sample_task(db: Session, to_update):
+    global sampling_trending_words
+    if sampling_trending_words:
+        return
+
+    sampling_trending_words = True
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+    loop.run_until_complete(sample_trending_words(db, to_update))
+    loop.close()
+
+
+async def sample_trending_words(db: Session, to_update):
+    global sampling_trending_words
+    k = 3
+
+    db_stories = db.query(models.Story).all()
+    stop_words = set(stopwords.words("english"))
+    all_words = []
+
+    for story in db_stories:
+        for my_story in story.my_stories:
+            if not my_story.text:
+                continue
+
+            text = my_story.text.lower().translate(
+                str.maketrans("", "", string.punctuation)
+            )
+            for word in text.split():
+                if word not in stop_words:
+                    all_words.append(word)
+
+    fdist = FreqDist(all_words)
+    top = fdist.most_common(k)
+
+    if to_update:
+        update(to_update.id, {"data": top}, models.Trending, db)
+    else:
+        db_trending = models.Trending(data=top)
+        db.add(db_trending)
+        db.commit()
+
+    sampling_trending_words = False

backend/stories/models.py

@@ -103,3 +103,9 @@ class MyStory(Base):
 
     story = relationship("Story", back_populates="my_stories")
     comments = relationship("Comment", lazy="select")
+
+
+class Trending(Base):
+    __tablename__ = "trending_words"
+
+    data = Column(JSON)