Skip to content

Commit

Permalink
Add --match arg to update tools to enforce a subset of objects
Browse files Browse the repository at this point in the history
  • Loading branch information
booxter committed Dec 6, 2024
1 parent 6173e69 commit 147479c
Show file tree
Hide file tree
Showing 2 changed files with 35 additions and 14 deletions.
43 changes: 33 additions & 10 deletions src/letsrolld/cmd/update.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import time
import traceback

from sqlalchemy import func, select, or_
from sqlalchemy import func, select, or_, and_
from sqlalchemy.orm import sessionmaker

from letsrolld import db
Expand All @@ -31,24 +31,35 @@
}


def _get_obj_to_update_query(model, threshold, last_checked_field):
def _get_obj_to_update_query(model, threshold, last_checked_field, match):
field = getattr(model, last_checked_field)
return or_(
query_filter = or_(
field < _NOW - threshold,
field > _NOW,
field == None, # noqa
)
if match:
return and_(
query_filter,
or_(
model.name.ilike(f"%{match}%"),
model.lb_url.ilike(f"%{match}%"),
),
)
return query_filter


def _seen_obj_query(model, seen):
return model.id.notin_(seen)


def get_obj_to_update(session, model, threshold, last_checked_field, seen):
def get_obj_to_update(session, model, threshold, last_checked_field, seen, match):
return (
session.execute(
select(model)
.filter(_get_obj_to_update_query(model, threshold, last_checked_field))
.filter(
_get_obj_to_update_query(model, threshold, last_checked_field, match)
)
.filter(_seen_obj_query(model, seen))
.limit(1)
)
Expand All @@ -57,12 +68,14 @@ def get_obj_to_update(session, model, threshold, last_checked_field, seen):
)


def get_number_of_objs_to_update(session, model, threshold, last_checked_field):
def get_number_of_objs_to_update(session, model, threshold, last_checked_field, match):
try:
return session.scalar(
select(func.count())
.select_from(model)
.filter(_get_obj_to_update_query(model, threshold, last_checked_field))
.filter(
_get_obj_to_update_query(model, threshold, last_checked_field, match)
)
)
finally:
session.close()
Expand Down Expand Up @@ -295,10 +308,13 @@ def run_update(
last_checked_field,
last_updated_field,
dry_run=False,
match=None,
):
model_name = model.__name__

n_objs = get_number_of_objs_to_update(session, model, threshold, last_checked_field)
n_objs = get_number_of_objs_to_update(
session, model, threshold, last_checked_field, match
)

i = 1
seen = set()
Expand Down Expand Up @@ -327,7 +343,9 @@ def loop_housekeeping(session, obj, updated=False):
i += 1

while True:
obj = get_obj_to_update(session, model, threshold, last_checked_field, seen)
obj = get_obj_to_update(
session, model, threshold, last_checked_field, seen, match
)
if obj is None:
break

Expand All @@ -354,6 +372,7 @@ def parse_args():
parser.add_argument("--debug", action="store_true")
parser.add_argument("--dry-run", action="store_true")
parser.add_argument("--force", action="store_true")
parser.add_argument("--match")
return parser.parse_args()


Expand All @@ -377,7 +396,9 @@ def main(
while True:
try:
threshold = (
datetime.timedelta(0) if args.force else _MODEL_TO_THRESHOLD[model]
datetime.timedelta(0)
if (args.force or args.match)
else _MODEL_TO_THRESHOLD[model]
)
run_update(
get_session(),
Expand All @@ -389,6 +410,7 @@ def main(
last_checked_field,
last_updated_field,
dry_run=args.dry_run,
match=args.match,
)
break
except Exception as e:
Expand Down Expand Up @@ -417,6 +439,7 @@ def offers_main():
)


# TODO: move this to cleanup script?
# TODO: reuse generic main() machinery for services_main()
def services_main():
session = get_session()
Expand Down
6 changes: 2 additions & 4 deletions src/letsrolld/db/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from sqlalchemy import Enum, Integer, String, Numeric, DateTime
from sqlalchemy import Column, Table, ForeignKey
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import Mapped, relationship, mapped_column
from sqlalchemy.orm import Mapped, relationship, mapped_column, synonym

Base = declarative_base()

Expand Down Expand Up @@ -110,9 +110,7 @@ class Film(Base): # type: ignore[valid-type,misc]
back_populates="films",
)

@property
def name(self):
return self.title
name = synonym("title")


class Director(Base): # type: ignore[valid-type,misc]
Expand Down

0 comments on commit 147479c

Please sign in to comment.