From f5c6d9526c263ee71ab91059596bdc8889958eb5 Mon Sep 17 00:00:00 2001 From: Ihar Hrachyshka Date: Sat, 30 Nov 2024 11:43:07 -0500 Subject: [PATCH 1/3] Add amazonamcplus to stream --- src/letsrolld/film.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/letsrolld/film.py b/src/letsrolld/film.py index 1000ae3..a659d28 100644 --- a/src/letsrolld/film.py +++ b/src/letsrolld/film.py @@ -18,6 +18,7 @@ HOOPLA = "hoopla" AMAZONPRIME = "amazonprime" AMAZONPRIMEWITHADS = "amazonprimevideowithads" +AMAZONAMCPLUS = "amazonamcplus" AMAZON = "amazon" YOUTUBE = "youtube" CRITERION = "criterionchannel" @@ -44,6 +45,7 @@ HOOPLA, AMAZONPRIME, AMAZONPRIMEWITHADS, + AMAZONAMCPLUS, AMAZON, YOUTUBE, CRITERION, @@ -81,6 +83,7 @@ ] STREAM_SERVICES = FREE_SERVICES + [ AMAZON, + AMAZONAMCPLUS, YOUTUBE, CRITERION, METROGRAPH, From 0a080a252ab8bf30fc131f9d688d9fef0ac14464 Mon Sep 17 00:00:00 2001 From: Ihar Hrachyshka Date: Sat, 30 Nov 2024 11:44:25 -0500 Subject: [PATCH 2/3] Add amazonmubi --- src/letsrolld/film.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/letsrolld/film.py b/src/letsrolld/film.py index a659d28..3088b91 100644 --- a/src/letsrolld/film.py +++ b/src/letsrolld/film.py @@ -19,6 +19,7 @@ AMAZONPRIME = "amazonprime" AMAZONPRIMEWITHADS = "amazonprimevideowithads" AMAZONAMCPLUS = "amazonamcplus" +AMAZONMUBI = "amazonmubi" AMAZON = "amazon" YOUTUBE = "youtube" CRITERION = "criterionchannel" @@ -45,6 +46,7 @@ HOOPLA, AMAZONPRIME, AMAZONPRIMEWITHADS, + AMAZONMUBI, AMAZONAMCPLUS, AMAZON, YOUTUBE, @@ -84,6 +86,7 @@ STREAM_SERVICES = FREE_SERVICES + [ AMAZON, AMAZONAMCPLUS, + AMAZONMUBI, YOUTUBE, CRITERION, METROGRAPH, From 06a34646b404aaadf0a848299286d8b42503fc56 Mon Sep 17 00:00:00 2001 From: Ihar Hrachyshka Date: Sat, 30 Nov 2024 13:53:40 -0500 Subject: [PATCH 3/3] Add monetization-type --- Makefile | 5 +- .../8e251e9b7a42_add_monetization_type.py | 44 +++++++ configs/default.json | 14 +-- js/docs/DirectorFilmsInnerOffersInner.md | 1 + js/src/model/DirectorFilmsInnerOffersInner.js | 22 +++- .../DirectorFilmsInnerOffersInner.spec.js | 6 + ...f_directors_item_films_item_offers_item.py | 8 ++ .../models/array_of_films_item_offers_item.py | 8 ++ ...em_sections_item_films_item_offers_item.py | 8 ++ .../models/director_films_item_offers_item.py | 8 ++ .../models/film_offers_item.py | 8 ++ ...rt_sections_item_films_item_offers_item.py | 8 ++ pyproject.toml | 1 + src/letsrolld/cmd/cleanup.py | 23 ++++ src/letsrolld/cmd/update.py | 61 ++++++++- src/letsrolld/db/models.py | 20 ++- src/letsrolld/film.py | 118 ++---------------- src/letsrolld/webapi/app.py | 24 ++-- src/letsrolld/webapi/models.py | 3 +- src/letsrolld/webcli/cli.py | 16 ++- src/letsrolld/webcli/templates/film-full.j2 | 13 +- swagger.json | 60 +++++++-- tests/test_film.py | 34 ----- ts/model/directorFilmsInnerOffersInner.ts | 6 + 24 files changed, 332 insertions(+), 187 deletions(-) create mode 100644 alembic/versions/8e251e9b7a42_add_monetization_type.py diff --git a/Makefile b/Makefile index 3406d68..7909197 100644 --- a/Makefile +++ b/Makefile @@ -28,10 +28,13 @@ run-update-films: run-update-offers: pdm run update-offers $(ARGS) | $(RUN_LOG_CMD) +run-update-services: + pdm run update-services $(ARGS) | $(RUN_LOG_CMD) + run-cleanup: pdm run cleanup $(ARGS) | $(RUN_LOG_CMD) -run-all: run-update-directors run-update-films run-update-offers run-cleanup +run-all: run-update-directors run-update-films run-update-offers run-update-services run-cleanup run-db-upgrade: pdm run alembic upgrade head diff --git a/alembic/versions/8e251e9b7a42_add_monetization_type.py b/alembic/versions/8e251e9b7a42_add_monetization_type.py new file mode 100644 index 0000000..f57e3dd --- /dev/null +++ b/alembic/versions/8e251e9b7a42_add_monetization_type.py @@ -0,0 +1,44 @@ +"""add-monetization-type + +Revision ID: 8e251e9b7a42 +Revises: 853345b1c2f1 +Create Date: 2024-11-30 12:50:24.880021 + +""" + +from typing import Sequence, Union + +# TODO: Fix type ignore by moving alembic/ directory? +from alembic import op # type: ignore[attr-defined] +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "8e251e9b7a42" +down_revision: Union[str, None] = "853345b1c2f1" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "offers", + sa.Column( + "monetization_type", + sa.Enum( + "FREE", + "FLATRATE", + "RENT", + "BUY", + "CINEMA", + "ADS", + "FAST", + "DISC", + name="monetizationtype", + ), + ), + ) + + +def downgrade() -> None: + op.drop_column("offers", "monetization_type") diff --git a/configs/default.json b/configs/default.json index ab72e83..2f0f1b6 100644 --- a/configs/default.json +++ b/configs/default.json @@ -1,14 +1,14 @@ { "Top Movies": { "max_movies": 7, - "services": ["STREAM"], + "services": ["FREE", "ADS", "FLATRATE", "RENT", "BUY"], "min_rating": "3.85", "min_length": 60, "exclude_genres": ["documentary", "animation"] }, "International": { "max_movies": 5, - "services": ["STREAM"], + "services": ["FREE", "ADS", "FLATRATE", "RENT", "BUY"], "min_rating": "3.85", "min_length": 60, "exclude_genres": ["documentary", "animation"], @@ -16,7 +16,7 @@ }, "Horrors": { "max_movies": 3, - "services": ["STREAM"], + "services": ["FREE", "ADS", "FLATRATE", "RENT", "BUY"], "min_rating": "3.75", "genre": "horror", "min_length": 60, @@ -24,7 +24,7 @@ }, "Classics": { "max_movies": 3, - "services": ["STREAM"], + "services": ["FREE", "ADS", "FLATRATE", "RENT", "BUY"], "min_rating": "3.90", "min_year": 1900, "max_year": 1970, @@ -33,7 +33,7 @@ }, "Animation": { "max_movies": 2, - "services": ["STREAM"], + "services": ["FREE", "ADS", "FLATRATE", "RENT", "BUY"], "min_rating": "3.75", "min_length": 60, "genre": "animation", @@ -41,7 +41,7 @@ }, "Anime": { "max_movies": 2, - "services": ["STREAM"], + "services": ["FREE", "ADS", "FLATRATE", "RENT", "BUY"], "min_rating": "3.75", "min_length": 60, "genre": "animation", @@ -49,7 +49,7 @@ }, "Documentaries": { "max_movies": 3, - "services": ["STREAM"], + "services": ["FREE", "ADS", "FLATRATE", "RENT", "BUY"], "min_rating": "3.75", "min_length": 60, "genre": "documentary" diff --git a/js/docs/DirectorFilmsInnerOffersInner.md b/js/docs/DirectorFilmsInnerOffersInner.md index 0edcdd9..cd95e4d 100644 --- a/js/docs/DirectorFilmsInnerOffersInner.md +++ b/js/docs/DirectorFilmsInnerOffersInner.md @@ -5,6 +5,7 @@ Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- **name** | **String** | | +**monetizationType** | **String** | | **url** | **String** | | diff --git a/js/src/model/DirectorFilmsInnerOffersInner.js b/js/src/model/DirectorFilmsInnerOffersInner.js index c1fca94..423c930 100644 --- a/js/src/model/DirectorFilmsInnerOffersInner.js +++ b/js/src/model/DirectorFilmsInnerOffersInner.js @@ -23,11 +23,12 @@ class DirectorFilmsInnerOffersInner { * Constructs a new DirectorFilmsInnerOffersInner. * @alias module:model/DirectorFilmsInnerOffersInner * @param name {String} + * @param monetizationType {String} * @param url {String} */ - constructor(name, url) { + constructor(name, monetizationType, url) { - DirectorFilmsInnerOffersInner.initialize(this, name, url); + DirectorFilmsInnerOffersInner.initialize(this, name, monetizationType, url); } /** @@ -35,8 +36,9 @@ class DirectorFilmsInnerOffersInner { * This method is used by the constructors of any subclasses, in order to implement multiple inheritance (mix-ins). * Only for internal use. */ - static initialize(obj, name, url) { + static initialize(obj, name, monetizationType, url) { obj['name'] = name; + obj['monetization_type'] = monetizationType; obj['url'] = url; } @@ -54,6 +56,9 @@ class DirectorFilmsInnerOffersInner { if (data.hasOwnProperty('name')) { obj['name'] = ApiClient.convertToType(data['name'], 'String'); } + if (data.hasOwnProperty('monetization_type')) { + obj['monetization_type'] = ApiClient.convertToType(data['monetization_type'], 'String'); + } if (data.hasOwnProperty('url')) { obj['url'] = ApiClient.convertToType(data['url'], 'String'); } @@ -78,6 +83,10 @@ class DirectorFilmsInnerOffersInner { throw new Error("Expected the field `name` to be a primitive type in the JSON string but got " + data['name']); } // ensure the json data is a string + if (data['monetization_type'] && !(typeof data['monetization_type'] === 'string' || data['monetization_type'] instanceof String)) { + throw new Error("Expected the field `monetization_type` to be a primitive type in the JSON string but got " + data['monetization_type']); + } + // ensure the json data is a string if (data['url'] && !(typeof data['url'] === 'string' || data['url'] instanceof String)) { throw new Error("Expected the field `url` to be a primitive type in the JSON string but got " + data['url']); } @@ -88,13 +97,18 @@ class DirectorFilmsInnerOffersInner { } -DirectorFilmsInnerOffersInner.RequiredProperties = ["name", "url"]; +DirectorFilmsInnerOffersInner.RequiredProperties = ["name", "monetization_type", "url"]; /** * @member {String} name */ DirectorFilmsInnerOffersInner.prototype['name'] = undefined; +/** + * @member {String} monetization_type + */ +DirectorFilmsInnerOffersInner.prototype['monetization_type'] = undefined; + /** * @member {String} url */ diff --git a/js/test/model/DirectorFilmsInnerOffersInner.spec.js b/js/test/model/DirectorFilmsInnerOffersInner.spec.js index 372d10c..e89e248 100644 --- a/js/test/model/DirectorFilmsInnerOffersInner.spec.js +++ b/js/test/model/DirectorFilmsInnerOffersInner.spec.js @@ -60,6 +60,12 @@ //expect(instance).to.be(); }); + it('should have the property monetizationType (base name: "monetization_type")', function() { + // uncomment below and update the code to test the property monetizationType + //var instance = new LetsrolldApi.DirectorFilmsInnerOffersInner(); + //expect(instance).to.be(); + }); + it('should have the property url (base name: "url")', function() { // uncomment below and update the code to test the property url //var instance = new LetsrolldApi.DirectorFilmsInnerOffersInner(); diff --git a/letsrolld-api-client/letsrolld_api_client/models/array_of_directors_item_films_item_offers_item.py b/letsrolld-api-client/letsrolld_api_client/models/array_of_directors_item_films_item_offers_item.py index 2ee2f92..3b691af 100644 --- a/letsrolld-api-client/letsrolld_api_client/models/array_of_directors_item_films_item_offers_item.py +++ b/letsrolld-api-client/letsrolld_api_client/models/array_of_directors_item_films_item_offers_item.py @@ -11,16 +11,20 @@ class ArrayOfDirectorsItemFilmsItemOffersItem: """ Attributes: name (str): + monetization_type (str): url (Union[None, str]): """ name: str + monetization_type: str url: Union[None, str] additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> Dict[str, Any]: name = self.name + monetization_type = self.monetization_type + url: Union[None, str] url = self.url @@ -29,6 +33,7 @@ def to_dict(self) -> Dict[str, Any]: field_dict.update( { "name": name, + "monetization_type": monetization_type, "url": url, } ) @@ -40,6 +45,8 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: d = src_dict.copy() name = d.pop("name") + monetization_type = d.pop("monetization_type") + def _parse_url(data: object) -> Union[None, str]: if data is None: return data @@ -49,6 +56,7 @@ def _parse_url(data: object) -> Union[None, str]: array_of_directors_item_films_item_offers_item = cls( name=name, + monetization_type=monetization_type, url=url, ) diff --git a/letsrolld-api-client/letsrolld_api_client/models/array_of_films_item_offers_item.py b/letsrolld-api-client/letsrolld_api_client/models/array_of_films_item_offers_item.py index 36a1692..84f33cc 100644 --- a/letsrolld-api-client/letsrolld_api_client/models/array_of_films_item_offers_item.py +++ b/letsrolld-api-client/letsrolld_api_client/models/array_of_films_item_offers_item.py @@ -11,16 +11,20 @@ class ArrayOfFilmsItemOffersItem: """ Attributes: name (str): + monetization_type (str): url (Union[None, str]): """ name: str + monetization_type: str url: Union[None, str] additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> Dict[str, Any]: name = self.name + monetization_type = self.monetization_type + url: Union[None, str] url = self.url @@ -29,6 +33,7 @@ def to_dict(self) -> Dict[str, Any]: field_dict.update( { "name": name, + "monetization_type": monetization_type, "url": url, } ) @@ -40,6 +45,8 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: d = src_dict.copy() name = d.pop("name") + monetization_type = d.pop("monetization_type") + def _parse_url(data: object) -> Union[None, str]: if data is None: return data @@ -49,6 +56,7 @@ def _parse_url(data: object) -> Union[None, str]: array_of_films_item_offers_item = cls( name=name, + monetization_type=monetization_type, url=url, ) diff --git a/letsrolld-api-client/letsrolld_api_client/models/array_of_reports_item_sections_item_films_item_offers_item.py b/letsrolld-api-client/letsrolld_api_client/models/array_of_reports_item_sections_item_films_item_offers_item.py index 50fb545..547ce1d 100644 --- a/letsrolld-api-client/letsrolld_api_client/models/array_of_reports_item_sections_item_films_item_offers_item.py +++ b/letsrolld-api-client/letsrolld_api_client/models/array_of_reports_item_sections_item_films_item_offers_item.py @@ -11,16 +11,20 @@ class ArrayOfReportsItemSectionsItemFilmsItemOffersItem: """ Attributes: name (str): + monetization_type (str): url (Union[None, str]): """ name: str + monetization_type: str url: Union[None, str] additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> Dict[str, Any]: name = self.name + monetization_type = self.monetization_type + url: Union[None, str] url = self.url @@ -29,6 +33,7 @@ def to_dict(self) -> Dict[str, Any]: field_dict.update( { "name": name, + "monetization_type": monetization_type, "url": url, } ) @@ -40,6 +45,8 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: d = src_dict.copy() name = d.pop("name") + monetization_type = d.pop("monetization_type") + def _parse_url(data: object) -> Union[None, str]: if data is None: return data @@ -49,6 +56,7 @@ def _parse_url(data: object) -> Union[None, str]: array_of_reports_item_sections_item_films_item_offers_item = cls( name=name, + monetization_type=monetization_type, url=url, ) diff --git a/letsrolld-api-client/letsrolld_api_client/models/director_films_item_offers_item.py b/letsrolld-api-client/letsrolld_api_client/models/director_films_item_offers_item.py index 1b0cee7..8760eb1 100644 --- a/letsrolld-api-client/letsrolld_api_client/models/director_films_item_offers_item.py +++ b/letsrolld-api-client/letsrolld_api_client/models/director_films_item_offers_item.py @@ -11,16 +11,20 @@ class DirectorFilmsItemOffersItem: """ Attributes: name (str): + monetization_type (str): url (Union[None, str]): """ name: str + monetization_type: str url: Union[None, str] additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> Dict[str, Any]: name = self.name + monetization_type = self.monetization_type + url: Union[None, str] url = self.url @@ -29,6 +33,7 @@ def to_dict(self) -> Dict[str, Any]: field_dict.update( { "name": name, + "monetization_type": monetization_type, "url": url, } ) @@ -40,6 +45,8 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: d = src_dict.copy() name = d.pop("name") + monetization_type = d.pop("monetization_type") + def _parse_url(data: object) -> Union[None, str]: if data is None: return data @@ -49,6 +56,7 @@ def _parse_url(data: object) -> Union[None, str]: director_films_item_offers_item = cls( name=name, + monetization_type=monetization_type, url=url, ) diff --git a/letsrolld-api-client/letsrolld_api_client/models/film_offers_item.py b/letsrolld-api-client/letsrolld_api_client/models/film_offers_item.py index 3f8c04e..14b4363 100644 --- a/letsrolld-api-client/letsrolld_api_client/models/film_offers_item.py +++ b/letsrolld-api-client/letsrolld_api_client/models/film_offers_item.py @@ -11,16 +11,20 @@ class FilmOffersItem: """ Attributes: name (str): + monetization_type (str): url (Union[None, str]): """ name: str + monetization_type: str url: Union[None, str] additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> Dict[str, Any]: name = self.name + monetization_type = self.monetization_type + url: Union[None, str] url = self.url @@ -29,6 +33,7 @@ def to_dict(self) -> Dict[str, Any]: field_dict.update( { "name": name, + "monetization_type": monetization_type, "url": url, } ) @@ -40,6 +45,8 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: d = src_dict.copy() name = d.pop("name") + monetization_type = d.pop("monetization_type") + def _parse_url(data: object) -> Union[None, str]: if data is None: return data @@ -49,6 +56,7 @@ def _parse_url(data: object) -> Union[None, str]: film_offers_item = cls( name=name, + monetization_type=monetization_type, url=url, ) diff --git a/letsrolld-api-client/letsrolld_api_client/models/report_sections_item_films_item_offers_item.py b/letsrolld-api-client/letsrolld_api_client/models/report_sections_item_films_item_offers_item.py index 3101ee6..7df5dca 100644 --- a/letsrolld-api-client/letsrolld_api_client/models/report_sections_item_films_item_offers_item.py +++ b/letsrolld-api-client/letsrolld_api_client/models/report_sections_item_films_item_offers_item.py @@ -11,16 +11,20 @@ class ReportSectionsItemFilmsItemOffersItem: """ Attributes: name (str): + monetization_type (str): url (Union[None, str]): """ name: str + monetization_type: str url: Union[None, str] additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> Dict[str, Any]: name = self.name + monetization_type = self.monetization_type + url: Union[None, str] url = self.url @@ -29,6 +33,7 @@ def to_dict(self) -> Dict[str, Any]: field_dict.update( { "name": name, + "monetization_type": monetization_type, "url": url, } ) @@ -40,6 +45,8 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: d = src_dict.copy() name = d.pop("name") + monetization_type = d.pop("monetization_type") + def _parse_url(data: object) -> Union[None, str]: if data is None: return data @@ -49,6 +56,7 @@ def _parse_url(data: object) -> Union[None, str]: report_sections_item_films_item_offers_item = cls( name=name, + monetization_type=monetization_type, url=url, ) diff --git a/pyproject.toml b/pyproject.toml index ff60bd0..240a729 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ populate-directors = "letsrolld.cmd.populate_directors:main" update-directors = "letsrolld.cmd.update:directors_main" update-films = "letsrolld.cmd.update:films_main" update-offers = "letsrolld.cmd.update:offers_main" +update-services = "letsrolld.cmd.update:services_main" cleanup = "letsrolld.cmd.cleanup:main" # webapi diff --git a/src/letsrolld/cmd/cleanup.py b/src/letsrolld/cmd/cleanup.py index 56e96bf..a6c6152 100644 --- a/src/letsrolld/cmd/cleanup.py +++ b/src/letsrolld/cmd/cleanup.py @@ -54,6 +54,25 @@ def delete_orphaned_films(session, model, dry_run=False): session.rollback() +def delete_orphaned_offers(session, model, dry_run=False): + try: + for offer in session.query(model).all(): + film = ( + session.query(models.Film) + .join(models.Film.offers) + .filter(models.Offer.id == offer.id) + .first() + ) + if film is None: + print(f"Deleting orphaned offer: {offer.name}") + session.delete(offer) + finally: + if not dry_run: + session.commit() + else: + session.rollback() + + # TODO: abstract dry_run handling away def nullify_zero_years(session, model, dry_run=False): try: @@ -93,6 +112,10 @@ def nullify_one_runtime(session, model, dry_run=False): models.Film, delete_orphaned_films, ), + ( + models.Offer, + delete_orphaned_offers, + ), # ( # models.Film, # nullify_zero_years, diff --git a/src/letsrolld/cmd/update.py b/src/letsrolld/cmd/update.py index 380fa80..3f76884 100644 --- a/src/letsrolld/cmd/update.py +++ b/src/letsrolld/cmd/update.py @@ -111,7 +111,19 @@ def update_countries(session, countries): def update_offers(session, offers): - return update_objs(session, models.Offer, {o.technical_name for o in offers}) + objs = [] + for offer in offers: + model = models.Offer + name = offer.technical_name + db_obj = session.query(model).filter_by(name=name).first() + if db_obj is not None: + objs.append(db_obj) + continue + db_obj = model(name=name, monetization_type=offer.monetization_type) + session.add(db_obj) + objs.append(db_obj) + print(f"Adding {model.__name__.lower()}: {name}") + return objs def add_films(session, films): @@ -249,7 +261,6 @@ def refresh_film(session, db_obj, api_obj): def refresh_offers(session, db_obj, api_obj): - # just in case genres or countries or offers changed db_offers = update_offers(session, api_obj.available_services) def get_offer_id(offer): @@ -346,6 +357,10 @@ def parse_args(): return parser.parse_args() +def get_session(): + return sessionmaker(bind=db.create_engine())() + + def main( model, api_cls, @@ -365,7 +380,7 @@ def main( datetime.timedelta(0) if args.force else _MODEL_TO_THRESHOLD[model] ) run_update( - sessionmaker(bind=db.create_engine())(), + get_session(), model, api_cls, refresh_func, @@ -400,3 +415,43 @@ def offers_main(): "last_offers_checked", "last_offers_updated", ) + + +# TODO: reuse generic main() machinery for services_main() +def services_main(): + session = get_session() + done = set() + while True: + offers = ( + session.query(models.Offer) + .filter(models.Offer.is_(None)) + .filter(models.Offer.id.notin_(done)) + .all() + ) + if not offers: + break + for offer in offers: + print(f"Offer {offer.name} has no monetization type, fixing...") + film = ( + session.query(models.Film) + .join(models.FilmOffer) + .filter(models.FilmOffer.offer_id == offer.id) + .order_by(func.random()) + .limit(1) + .first() + ) + if film is None: + print(f"Offer {offer.name} has no film, skipping...") + done.add(offer.id) + continue + print(f"Offer {offer.name} is associated with film {film.name}, fixing...") + for api_offer in film_obj.Film(film.lb_url).available_services: + if api_offer.technical_name == offer.name: + offer.monetization_type = api_offer.monetization_type + print( + f"Offer {offer.name} is now of type {offer.monetization_type}" + ) + session.add(offer) + done.add(offer.id) + break + session.commit() diff --git a/src/letsrolld/db/models.py b/src/letsrolld/db/models.py index 0192477..2f166f5 100644 --- a/src/letsrolld/db/models.py +++ b/src/letsrolld/db/models.py @@ -1,4 +1,6 @@ -from sqlalchemy import Integer, String, Numeric, DateTime +import enum + +from sqlalchemy import Enum, Integer, String, Numeric, DateTime from sqlalchemy import Column, Table, ForeignKey from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import Mapped, relationship, mapped_column @@ -45,11 +47,27 @@ class FilmOffer(Base): # type: ignore[valid-type,misc] url = Column(String, nullable=True) +class MonetizationType(enum.Enum): + FREE = "FREE" + FLATRATE = "FLATRATE" + RENT = "RENT" + BUY = "BUY" + ADS = "ADS" + # TODO: what is that? should I expose it in render? + FAST = "FAST" + CINEMA = "CINEMA" + DISC = "DISC" + + def __str__(self): + return str(self.value) + + class Offer(Base): # type: ignore[valid-type,misc] __tablename__ = "offers" id = Column(Integer, primary_key=True) name = Column(String, unique=True) + monetization_type = Column(Enum(MonetizationType)) # type: ignore[var-annotated] director_film_association_table = Table( diff --git a/src/letsrolld/film.py b/src/letsrolld/film.py index 3088b91..e93a27f 100644 --- a/src/letsrolld/film.py +++ b/src/letsrolld/film.py @@ -11,111 +11,7 @@ from letsrolld.base import BaseObject -Offer = namedtuple("Offer", ["technical_name", "url"]) - - -KANOPY = "kanopy" -HOOPLA = "hoopla" -AMAZONPRIME = "amazonprime" -AMAZONPRIMEWITHADS = "amazonprimevideowithads" -AMAZONAMCPLUS = "amazonamcplus" -AMAZONMUBI = "amazonmubi" -AMAZON = "amazon" -YOUTUBE = "youtube" -CRITERION = "criterionchannel" -METROGRAPH = "metrograph" -PLEX = "plex" -JUSTWATCHPLEX = "justwatchplexchannel" -PLUTO = "pluto" -PLUTOTV = "plutotv" -TUBITV = "tubitv" -FANDOR = "amazonfandor" -NETFLIX = "netflix" -DISNEYPLUS = "disneyplus" -OVID = "ovid" -KLASSIKI = "klassiki" -DAFILMS = "dafilms" -GUIDEDOC = "guidedoc" -HULU = "hulu" - -PHYSICAL = "physical" - -# TODO: make these sets? -SERVICES = [ - KANOPY, - HOOPLA, - AMAZONPRIME, - AMAZONPRIMEWITHADS, - AMAZONMUBI, - AMAZONAMCPLUS, - AMAZON, - YOUTUBE, - CRITERION, - METROGRAPH, - PLEX, - JUSTWATCHPLEX, - PLUTO, - PLUTOTV, - TUBITV, - FANDOR, - NETFLIX, - DISNEYPLUS, - OVID, - KLASSIKI, - DAFILMS, - GUIDEDOC, - HULU, - PHYSICAL, -] - -FREE_ALIAS = "FREE" -STREAM_ALIAS = "STREAM" -ANY_ALIAS = "ANY" - -FREE_SERVICES = [ - KANOPY, - HOOPLA, - AMAZONPRIME, - AMAZONPRIMEWITHADS, - PLEX, - JUSTWATCHPLEX, - PLUTO, - PLUTOTV, - TUBITV, -] -STREAM_SERVICES = FREE_SERVICES + [ - AMAZON, - AMAZONAMCPLUS, - AMAZONMUBI, - YOUTUBE, - CRITERION, - METROGRAPH, - FANDOR, - NETFLIX, - DISNEYPLUS, - OVID, - KLASSIKI, - DAFILMS, - GUIDEDOC, - HULU, -] -ANY_SERVICES = STREAM_SERVICES + [PHYSICAL] - -SERVICE_ALIASES = { - FREE_ALIAS: FREE_SERVICES, - STREAM_ALIAS: STREAM_SERVICES, - ANY_ALIAS: ANY_SERVICES, -} - - -def get_services(services): - res = set() - for s in services or []: - if s in SERVICE_ALIASES: - res.update(SERVICE_ALIASES[s]) - elif s in SERVICES: - res.add(s) - return res +Offer = namedtuple("Offer", ["technical_name", "url", "monetization_type"]) class Film(BaseObject): @@ -184,11 +80,19 @@ def available_physical(self): @property def available_services(self): services = [ - Offer(technical_name=offer.package.technical_name, url=offer.url) + # TODO: confirm that api returns one of the known monetization types + Offer( + technical_name=offer.package.technical_name, + url=offer.url, + monetization_type=offer.monetization_type, + ) for offer in self.offers ] if self.available_physical(): - services.append(Offer(technical_name=PHYSICAL, url=None)) + # TODO: use const for monetization type + services.append( + Offer(technical_name="physical", url=None, monetization_type="DISC") + ) return services @functools.cached_property diff --git a/src/letsrolld/webapi/app.py b/src/letsrolld/webapi/app.py index d2b847b..a2c78bb 100644 --- a/src/letsrolld/webapi/app.py +++ b/src/letsrolld/webapi/app.py @@ -7,13 +7,13 @@ from flask_sqlalchemy import SQLAlchemy import pycountry +from sqlalchemy import or_ from sqlalchemy.sql.expression import func from letsrolld import config as lconfig from letsrolld import db -from letsrolld import film -from letsrolld import filmlist from letsrolld.db import models +from letsrolld import filmlist from letsrolld.webapi import models as webapi_models import logging @@ -62,7 +62,9 @@ def _get_flag(country): # TODO: this is ugly; reimplement it as association proxy if possible def _get_offers(session, f): return list( - session.query(models.Offer.name, models.FilmOffer.url) + session.query( + models.Offer.name, models.FilmOffer.url, models.Offer.monetization_type + ) .join(models.FilmOffer) .filter(models.FilmOffer.film_id == f.id) .all() @@ -74,7 +76,10 @@ def _get_film(session, f): webapi_models.Country(name=c.name, flag=_get_flag(c.name)) for c in f.countries ] offers = [ - webapi_models.Offer(name=name, url=url) for name, url in _get_offers(session, f) + webapi_models.Offer( + name=name, url=url, monetization_type=str(monetization_type) + ) + for name, url, monetization_type in _get_offers(session, f) ] return webapi_models.Film( id=f.id, @@ -266,14 +271,17 @@ def _execute_section_plan(db, config, seen_films): if config.services: query = query.join(models.Film.offers).filter( - models.Offer.name.in_(film.get_services(config.services)) + or_( + models.Offer.name.in_(config.services), + models.Offer.monetization_type.in_(config.services), + ) ) if config.exclude_services: query = query.join(models.Film.offers).filter( ~models.Film.offers.any( - models.Offer.name.in_( - film.get_services(config.exclude_services) - - film.get_services(config.services) + or_( + models.Offer.name.in_(config.exclude_services), + models.Offer.monetization_type.in_(config.exclude_services), ) ) ) diff --git a/src/letsrolld/webapi/models.py b/src/letsrolld/webapi/models.py index 86b28b1..e3abb23 100644 --- a/src/letsrolld/webapi/models.py +++ b/src/letsrolld/webapi/models.py @@ -32,9 +32,10 @@ class Country(Schema): class Offer(Schema): properties = { "name": {"type": "string"}, + "monetization_type": {"type": "string"}, "url": NullableURL, } - required = ["name", "url"] + required = ["name", "url", "monetization_type"] class DirectorInfo(Schema): diff --git a/src/letsrolld/webcli/cli.py b/src/letsrolld/webcli/cli.py index ecffdde..05992bd 100644 --- a/src/letsrolld/webcli/cli.py +++ b/src/letsrolld/webcli/cli.py @@ -1,7 +1,8 @@ +from collections import defaultdict + import click from jinja2 import Environment, PackageLoader -from letsrolld import film as lfilm from letsrolld_api_client import Client from letsrolld_api_client.api.default import get_directors @@ -34,15 +35,12 @@ def list_director(director): def _get_services_to_report(film): - offers_to_report = [] + offers_to_report = defaultdict(list) urls_seen = set() - for service in lfilm.STREAM_SERVICES: - for o in film.offers: - if o.name != service: - continue - if o.url not in urls_seen: - offers_to_report.append(o.name) - urls_seen.add(o.url) + for o in film.offers: + if o.url not in urls_seen: + offers_to_report[o.monetization_type].append(o) + urls_seen.add(o.url) return offers_to_report diff --git a/src/letsrolld/webcli/templates/film-full.j2 b/src/letsrolld/webcli/templates/film-full.j2 index 2683084..8879f4b 100644 --- a/src/letsrolld/webcli/templates/film-full.j2 +++ b/src/letsrolld/webcli/templates/film-full.j2 @@ -14,8 +14,15 @@ {{ film.description|wordwrap(70)|indent(2, first=True) }} --- Available @ - {% for offer in film.offers|selectattr("name", "in", offers) -%} - {%- if offer.url -%} - {{ offer.name }}: {{ offer.url }} + {% for monetization_type in ('FREE', 'ADS', 'FLATRATE', 'RENT', 'BUY', 'DISC') -%} + {%- if monetization_type in offers -%} + {{ monetization_type }}: + {% for offer in offers[monetization_type] -%} + {%- if offer.url -%} + {{ offer.name }}: {{ offer.url }} + {%- else -%} + {{ offer.name }} + {% endif %} + {% endfor %} {% endif -%} {% endfor -%} diff --git a/swagger.json b/swagger.json index f6da026..2fbf505 100644 --- a/swagger.json +++ b/swagger.json @@ -108,6 +108,9 @@ "name": { "type": "string" }, + "monetization_type": { + "type": "string" + }, "url": { "type": "string", "format": "url", @@ -116,7 +119,8 @@ }, "required": [ "name", - "url" + "url", + "monetization_type" ], "type": "object" } @@ -246,6 +250,9 @@ "name": { "type": "string" }, + "monetization_type": { + "type": "string" + }, "url": { "type": "string", "format": "url", @@ -254,7 +261,8 @@ }, "required": [ "name", - "url" + "url", + "monetization_type" ], "type": "object" } @@ -390,6 +398,9 @@ "name": { "type": "string" }, + "monetization_type": { + "type": "string" + }, "url": { "type": "string", "format": "url", @@ -398,7 +409,8 @@ }, "required": [ "name", - "url" + "url", + "monetization_type" ], "type": "object" } @@ -510,6 +522,9 @@ "name": { "type": "string" }, + "monetization_type": { + "type": "string" + }, "url": { "type": "string", "format": "url", @@ -518,7 +533,8 @@ }, "required": [ "name", - "url" + "url", + "monetization_type" ], "type": "object" } @@ -617,6 +633,9 @@ "name": { "type": "string" }, + "monetization_type": { + "type": "string" + }, "url": { "type": "string", "format": "url", @@ -625,7 +644,8 @@ }, "required": [ "name", - "url" + "url", + "monetization_type" ], "type": "object" } @@ -730,6 +750,9 @@ "name": { "type": "string" }, + "monetization_type": { + "type": "string" + }, "url": { "type": "string", "format": "url", @@ -738,7 +761,8 @@ }, "required": [ "name", - "url" + "url", + "monetization_type" ], "type": "object" } @@ -861,6 +885,9 @@ "name": { "type": "string" }, + "monetization_type": { + "type": "string" + }, "url": { "type": "string", "format": "url", @@ -869,7 +896,8 @@ }, "required": [ "name", - "url" + "url", + "monetization_type" ], "type": "object" } @@ -1001,6 +1029,9 @@ "name": { "type": "string" }, + "monetization_type": { + "type": "string" + }, "url": { "type": "string", "format": "url", @@ -1009,7 +1040,8 @@ }, "required": [ "name", - "url" + "url", + "monetization_type" ], "type": "object" } @@ -1148,6 +1180,9 @@ "name": { "type": "string" }, + "monetization_type": { + "type": "string" + }, "url": { "type": "string", "format": "url", @@ -1156,7 +1191,8 @@ }, "required": [ "name", - "url" + "url", + "monetization_type" ], "type": "object" } @@ -1261,6 +1297,7 @@ "offers": [ { "name": "string", + "monetization_type": "string", "url": "string" } ], @@ -1333,6 +1370,7 @@ "offers": [ { "name": "string", + "monetization_type": "string", "url": "string" } ], @@ -1426,6 +1464,7 @@ "offers": [ { "name": "string", + "monetization_type": "string", "url": "string" } ], @@ -1489,6 +1528,7 @@ "offers": [ { "name": "string", + "monetization_type": "string", "url": "string" } ], @@ -1549,6 +1589,7 @@ "offers": [ { "name": "string", + "monetization_type": "string", "url": "string" } ], @@ -1623,6 +1664,7 @@ "offers": [ { "name": "string", + "monetization_type": "string", "url": "string" } ], diff --git a/tests/test_film.py b/tests/test_film.py index c67c4e9..efe1ab7 100644 --- a/tests/test_film.py +++ b/tests/test_film.py @@ -1,39 +1,5 @@ from letsrolld import film -def test_get_services_default(): - assert film.get_services(None) == set() - - -def test_get_services_empty(): - assert film.get_services([]) == set() - - -def test_get_services_single(): - assert film.get_services([film.AMAZONPRIME]) == {film.AMAZONPRIME} - - -def test_get_services_multiple(): - expected = {film.AMAZONPRIME, film.CRITERION} - assert film.get_services([film.AMAZONPRIME, film.CRITERION]) == expected - - -def test_get_services_alias_FREE_kanopy(): - assert film.KANOPY in film.get_services([film.FREE_ALIAS]) - - -def test_get_services_alias_FREE_amazon(): - assert film.AMAZON not in film.get_services([film.FREE_ALIAS]) - - -def test_get_services_alias_FREE_plus_explicit_entry(): - expected = set(film.FREE_SERVICES) | {film.AMAZON} - assert film.get_services([film.FREE_ALIAS, film.AMAZON]) == expected - - -def test_get_services_unknown_service(): - assert film.get_services(["unknown-service"]) == set() - - def test_Film(): film.Film("https://url.com/movie") diff --git a/ts/model/directorFilmsInnerOffersInner.ts b/ts/model/directorFilmsInnerOffersInner.ts index 36b70ce..21c8a4c 100644 --- a/ts/model/directorFilmsInnerOffersInner.ts +++ b/ts/model/directorFilmsInnerOffersInner.ts @@ -14,6 +14,7 @@ import { RequestFile } from './models'; export class DirectorFilmsInnerOffersInner { 'name': string; + 'monetizationType': string; 'url': string | null; static discriminator: string | undefined = undefined; @@ -24,6 +25,11 @@ export class DirectorFilmsInnerOffersInner { "baseName": "name", "type": "string" }, + { + "name": "monetizationType", + "baseName": "monetization_type", + "type": "string" + }, { "name": "url", "baseName": "url",