From f62f1ba804bee541847e77d8c2cb491fbb831e53 Mon Sep 17 00:00:00 2001 From: Johnny JTH Date: Thu, 8 Feb 2024 20:02:47 +0100 Subject: [PATCH] Change search_term to search_url --- README.md | 18 ++++++++++++++++-- example.config.json | 2 +- main.py | 16 +++++++++++----- plugins/common.py | 7 +++++-- plugins/dba.py | 16 +++++++--------- static/schema.json | 23 ++++++++++------------- 6 files changed, 50 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index e860045..f0f2363 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,22 @@ The `config.json` file contains the following fields: ### Plugins Each plugin has the following fields: - `name`: The name of the plugin. This can be one of the following: - - `dba` -- `search_term`: The search term you want to be notified about. + - [`dba`](#dba) +- `search_url`: The URL of the search query you want to monitor. Plugin specific. - `webhook_url`: The URL of the Discord Webhook you want to send the message to. - `message_content`: The message you want to be sent to the Discord channel alongside the listings. - For Discord mentions you have to write the mention in a channel and put a backslash before the @ symbol. Example: `\@JohnnyJTH`. When you send the message, it will look something like this: `<@&1001569699735273502>`. You can copy this and paste it into the `message_content` field. + +> [!NOTE] +> In most cases, the correct `search_url` can be found by going to the marketplace and searching for the item you want to monitor, with the filters you want. Then copy the URL and paste it into the `search_url` field. + +> [!IMPORTANT] +> You should always sort the listings by the newest first for the best results. + +#### DBA +For the `dba` plugin, the `search_url` field should be the URL of the search query you want to monitor. For example, if you want to monitor new analogue cameras, the `search_url` field could be `https://www.dba.dk/billede-og-lyd/analoge-kameraer/andre-analoge-kameraer/?sort=listingdate-desc`. + +It can also be a normal search query, for example: `https://www.dba.dk/soeg/?soeg=analoge+kameraer&sort=listingdate-desc`. The `sort` parameter is important, as it sorts the listings by the date they were added. The `listingdate-desc` parameter sorts the listings by the date they were added, in descending order. This ensures that the newest listings are always shown first. + +#### eBay +For the `ebay` plugin, the `search_url` field should be the URL of the search query you want to monitor. For example, if you want to monitor new analogue cameras, the `search_url` field could be `https://www.ebay.com/sch/i.html?_nkw=analog+camera&_sop=10`. This means that the search query is `analog camera` and the listings are sorted by the newest first. diff --git a/example.config.json b/example.config.json index 9d18a08..f5c0d35 100644 --- a/example.config.json +++ b/example.config.json @@ -3,7 +3,7 @@ "plugins": [ { "name": "dba", - "search_term": "camera", + "search_url": "https://www.dba.dk/billede-og-lyd/analoge-kameraer/andre-analoge-kameraer/?sort=listingdate-desc", "webhook_url": "", "message_content": "Hello World!" } diff --git a/main.py b/main.py index a0abfbd..b0b285a 100644 --- a/main.py +++ b/main.py @@ -32,11 +32,14 @@ async def main(): config = json.loads(config_text) validate(instance=config, schema=schema) except FileNotFoundError: - return print("No config file found.") + print("No config file found.") + return except ValueError: - return print("Config file contains invalid JSON.") + print("Config file contains invalid JSON.") + return except ValidationError as e: - return print(f"Configuration is invalid.\n{e.message}") + print(f"Configuration is invalid.\n{e.message}") + return while True: for plugin_config in config["plugins"]: @@ -46,12 +49,15 @@ async def main(): ) plugin_class = getattr(plugin_module, "Plugin") plugin: MarketplacePlugin = plugin_class( - session=session, search_term=plugin_config["search_term"] + session=session, search_url=plugin_config["search_url"] ) except ImportError as e: print(f"Error loading plugin: {e}") + continue except AttributeError as e: print(f"Error loading plugin class: {e}") + continue + listings = await plugin.fetch_listings() if not listings: continue @@ -83,7 +89,7 @@ async def main(): ) except Exception as e: print(f"[{plugin.name}] Error while sending message: {e}") - return + continue await asyncio.sleep(config["interval"]) diff --git a/plugins/common.py b/plugins/common.py index 5cba1a8..40ffec4 100644 --- a/plugins/common.py +++ b/plugins/common.py @@ -21,12 +21,12 @@ def __str__(self) -> str: class MarketplacePlugin: def __init__( - self, name: str, color: int, session: ClientSession, search_term: str + self, name: str, color: int, session: ClientSession, search_url: str ) -> None: self.name = name self.color = color self.session = session - self.search_term = search_term + self.search_url = search_url async def fetch_listings(self) -> list[MarketplaceListing]: """Returns a list of new listings since last run.""" @@ -47,3 +47,6 @@ async def save_listings(self, listings: list[MarketplaceListing]) -> None: file_name = f"listing_data/{self.name}{self.search_term}.json" async with aiofiles.open(file_name, mode="w") as f: await f.write(MarketplaceListing.schema().dumps(listings, many=True)) + + def log(self, message: str) -> None: + print(f"[{self.name}] {message}") diff --git a/plugins/dba.py b/plugins/dba.py index 96c1588..541c7cd 100644 --- a/plugins/dba.py +++ b/plugins/dba.py @@ -6,20 +6,18 @@ class Plugin(MarketplacePlugin): - def __init__(self, session: ClientSession, search_term: str) -> None: + def __init__(self, session: ClientSession, search_url: str) -> None: name = "DBA" color = 0x010C8D - super().__init__(name, color, session, search_term) + super().__init__(name, color, session, search_url) async def fetch_listings(self): existing_listings = await self.get_saved_listings() try: - response = await self.session.get( - f"https://www.dba.dk/soeg/?soeg={self.search_term}" - ) + response = await self.session.get(self.search_url) except Exception as e: - print(f"[{self.name}] Error when requesting DBA site: {e}") - return + self.log(f"Error when requesting DBA site: {e}") + return [] text = await response.text() # Parse HTML @@ -32,8 +30,8 @@ async def fetch_listings(self): try: json_data = json.loads(text) except json.JSONDecodeError as e: - print(text) - print(f"[{self.name}] Error decoding JSON: {e}") + self.log(f"Error decoding JSON: {e}") + return [] name = json_data.get("name") image_url = json_data.get("image").split("?")[0] + "?class=S1600X1600" diff --git a/static/schema.json b/static/schema.json index 2dc2751..76cff05 100644 --- a/static/schema.json +++ b/static/schema.json @@ -17,23 +17,20 @@ "title": "A plugin.", "required": [ "name", - "search_term", + "search_url", "webhook_url" ], "properties": { "name": { "type": "string", - "enum": ["dba"], + "enum": ["dba", "ebay"], "default": "dba", "title": "The name of the plugin." }, - "search_term": { + "search_url": { "type": "string", "default": "", - "title": "The search term to use.", - "examples": [ - "camera" - ] + "title": "The url to search for listings. See README." }, "webhook_url": { "type": "string", @@ -52,16 +49,16 @@ } }, "examples": [{ - "name": "DBA", - "search_term": "camera", + "name": "dba", + "search_url": "https://www.dba.dk/billede-og-lyd/analoge-kameraer/andre-analoge-kameraer/?sort=listingdate-desc", "webhook_url": "webhook_url_here", "message_content": "Hello World!" }] }, "examples": [ [{ - "name": "DBA", - "search_term": "camera", + "name": "dba", + "search_url": "https://www.dba.dk/billede-og-lyd/analoge-kameraer/andre-analoge-kameraer/?sort=listingdate-desc", "webhook_url": "webhook_url_here", "message_content": "Hello World!" }] @@ -78,8 +75,8 @@ }, "examples": [{ "plugins": [{ - "name": "DBA", - "search_term": "camera", + "name": "dba", + "search_url": "https://www.dba.dk/billede-og-lyd/analoge-kameraer/andre-analoge-kameraer/?sort=listingdate-desc", "webhook_url": "webhook_url_here", "message_content": "Hello World!" }],