add #540, prevent starting scans when credentials are not correct, pe…

…riodically check credentials for this

dashboard/internet_nl_dashboard/fixtures/dashboard_production_periodic_tasks.json

@@ -35,6 +35,18 @@
       "timezone": "UTC"
     }
   },
+  {
+    "model": "django_celery_beat.crontabschedule",
+    "pk": 37,
+    "fields": {
+      "minute": "0",
+      "hour": "1",
+      "day_of_week": "*",
+      "day_of_month": "*",
+      "month_of_year": "*",
+      "timezone": "UTC"
+    }
+  },
   {
     "model": "django_celery_beat.intervalschedule",
     "pk": 2,
@@ -181,5 +193,33 @@
       "date_changed": "2024-06-18T15:10:04.162Z",
       "description": ""
     }
+  },
+    {
+    "model": "django_celery_beat.periodictask",
+    "pk": 140,
+    "fields": {
+      "name": "Check account credential validity to prevent starting scheduled scans that cannot finish",
+      "task": "dashboard.internet_nl_dashboard.tasks.update_account_access_to_api",
+      "interval": null,
+      "crontab": 37,
+      "solar": null,
+      "clocked": null,
+      "args": "[]",
+      "kwargs": "{}",
+      "queue": null,
+      "exchange": null,
+      "routing_key": null,
+      "headers": "{}",
+      "priority": null,
+      "expires": null,
+      "expire_seconds": null,
+      "one_off": false,
+      "start_time": null,
+      "enabled": true,
+      "last_run_at": "2024-06-18T15:10:00.009Z",
+      "total_run_count": 39348,
+      "date_changed": "2024-06-18T15:10:04.162Z",
+      "description": ""
+    }
   }
 ]

dashboard/internet_nl_dashboard/tasks.py

@@ -8,7 +8,7 @@
 from constance import config
 
 from dashboard.celery import app
-from dashboard.internet_nl_dashboard.models import UrlList, UrlListReport
+from dashboard.internet_nl_dashboard.models import Account, UrlList, UrlListReport
 from dashboard.internet_nl_dashboard.scanners import scan_internet_nl_per_account, subdomains
 from dashboard.internet_nl_dashboard.scanners.scan_internet_nl_per_account import initialize_scan
 
@@ -35,6 +35,16 @@ def start_scans_for_lists_who_are_up_for_scanning() -> Task:
             urllist.renew_scan_moment()
             continue
 
+        # Prevent starting scans when credentials are not valid.
+        # https://github.com/internetstandards/Internet.nl-dashboard/issues/540
+        # the validation process is separated from this call as it can take an unknown amount of time before
+        # the entire validation is done. Also: we don't want to validate per list, as will re-validate
+        # every account every time, which is wasteful and even slower :)
+        # see update_account_access_to_api. Update the scan moment so this isn't called every time...
+        if not urllist.account.can_connect_to_internet_nl_api:
+            urllist.renew_scan_moment()
+            continue
+
         if urllist.is_due_for_scanning():
             tasks.append(initialize_scan.si(urllist.id))
 
@@ -68,3 +78,18 @@ def autoshare_report_to_front_page():
         is_publicly_shared=True,
         is_shared_on_homepage=True,
     )
+
+
+@app.task(queue="storage", ignore_result=True)
+def update_account_access_to_api():
+    """
+    This is performed in a separate task to prevent long start times from automatically started scans in
+    start_scans_for_lists_who_are_up_for_scanning. Otherwise a network request is needed 2000x times a month that
+    takes unknown amounts of time. This can result in a flood when calling that method too frequently (which
+    will happen at some point as the amount of lists/accounts grows).
+    """
+    for account in Account.objects.all().filter():
+        username = account.internet_nl_api_username
+        password = account.decrypt_password()
+        account.can_connect_to_internet_nl_api = account.connect_to_internet_nl_api(username, password)
+        account.save()