feat: Add REQUESTS_TIMEOUT setting

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+### Added
+
+- Set the `REQUESTS_TIMEOUT` setting, to prevent source URLs from causing a denial of service, whether accidentally or maliciously.
+
 ## [0.30.4] - 2024-06-28
 
 ### Fixed

cove/input/models.py

@@ -69,7 +69,11 @@ def is_google_doc(self):
 
     def download(self):
         if self.source_url:
-            r = requests.get(self.source_url, headers={'User-Agent': 'Cove (cove.opendataservice.coop)'})
+            r = requests.get(
+                self.source_url,
+                headers={'User-Agent': 'Cove (cove.opendataservice.coop)'},
+                timeout=getattr(settings, "REQUESTS_TIMEOUT", None),
+            )
             r.raise_for_status()
             content_type = r.headers.get('content-type', '').split(';')[0].lower()
             file_extension = CONTENT_TYPE_MAP.get(content_type)

cove/input/views.py

@@ -83,6 +83,16 @@ def data_input(request, form_classes=default_form_classes, text_file_name='test.
                         'link_text': _('Try Again'),
                         'msg': str(err)
                     })
+                except requests.Timeout as err:
+                    return render(request, 'error.html', context={
+                        'sub_title': _(
+                            "The request timed out after %(timeout)s seconds",
+                            getattr(settings, "REQUESTS_TIMEOUT", "indefinite"),
+                        ),
+                        'link': 'index',
+                        'link_text': _('Try Again'),
+                        'msg': str(err)
+                    })
                 except requests.ConnectionError as err:
                     return render(request, 'error.html', context={
                         'sub_title': _("Sorry we got a ConnectionError whilst trying to download that file"),

cove/locale/en/LC_MESSAGES/django.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2024-05-04 04:28+0000\n"
+"POT-Creation-Date: 2024-09-25 21:29+0000\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <[email protected]>\n"
@@ -209,26 +209,31 @@ msgstr ""
 msgid "That URL is invalid"
 msgstr ""
 
-#: cove/input/views.py:83 cove/input/views.py:90 cove/input/views.py:98
-#: cove/views.py:28 cove/views.py:39 cove/views.py:49
+#: cove/input/views.py:83 cove/input/views.py:93 cove/input/views.py:100
+#: cove/input/views.py:109 cove/views.py:29 cove/views.py:40 cove/views.py:50
 msgid "Try Again"
 msgstr ""
 
-#: cove/input/views.py:88
+#: cove/input/views.py:89
+#, python-format
+msgid "The request timed out after %(timeout)s seconds"
+msgstr ""
+
+#: cove/input/views.py:98
 msgid "Sorry we got a ConnectionError whilst trying to download that file"
 msgstr ""
 
-#: cove/input/views.py:91
+#: cove/input/views.py:102
 msgid ""
 "Common reasons for this error include supplying a local development url that "
 "our servers can't access, or misconfigured SSL certificates."
 msgstr ""
 
-#: cove/input/views.py:96
+#: cove/input/views.py:107
 msgid "Sorry we got a HTTP Error whilst trying to download that file"
 msgstr ""
 
-#: cove/input/views.py:99
+#: cove/input/views.py:111
 msgid ""
 "If you can access the file through a browser then the problem may be related "
 "to permissions, or you may be blocking certain user agents."
@@ -528,44 +533,44 @@ msgstr ""
 msgid "Column:"
 msgstr ""
 
-#: cove/views.py:26 cove/views.py:37 cove/views.py:47
+#: cove/views.py:27 cove/views.py:38 cove/views.py:48
 msgid "Sorry, we can't process that data"
 msgstr ""
 
-#: cove/views.py:30
+#: cove/views.py:31
 msgid ""
 "We think you tried to upload a JSON file, but it is not well formed JSON.\n"
 "\n"
 "Error message: {}"
 msgstr ""
 
-#: cove/views.py:41
+#: cove/views.py:42
 msgid ""
 "We did not recognise the file type.\n"
 "\n"
 "We can only process json, csv, ods and xlsx files."
 msgstr ""
 
-#: cove/views.py:51
+#: cove/views.py:52
 msgid ""
 "We think you tried to supply a spreadsheet, but we failed to convert it.\n"
 "\n"
 "Error message: {}"
 msgstr ""
 
-#: cove/views.py:75 cove/views.py:87
+#: cove/views.py:77 cove/views.py:90
 msgid "Sorry, the page you are looking for is not available"
 msgstr ""
 
-#: cove/views.py:77 cove/views.py:89
+#: cove/views.py:79 cove/views.py:92
 msgid "Go to Home page"
 msgstr ""
 
-#: cove/views.py:78
+#: cove/views.py:81
 msgid "We don't seem to be able to find the data you requested."
 msgstr ""
 
-#: cove/views.py:90
+#: cove/views.py:94
 msgid ""
 "The data you were hoping to explore no longer exists.\n"
 "\n"

cove/locale/es/LC_MESSAGES/django.po

@@ -17,7 +17,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2024-05-04 04:28+0000\n"
+"POT-Creation-Date: 2024-09-26 01:52+0000\n"
 "PO-Revision-Date: 2018-11-30 11:12+0000\n"
 "Last-Translator: Bibiana Cristòfol <[email protected]>, 2021\n"
 "Language-Team: Spanish (https://www.transifex.com/OpenDataServices/"
@@ -236,18 +236,23 @@ msgstr "Pegar (sólo JSON)"
 msgid "That URL is invalid"
 msgstr "Esa URL no es válida"
 
-#: cove/input/views.py:83 cove/input/views.py:90 cove/input/views.py:98
-#: cove/views.py:28 cove/views.py:39 cove/views.py:49
+#: cove/input/views.py:83 cove/input/views.py:93 cove/input/views.py:100
+#: cove/input/views.py:109 cove/views.py:29 cove/views.py:40 cove/views.py:50
 msgid "Try Again"
 msgstr "Inténtelo de nuevo"
 
-#: cove/input/views.py:88
+#: cove/input/views.py:89
+#, python-format
+msgid "The request timed out after %(timeout)s seconds"
+msgstr "La solicitud expiró luego de %(timeout)s segundos"
+
+#: cove/input/views.py:98
 msgid "Sorry we got a ConnectionError whilst trying to download that file"
 msgstr ""
 "Lo sentimos, al tratar de descargar el archivo hemos encontrado un "
 "ConnectionError"
 
-#: cove/input/views.py:91
+#: cove/input/views.py:102
 msgid ""
 "Common reasons for this error include supplying a local development url that "
 "our servers can't access, or misconfigured SSL certificates."
@@ -256,12 +261,12 @@ msgstr ""
 "desarrollo local a la que nuestros servidores no pueden acceder o "
 "certificados SSL mal configurados."
 
-#: cove/input/views.py:96
+#: cove/input/views.py:107
 msgid "Sorry we got a HTTP Error whilst trying to download that file"
 msgstr ""
 "Lo sentimos, al tratar de descargar el archivo hemos encontrado un HTTP Error"
 
-#: cove/input/views.py:99
+#: cove/input/views.py:111
 msgid ""
 "If you can access the file through a browser then the problem may be related "
 "to permissions, or you may be blocking certain user agents."
@@ -575,11 +580,11 @@ msgstr "(más información)"
 msgid "Column:"
 msgstr "Columna:"
 
-#: cove/views.py:26 cove/views.py:37 cove/views.py:47
+#: cove/views.py:27 cove/views.py:38 cove/views.py:48
 msgid "Sorry, we can't process that data"
 msgstr "Lo sentimos, no podemos procesar esos datos"
 
-#: cove/views.py:30
+#: cove/views.py:31
 msgid ""
 "We think you tried to upload a JSON file, but it is not well formed JSON.\n"
 "\n"
@@ -590,7 +595,7 @@ msgstr ""
 "\n"
 "Mensaje del error: {}"
 
-#: cove/views.py:41
+#: cove/views.py:42
 msgid ""
 "We did not recognise the file type.\n"
 "\n"
@@ -600,7 +605,7 @@ msgstr ""
 "\n"
 "Solo podemos procesar archivos son, csv, ods y xlsx."
 
-#: cove/views.py:51
+#: cove/views.py:52
 msgid ""
 "We think you tried to supply a spreadsheet, but we failed to convert it.\n"
 "\n"
@@ -611,19 +616,19 @@ msgstr ""
 "\n"
 "Mensaje del error: {}"
 
-#: cove/views.py:75 cove/views.py:87
+#: cove/views.py:77 cove/views.py:90
 msgid "Sorry, the page you are looking for is not available"
 msgstr "Lo sentimos, la página que está buscando no está disponible"
 
-#: cove/views.py:77 cove/views.py:89
+#: cove/views.py:79 cove/views.py:92
 msgid "Go to Home page"
 msgstr "Ir a la Página de Inicio"
 
-#: cove/views.py:78
+#: cove/views.py:81
 msgid "We don't seem to be able to find the data you requested."
 msgstr "Parece que no podemos encontrar los datos que usted solicitó."
 
-#: cove/views.py:90
+#: cove/views.py:94
 msgid ""
 "The data you were hoping to explore no longer exists.\n"
 "\n"

cove/settings.py

@@ -39,6 +39,7 @@
 VALIDATION_ERROR_LOCATIONS_LENGTH = int(os.getenv("VALIDATION_ERROR_LOCATIONS_LENGTH", 1000))
 VALIDATION_ERROR_LOCATIONS_SAMPLE = os.getenv("VALIDATION_ERROR_LOCATIONS_SAMPLE", '').lower() in boolean_true_strings
 
+REQUESTS_TIMEOUT = int(os.getenv("REQUESTS_TIMEOUT", 0)) or None
 DELETE_FILES_AFTER_DAYS = int(os.getenv("DELETE_FILES_AFTER_DAYS", 7))
 
 MEDIA_ROOT = os.path.join(BASE_DIR, 'media')