fix: add robots.txt

openfoodfacts · Aug 11, 2023 · 226b41a · 226b41a
1 parent 812450f
commit 226b41a
Showing 1 changed file with 10 additions and 0 deletions.
diff --git a/robotoff/app/api.py b/robotoff/app/api.py
@@ -1669,6 +1669,15 @@ def on_get(self, req: falcon.Request, resp: falcon.Response):
         resp.media = response
 
 
+class RobotsTxtResource:
+    def on_get(self, req: falcon.Request, resp: falcon.Response):
+        # Disallow completely indexation: otherwise web crawlers send millions
+        # of requests to Robotoff (420k requests/day by Google alone)
+        resp.body = "User-agent: *\nDisallow: /\n"
+        resp.content_type = falcon.MEDIA_TEXT
+        resp.status = falcon.HTTP_200
+
+
 cors = CORS(
     allow_all_origins=True,
     allow_all_headers=True,
@@ -1731,3 +1740,4 @@ def on_get(self, req: falcon.Request, resp: falcon.Response):
 api.add_route("/api/v1/users/statistics/{username}", UserStatisticsResource())
 api.add_route("/api/v1/predictions", PredictionCollection())
 api.add_route("/api/v1/annotation/collection", LogoAnnotationCollection())
+api.add_route("/robots.txt", RobotsTxtResource())