Include robots.txt in UI and API for handling of web crawlers

.gitignore

@@ -125,5 +125,5 @@ dmypy.json
 
 # Editor settings
 .vscode
-
+.idea/
 .DS_Store

dandiapi/api/views/__init__.py

@@ -5,6 +5,7 @@
 from .dandiset import DandisetViewSet
 from .dashboard import DashboardView, mailchimp_csv_view, user_approval_view
 from .info import info_view
+from .robots import robots_txt_view
 from .root import root_content_view
 from .stats import stats_view
 from .upload import (
@@ -36,4 +37,5 @@
     'stats_view',
     'info_view',
     'root_content_view',
+    'robots_txt_view'
 ]

dandiapi/api/views/robots.py

@@ -0,0 +1,10 @@
+from django.views.decorators.cache import cache_page
+from django.http import HttpResponse
+
+@cache_page(60 * 60 * 24)  # 24 hour cache
+def robots_txt_view(request):
+    content = """
+    User-agent: *
+    Disallow: /
+    """
+    return HttpResponse(content, content_type="text/plain")

dandiapi/urls.py

@@ -19,6 +19,7 @@
     blob_read_view,
     info_view,
     mailchimp_csv_view,
+    robots_txt_view,
     root_content_view,
     stats_view,
     upload_complete_view,
@@ -80,9 +81,11 @@ def to_url(self, value):
 register_converter(DandisetIDConverter, 'dandiset_id')
 urlpatterns = [
     path('', root_content_view),
+    path("robots.txt", robots_txt_view),
     path('api/', include(router.urls)),
     path('api/auth/token/', auth_token_view, name='auth-token'),
     path('api/stats/', stats_view),
+    path('api/stats/', stats_view),
     path('api/info/', info_view),
     path('api/blobs/digest/', blob_read_view, name='blob-read'),
     path('api/uploads/initialize/', upload_initialize_view, name='upload-initialize'),

web/public/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /