Add robots.txt

feeds/urls.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from django.urls import URLPattern, path
+from django.views.decorators.cache import cache_page
 
 from feeds import views
 
@@ -15,6 +16,7 @@ urlpatterns: list[URLPattern] = [
     path(route="add", view=views.AddView.as_view(), name="add"),
     path(route="upload", view=views.UploadView.as_view(), name="upload"),
     path(route="api/", view=APIView.as_view(), name="api"),
+    path(route="robots.txt", view=cache_page(timeout=60 * 60 * 365)(views.RobotsView.as_view()), name="robots"),
 ]
 
 # Account urls

feeds/views.py

@@ -259,3 +259,14 @@ class APIView(View):
             "title": "API Documentation",
         }
         return HttpResponse(content=template.render(context=context, request=request))
+
+
+class RobotsView(View):
+    """Robots.txt view."""
+
+    def get(self, request: HttpRequest) -> HttpResponse:  # noqa: ARG002
+        """Load the robots.txt file."""
+        return HttpResponse(
+            content="""User-agent: *\nDisallow: /add\nDisallow: /upload\nDisallow: /accounts/""",
+            content_type="text/plain",
+        )