config/tests/test_urls.py

@@ -34,6 +34,7 @@ def _reload_urls_with(**overrides) -> ModuleType:
 def test_top_level_named_routes_available() -> None:
     """Top-level routes defined in `config.urls` are reversible."""
     assert reverse("sitemap") == "/sitemap.xml"
+    assert reverse("robots") == "/robots.txt"
     # ensure the included `twitch` namespace is present
     assert reverse("twitch:dashboard") == "/"
 

config/urls.py

@@ -12,7 +12,8 @@ if TYPE_CHECKING:
     from django.urls.resolvers import URLResolver
 
 urlpatterns: list[URLPattern | URLResolver] = [
-    path(route="sitemap.xml", view=twitch_views.sitemap_view, name="sitemap"),
+    path("sitemap.xml", twitch_views.sitemap_view, name="sitemap"),
+    path("robots.txt", twitch_views.robots_txt_view, name="robots"),
     path(route="", view=include("twitch.urls", namespace="twitch")),
 ]
 

static/robots.xml

@@ -1,5 +0,0 @@
-User-agent: *
-Allow: /
-
-# Sitemap location
-Sitemap: https://ttvdrops.lovinator.space/sitemap.xml

templates/base.html

@@ -239,20 +239,5 @@
         {% block content %}
             <!-- Main content will be injected here -->
         {% endblock content %}
-        <footer>
-            <div style="text-align: center; font-size: 0.9em; color: #6a737d">
-                <img src="{% static 'toycar_driving_lc.gif' %}"
-                     alt="Animated toy car driving"
-                     width="156"
-                     height="156"
-                     loading="lazy" />
-                <br />
-                CC0; Information wants to be free.
-                <br />
-                Data retrival possible because of <a href="https://github.com/DevilXD/TwitchDropsMiner">DevilXD/TwitchDropsMiner</a>.
-                <br />
-                Data fetched at :01, :16, :31, and :46.
-            </div>
-        </footer>
     </body>
 </html>

twitch/tests/test_views.py

@@ -1452,6 +1452,60 @@ class TestSitemapView:
         assert "<lastmod>" in content
 
 
+@pytest.mark.django_db
+class TestRobotsTxtView:
+    """Tests for the robots.txt view."""
+
+    def test_robots_txt_returns_text(self, client: Client) -> None:
+        """Test robots.txt view returns text content."""
+        response: _MonkeyPatchedWSGIResponse = client.get("/robots.txt")
+        assert response.status_code == 200
+        assert response["Content-Type"] in {"text/plain", "text/plain; charset=utf-8"}
+
+    def test_robots_txt_user_agent(self, client: Client) -> None:
+        """Test robots.txt contains user-agent."""
+        response: _MonkeyPatchedWSGIResponse = client.get("/robots.txt")
+        content: str = response.content.decode()
+        assert "User-agent: *" in content
+
+    def test_robots_txt_allow_root(self, client: Client) -> None:
+        """Test robots.txt allows root path."""
+        response: _MonkeyPatchedWSGIResponse = client.get("/robots.txt")
+        content: str = response.content.decode()
+        assert "Allow: /" in content
+
+    def test_robots_txt_disallow_admin(self, client: Client) -> None:
+        """Test robots.txt disallows /admin/."""
+        response: _MonkeyPatchedWSGIResponse = client.get("/robots.txt")
+        content: str = response.content.decode()
+        assert "Disallow: /admin/" in content
+
+    def test_robots_txt_disallow_debug(self, client: Client) -> None:
+        """Test robots.txt disallows /debug/."""
+        response: _MonkeyPatchedWSGIResponse = client.get("/robots.txt")
+        content: str = response.content.decode()
+        assert "Disallow: /debug/" in content
+
+    def test_robots_txt_disallow_datasets(self, client: Client) -> None:
+        """Test robots.txt disallows /datasets/."""
+        response: _MonkeyPatchedWSGIResponse = client.get("/robots.txt")
+        content: str = response.content.decode()
+        assert "Disallow: /datasets/" in content
+
+    def test_robots_txt_sitemap_reference(self, client: Client) -> None:
+        """Test robots.txt references sitemap."""
+        response: _MonkeyPatchedWSGIResponse = client.get("/robots.txt")
+        content: str = response.content.decode()
+        assert "Sitemap:" in content
+        assert "/sitemap.xml" in content
+
+    def test_robots_txt_disallow_export(self, client: Client) -> None:
+        """Test robots.txt disallows /export/."""
+        response: _MonkeyPatchedWSGIResponse = client.get("/robots.txt")
+        content: str = response.content.decode()
+        assert "Disallow: /export/" in content
+
+
 @pytest.mark.django_db
 class TestSEOPaginationLinks:
     """Tests for SEO pagination links in views."""

twitch/views.py

@@ -2753,3 +2753,30 @@ def sitemap_view(request: HttpRequest) -> HttpResponse:  # noqa: PLR0915
     xml_content += "</urlset>"
 
     return HttpResponse(xml_content, content_type="application/xml")
+
+
+# MARK: /robots.txt
+def robots_txt_view(request: HttpRequest) -> HttpResponse:
+    """Generate robots.txt for search engine crawlers.
+
+    Args:
+        request: The HTTP request.
+
+    Returns:
+        HttpResponse: robots.txt content.
+    """
+    base_url: str = f"{request.scheme}://{request.get_host()}"
+
+    robots_content: str = f"""User-agent: *
+Allow: /
+Disallow: /admin/
+Disallow: /debug/
+Disallow: /datasets/
+Disallow: /docs/rss/
+Disallow: /export/
+
+# Sitemap location
+Sitemap: {base_url}/sitemap.xml
+"""
+
+    return HttpResponse(robots_content, content_type="text/plain")