Improve sitemaps

twitch/tests/test_views.py

@@ -2,12 +2,17 @@ from __future__ import annotations
 
 import datetime
 import json
+import os
+from datetime import UTC
 from datetime import timedelta
+from pathlib import Path
 from typing import TYPE_CHECKING
 from typing import Any
 from typing import Literal
 
 import pytest
+from django.conf import settings
+from django.core.cache import cache
 from django.core.handlers.wsgi import WSGIRequest
 from django.core.paginator import Paginator
 from django.test import RequestFactory
@@ -21,6 +26,7 @@ from twitch.models import DropBenefit
 from twitch.models import DropCampaign
 from twitch.models import Game
 from twitch.models import Organization
+from twitch.models import RewardCampaign
 from twitch.models import TimeBasedDrop
 from twitch.views import Page
 from twitch.views import _build_breadcrumb_schema
@@ -29,6 +35,8 @@ from twitch.views import _build_seo_context
 from twitch.views import _truncate_description
 
 if TYPE_CHECKING:
+    from pathlib import Path
+
     from django.core.handlers.wsgi import WSGIRequest
     from django.test import Client
     from django.test.client import _MonkeyPatchedWSGIResponse
@@ -423,7 +431,7 @@ class TestChannelListView:
     @pytest.mark.django_db
     def test_dashboard_dedupes_campaigns_for_multi_owner_game(self, client: Client) -> None:
         """Dashboard should not render duplicate campaign cards when a game has multiple owners."""
-        now = timezone.now()
+        now: datetime.datetime = timezone.now()
         org1: Organization = Organization.objects.create(twitch_id="org_a", name="Org A")
         org2: Organization = Organization.objects.create(twitch_id="org_b", name="Org B")
         game: Game = Game.objects.create(twitch_id="game_multi_owner", name="game", display_name="Multi Owner")
@@ -474,7 +482,7 @@ class TestChannelListView:
         now: datetime.datetime = timezone.now()
 
         # Create 150 campaigns to test pagination
-        campaigns = [
+        campaigns: list[DropCampaign] = [
             DropCampaign(
                 twitch_id=f"c{i}",
                 name=f"Campaign {i}",
@@ -698,7 +706,7 @@ class TestChannelListView:
         now: datetime.datetime = timezone.now()
 
         # Create 150 active campaigns for game g1
-        campaigns = [
+        campaigns: list[DropCampaign] = [
             DropCampaign(
                 twitch_id=f"c{i}",
                 name=f"Campaign {i}",
@@ -752,7 +760,7 @@ class TestChannelListView:
             operation_names=["DropCampaignDetails"],
         )
 
-        drop = TimeBasedDrop.objects.create(
+        drop: TimeBasedDrop = TimeBasedDrop.objects.create(
             twitch_id="d1",
             name="Drop",
             campaign=campaign,
@@ -760,14 +768,14 @@ class TestChannelListView:
             required_subs=1,
         )
 
-        benefit = DropBenefit.objects.create(
+        benefit: DropBenefit = DropBenefit.objects.create(
             twitch_id="b1",
             name="Diana",
             distribution_type="BADGE",
         )
         drop.benefits.add(benefit)
 
-        badge_set = ChatBadgeSet.objects.create(set_id="diana")
+        badge_set: ChatBadgeSet = ChatBadgeSet.objects.create(set_id="diana")
         ChatBadge.objects.create(
             badge_set=badge_set,
             badge_id="1",
@@ -783,7 +791,7 @@ class TestChannelListView:
         assert response.status_code == 200
 
         # The campaign detail page prints a syntax-highlighted JSON block; the badge description should be present.
-        html = response.content.decode("utf-8")
+        html: str = response.content.decode("utf-8")
         assert "This badge was earned by subscribing." in html
 
     @pytest.mark.django_db
@@ -1007,11 +1015,12 @@ class TestSEOMetaTags:
     ) -> None:
         """Test campaign detail view has breadcrumb schema."""
         campaign: DropCampaign = game_with_campaign["campaign"]
-        url = reverse("twitch:campaign_detail", args=[campaign.twitch_id])
+        url: str = reverse("twitch:campaign_detail", args=[campaign.twitch_id])
         response: _MonkeyPatchedWSGIResponse = client.get(url)
 
         assert response.status_code == 200
         assert "breadcrumb_schema" in response.context
+
         # breadcrumb_schema is JSON-dumped in context
         breadcrumb_str = response.context["breadcrumb_schema"]
         breadcrumb = json.loads(breadcrumb_str)
@@ -1025,7 +1034,7 @@ class TestSEOMetaTags:
     ) -> None:
         """Test campaign detail view has modified_date."""
         campaign: DropCampaign = game_with_campaign["campaign"]
-        url = reverse("twitch:campaign_detail", args=[campaign.twitch_id])
+        url: str = reverse("twitch:campaign_detail", args=[campaign.twitch_id])
         response: _MonkeyPatchedWSGIResponse = client.get(url)
 
         assert response.status_code == 200
@@ -1075,16 +1084,18 @@ class TestSEOMetaTags:
 
 
 @pytest.mark.django_db
-class TestSitemapView:
-    """Tests for the sitemap.xml view."""
+class TestSitemapViews:
+    """Tests for the split sitemap index and section files."""
 
     @pytest.fixture
-    def sample_entities(self) -> dict[str, Any]:
-        """Create sample entities for sitemap testing.
+    def sample_entities(
+        self,
+    ) -> dict[str, Organization | Game | Channel | DropCampaign | RewardCampaign | ChatBadgeSet]:
+        """Fixture to create sample entities for testing sitemap sections.
 
         Returns:
-            dict[str, Any]: A dictionary containing the created organization, game, channel, campaign, and badge set.
-        """
+            dict[str, Organization | Game | Channel | DropCampaign | RewardCampaign | ChatBadgeSet]: Dictionary of sample entities created for testing.
+        """  # noqa: E501
         org: Organization = Organization.objects.create(twitch_id="org1", name="Test Org")
         game: Game = Game.objects.create(
             twitch_id="game1",
@@ -1092,7 +1103,11 @@ class TestSitemapView:
             display_name="Test Game",
         )
         game.owners.add(org)
-        channel: Channel = Channel.objects.create(twitch_id="ch1", name="ch1", display_name="Channel 1")
+        channel: Channel = Channel.objects.create(
+            twitch_id="ch1",
+            name="ch1",
+            display_name="Channel 1",
+        )
         campaign: DropCampaign = DropCampaign.objects.create(
             twitch_id="camp1",
             name="Test Campaign",
@@ -1100,118 +1115,122 @@ class TestSitemapView:
             game=game,
             operation_names=["DropCampaignDetails"],
         )
+        reward: RewardCampaign = RewardCampaign.objects.create(
+            twitch_id="reward1",
+            name="Test Reward",
+        )
         badge: ChatBadgeSet = ChatBadgeSet.objects.create(set_id="badge1")
         return {
             "org": org,
             "game": game,
             "channel": channel,
             "campaign": campaign,
+            "reward": reward,
             "badge": badge,
         }
 
-    def test_sitemap_view_returns_xml(self, client: Client, sample_entities: dict[str, Any]) -> None:
-        """Test sitemap view returns XML content."""
+    def test_index_contains_sections(self, client: Client) -> None:
+        """Test that the sitemap index references all expected sections.
+
+        Args:
+            client (Client): Django test client fixture for making HTTP requests.
+        """
         response: _MonkeyPatchedWSGIResponse = client.get("/sitemap.xml")
         assert response.status_code == 200
         assert response["Content-Type"] == "application/xml"
-
-    def test_sitemap_contains_xml_declaration(self, client: Client, sample_entities: dict[str, Any]) -> None:
-        """Test sitemap contains proper XML declaration."""
-        response: _MonkeyPatchedWSGIResponse = client.get("/sitemap.xml")
-        content = response.content.decode()
-        assert content.startswith('<?xml version="1.0" encoding="UTF-8"?>')
-
-    def test_sitemap_contains_urlset(self, client: Client, sample_entities: dict[str, Any]) -> None:
-        """Test sitemap contains urlset element."""
-        response: _MonkeyPatchedWSGIResponse = client.get("/sitemap.xml")
         content: str = response.content.decode()
-        assert "<urlset" in content
-        assert "</urlset>" in content
+        assert "<sitemapindex" in content
+        for section in [
+            "sitemap-static.xml",
+            "sitemap-games.xml",
+            "sitemap-campaigns.xml",
+            "sitemap-organizations.xml",
+            "sitemap-channels.xml",
+            "sitemap-badges.xml",
+            "sitemap-reward-campaigns.xml",
+        ]:
+            assert section in content
 
-    def test_sitemap_contains_static_pages(self, client: Client, sample_entities: dict[str, Any]) -> None:
-        """Test sitemap includes static pages."""
-        response: _MonkeyPatchedWSGIResponse = client.get("/sitemap.xml")
-        content: str = response.content.decode()
-        # Check for some static pages
-        assert "<loc>http://testserver/</loc>" in content or "<loc>http://localhost:8000/</loc>" in content
-        assert "/campaigns/" in content
-        assert "/games/" in content
+    def test_sections_provide_expected_urls(self, client: Client, sample_entities: dict[str, Any]) -> None:
+        """Test that each sitemap section returns expected URLs for the entities created in the fixture.
 
-    def test_sitemap_contains_game_detail_pages(
-        self,
-        client: Client,
-        sample_entities: dict[str, Any],
-    ) -> None:
-        """Test sitemap includes game detail pages."""
+        Args:
+            client (Client): Django test client fixture for making HTTP requests.
+            sample_entities (dict[str, Any]): Dictionary of sample entities created in the fixture.
+        """
+        # games
         game: Game = sample_entities["game"]
-        response: _MonkeyPatchedWSGIResponse = client.get("/sitemap.xml")
-        content: str = response.content.decode()
-        assert f"/games/{game.twitch_id}/" in content
+        response: _MonkeyPatchedWSGIResponse = client.get("/sitemap-games.xml")
+        assert response.status_code == 200
+        assert "<urlset" in response.content.decode()
+        assert f"/games/{game.twitch_id}/" in response.content.decode()
 
-    def test_sitemap_contains_campaign_detail_pages(
-        self,
-        client: Client,
-        sample_entities: dict[str, Any],
-    ) -> None:
-        """Test sitemap includes campaign detail pages."""
+        # campaigns
         campaign: DropCampaign = sample_entities["campaign"]
-        response: _MonkeyPatchedWSGIResponse = client.get("/sitemap.xml")
-        content: str = response.content.decode()
-        assert f"/campaigns/{campaign.twitch_id}/" in content
+        response: _MonkeyPatchedWSGIResponse = client.get("/sitemap-campaigns.xml")
+        assert f"/campaigns/{campaign.twitch_id}/" in response.content.decode()
 
-    def test_sitemap_contains_organization_detail_pages(
-        self,
-        client: Client,
-        sample_entities: dict[str, Any],
-    ) -> None:
-        """Test sitemap includes organization detail pages."""
+        # organizations
         org: Organization = sample_entities["org"]
-        response: _MonkeyPatchedWSGIResponse = client.get("/sitemap.xml")
-        content: str = response.content.decode()
-        assert f"/organizations/{org.twitch_id}/" in content
+        response: _MonkeyPatchedWSGIResponse = client.get("/sitemap-organizations.xml")
+        assert f"/organizations/{org.twitch_id}/" in response.content.decode()
 
-    def test_sitemap_contains_channel_detail_pages(
-        self,
-        client: Client,
-        sample_entities: dict[str, Any],
-    ) -> None:
-        """Test sitemap includes channel detail pages."""
+        # channels
         channel: Channel = sample_entities["channel"]
-        response: _MonkeyPatchedWSGIResponse = client.get("/sitemap.xml")
-        content: str = response.content.decode()
-        assert f"/channels/{channel.twitch_id}/" in content
+        response: _MonkeyPatchedWSGIResponse = client.get("/sitemap-channels.xml")
+        assert f"/channels/{channel.twitch_id}/" in response.content.decode()
 
-    def test_sitemap_contains_badge_detail_pages(
+        # badges
+        badge: ChatBadgeSet = sample_entities["badge"]
+        response: _MonkeyPatchedWSGIResponse = client.get("/sitemap-badges.xml")
+        assert f"/badges/{badge.set_id}/" in response.content.decode()
+
+        # reward campaigns
+        reward: RewardCampaign = sample_entities["reward"]
+        response: _MonkeyPatchedWSGIResponse = client.get("/sitemap-reward-campaigns.xml")
+        assert f"/reward-campaigns/{reward.twitch_id}/" in response.content.decode()
+
+        # static
+        response: _MonkeyPatchedWSGIResponse = client.get("/sitemap-static.xml")
+        static_content: str = response.content.decode()
+        assert "<loc>http://testserver/</loc>" in static_content
+        assert "/campaigns/" in static_content
+        assert "/games/" in static_content
+
+    def test_static_sitemap_lastmod_and_docs_rss(
         self,
         client: Client,
-        sample_entities: dict[str, Any],
+        tmp_path: Path,
+        monkeypatch: pytest.MonkeyPatch,
     ) -> None:
-        """Test sitemap includes badge detail pages."""
-        badge: ChatBadge = sample_entities["badge"]
-        response: _MonkeyPatchedWSGIResponse = client.get("/sitemap.xml")
-        content: str = response.content.decode()
-        assert f"/badges/{badge.set_id}/" in content  # pyright: ignore[reportAttributeAccessIssue]
+        """Ensure the XML output contains correct lastmod for backups and skips docs RSS."""
+        # configure a fake DATA_DIR with backup files
+        cache.clear()
+        monkeypatch.setattr(settings, "DATA_DIR", tmp_path)
+        datasets: Path = tmp_path / "datasets"
+        datasets.mkdir()
+        older: Path = datasets / "dataset_backup_old.zip"
+        newer: Path = datasets / "dataset_backup_new.zip"
+        older.write_text("old", encoding="utf-8")
+        newer.write_text("new", encoding="utf-8")
+        os.utime(older, (1_000, 1_000))
+        os.utime(newer, (2_000, 2_000))
 
-    def test_sitemap_includes_priority(self, client: Client, sample_entities: dict[str, Any]) -> None:
-        """Test sitemap includes priority values."""
-        response: _MonkeyPatchedWSGIResponse = client.get("/sitemap.xml")
+        response: _MonkeyPatchedWSGIResponse = client.get("/sitemap-static.xml")
         content: str = response.content.decode()
-        assert "<priority>" in content
-        assert "</priority>" in content
 
-    def test_sitemap_includes_changefreq(self, client: Client, sample_entities: dict[str, Any]) -> None:
-        """Test sitemap includes changefreq values."""
-        response: _MonkeyPatchedWSGIResponse = client.get("/sitemap.xml")
-        content: str = response.content.decode()
-        assert "<changefreq>" in content
-        assert "</changefreq>" in content
+        # lastmod should match the newer file's timestamp.  Django's
+        # sitemap renderer outputs only the date portion, so check for that.
+        expected_date: str = datetime.datetime.fromtimestamp(2_000, tz=UTC).date().isoformat()
+        assert f"<lastmod>{expected_date}</lastmod>" in content
 
-    def test_sitemap_includes_lastmod(self, client: Client, sample_entities: dict[str, Any]) -> None:
-        """Test sitemap includes lastmod for detail pages."""
-        response: _MonkeyPatchedWSGIResponse = client.get("/sitemap.xml")
-        content: str = response.content.decode()
-        # Check for lastmod in game or campaign entries
-        assert "<lastmod>" in content
+        # docs RSS entry must not include a lastmod element
+        # find the docs_rss loc and assert no <lastmod> on the next line
+        assert "<loc>http://testserver/docs/rss/</loc>" in content
+        sections: list[str] = content.split("<loc>http://testserver/docs/rss/</loc>")
+        assert len(sections) >= 2
+        after: str = sections[1]
+        assert "<lastmod>" not in after.split("</url>", maxsplit=1)[0]
 
 
 @pytest.mark.django_db