Refactor RSS feed documentation and optimize query limits in views

templates/twitch/docs_rss.html

@@ -5,35 +5,32 @@
 {% endblock title %}
 {% block content %}
     <main>
-        <h1 id="page-title">RSS Feeds Documentation</h1>
+        <h1>RSS Feeds Documentation</h1>
         <p>This page lists all available RSS feeds for TTVDrops.</p>
         <section>
-            <h2 id="available-feeds-header">Global RSS Feeds</h2>
+            <h2>Global RSS Feeds</h2>
             <p>These feeds contain all items across the entire site:</p>
-            <ul id="feeds-list">
+            <ul>
                 {% for feed in feeds %}
-                    <li id="feed-{{ forloop.counter }}">
+                    <li>
                         <h3>{{ feed.title }}</h3>
                         <p>{{ feed.description }}</p>
                         <p>
                             <a href="{{ feed.url }}">Subscribe to {{ feed.title }} RSS Feed</a>
                         </p>
-                        <details>
+                        <pre><code class="language-xml">{% if feed.example_xml %}{{ feed.example_xml|escape }}{% else %}No example XML available yet.{% endif %}</code></pre>
-                            <summary>Example XML</summary>
-                            <pre><code class="language-xml">{{ feed.example_xml|escape }}</code></pre>
-                        </details>
                     </li>
                 {% endfor %}
             </ul>
         </section>
-        <section style="margin-top: 2rem;">
+        <section>
-            <h2 id="filtered-feeds-header">Filtered RSS Feeds</h2>
+            <h2>Filtered RSS Feeds</h2>
             <p>
                 You can subscribe to RSS feeds scoped to a specific game or organization. When available, links below point to live examples; otherwise use the endpoint template.
             </p>
-            <ul id="filtered-feeds-list">
+            <ul>
                 {% for feed in filtered_feeds %}
-                    <li id="filtered-feed-{{ forloop.counter }}">
+                    <li>
                         <h3>{{ feed.title }}</h3>
                         <p>{{ feed.description }}</p>
                         <p>
@@ -44,19 +41,13 @@
                                 <a href="{{ feed.url }}">View a live example</a>
                             </p>
                         {% endif %}
-                        <details>
+                        <pre><code class="language-xml">{% if feed.example_xml %}{{ feed.example_xml|escape }}{% else %}No example XML available yet.{% endif %}</code></pre>
-                            <summary>Example XML</summary>
-                            <pre><code class="language-xml">{{ feed.example_xml|escape }}</code></pre>
-                        </details>
                     </li>
                 {% endfor %}
             </ul>
-            <p>
-                Versioned paths under <code>/rss/v1/</code> are available and return the same XML structure.
-            </p>
         </section>
-        <section style="margin-top: 2rem;">
+        <section>
-            <h2 id="usage-header">How to Use RSS Feeds</h2>
+            <h2>How to Use RSS Feeds</h2>
             <p>
                 RSS feeds allow you to stay updated with new content. You can use any RSS reader application to subscribe to these feeds.
             </p>

twitch/feeds.py

@@ -32,6 +32,7 @@ if TYPE_CHECKING:
     from django.db.models import Model
     from django.db.models import QuerySet
     from django.http import HttpRequest
+    from django.http import HttpResponse
 
 logger: logging.Logger = logging.getLogger("ttvdrops")
 
@@ -308,10 +309,30 @@ class OrganizationRSSFeed(Feed):
     link: str = "/organizations/"
     description: str = "Latest organizations on TTVDrops"
     feed_copyright: str = "Information wants to be free."
+    _limit: int | None = None
+
+    def __call__(self, request: HttpRequest, *args: object, **kwargs: object) -> HttpResponse:
+        """Override to capture limit parameter from request.
+
+        Args:
+            request (HttpRequest): The incoming HTTP request, potentially containing a 'limit' query parameter.
+            *args: Additional positional arguments.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            HttpResponse: The HTTP response generated by the parent Feed class after processing the request.
+        """
+        if request.GET.get("limit"):
+            try:
+                self._limit = int(request.GET.get("limit", 200))
+            except ValueError, TypeError:
+                self._limit = None
+        return super().__call__(request, *args, **kwargs)
 
     def items(self) -> QuerySet[Organization, Organization]:
-        """Return the latest 200 organizations."""
+        """Return the latest organizations (default 200, or limited by ?limit query param)."""
-        return Organization.objects.order_by("-added_at")[:200]
+        limit: int = self._limit if self._limit is not None else 200
+        return Organization.objects.order_by("-added_at")[:limit]
 
     def item_title(self, item: Organization) -> SafeText:
         """Return the organization name as the item title."""
@@ -355,10 +376,30 @@ class GameFeed(Feed):
     link: str = "/games/"
     description: str = "Latest games on TTVDrops"
     feed_copyright: str = "Information wants to be free."
+    _limit: int | None = None
+
+    def __call__(self, request: HttpRequest, *args: object, **kwargs: object) -> HttpResponse:
+        """Override to capture limit parameter from request.
+
+        Args:
+            request (HttpRequest): The incoming HTTP request, potentially containing a 'limit' query parameter.
+            *args: Additional positional arguments.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            HttpResponse: The HTTP response generated by the parent Feed class after processing the request.
+        """
+        if request.GET.get("limit"):
+            try:
+                self._limit = int(request.GET.get("limit", 200))
+            except ValueError, TypeError:
+                self._limit = None
+        return super().__call__(request, *args, **kwargs)
 
     def items(self) -> list[Game]:
-        """Return the latest 200 games."""
+        """Return the latest games (default 200, or limited by ?limit query param)."""
-        return list(Game.objects.order_by("-added_at")[:200])
+        limit: int = self._limit if self._limit is not None else 200
+        return list(Game.objects.order_by("-added_at")[:limit])
 
     def item_title(self, item: Model) -> SafeText:
         """Return the game name as the item title (SafeText for RSS)."""
@@ -472,11 +513,31 @@ class DropCampaignFeed(Feed):
     description: str = "Latest Twitch drop campaigns on TTVDrops"
     feed_url: str = "/rss/campaigns/"
     feed_copyright: str = "Information wants to be free."
+    _limit: int | None = None
+
+    def __call__(self, request: HttpRequest, *args: object, **kwargs: object) -> HttpResponse:
+        """Override to capture limit parameter from request.
+
+        Args:
+            request (HttpRequest): The incoming HTTP request, potentially containing a 'limit' query parameter.
+            *args: Additional positional arguments.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            HttpResponse: The HTTP response generated by the parent Feed class after processing the request.
+        """
+        if request.GET.get("limit"):
+            try:
+                self._limit = int(request.GET.get("limit", 200))
+            except ValueError, TypeError:
+                self._limit = None
+        return super().__call__(request, *args, **kwargs)
 
     def items(self) -> list[DropCampaign]:
-        """Return the latest 200 drop campaigns ordered by most recent start date."""
+        """Return the latest drop campaigns ordered by most recent start date (default 200, or limited by ?limit query param)."""  # noqa: E501
+        limit: int = self._limit if self._limit is not None else 200
         queryset: QuerySet[DropCampaign] = DropCampaign.objects.order_by("-start_at")
-        return list(_with_campaign_related(queryset)[:200])
+        return list(_with_campaign_related(queryset)[:limit])
 
     def item_title(self, item: Model) -> SafeText:
         """Return the campaign name as the item title (SafeText for RSS)."""
@@ -593,6 +654,25 @@ class GameCampaignFeed(Feed):
     """RSS feed for the latest drop campaigns of a specific game."""
 
     feed_copyright: str = "Information wants to be free."
+    _limit: int | None = None
+
+    def __call__(self, request: HttpRequest, *args: object, **kwargs: object) -> HttpResponse:
+        """Override to capture limit parameter from request.
+
+        Args:
+            request (HttpRequest): The incoming HTTP request, potentially containing a 'limit' query parameter
+            *args: Additional positional arguments.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            HttpResponse: The HTTP response generated by the parent Feed class after processing the request.
+        """
+        if request.GET.get("limit"):
+            try:
+                self._limit = int(request.GET.get("limit", 200))
+            except ValueError, TypeError:
+                self._limit = None
+        return super().__call__(request, *args, **kwargs)
 
     def get_object(self, request: HttpRequest, twitch_id: str) -> Game:  # noqa: ARG002
         """Retrieve the Game instance for the given Twitch ID.
@@ -623,9 +703,10 @@ class GameCampaignFeed(Feed):
         return reverse("twitch:game_campaign_feed", args=[obj.twitch_id])
 
     def items(self, obj: Game) -> list[DropCampaign]:
-        """Return the latest 200 drop campaigns for this game, ordered by most recent start date."""
+        """Return the latest drop campaigns for this game, ordered by most recent start date (default 200, or limited by ?limit query param)."""  # noqa: E501
+        limit: int = self._limit if self._limit is not None else 200
         queryset: QuerySet[DropCampaign] = DropCampaign.objects.filter(game=obj).order_by("-start_at")
-        return list(_with_campaign_related(queryset)[:200])
+        return list(_with_campaign_related(queryset)[:limit])
 
     def item_title(self, item: Model) -> SafeText:
         """Return the campaign name as the item title (SafeText for RSS)."""
@@ -738,6 +819,26 @@ class GameCampaignFeed(Feed):
 class OrganizationCampaignFeed(Feed):
     """RSS feed for campaigns of a specific organization."""
 
+    _limit: int | None = None
+
+    def __call__(self, request: HttpRequest, *args: object, **kwargs: object) -> HttpResponse:
+        """Override to capture limit parameter from request.
+
+        Args:
+            request (HttpRequest): The incoming HTTP request, potentially containing a 'limit' query parameter
+            *args: Additional positional arguments.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            HttpResponse: The HTTP response generated by the parent Feed class after processing the request.
+        """
+        if request.GET.get("limit"):
+            try:
+                self._limit = int(request.GET.get("limit", 200))
+            except ValueError, TypeError:
+                self._limit = None
+        return super().__call__(request, *args, **kwargs)
+
     def get_object(self, request: HttpRequest, twitch_id: str) -> Organization:  # noqa: ARG002
         """Retrieve the Organization instance for the given Twitch ID.
 
@@ -763,9 +864,10 @@ class OrganizationCampaignFeed(Feed):
         return f"Latest drop campaigns for organization {obj.name}"
 
     def items(self, obj: Organization) -> list[DropCampaign]:
-        """Return the latest 200 drop campaigns for this organization, ordered by most recent start date."""
+        """Return the latest drop campaigns for this organization, ordered by most recent start date (default 200, or limited by ?limit query param)."""  # noqa: E501
+        limit: int = self._limit if self._limit is not None else 200
         queryset: QuerySet[DropCampaign] = DropCampaign.objects.filter(game__owners=obj).order_by("-start_at")
-        return list(_with_campaign_related(queryset)[:200])
+        return list(_with_campaign_related(queryset)[:limit])
 
     def item_author_name(self, item: DropCampaign) -> str:
         """Return the author name for the campaign, typically the game name."""
@@ -874,11 +976,31 @@ class RewardCampaignFeed(Feed):
     description: str = "Latest Twitch reward campaigns (Quest rewards) on TTVDrops"
     feed_url: str = "/rss/reward-campaigns/"
     feed_copyright: str = "Information wants to be free."
+    _limit: int | None = None
+
+    def __call__(self, request: HttpRequest, *args: object, **kwargs: object) -> HttpResponse:
+        """Override to capture limit parameter from request.
+
+        Args:
+            request (HttpRequest): The incoming HTTP request, potentially containing a 'limit' query parameter.
+            *args: Additional positional arguments.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            HttpResponse: The HTTP response generated by the parent Feed class after processing the request.
+        """
+        if request.GET.get("limit"):
+            try:
+                self._limit = int(request.GET.get("limit", 200))
+            except ValueError, TypeError:
+                self._limit = None
+        return super().__call__(request, *args, **kwargs)
 
     def items(self) -> list[RewardCampaign]:
-        """Return the latest 200 reward campaigns."""
+        """Return the latest reward campaigns (default 200, or limited by ?limit query param)."""
+        limit: int = self._limit if self._limit is not None else 200
         return list(
-            RewardCampaign.objects.select_related("game").order_by("-added_at")[:200],
+            RewardCampaign.objects.select_related("game").order_by("-added_at")[:limit],
         )
 
     def item_title(self, item: Model) -> SafeText:

twitch/tests/test_feeds.py

@@ -274,7 +274,8 @@ def test_campaign_feed_queries_bounded(client: Client, django_assert_num_queries
         _build_campaign(game, i)
 
     url: str = reverse("twitch:campaign_feed")
-    with django_assert_num_queries(20, exact=False):
+    # TODO(TheLovinator): 14 queries is still quite high for a feed - we should be able to optimize this further, but this is a good starting point to prevent regressions for now.  # noqa: E501, TD003
+    with django_assert_num_queries(14, exact=False):
         response: _MonkeyPatchedWSGIResponse = client.get(url)
 
     assert response.status_code == 200
@@ -299,7 +300,9 @@ def test_game_campaign_feed_queries_bounded(client: Client, django_assert_num_qu
         _build_campaign(game, i)
 
     url: str = reverse("twitch:game_campaign_feed", args=[game.twitch_id])
-    with django_assert_num_queries(22, exact=False):
+
+    # TODO(TheLovinator): 15 queries is still quite high for a feed - we should be able to optimize this further, but this is a good starting point to prevent regressions for now.  # noqa: E501, TD003
+    with django_assert_num_queries(15, exact=False):
         response: _MonkeyPatchedWSGIResponse = client.get(url)
 
     assert response.status_code == 200
@@ -315,7 +318,7 @@ def test_organization_feed_queries_bounded(client: Client, django_assert_num_que
         )
 
     url: str = reverse("twitch:organization_feed")
-    with django_assert_num_queries(6, exact=False):
+    with django_assert_num_queries(1, exact=True):
         response: _MonkeyPatchedWSGIResponse = client.get(url)
 
     assert response.status_code == 200
@@ -339,7 +342,7 @@ def test_game_feed_queries_bounded(client: Client, django_assert_num_queries: Qu
         game.owners.add(org)
 
     url: str = reverse("twitch:game_feed")
-    with django_assert_num_queries(10, exact=False):
+    with django_assert_num_queries(1, exact=True):
         response: _MonkeyPatchedWSGIResponse = client.get(url)
 
     assert response.status_code == 200
@@ -364,7 +367,8 @@ def test_organization_campaign_feed_queries_bounded(client: Client, django_asser
         _build_campaign(game, i)
 
     url: str = reverse("twitch:organization_campaign_feed", args=[org.twitch_id])
-    with django_assert_num_queries(22, exact=False):
+    # TODO(TheLovinator): 15 queries is still quite high for a feed - we should be able to optimize this further, but this is a good starting point to prevent regressions for now.  # noqa: E501, TD003
+    with django_assert_num_queries(15, exact=True):
         response: _MonkeyPatchedWSGIResponse = client.get(url)
 
     assert response.status_code == 200
@@ -389,7 +393,7 @@ def test_reward_campaign_feed_queries_bounded(client: Client, django_assert_num_
         _build_reward_campaign(game, i)
 
     url: str = reverse("twitch:reward_campaign_feed")
-    with django_assert_num_queries(8, exact=False):
+    with django_assert_num_queries(1, exact=True):
         response: _MonkeyPatchedWSGIResponse = client.get(url)
 
     assert response.status_code == 200
@@ -397,7 +401,11 @@ def test_reward_campaign_feed_queries_bounded(client: Client, django_assert_num_
 
 @pytest.mark.django_db
 def test_docs_rss_queries_bounded(client: Client, django_assert_num_queries: QueryAsserter) -> None:
-    """Docs RSS page should stay within a reasonable query budget."""
+    """Docs RSS page should stay within a reasonable query budget.
+
+    With limit=1 for documentation examples, we should have dramatically fewer queries
+    than if we were rendering 200+ items per feed.
+    """
     org: Organization = Organization.objects.create(
         twitch_id="docs-org",
         name="Docs Org",
@@ -415,7 +423,9 @@ def test_docs_rss_queries_bounded(client: Client, django_assert_num_queries: Que
         _build_reward_campaign(game, i)
 
     url: str = reverse("twitch:docs_rss")
-    with django_assert_num_queries(60, exact=False):
+
+    # TODO(TheLovinator): 31 queries is still quite high for a feed - we should be able to optimize this further, but this is a good starting point to prevent regressions for now.  # noqa: E501, TD003
+    with django_assert_num_queries(31, exact=False):
         response: _MonkeyPatchedWSGIResponse = client.get(url)
 
     assert response.status_code == 200

twitch/views.py

@@ -5,6 +5,7 @@ import json
 import logging
 from collections import OrderedDict
 from collections import defaultdict
+from copy import copy
 from typing import TYPE_CHECKING
 from typing import Any
 from typing import Literal
@@ -52,6 +53,7 @@ from twitch.models import TimeBasedDrop
 if TYPE_CHECKING:
     from collections.abc import Callable
 
+    from debug_toolbar.utils import QueryDict
     from django.db.models.query import QuerySet
 
 logger: logging.Logger = logging.getLogger("ttvdrops.views")
@@ -1042,7 +1044,13 @@ def docs_rss_view(request: HttpRequest) -> HttpResponse:
 
     def render_feed(feed_view: Callable[..., HttpResponse], *args: object) -> str:
         try:
-            response: HttpResponse = feed_view(request, *args)
+            limited_request: HttpRequest = copy(request)
+            # Add limit=1 to GET parameters
+            get_data: QueryDict = request.GET.copy()
+            get_data["limit"] = "1"
+            limited_request.GET = get_data  # pyright: ignore[reportAttributeAccessIssue]
+
+            response: HttpResponse = feed_view(limited_request, *args)
             return _pretty_example(response.content.decode("utf-8"))
         except Exception:  # pragma: no cover - defensive logging for docs only
             logger.exception("Failed to render %s for RSS docs", feed_view.__class__.__name__)
@@ -1075,8 +1083,10 @@ def docs_rss_view(request: HttpRequest) -> HttpResponse:
         },
     ]
 
-    sample_game: Game | None = Game.objects.first()
+    sample_game: Game | None = Game.objects.order_by("-added_at").first()
-    sample_org: Organization | None = Organization.objects.first()
+    sample_org: Organization | None = Organization.objects.order_by("-added_at").first()
+    if sample_org is None and sample_game is not None:
+        sample_org = sample_game.owners.order_by("-pk").first()
 
     filtered_feeds: list[dict[str, str | bool]] = [
         {