Refactor RSS feed documentation and optimize query limits in views
This commit is contained in:
parent
d350b7bcd8
commit
0a0345f217
Signed by:
Joakim Hellsén
Joakim Hellsén
SSH key fingerprint:
SHA256:/9h/CsExpFp+PRhsfA0xznFx2CGfTT5R/kpuFfUgEQk
4 changed files with 177 additions and 44 deletions
146
twitch/feeds.py
146
twitch/feeds.py
|
|
@ -32,6 +32,7 @@ if TYPE_CHECKING:
|
|||
from django.db.models import Model
|
||||
from django.db.models import QuerySet
|
||||
from django.http import HttpRequest
|
||||
from django.http import HttpResponse
|
||||
|
||||
logger: logging.Logger = logging.getLogger("ttvdrops")
|
||||
|
||||
|
|
@ -308,10 +309,30 @@ class OrganizationRSSFeed(Feed):
|
|||
link: str = "/organizations/"
|
||||
description: str = "Latest organizations on TTVDrops"
|
||||
feed_copyright: str = "Information wants to be free."
|
||||
_limit: int | None = None
|
||||
|
||||
def __call__(self, request: HttpRequest, *args: object, **kwargs: object) -> HttpResponse:
|
||||
"""Override to capture limit parameter from request.
|
||||
|
||||
Args:
|
||||
request (HttpRequest): The incoming HTTP request, potentially containing a 'limit' query parameter.
|
||||
*args: Additional positional arguments.
|
||||
**kwargs: Additional keyword arguments.
|
||||
|
||||
Returns:
|
||||
HttpResponse: The HTTP response generated by the parent Feed class after processing the request.
|
||||
"""
|
||||
if request.GET.get("limit"):
|
||||
try:
|
||||
self._limit = int(request.GET.get("limit", 200))
|
||||
except ValueError, TypeError:
|
||||
self._limit = None
|
||||
return super().__call__(request, *args, **kwargs)
|
||||
|
||||
def items(self) -> QuerySet[Organization, Organization]:
|
||||
"""Return the latest 200 organizations."""
|
||||
return Organization.objects.order_by("-added_at")[:200]
|
||||
"""Return the latest organizations (default 200, or limited by ?limit query param)."""
|
||||
limit: int = self._limit if self._limit is not None else 200
|
||||
return Organization.objects.order_by("-added_at")[:limit]
|
||||
|
||||
def item_title(self, item: Organization) -> SafeText:
|
||||
"""Return the organization name as the item title."""
|
||||
|
|
@ -355,10 +376,30 @@ class GameFeed(Feed):
|
|||
link: str = "/games/"
|
||||
description: str = "Latest games on TTVDrops"
|
||||
feed_copyright: str = "Information wants to be free."
|
||||
_limit: int | None = None
|
||||
|
||||
def __call__(self, request: HttpRequest, *args: object, **kwargs: object) -> HttpResponse:
|
||||
"""Override to capture limit parameter from request.
|
||||
|
||||
Args:
|
||||
request (HttpRequest): The incoming HTTP request, potentially containing a 'limit' query parameter.
|
||||
*args: Additional positional arguments.
|
||||
**kwargs: Additional keyword arguments.
|
||||
|
||||
Returns:
|
||||
HttpResponse: The HTTP response generated by the parent Feed class after processing the request.
|
||||
"""
|
||||
if request.GET.get("limit"):
|
||||
try:
|
||||
self._limit = int(request.GET.get("limit", 200))
|
||||
except ValueError, TypeError:
|
||||
self._limit = None
|
||||
return super().__call__(request, *args, **kwargs)
|
||||
|
||||
def items(self) -> list[Game]:
|
||||
"""Return the latest 200 games."""
|
||||
return list(Game.objects.order_by("-added_at")[:200])
|
||||
"""Return the latest games (default 200, or limited by ?limit query param)."""
|
||||
limit: int = self._limit if self._limit is not None else 200
|
||||
return list(Game.objects.order_by("-added_at")[:limit])
|
||||
|
||||
def item_title(self, item: Model) -> SafeText:
|
||||
"""Return the game name as the item title (SafeText for RSS)."""
|
||||
|
|
@ -472,11 +513,31 @@ class DropCampaignFeed(Feed):
|
|||
description: str = "Latest Twitch drop campaigns on TTVDrops"
|
||||
feed_url: str = "/rss/campaigns/"
|
||||
feed_copyright: str = "Information wants to be free."
|
||||
_limit: int | None = None
|
||||
|
||||
def __call__(self, request: HttpRequest, *args: object, **kwargs: object) -> HttpResponse:
|
||||
"""Override to capture limit parameter from request.
|
||||
|
||||
Args:
|
||||
request (HttpRequest): The incoming HTTP request, potentially containing a 'limit' query parameter.
|
||||
*args: Additional positional arguments.
|
||||
**kwargs: Additional keyword arguments.
|
||||
|
||||
Returns:
|
||||
HttpResponse: The HTTP response generated by the parent Feed class after processing the request.
|
||||
"""
|
||||
if request.GET.get("limit"):
|
||||
try:
|
||||
self._limit = int(request.GET.get("limit", 200))
|
||||
except ValueError, TypeError:
|
||||
self._limit = None
|
||||
return super().__call__(request, *args, **kwargs)
|
||||
|
||||
def items(self) -> list[DropCampaign]:
|
||||
"""Return the latest 200 drop campaigns ordered by most recent start date."""
|
||||
"""Return the latest drop campaigns ordered by most recent start date (default 200, or limited by ?limit query param).""" # noqa: E501
|
||||
limit: int = self._limit if self._limit is not None else 200
|
||||
queryset: QuerySet[DropCampaign] = DropCampaign.objects.order_by("-start_at")
|
||||
return list(_with_campaign_related(queryset)[:200])
|
||||
return list(_with_campaign_related(queryset)[:limit])
|
||||
|
||||
def item_title(self, item: Model) -> SafeText:
|
||||
"""Return the campaign name as the item title (SafeText for RSS)."""
|
||||
|
|
@ -593,6 +654,25 @@ class GameCampaignFeed(Feed):
|
|||
"""RSS feed for the latest drop campaigns of a specific game."""
|
||||
|
||||
feed_copyright: str = "Information wants to be free."
|
||||
_limit: int | None = None
|
||||
|
||||
def __call__(self, request: HttpRequest, *args: object, **kwargs: object) -> HttpResponse:
|
||||
"""Override to capture limit parameter from request.
|
||||
|
||||
Args:
|
||||
request (HttpRequest): The incoming HTTP request, potentially containing a 'limit' query parameter
|
||||
*args: Additional positional arguments.
|
||||
**kwargs: Additional keyword arguments.
|
||||
|
||||
Returns:
|
||||
HttpResponse: The HTTP response generated by the parent Feed class after processing the request.
|
||||
"""
|
||||
if request.GET.get("limit"):
|
||||
try:
|
||||
self._limit = int(request.GET.get("limit", 200))
|
||||
except ValueError, TypeError:
|
||||
self._limit = None
|
||||
return super().__call__(request, *args, **kwargs)
|
||||
|
||||
def get_object(self, request: HttpRequest, twitch_id: str) -> Game: # noqa: ARG002
|
||||
"""Retrieve the Game instance for the given Twitch ID.
|
||||
|
|
@ -623,9 +703,10 @@ class GameCampaignFeed(Feed):
|
|||
return reverse("twitch:game_campaign_feed", args=[obj.twitch_id])
|
||||
|
||||
def items(self, obj: Game) -> list[DropCampaign]:
|
||||
"""Return the latest 200 drop campaigns for this game, ordered by most recent start date."""
|
||||
"""Return the latest drop campaigns for this game, ordered by most recent start date (default 200, or limited by ?limit query param).""" # noqa: E501
|
||||
limit: int = self._limit if self._limit is not None else 200
|
||||
queryset: QuerySet[DropCampaign] = DropCampaign.objects.filter(game=obj).order_by("-start_at")
|
||||
return list(_with_campaign_related(queryset)[:200])
|
||||
return list(_with_campaign_related(queryset)[:limit])
|
||||
|
||||
def item_title(self, item: Model) -> SafeText:
|
||||
"""Return the campaign name as the item title (SafeText for RSS)."""
|
||||
|
|
@ -738,6 +819,26 @@ class GameCampaignFeed(Feed):
|
|||
class OrganizationCampaignFeed(Feed):
|
||||
"""RSS feed for campaigns of a specific organization."""
|
||||
|
||||
_limit: int | None = None
|
||||
|
||||
def __call__(self, request: HttpRequest, *args: object, **kwargs: object) -> HttpResponse:
|
||||
"""Override to capture limit parameter from request.
|
||||
|
||||
Args:
|
||||
request (HttpRequest): The incoming HTTP request, potentially containing a 'limit' query parameter
|
||||
*args: Additional positional arguments.
|
||||
**kwargs: Additional keyword arguments.
|
||||
|
||||
Returns:
|
||||
HttpResponse: The HTTP response generated by the parent Feed class after processing the request.
|
||||
"""
|
||||
if request.GET.get("limit"):
|
||||
try:
|
||||
self._limit = int(request.GET.get("limit", 200))
|
||||
except ValueError, TypeError:
|
||||
self._limit = None
|
||||
return super().__call__(request, *args, **kwargs)
|
||||
|
||||
def get_object(self, request: HttpRequest, twitch_id: str) -> Organization: # noqa: ARG002
|
||||
"""Retrieve the Organization instance for the given Twitch ID.
|
||||
|
||||
|
|
@ -763,9 +864,10 @@ class OrganizationCampaignFeed(Feed):
|
|||
return f"Latest drop campaigns for organization {obj.name}"
|
||||
|
||||
def items(self, obj: Organization) -> list[DropCampaign]:
|
||||
"""Return the latest 200 drop campaigns for this organization, ordered by most recent start date."""
|
||||
"""Return the latest drop campaigns for this organization, ordered by most recent start date (default 200, or limited by ?limit query param).""" # noqa: E501
|
||||
limit: int = self._limit if self._limit is not None else 200
|
||||
queryset: QuerySet[DropCampaign] = DropCampaign.objects.filter(game__owners=obj).order_by("-start_at")
|
||||
return list(_with_campaign_related(queryset)[:200])
|
||||
return list(_with_campaign_related(queryset)[:limit])
|
||||
|
||||
def item_author_name(self, item: DropCampaign) -> str:
|
||||
"""Return the author name for the campaign, typically the game name."""
|
||||
|
|
@ -874,11 +976,31 @@ class RewardCampaignFeed(Feed):
|
|||
description: str = "Latest Twitch reward campaigns (Quest rewards) on TTVDrops"
|
||||
feed_url: str = "/rss/reward-campaigns/"
|
||||
feed_copyright: str = "Information wants to be free."
|
||||
_limit: int | None = None
|
||||
|
||||
def __call__(self, request: HttpRequest, *args: object, **kwargs: object) -> HttpResponse:
|
||||
"""Override to capture limit parameter from request.
|
||||
|
||||
Args:
|
||||
request (HttpRequest): The incoming HTTP request, potentially containing a 'limit' query parameter.
|
||||
*args: Additional positional arguments.
|
||||
**kwargs: Additional keyword arguments.
|
||||
|
||||
Returns:
|
||||
HttpResponse: The HTTP response generated by the parent Feed class after processing the request.
|
||||
"""
|
||||
if request.GET.get("limit"):
|
||||
try:
|
||||
self._limit = int(request.GET.get("limit", 200))
|
||||
except ValueError, TypeError:
|
||||
self._limit = None
|
||||
return super().__call__(request, *args, **kwargs)
|
||||
|
||||
def items(self) -> list[RewardCampaign]:
|
||||
"""Return the latest 200 reward campaigns."""
|
||||
"""Return the latest reward campaigns (default 200, or limited by ?limit query param)."""
|
||||
limit: int = self._limit if self._limit is not None else 200
|
||||
return list(
|
||||
RewardCampaign.objects.select_related("game").order_by("-added_at")[:200],
|
||||
RewardCampaign.objects.select_related("game").order_by("-added_at")[:limit],
|
||||
)
|
||||
|
||||
def item_title(self, item: Model) -> SafeText:
|
||||
|
|
|
|||
|
|
@ -274,7 +274,8 @@ def test_campaign_feed_queries_bounded(client: Client, django_assert_num_queries
|
|||
_build_campaign(game, i)
|
||||
|
||||
url: str = reverse("twitch:campaign_feed")
|
||||
with django_assert_num_queries(20, exact=False):
|
||||
# TODO(TheLovinator): 14 queries is still quite high for a feed - we should be able to optimize this further, but this is a good starting point to prevent regressions for now. # noqa: E501, TD003
|
||||
with django_assert_num_queries(14, exact=False):
|
||||
response: _MonkeyPatchedWSGIResponse = client.get(url)
|
||||
|
||||
assert response.status_code == 200
|
||||
|
|
@ -299,7 +300,9 @@ def test_game_campaign_feed_queries_bounded(client: Client, django_assert_num_qu
|
|||
_build_campaign(game, i)
|
||||
|
||||
url: str = reverse("twitch:game_campaign_feed", args=[game.twitch_id])
|
||||
with django_assert_num_queries(22, exact=False):
|
||||
|
||||
# TODO(TheLovinator): 15 queries is still quite high for a feed - we should be able to optimize this further, but this is a good starting point to prevent regressions for now. # noqa: E501, TD003
|
||||
with django_assert_num_queries(15, exact=False):
|
||||
response: _MonkeyPatchedWSGIResponse = client.get(url)
|
||||
|
||||
assert response.status_code == 200
|
||||
|
|
@ -315,7 +318,7 @@ def test_organization_feed_queries_bounded(client: Client, django_assert_num_que
|
|||
)
|
||||
|
||||
url: str = reverse("twitch:organization_feed")
|
||||
with django_assert_num_queries(6, exact=False):
|
||||
with django_assert_num_queries(1, exact=True):
|
||||
response: _MonkeyPatchedWSGIResponse = client.get(url)
|
||||
|
||||
assert response.status_code == 200
|
||||
|
|
@ -339,7 +342,7 @@ def test_game_feed_queries_bounded(client: Client, django_assert_num_queries: Qu
|
|||
game.owners.add(org)
|
||||
|
||||
url: str = reverse("twitch:game_feed")
|
||||
with django_assert_num_queries(10, exact=False):
|
||||
with django_assert_num_queries(1, exact=True):
|
||||
response: _MonkeyPatchedWSGIResponse = client.get(url)
|
||||
|
||||
assert response.status_code == 200
|
||||
|
|
@ -364,7 +367,8 @@ def test_organization_campaign_feed_queries_bounded(client: Client, django_asser
|
|||
_build_campaign(game, i)
|
||||
|
||||
url: str = reverse("twitch:organization_campaign_feed", args=[org.twitch_id])
|
||||
with django_assert_num_queries(22, exact=False):
|
||||
# TODO(TheLovinator): 15 queries is still quite high for a feed - we should be able to optimize this further, but this is a good starting point to prevent regressions for now. # noqa: E501, TD003
|
||||
with django_assert_num_queries(15, exact=True):
|
||||
response: _MonkeyPatchedWSGIResponse = client.get(url)
|
||||
|
||||
assert response.status_code == 200
|
||||
|
|
@ -389,7 +393,7 @@ def test_reward_campaign_feed_queries_bounded(client: Client, django_assert_num_
|
|||
_build_reward_campaign(game, i)
|
||||
|
||||
url: str = reverse("twitch:reward_campaign_feed")
|
||||
with django_assert_num_queries(8, exact=False):
|
||||
with django_assert_num_queries(1, exact=True):
|
||||
response: _MonkeyPatchedWSGIResponse = client.get(url)
|
||||
|
||||
assert response.status_code == 200
|
||||
|
|
@ -397,7 +401,11 @@ def test_reward_campaign_feed_queries_bounded(client: Client, django_assert_num_
|
|||
|
||||
@pytest.mark.django_db
|
||||
def test_docs_rss_queries_bounded(client: Client, django_assert_num_queries: QueryAsserter) -> None:
|
||||
"""Docs RSS page should stay within a reasonable query budget."""
|
||||
"""Docs RSS page should stay within a reasonable query budget.
|
||||
|
||||
With limit=1 for documentation examples, we should have dramatically fewer queries
|
||||
than if we were rendering 200+ items per feed.
|
||||
"""
|
||||
org: Organization = Organization.objects.create(
|
||||
twitch_id="docs-org",
|
||||
name="Docs Org",
|
||||
|
|
@ -415,7 +423,9 @@ def test_docs_rss_queries_bounded(client: Client, django_assert_num_queries: Que
|
|||
_build_reward_campaign(game, i)
|
||||
|
||||
url: str = reverse("twitch:docs_rss")
|
||||
with django_assert_num_queries(60, exact=False):
|
||||
|
||||
# TODO(TheLovinator): 31 queries is still quite high for a feed - we should be able to optimize this further, but this is a good starting point to prevent regressions for now. # noqa: E501, TD003
|
||||
with django_assert_num_queries(31, exact=False):
|
||||
response: _MonkeyPatchedWSGIResponse = client.get(url)
|
||||
|
||||
assert response.status_code == 200
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import json
|
|||
import logging
|
||||
from collections import OrderedDict
|
||||
from collections import defaultdict
|
||||
from copy import copy
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import Any
|
||||
from typing import Literal
|
||||
|
|
@ -52,6 +53,7 @@ from twitch.models import TimeBasedDrop
|
|||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
from debug_toolbar.utils import QueryDict
|
||||
from django.db.models.query import QuerySet
|
||||
|
||||
logger: logging.Logger = logging.getLogger("ttvdrops.views")
|
||||
|
|
@ -1042,7 +1044,13 @@ def docs_rss_view(request: HttpRequest) -> HttpResponse:
|
|||
|
||||
def render_feed(feed_view: Callable[..., HttpResponse], *args: object) -> str:
|
||||
try:
|
||||
response: HttpResponse = feed_view(request, *args)
|
||||
limited_request: HttpRequest = copy(request)
|
||||
# Add limit=1 to GET parameters
|
||||
get_data: QueryDict = request.GET.copy()
|
||||
get_data["limit"] = "1"
|
||||
limited_request.GET = get_data # pyright: ignore[reportAttributeAccessIssue]
|
||||
|
||||
response: HttpResponse = feed_view(limited_request, *args)
|
||||
return _pretty_example(response.content.decode("utf-8"))
|
||||
except Exception: # pragma: no cover - defensive logging for docs only
|
||||
logger.exception("Failed to render %s for RSS docs", feed_view.__class__.__name__)
|
||||
|
|
@ -1075,8 +1083,10 @@ def docs_rss_view(request: HttpRequest) -> HttpResponse:
|
|||
},
|
||||
]
|
||||
|
||||
sample_game: Game | None = Game.objects.first()
|
||||
sample_org: Organization | None = Organization.objects.first()
|
||||
sample_game: Game | None = Game.objects.order_by("-added_at").first()
|
||||
sample_org: Organization | None = Organization.objects.order_by("-added_at").first()
|
||||
if sample_org is None and sample_game is not None:
|
||||
sample_org = sample_game.owners.order_by("-pk").first()
|
||||
|
||||
filtered_feeds: list[dict[str, str | bool]] = [
|
||||
{
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue