From 1782db484024e8491420e5db9bc8e85f8ed63394 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Hells=C3=A9n?= Date: Fri, 10 Apr 2026 23:54:10 +0200 Subject: [PATCH] Add smoke tests for endpoints and optimize database queries --- config/tests/test_site_endpoint_smoke.py | 349 +++++++++++++++++++++++ kick/models.py | 19 +- kick/tests/test_kick.py | 311 ++++++++++++++++++++ kick/views.py | 2 +- templates/twitch/dashboard.html | 2 +- twitch/models.py | 145 +++++++++- twitch/tests/test_views.py | 222 ++++++++++++++ twitch/views.py | 42 +-- 8 files changed, 1044 insertions(+), 48 deletions(-) create mode 100644 config/tests/test_site_endpoint_smoke.py diff --git a/config/tests/test_site_endpoint_smoke.py b/config/tests/test_site_endpoint_smoke.py new file mode 100644 index 0000000..90537c0 --- /dev/null +++ b/config/tests/test_site_endpoint_smoke.py @@ -0,0 +1,349 @@ +from __future__ import annotations + +from datetime import timedelta +from pathlib import Path +from typing import TYPE_CHECKING + +from django.conf import settings +from django.test import TestCase +from django.urls import reverse +from django.utils import timezone + +from chzzk.models import ChzzkCampaign +from kick.models import KickCategory +from kick.models import KickChannel +from kick.models import KickDropCampaign +from kick.models import KickOrganization +from kick.models import KickReward +from kick.models import KickUser +from twitch.models import Channel +from twitch.models import ChatBadge +from twitch.models import ChatBadgeSet +from twitch.models import DropBenefit +from twitch.models import DropCampaign +from twitch.models import Game +from twitch.models import Organization +from twitch.models import RewardCampaign +from twitch.models import TimeBasedDrop + +if TYPE_CHECKING: + from datetime import datetime + from pathlib import Path + + from django.test.client import _MonkeyPatchedWSGIResponse + + +class SiteEndpointSmokeTest(TestCase): + """Smoke-test all named site endpoints with realistic fixture data.""" + + def setUp(self) -> None: + """Set up representative Twitch, Kick, and CHZZK data for endpoint smoke tests.""" + now: datetime = timezone.now() + + # Twitch fixtures + self.twitch_org: Organization = Organization.objects.create( + twitch_id="smoke-org-1", + name="Smoke Organization", + ) + self.twitch_game: Game = Game.objects.create( + twitch_id="smoke-game-1", + slug="smoke-game", + name="Smoke Game", + display_name="Smoke Game", + box_art="https://example.com/smoke-game.png", + ) + self.twitch_game.owners.add(self.twitch_org) + + self.twitch_channel: Channel = Channel.objects.create( + twitch_id="smoke-channel-1", + name="smokechannel", + display_name="SmokeChannel", + ) + + self.twitch_campaign: DropCampaign = DropCampaign.objects.create( + twitch_id="smoke-campaign-1", + name="Smoke Campaign", + description="Smoke campaign description", + game=self.twitch_game, + image_url="https://example.com/smoke-campaign.png", + start_at=now - timedelta(days=1), + end_at=now + timedelta(days=1), + operation_names=["DropCampaignDetails"], + is_fully_imported=True, + ) + self.twitch_campaign.allow_channels.add(self.twitch_channel) + + self.twitch_drop: TimeBasedDrop = TimeBasedDrop.objects.create( + twitch_id="smoke-drop-1", + name="Smoke Drop", + campaign=self.twitch_campaign, + required_minutes_watched=15, + start_at=now - timedelta(days=1), + end_at=now + timedelta(days=1), + ) + self.twitch_benefit: DropBenefit = DropBenefit.objects.create( + twitch_id="smoke-benefit-1", + name="Smoke Benefit", + image_asset_url="https://example.com/smoke-benefit.png", + ) + self.twitch_drop.benefits.add(self.twitch_benefit) + + self.twitch_reward_campaign: RewardCampaign = RewardCampaign.objects.create( + twitch_id="smoke-reward-campaign-1", + name="Smoke Reward Campaign", + brand="Smoke Brand", + starts_at=now - timedelta(days=1), + ends_at=now + timedelta(days=2), + status="ACTIVE", + summary="Smoke reward summary", + external_url="https://example.com/smoke-reward", + is_sitewide=False, + game=self.twitch_game, + ) + + self.badge_set: ChatBadgeSet = ChatBadgeSet.objects.create( + set_id="smoke-badge-set", + ) + ChatBadge.objects.create( + badge_set=self.badge_set, + badge_id="1", + image_url_1x="https://example.com/badge-1x.png", + image_url_2x="https://example.com/badge-2x.png", + image_url_4x="https://example.com/badge-4x.png", + title="Smoke Badge", + description="Smoke badge description", + ) + + # Kick fixtures + self.kick_org: KickOrganization = KickOrganization.objects.create( + kick_id="smoke-kick-org-1", + name="Smoke Kick Organization", + ) + self.kick_category: KickCategory = KickCategory.objects.create( + kick_id=9101, + name="Smoke Kick Category", + slug="smoke-kick-category", + image_url="https://example.com/smoke-kick-category.png", + ) + self.kick_campaign: KickDropCampaign = KickDropCampaign.objects.create( + kick_id="smoke-kick-campaign-1", + name="Smoke Kick Campaign", + status="active", + starts_at=now - timedelta(days=1), + ends_at=now + timedelta(days=1), + organization=self.kick_org, + category=self.kick_category, + rule_id=1, + rule_name="Watch to redeem", + is_fully_imported=True, + ) + kick_user: KickUser = KickUser.objects.create( + kick_id=990001, + username="smokekickuser", + ) + kick_channel: KickChannel = KickChannel.objects.create( + kick_id=990002, + slug="smokekickchannel", + user=kick_user, + ) + self.kick_campaign.channels.add(kick_channel) + KickReward.objects.create( + kick_id="smoke-kick-reward-1", + name="Smoke Kick Reward", + image_url="drops/reward-image/smoke-kick-reward.png", + required_units=20, + campaign=self.kick_campaign, + category=self.kick_category, + organization=self.kick_org, + ) + + # CHZZK fixtures + self.chzzk_campaign: ChzzkCampaign = ChzzkCampaign.objects.create( + campaign_no=9901, + title="Smoke CHZZK Campaign", + description="Smoke CHZZK description", + category_type="game", + category_id="1", + category_value="SmokeGame", + service_id="chzzk", + state="ACTIVE", + start_date=now - timedelta(days=1), + end_date=now + timedelta(days=1), + has_ios_based_reward=False, + drops_campaign_not_started=False, + source_api="unit-test", + raw_json_v1={"ok": True}, + ) + self.chzzk_campaign.rewards.create( # pyright: ignore[reportAttributeAccessIssue] + reward_no=991, + title="Smoke CHZZK Reward", + reward_type="ITEM", + campaign_reward_type="Standard", + condition_type="watch", + condition_for_minutes=10, + ios_based_reward=False, + code_remaining_count=100, + ) + + # Core dataset download fixture + self.dataset_dir: Path = settings.DATA_DIR / "datasets" + self.dataset_dir.mkdir(parents=True, exist_ok=True) + self.dataset_name = "smoke-dataset.zst" + (self.dataset_dir / self.dataset_name).write_bytes(b"smoke") + + def tearDown(self) -> None: + """Clean up any files created for testing.""" + dataset_path: Path = self.dataset_dir / self.dataset_name + if dataset_path.exists(): + dataset_path.unlink() + + def test_all_site_endpoints_return_success(self) -> None: + """Test that all named site endpoints return a successful response with representative data.""" + endpoints: list[tuple[str, dict[str, str | int], int]] = [ + # Top-level config endpoints + ("sitemap", {}, 200), + ("sitemap-static", {}, 200), + ("sitemap-twitch-channels", {}, 200), + ("sitemap-twitch-drops", {}, 200), + ("sitemap-twitch-others", {}, 200), + ("sitemap-kick", {}, 200), + ("sitemap-youtube", {}, 200), + # Core endpoints + ("core:dashboard", {}, 200), + ("core:search", {}, 200), + ("core:debug", {}, 200), + ("core:dataset_backups", {}, 200), + ("core:dataset_backup_download", {"relative_path": self.dataset_name}, 200), + ("core:docs_rss", {}, 200), + ("core:campaign_feed", {}, 200), + ("core:game_feed", {}, 200), + ("core:game_campaign_feed", {"twitch_id": self.twitch_game.twitch_id}, 200), + ("core:organization_feed", {}, 200), + ("core:reward_campaign_feed", {}, 200), + ("core:campaign_feed_atom", {}, 200), + ("core:game_feed_atom", {}, 200), + ( + "core:game_campaign_feed_atom", + {"twitch_id": self.twitch_game.twitch_id}, + 200, + ), + ("core:organization_feed_atom", {}, 200), + ("core:reward_campaign_feed_atom", {}, 200), + ("core:campaign_feed_discord", {}, 200), + ("core:game_feed_discord", {}, 200), + ( + "core:game_campaign_feed_discord", + {"twitch_id": self.twitch_game.twitch_id}, + 200, + ), + ("core:organization_feed_discord", {}, 200), + ("core:reward_campaign_feed_discord", {}, 200), + # Twitch endpoints + ("twitch:dashboard", {}, 200), + ("twitch:badge_list", {}, 200), + ("twitch:badge_set_detail", {"set_id": self.badge_set.set_id}, 200), + ("twitch:campaign_list", {}, 200), + ( + "twitch:campaign_detail", + {"twitch_id": self.twitch_campaign.twitch_id}, + 200, + ), + ("twitch:channel_list", {}, 200), + ( + "twitch:channel_detail", + {"twitch_id": self.twitch_channel.twitch_id}, + 200, + ), + ("twitch:emote_gallery", {}, 200), + ("twitch:games_grid", {}, 200), + ("twitch:games_list", {}, 200), + ("twitch:game_detail", {"twitch_id": self.twitch_game.twitch_id}, 200), + ("twitch:org_list", {}, 200), + ( + "twitch:organization_detail", + {"twitch_id": self.twitch_org.twitch_id}, + 200, + ), + ("twitch:reward_campaign_list", {}, 200), + ( + "twitch:reward_campaign_detail", + {"twitch_id": self.twitch_reward_campaign.twitch_id}, + 200, + ), + ("twitch:export_campaigns_csv", {}, 200), + ("twitch:export_campaigns_json", {}, 200), + ("twitch:export_games_csv", {}, 200), + ("twitch:export_games_json", {}, 200), + ("twitch:export_organizations_csv", {}, 200), + ("twitch:export_organizations_json", {}, 200), + # Kick endpoints + ("kick:dashboard", {}, 200), + ("kick:campaign_list", {}, 200), + ("kick:campaign_detail", {"kick_id": self.kick_campaign.kick_id}, 200), + ("kick:game_list", {}, 200), + ("kick:game_detail", {"kick_id": self.kick_category.kick_id}, 200), + ("kick:category_list", {}, 200), + ("kick:category_detail", {"kick_id": self.kick_category.kick_id}, 200), + ("kick:organization_list", {}, 200), + ("kick:organization_detail", {"kick_id": self.kick_org.kick_id}, 200), + ("kick:campaign_feed", {}, 200), + ("kick:game_feed", {}, 200), + ("kick:game_campaign_feed", {"kick_id": self.kick_category.kick_id}, 200), + ("kick:category_feed", {}, 200), + ( + "kick:category_campaign_feed", + {"kick_id": self.kick_category.kick_id}, + 200, + ), + ("kick:organization_feed", {}, 200), + ("kick:campaign_feed_atom", {}, 200), + ("kick:game_feed_atom", {}, 200), + ( + "kick:game_campaign_feed_atom", + {"kick_id": self.kick_category.kick_id}, + 200, + ), + ("kick:category_feed_atom", {}, 200), + ( + "kick:category_campaign_feed_atom", + {"kick_id": self.kick_category.kick_id}, + 200, + ), + ("kick:organization_feed_atom", {}, 200), + ("kick:campaign_feed_discord", {}, 200), + ("kick:game_feed_discord", {}, 200), + ( + "kick:game_campaign_feed_discord", + {"kick_id": self.kick_category.kick_id}, + 200, + ), + ("kick:category_feed_discord", {}, 200), + ( + "kick:category_campaign_feed_discord", + {"kick_id": self.kick_category.kick_id}, + 200, + ), + ("kick:organization_feed_discord", {}, 200), + # CHZZK endpoints + ("chzzk:dashboard", {}, 200), + ("chzzk:campaign_list", {}, 200), + ( + "chzzk:campaign_detail", + {"campaign_no": self.chzzk_campaign.campaign_no}, + 200, + ), + ("chzzk:campaign_feed", {}, 200), + ("chzzk:campaign_feed_atom", {}, 200), + ("chzzk:campaign_feed_discord", {}, 200), + # YouTube endpoint + ("youtube:index", {}, 200), + ] + + for route_name, kwargs, expected_status in endpoints: + response: _MonkeyPatchedWSGIResponse = self.client.get( + reverse(route_name, kwargs=kwargs), + ) + assert response.status_code == expected_status, ( + f"{route_name} returned {response.status_code}, expected {expected_status}" + ) + response.close() diff --git a/kick/models.py b/kick/models.py index d58a379..450434f 100644 --- a/kick/models.py +++ b/kick/models.py @@ -361,7 +361,24 @@ class KickDropCampaign(auto_prefetch.Model): If both a base reward and a "(Con)" variant exist, prefer the base reward name. """ rewards_by_name: dict[str, KickReward] = {} - for reward in self.rewards.all().order_by("required_units", "name", "kick_id"): # pyright: ignore[reportAttributeAccessIssue] + prefetched_rewards: list[KickReward] | None = getattr( + self, + "_prefetched_objects_cache", + {}, + ).get("rewards") + if prefetched_rewards is not None: + rewards_iterable = sorted( + prefetched_rewards, + key=lambda reward: (reward.required_units, reward.name, reward.kick_id), + ) + else: + rewards_iterable = self.rewards.all().order_by( # pyright: ignore[reportAttributeAccessIssue] + "required_units", + "name", + "kick_id", + ) + + for reward in rewards_iterable: key: str = self._normalized_reward_name(reward.name) existing: KickReward | None = rewards_by_name.get(key) if existing is None: diff --git a/kick/tests/test_kick.py b/kick/tests/test_kick.py index 357f604..84428c2 100644 --- a/kick/tests/test_kick.py +++ b/kick/tests/test_kick.py @@ -12,8 +12,10 @@ from unittest.mock import patch import httpx import pytest from django.core.management import call_command +from django.db import connection from django.test import Client from django.test import TestCase +from django.test.utils import CaptureQueriesContext from django.urls import reverse from django.utils import timezone from pydantic import ValidationError @@ -414,6 +416,37 @@ class KickDropCampaignMergedRewardsTest(TestCase): assert len(merged) == 1 assert merged[0].name == "9th Anniversary Cake & Confetti" + def test_uses_prefetched_rewards_without_extra_queries(self) -> None: + """When rewards are prefetched, merged_rewards should not hit the database again.""" + campaign: KickDropCampaign = self._make_campaign() + KickReward.objects.create( + kick_id="reward-prefetch-a", + name="Alpha Reward", + image_url="drops/reward-image/alpha.png", + required_units=10, + campaign=campaign, + category=campaign.category, + organization=campaign.organization, + ) + KickReward.objects.create( + kick_id="reward-prefetch-b", + name="Alpha Reward (Con)", + image_url="drops/reward-image/alpha-con.png", + required_units=10, + campaign=campaign, + category=campaign.category, + organization=campaign.organization, + ) + + campaign = KickDropCampaign.objects.prefetch_related("rewards").get( + pk=campaign.pk, + ) + with self.assertNumQueries(0): + merged: list[KickReward] = campaign.merged_rewards + + assert len(merged) == 1 + assert merged[0].name == "Alpha Reward" + # MARK: Management command tests class ImportKickDropsCommandTest(TestCase): @@ -546,6 +579,92 @@ class KickDashboardViewTest(TestCase): ) assert campaign.name in response.content.decode() + def test_dashboard_query_count_stays_flat_with_more_campaigns(self) -> None: + """Dashboard SELECT query count should stay flat as active campaign count grows.""" + + def _create_active_campaign(index: int) -> KickDropCampaign: + org: KickOrganization = KickOrganization.objects.create( + kick_id=f"org-qc-{index}", + name=f"Org QC {index}", + ) + cat: KickCategory = KickCategory.objects.create( + kick_id=10000 + index, + name=f"Cat QC {index}", + slug=f"cat-qc-{index}", + ) + campaign: KickDropCampaign = KickDropCampaign.objects.create( + kick_id=f"camp-qc-{index}", + name=f"Campaign QC {index}", + status="active", + starts_at=dt(2020, 1, 1, tzinfo=UTC), + ends_at=dt(2099, 12, 31, tzinfo=UTC), + organization=org, + category=cat, + rule_id=1, + rule_name="Watch to redeem", + is_fully_imported=True, + ) + + user: KickUser = KickUser.objects.create( + kick_id=3000000 + index, + username=f"qcuser{index}", + ) + channel: KickChannel = KickChannel.objects.create( + kick_id=2000000 + index, + slug=f"qc-channel-{index}", + user=user, + ) + campaign.channels.add(channel) + + KickReward.objects.create( + kick_id=f"reward-qc-{index}-a", + name="Alpha Reward", + image_url="drops/reward-image/alpha.png", + required_units=30, + campaign=campaign, + category=cat, + organization=org, + ) + KickReward.objects.create( + kick_id=f"reward-qc-{index}-b", + name="Alpha Reward (Con)", + image_url="drops/reward-image/alpha-con.png", + required_units=30, + campaign=campaign, + category=cat, + organization=org, + ) + + return campaign + + def _capture_dashboard_select_count() -> int: + with CaptureQueriesContext(connection) as queries: + response: _MonkeyPatchedWSGIResponse = self.client.get( + reverse("kick:dashboard"), + ) + assert response.status_code == 200 + + select_queries: list[str] = [ + query_info["sql"] + for query_info in queries.captured_queries + if query_info["sql"].lstrip().upper().startswith("SELECT") + ] + return len(select_queries) + + _create_active_campaign(1) + baseline_select_count: int = _capture_dashboard_select_count() + + for i in range(2, 12): + _create_active_campaign(i) + + scaled_select_count: int = _capture_dashboard_select_count() + + assert scaled_select_count <= baseline_select_count + 2, ( + "Kick dashboard SELECT query count grew with campaign volume; " + f"possible N+1 regression. baseline={baseline_select_count}, " + f"scaled={scaled_select_count}" + ) + class KickCampaignListViewTest(TestCase): """Tests for the kick campaign list view.""" @@ -811,6 +930,95 @@ class KickOrganizationDetailViewTest(TestCase): ) assert response.status_code == 404 + def test_organization_detail_query_count_stays_flat_with_more_campaigns( + self, + ) -> None: + """Organization detail SELECT query count should stay flat as campaign count grows.""" + org: KickOrganization = KickOrganization.objects.create( + kick_id="org-orgdet-qc", + name="Orgdet Query Count", + ) + + def _create_org_campaign(index: int) -> None: + cat: KickCategory = KickCategory.objects.create( + kick_id=17000 + index, + name=f"Orgdet QC Cat {index}", + slug=f"orgdet-qc-cat-{index}", + ) + campaign: KickDropCampaign = KickDropCampaign.objects.create( + kick_id=f"camp-orgdet-qc-{index}", + name=f"Orgdet QC Campaign {index}", + status="active", + starts_at=dt(2020, 1, 1, tzinfo=UTC), + ends_at=dt(2099, 12, 31, tzinfo=UTC), + organization=org, + category=cat, + rule_id=1, + rule_name="Watch to redeem", + is_fully_imported=True, + ) + + user: KickUser = KickUser.objects.create( + kick_id=3700000 + index, + username=f"orgdetqcuser{index}", + ) + channel: KickChannel = KickChannel.objects.create( + kick_id=2700000 + index, + slug=f"orgdet-qc-channel-{index}", + user=user, + ) + campaign.channels.add(channel) + + KickReward.objects.create( + kick_id=f"reward-orgdet-qc-{index}-a", + name="Org Reward", + image_url="drops/reward-image/org.png", + required_units=30, + campaign=campaign, + category=cat, + organization=org, + ) + KickReward.objects.create( + kick_id=f"reward-orgdet-qc-{index}-b", + name="Org Reward (Con)", + image_url="drops/reward-image/org-con.png", + required_units=30, + campaign=campaign, + category=cat, + organization=org, + ) + + def _capture_org_detail_select_count() -> int: + with CaptureQueriesContext(connection) as queries: + response: _MonkeyPatchedWSGIResponse = self.client.get( + reverse( + "kick:organization_detail", + kwargs={"kick_id": org.kick_id}, + ), + ) + assert response.status_code == 200 + + select_queries: list[str] = [ + query_info["sql"] + for query_info in queries.captured_queries + if query_info["sql"].lstrip().upper().startswith("SELECT") + ] + return len(select_queries) + + _create_org_campaign(1) + baseline_select_count: int = _capture_org_detail_select_count() + + for i in range(2, 12): + _create_org_campaign(i) + + scaled_select_count: int = _capture_org_detail_select_count() + + assert scaled_select_count <= baseline_select_count + 2, ( + "Organization detail SELECT query count grew with campaign volume; " + f"possible N+1 regression. baseline={baseline_select_count}, " + f"scaled={scaled_select_count}" + ) + class KickFeedsTest(TestCase): """Tests for Kick RSS/Atom/Discord feed endpoints.""" @@ -942,6 +1150,109 @@ class KickFeedsTest(TestCase): assert not str(discord_timestamp(None)) +class KickEndpointCoverageTest(TestCase): + """Endpoint smoke coverage for all Kick routes in kick.urls.""" + + def setUp(self) -> None: + """Create shared fixtures used by detail and feed endpoints.""" + self.org: KickOrganization = KickOrganization.objects.create( + kick_id="org-endpoint-1", + name="Endpoint Org", + logo_url="https://example.com/org-endpoint.png", + ) + self.category: KickCategory = KickCategory.objects.create( + kick_id=9123, + name="Endpoint Category", + slug="endpoint-category", + image_url="https://example.com/endpoint-category.png", + ) + self.campaign: KickDropCampaign = KickDropCampaign.objects.create( + kick_id="camp-endpoint-1", + name="Endpoint Campaign", + status="active", + starts_at=timezone.now() - timedelta(days=1), + ends_at=timezone.now() + timedelta(days=1), + organization=self.org, + category=self.category, + connect_url="https://example.com/connect", + url="https://example.com/campaign", + rule_id=1, + rule_name="Watch to redeem", + is_fully_imported=True, + ) + + user: KickUser = KickUser.objects.create( + kick_id=5551001, + username="endpointuser", + ) + channel: KickChannel = KickChannel.objects.create( + kick_id=5551002, + slug="endpointchannel", + user=user, + ) + self.campaign.channels.add(channel) + + KickReward.objects.create( + kick_id="reward-endpoint-1", + name="Endpoint Reward", + image_url="drops/reward-image/endpoint.png", + required_units=20, + campaign=self.campaign, + category=self.category, + organization=self.org, + ) + + def test_all_kick_html_endpoints_return_success(self) -> None: + """All Kick HTML endpoints should render successfully with populated fixtures.""" + html_routes: list[tuple[str, dict[str, str | int]]] = [ + ("kick:dashboard", {}), + ("kick:campaign_list", {}), + ("kick:campaign_detail", {"kick_id": self.campaign.kick_id}), + ("kick:game_list", {}), + ("kick:game_detail", {"kick_id": self.category.kick_id}), + ("kick:category_list", {}), + ("kick:category_detail", {"kick_id": self.category.kick_id}), + ("kick:organization_list", {}), + ("kick:organization_detail", {"kick_id": self.org.kick_id}), + ] + + for route_name, kwargs in html_routes: + response: _MonkeyPatchedWSGIResponse = self.client.get( + reverse(route_name, kwargs=kwargs), + ) + assert response.status_code == 200, route_name + + def test_all_kick_feed_endpoints_return_success(self) -> None: + """All Kick RSS/Atom/Discord feed endpoints should return XML responses.""" + feed_routes: list[tuple[str, dict[str, int]]] = [ + ("kick:campaign_feed", {}), + ("kick:game_feed", {}), + ("kick:game_campaign_feed", {"kick_id": self.category.kick_id}), + ("kick:category_feed", {}), + ("kick:category_campaign_feed", {"kick_id": self.category.kick_id}), + ("kick:organization_feed", {}), + ("kick:campaign_feed_atom", {}), + ("kick:game_feed_atom", {}), + ("kick:game_campaign_feed_atom", {"kick_id": self.category.kick_id}), + ("kick:category_feed_atom", {}), + ("kick:category_campaign_feed_atom", {"kick_id": self.category.kick_id}), + ("kick:organization_feed_atom", {}), + ("kick:campaign_feed_discord", {}), + ("kick:game_feed_discord", {}), + ("kick:game_campaign_feed_discord", {"kick_id": self.category.kick_id}), + ("kick:category_feed_discord", {}), + ("kick:category_campaign_feed_discord", {"kick_id": self.category.kick_id}), + ("kick:organization_feed_discord", {}), + ] + + for route_name, kwargs in feed_routes: + response: _MonkeyPatchedWSGIResponse = self.client.get( + reverse(route_name, kwargs=kwargs), + ) + assert response.status_code == 200, route_name + assert response["Content-Type"] == "application/xml; charset=utf-8" + + class KickDropCampaignFullyImportedTest(TestCase): """Tests for KickDropCampaign.is_fully_imported field and filtering.""" diff --git a/kick/views.py b/kick/views.py index b8cc126..7d89560 100644 --- a/kick/views.py +++ b/kick/views.py @@ -532,7 +532,7 @@ def organization_detail_view(request: HttpRequest, kick_id: str) -> HttpResponse KickDropCampaign.objects .filter(organization=org) .select_related("category") - .prefetch_related("rewards") + .prefetch_related("rewards", "channels__user") .order_by("-starts_at"), ) diff --git a/templates/twitch/dashboard.html b/templates/twitch/dashboard.html index 0689a2d..3d7aa55 100644 --- a/templates/twitch/dashboard.html +++ b/templates/twitch/dashboard.html @@ -68,7 +68,7 @@ flex-shrink: 0">
- {% picture campaign_data.campaign.image_best_url|default:campaign_data.campaign.image_url alt="Image for "|add:campaign_data.campaign.name width=120 %} + {% picture campaign_data.image_url alt="Image for "|add:campaign_data.campaign.name width=120 %}

{{ campaign_data.campaign.clean_name }}

 diff --git a/twitch/models.py b/twitch/models.py index 1ed2a57..bd5e9a1 100644 --- a/twitch/models.py +++ b/twitch/models.py @@ -1,5 +1,7 @@ import logging +from collections import OrderedDict from typing import TYPE_CHECKING +from typing import Any import auto_prefetch from django.conf import settings @@ -508,6 +510,93 @@ class DropCampaign(auto_prefetch.Model): def __str__(self) -> str: return self.name + @classmethod + def active_for_dashboard( + cls, + now: datetime.datetime, + ) -> models.QuerySet[DropCampaign]: + """Return active campaigns with relations needed by the dashboard. + + Args: + now: Current timestamp used to evaluate active campaigns. + + Returns: + QuerySet of active campaigns ordered by newest start date. + """ + return ( + cls.objects + .filter(start_at__lte=now, end_at__gte=now) + .only( + "twitch_id", + "name", + "image_url", + "image_file", + "start_at", + "end_at", + "allow_is_enabled", + "game", + "game__twitch_id", + "game__display_name", + "game__slug", + "game__box_art", + "game__box_art_file", + ) + .select_related("game") + .prefetch_related( + models.Prefetch( + "game__owners", + queryset=Organization.objects.only("twitch_id", "name"), + ), + models.Prefetch( + "allow_channels", + queryset=Channel.objects.only( + "twitch_id", + "name", + "display_name", + ).order_by("display_name"), + to_attr="channels_ordered", + ), + ) + .order_by("-start_at") + ) + + @staticmethod + def grouped_by_game( + campaigns: models.QuerySet[DropCampaign], + ) -> OrderedDict[str, dict[str, Any]]: + """Group campaigns by game for dashboard rendering. + + The grouping keeps insertion order and avoids duplicate per-game cards when + games have multiple owners. + + Args: + campaigns: Campaign queryset from active_for_dashboard(). + + Returns: + Ordered mapping keyed by game twitch_id. + """ + campaigns_by_game: OrderedDict[str, dict[str, Any]] = OrderedDict() + + for campaign in campaigns: + game: Game = campaign.game + game_id: str = game.twitch_id + + if game_id not in campaigns_by_game: + campaigns_by_game[game_id] = { + "name": game.display_name, + "box_art": game.box_art_best_url, + "owners": list(game.owners.all()), + "campaigns": [], + } + + campaigns_by_game[game_id]["campaigns"].append({ + "campaign": campaign, + "image_url": campaign.listing_image_url, + "allowed_channels": getattr(campaign, "channels_ordered", []), + }) + + return campaigns_by_game + @property def is_active(self) -> bool: """Check if the campaign is currently active.""" @@ -526,19 +615,21 @@ class DropCampaign(auto_prefetch.Model): "Skull & Bones - Closed Beta" -> "Closed Beta" (& is replaced with "and") """ - if not self.game or not self.game.display_name: + self_game: Game | None = self.game + + if not self_game or not self_game.display_name: return self.name - game_variations = [self.game.display_name] - if "&" in self.game.display_name: - game_variations.append(self.game.display_name.replace("&", "and")) - if "and" in self.game.display_name: - game_variations.append(self.game.display_name.replace("and", "&")) + game_variations: list[str] = [self_game.display_name] + if "&" in self_game.display_name: + game_variations.append(self_game.display_name.replace("&", "and")) + if "and" in self_game.display_name: + game_variations.append(self_game.display_name.replace("and", "&")) for game_name in game_variations: # Check for different separators after the game name for separator in [" - ", " | ", " "]: - prefix_to_check = game_name + separator + prefix_to_check: str = game_name + separator name: str = self.name if name.startswith(prefix_to_check): @@ -573,6 +664,20 @@ class DropCampaign(auto_prefetch.Model): return "" + @property + def listing_image_url(self) -> str: + """Return a campaign image URL optimized for list views. + + This intentionally avoids traversing drops/benefits to prevent N+1 queries + in list pages that render many campaigns. + """ + try: + if self.image_file and getattr(self.image_file, "url", None): + return self.image_file.url + except (AttributeError, OSError, ValueError) as exc: + logger.debug("Failed to resolve DropCampaign.image_file url: %s", exc) + return self.image_url or "" + @property def duration_iso(self) -> str: """Return the campaign duration in ISO 8601 format (e.g., 'P3DT4H30M'). @@ -1006,6 +1111,32 @@ class RewardCampaign(auto_prefetch.Model): """Return a string representation of the reward campaign.""" return f"{self.brand}: {self.name}" if self.brand else self.name + @classmethod + def active_for_dashboard( + cls, + now: datetime.datetime, + ) -> models.QuerySet[RewardCampaign]: + """Return active reward campaigns with only dashboard-needed fields.""" + return ( + cls.objects + .filter(starts_at__lte=now, ends_at__gte=now) + .only( + "twitch_id", + "name", + "brand", + "summary", + "external_url", + "starts_at", + "ends_at", + "is_sitewide", + "game", + "game__twitch_id", + "game__display_name", + ) + .select_related("game") + .order_by("-starts_at") + ) + @property def is_active(self) -> bool: """Check if the reward campaign is currently active.""" diff --git a/twitch/tests/test_views.py b/twitch/tests/test_views.py index 03f67d6..be951b2 100644 --- a/twitch/tests/test_views.py +++ b/twitch/tests/test_views.py @@ -9,8 +9,10 @@ from typing import Literal import pytest from django.core.handlers.wsgi import WSGIRequest from django.core.paginator import Paginator +from django.db import connection from django.db.models import Max from django.test import RequestFactory +from django.test.utils import CaptureQueriesContext from django.urls import reverse from django.utils import timezone @@ -34,6 +36,7 @@ from twitch.views import _truncate_description if TYPE_CHECKING: from django.core.handlers.wsgi import WSGIRequest + from django.db.models import QuerySet from django.test import Client from django.test.client import _MonkeyPatchedWSGIResponse from django.test.utils import ContextList @@ -537,6 +540,225 @@ class TestChannelListView: assert game.twitch_id in context["campaigns_by_game"] assert len(context["campaigns_by_game"][game.twitch_id]["campaigns"]) == 1 + @pytest.mark.django_db + def test_dashboard_queries_use_indexes(self) -> None: + """Dashboard source queries should use indexes for active-window filtering.""" + now: datetime.datetime = timezone.now() + + org: Organization = Organization.objects.create( + twitch_id="org_index_test", + name="Org Index Test", + ) + game: Game = Game.objects.create( + twitch_id="game_index_test", + name="Game Index Test", + display_name="Game Index Test", + ) + game.owners.add(org) + + # Add enough rows so the query planner has a reason to pick indexes. + campaigns: list[DropCampaign] = [] + for i in range(250): + campaigns.extend(( + DropCampaign( + twitch_id=f"inactive_old_{i}", + name=f"Inactive old {i}", + game=game, + operation_names=["DropCampaignDetails"], + start_at=now - timedelta(days=60), + end_at=now - timedelta(days=30), + ), + DropCampaign( + twitch_id=f"inactive_future_{i}", + name=f"Inactive future {i}", + game=game, + operation_names=["DropCampaignDetails"], + start_at=now + timedelta(days=30), + end_at=now + timedelta(days=60), + ), + )) + campaigns.append( + DropCampaign( + twitch_id="active_for_dashboard_index_test", + name="Active campaign", + game=game, + operation_names=["DropCampaignDetails"], + start_at=now - timedelta(hours=1), + end_at=now + timedelta(hours=1), + ), + ) + DropCampaign.objects.bulk_create(campaigns) + + reward_campaigns: list[RewardCampaign] = [] + for i in range(250): + reward_campaigns.extend(( + RewardCampaign( + twitch_id=f"reward_inactive_old_{i}", + name=f"Reward inactive old {i}", + game=game, + starts_at=now - timedelta(days=60), + ends_at=now - timedelta(days=30), + ), + RewardCampaign( + twitch_id=f"reward_inactive_future_{i}", + name=f"Reward inactive future {i}", + game=game, + starts_at=now + timedelta(days=30), + ends_at=now + timedelta(days=60), + ), + )) + reward_campaigns.append( + RewardCampaign( + twitch_id="reward_active_for_dashboard_index_test", + name="Active reward campaign", + game=game, + starts_at=now - timedelta(hours=1), + ends_at=now + timedelta(hours=1), + ), + ) + RewardCampaign.objects.bulk_create(reward_campaigns) + + active_campaigns_qs: QuerySet[DropCampaign] = DropCampaign.active_for_dashboard( + now, + ) + active_reward_campaigns_qs: QuerySet[RewardCampaign] = ( + RewardCampaign.objects + .filter(starts_at__lte=now, ends_at__gte=now) + .select_related("game") + .order_by("-starts_at") + ) + + campaigns_plan: str = active_campaigns_qs.explain() + reward_plan: str = active_reward_campaigns_qs.explain() + + if connection.vendor == "sqlite": + campaigns_uses_index: bool = "USING INDEX" in campaigns_plan.upper() + rewards_uses_index: bool = "USING INDEX" in reward_plan.upper() + elif connection.vendor == "postgresql": + campaigns_uses_index = ( + "INDEX SCAN" in campaigns_plan.upper() + or "BITMAP INDEX SCAN" in campaigns_plan.upper() + ) + rewards_uses_index = ( + "INDEX SCAN" in reward_plan.upper() + or "BITMAP INDEX SCAN" in reward_plan.upper() + ) + else: + pytest.skip( + f"Unsupported DB vendor for index-plan assertion: {connection.vendor}", + ) + + assert campaigns_uses_index, campaigns_plan + assert rewards_uses_index, reward_plan + + @pytest.mark.django_db + def test_dashboard_query_count_stays_flat_with_more_data( + self, + client: Client, + ) -> None: + """Dashboard should avoid N+1 queries as campaign volume grows.""" + now: datetime.datetime = timezone.now() + + org: Organization = Organization.objects.create( + twitch_id="org_query_count", + name="Org Query Count", + ) + game: Game = Game.objects.create( + twitch_id="game_query_count", + name="game_query_count", + display_name="Game Query Count", + ) + game.owners.add(org) + + def _capture_dashboard_select_count() -> int: + with CaptureQueriesContext(connection) as queries: + response: _MonkeyPatchedWSGIResponse = client.get( + reverse("twitch:dashboard"), + ) + assert response.status_code == 200 + + select_queries: list[str] = [ + query_info["sql"] + for query_info in queries.captured_queries + if query_info["sql"].lstrip().upper().startswith("SELECT") + ] + return len(select_queries) + + # Baseline: one active drop campaign and one active reward campaign. + base_campaign: DropCampaign = DropCampaign.objects.create( + twitch_id="baseline_campaign", + name="Baseline campaign", + game=game, + operation_names=["DropCampaignDetails"], + start_at=now - timedelta(hours=1), + end_at=now + timedelta(hours=1), + ) + base_channel: Channel = Channel.objects.create( + twitch_id="baseline_channel", + name="baselinechannel", + display_name="BaselineChannel", + ) + base_campaign.allow_channels.add(base_channel) + + RewardCampaign.objects.create( + twitch_id="baseline_reward_campaign", + name="Baseline reward campaign", + game=game, + starts_at=now - timedelta(hours=1), + ends_at=now + timedelta(hours=1), + summary="Baseline summary", + external_url="https://example.com/reward/baseline", + ) + + baseline_select_count: int = _capture_dashboard_select_count() + + # Scale up active dashboard data substantially. + extra_campaigns: list[DropCampaign] = [ + DropCampaign( + twitch_id=f"scaled_campaign_{i}", + name=f"Scaled campaign {i}", + game=game, + operation_names=["DropCampaignDetails"], + start_at=now - timedelta(hours=2), + end_at=now + timedelta(hours=2), + ) + for i in range(12) + ] + DropCampaign.objects.bulk_create(extra_campaigns) + + for i, campaign in enumerate( + DropCampaign.objects.filter( + twitch_id__startswith="scaled_campaign_", + ).order_by("twitch_id"), + ): + channel: Channel = Channel.objects.create( + twitch_id=f"scaled_channel_{i}", + name=f"scaledchannel{i}", + display_name=f"ScaledChannel{i}", + ) + campaign.allow_channels.add(channel) + + extra_rewards: list[RewardCampaign] = [ + RewardCampaign( + twitch_id=f"scaled_reward_{i}", + name=f"Scaled reward {i}", + game=game, + starts_at=now - timedelta(hours=2), + ends_at=now + timedelta(hours=2), + summary=f"Scaled summary {i}", + external_url=f"https://example.com/reward/{i}", + ) + for i in range(12) + ] + RewardCampaign.objects.bulk_create(extra_rewards) + + scaled_select_count: int = _capture_dashboard_select_count() + + assert scaled_select_count <= baseline_select_count + 2, ( + "Dashboard SELECT query count grew with data volume; possible N+1 regression. " + f"baseline={baseline_select_count}, scaled={scaled_select_count}" + ) + @pytest.mark.django_db def test_debug_view(self, client: Client) -> None: """Test debug view returns 200 and has games_without_owner in context.""" diff --git a/twitch/views.py b/twitch/views.py index 974f829..0fb8a74 100644 --- a/twitch/views.py +++ b/twitch/views.py @@ -1071,48 +1071,14 @@ def dashboard(request: HttpRequest) -> HttpResponse: HttpResponse: The rendered dashboard template. """ now: datetime.datetime = timezone.now() - active_campaigns: QuerySet[DropCampaign] = ( - DropCampaign.objects - .filter(start_at__lte=now, end_at__gte=now) - .select_related("game") - .prefetch_related("game__owners") - .prefetch_related( - Prefetch( - "allow_channels", - queryset=Channel.objects.order_by("display_name"), - to_attr="channels_ordered", - ), - ) - .order_by("-start_at") + active_campaigns: QuerySet[DropCampaign] = DropCampaign.active_for_dashboard(now) + campaigns_by_game: OrderedDict[str, dict[str, Any]] = DropCampaign.grouped_by_game( + active_campaigns, ) - # Preserve insertion order (newest campaigns first). - # Group by game so games with multiple owners don't render duplicate campaign cards. - campaigns_by_game: OrderedDict[str, dict[str, Any]] = OrderedDict() - - for campaign in active_campaigns: - game: Game = campaign.game - game_id: str = game.twitch_id - - if game_id not in campaigns_by_game: - campaigns_by_game[game_id] = { - "name": game.display_name, - "box_art": game.box_art_best_url, - "owners": list(game.owners.all()), - "campaigns": [], - } - - campaigns_by_game[game_id]["campaigns"].append({ - "campaign": campaign, - "allowed_channels": getattr(campaign, "channels_ordered", []), - }) - # Get active reward campaigns (Quest rewards) active_reward_campaigns: QuerySet[RewardCampaign] = ( - RewardCampaign.objects - .filter(starts_at__lte=now, ends_at__gte=now) - .select_related("game") - .order_by("-starts_at") + RewardCampaign.active_for_dashboard(now) ) # WebSite schema with SearchAction for sitelinks search box