From fb087a01c0599a870d0e27d75462222470b29404 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Hells=C3=A9n?= Date: Fri, 10 Apr 2026 22:32:38 +0200 Subject: [PATCH 1/6] Enhance performance by prefetching more --- chzzk/tests/test_views.py | 71 +++++++++++++++++++++++++++++++++++++++ chzzk/views.py | 1 + config/settings.py | 4 +++ kick/models.py | 17 +++++++--- pyproject.toml | 1 + twitch/views.py | 22 +++++++----- 6 files changed, 104 insertions(+), 12 deletions(-) diff --git a/chzzk/tests/test_views.py b/chzzk/tests/test_views.py index 38e84be..1af3cdd 100644 --- a/chzzk/tests/test_views.py +++ b/chzzk/tests/test_views.py @@ -1,7 +1,9 @@ from datetime import timedelta from typing import TYPE_CHECKING +from django.db import connection from django.test import TestCase +from django.test.utils import CaptureQueriesContext from django.urls import reverse from django.utils import timezone @@ -16,6 +18,75 @@ if TYPE_CHECKING: class ChzzkDashboardViewTests(TestCase): """Test cases for the dashboard view of the chzzk app.""" + def test_dashboard_view_no_n_plus_one_on_rewards(self) -> None: + """Test that the dashboard view does not trigger an N+1 query for rewards.""" + now = timezone.now() + base_kwargs = { + "category_type": "game", + "category_id": "1", + "category_value": "TestGame", + "service_id": "chzzk", + "state": "ACTIVE", + "start_date": now - timedelta(days=1), + "end_date": now + timedelta(days=1), + "has_ios_based_reward": False, + "drops_campaign_not_started": False, + "source_api": "unit-test", + } + reward_kwargs = { + "reward_type": "ITEM", + "campaign_reward_type": "Standard", + "condition_type": "watch", + "ios_based_reward": False, + "code_remaining_count": 100, + } + + campaign1 = ChzzkCampaign.objects.create( + campaign_no=9001, + title="C1", + **base_kwargs, + ) + campaign1.rewards.create( # pyright: ignore[reportAttributeAccessIssue] + reward_no=901, + title="R1", + condition_for_minutes=10, + **reward_kwargs, + ) # pyright: ignore[reportAttributeAccessIssue] + campaign2 = ChzzkCampaign.objects.create( + campaign_no=9002, + title="C2", + **base_kwargs, + ) + campaign2.rewards.create( # pyright: ignore[reportAttributeAccessIssue] + reward_no=902, + title="R2", + condition_for_minutes=20, + **reward_kwargs, + ) # pyright: ignore[reportAttributeAccessIssue] + + with CaptureQueriesContext(connection) as one_campaign_ctx: + self.client.get(reverse("chzzk:dashboard")) + query_count_two = len(one_campaign_ctx) + + campaign3 = ChzzkCampaign.objects.create( + campaign_no=9003, + title="C3", + **base_kwargs, + ) + campaign3.rewards.create( # pyright: ignore[reportAttributeAccessIssue] + reward_no=903, + title="R3", + condition_for_minutes=30, + **reward_kwargs, + ) # pyright: ignore[reportAttributeAccessIssue] + + with CaptureQueriesContext(connection) as three_campaign_ctx: + self.client.get(reverse("chzzk:dashboard")) + query_count_three = len(three_campaign_ctx) + + # With prefetch_related, adding more campaigns should not add extra queries per campaign. + assert query_count_two == query_count_three + def test_dashboard_view_excludes_testing_state_campaigns(self) -> None: """Test that the dashboard view excludes campaigns in the TESTING state.""" now: datetime = timezone.now() diff --git a/chzzk/views.py b/chzzk/views.py index 6146c31..16cd547 100644 --- a/chzzk/views.py +++ b/chzzk/views.py @@ -34,6 +34,7 @@ def dashboard_view(request: HttpRequest) -> HttpResponse: models.ChzzkCampaign.objects .filter(end_date__gte=timezone.now()) .exclude(state="TESTING") + .prefetch_related("rewards") .order_by("-start_date") ) return render( diff --git a/config/settings.py b/config/settings.py index 7fb2c0f..c65859b 100644 --- a/config/settings.py +++ b/config/settings.py @@ -224,6 +224,10 @@ DATABASES: dict[str, dict[str, Any]] = configure_databases( base_dir=BASE_DIR, ) +if DEBUG: + INSTALLED_APPS.append("zeal") + MIDDLEWARE.append("zeal.middleware.zeal_middleware") + if not TESTING: INSTALLED_APPS = [*INSTALLED_APPS, "debug_toolbar", "silk"] MIDDLEWARE = [ diff --git a/kick/models.py b/kick/models.py index 1c2b4c0..d58a379 100644 --- a/kick/models.py +++ b/kick/models.py @@ -292,10 +292,19 @@ class KickDropCampaign(auto_prefetch.Model): def image_url(self) -> str: """Return the image URL for the campaign.""" # Image from first drop - if self.rewards.exists(): # pyright: ignore[reportAttributeAccessIssue] - first_reward: KickReward | None = self.rewards.first() # pyright: ignore[reportAttributeAccessIssue] - if first_reward and first_reward.image_url: - return first_reward.full_image_url + rewards_prefetched: list[KickReward] | None = getattr( + self, + "rewards_ordered", + None, + ) + if rewards_prefetched is not None: + first_reward: KickReward | None = ( + rewards_prefetched[0] if rewards_prefetched else None + ) + else: + first_reward = self.rewards.first() # pyright: ignore[reportAttributeAccessIssue] + if first_reward and first_reward.image_url: + return first_reward.full_image_url if self.category and self.category.image_url: return self.category.image_url diff --git a/pyproject.toml b/pyproject.toml index 836228a..9fee185 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ dependencies = [ "setproctitle", "sitemap-parser", "tqdm", + "django-zeal>=2.1.0", ] diff --git a/twitch/views.py b/twitch/views.py index ed1c0c7..974f829 100644 --- a/twitch/views.py +++ b/twitch/views.py @@ -438,7 +438,13 @@ def drop_campaign_list_view(request: HttpRequest) -> HttpResponse: # noqa: PLR0 if game_filter: queryset = queryset.filter(game__twitch_id=game_filter) - queryset = queryset.prefetch_related("game__owners").order_by("-start_at") + queryset = queryset.prefetch_related( + "game__owners", + Prefetch( + "time_based_drops", + queryset=TimeBasedDrop.objects.prefetch_related("benefits"), + ), + ).order_by("-start_at") # Optionally filter by status (active, upcoming, expired) now: datetime.datetime = timezone.now() @@ -588,18 +594,18 @@ def drop_campaign_detail_view(request: HttpRequest, twitch_id: str) -> HttpRespo queryset=Channel.objects.order_by("display_name"), to_attr="channels_ordered", ), + Prefetch( + "time_based_drops", + queryset=TimeBasedDrop.objects.prefetch_related("benefits").order_by( + "required_minutes_watched", + ), + ), ).get(twitch_id=twitch_id) except DropCampaign.DoesNotExist as exc: msg = "No campaign found matching the query" raise Http404(msg) from exc - drops: QuerySet[TimeBasedDrop] = ( - TimeBasedDrop.objects - .filter(campaign=campaign) - .select_related("campaign") - .prefetch_related("benefits") - .order_by("required_minutes_watched") - ) + drops: QuerySet[TimeBasedDrop] = campaign.time_based_drops.all() # pyright: ignore[reportAttributeAccessIssue] now: datetime.datetime = timezone.now() enhanced_drops: list[dict[str, Any]] = _enhance_drops_with_context(drops, now) From 1782db484024e8491420e5db9bc8e85f8ed63394 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Hells=C3=A9n?= Date: Fri, 10 Apr 2026 23:54:10 +0200 Subject: [PATCH 2/6] Add smoke tests for endpoints and optimize database queries --- config/tests/test_site_endpoint_smoke.py | 349 +++++++++++++++++++++++ kick/models.py | 19 +- kick/tests/test_kick.py | 311 ++++++++++++++++++++ kick/views.py | 2 +- templates/twitch/dashboard.html | 2 +- twitch/models.py | 145 +++++++++- twitch/tests/test_views.py | 222 ++++++++++++++ twitch/views.py | 42 +-- 8 files changed, 1044 insertions(+), 48 deletions(-) create mode 100644 config/tests/test_site_endpoint_smoke.py diff --git a/config/tests/test_site_endpoint_smoke.py b/config/tests/test_site_endpoint_smoke.py new file mode 100644 index 0000000..90537c0 --- /dev/null +++ b/config/tests/test_site_endpoint_smoke.py @@ -0,0 +1,349 @@ +from __future__ import annotations + +from datetime import timedelta +from pathlib import Path +from typing import TYPE_CHECKING + +from django.conf import settings +from django.test import TestCase +from django.urls import reverse +from django.utils import timezone + +from chzzk.models import ChzzkCampaign +from kick.models import KickCategory +from kick.models import KickChannel +from kick.models import KickDropCampaign +from kick.models import KickOrganization +from kick.models import KickReward +from kick.models import KickUser +from twitch.models import Channel +from twitch.models import ChatBadge +from twitch.models import ChatBadgeSet +from twitch.models import DropBenefit +from twitch.models import DropCampaign +from twitch.models import Game +from twitch.models import Organization +from twitch.models import RewardCampaign +from twitch.models import TimeBasedDrop + +if TYPE_CHECKING: + from datetime import datetime + from pathlib import Path + + from django.test.client import _MonkeyPatchedWSGIResponse + + +class SiteEndpointSmokeTest(TestCase): + """Smoke-test all named site endpoints with realistic fixture data.""" + + def setUp(self) -> None: + """Set up representative Twitch, Kick, and CHZZK data for endpoint smoke tests.""" + now: datetime = timezone.now() + + # Twitch fixtures + self.twitch_org: Organization = Organization.objects.create( + twitch_id="smoke-org-1", + name="Smoke Organization", + ) + self.twitch_game: Game = Game.objects.create( + twitch_id="smoke-game-1", + slug="smoke-game", + name="Smoke Game", + display_name="Smoke Game", + box_art="https://example.com/smoke-game.png", + ) + self.twitch_game.owners.add(self.twitch_org) + + self.twitch_channel: Channel = Channel.objects.create( + twitch_id="smoke-channel-1", + name="smokechannel", + display_name="SmokeChannel", + ) + + self.twitch_campaign: DropCampaign = DropCampaign.objects.create( + twitch_id="smoke-campaign-1", + name="Smoke Campaign", + description="Smoke campaign description", + game=self.twitch_game, + image_url="https://example.com/smoke-campaign.png", + start_at=now - timedelta(days=1), + end_at=now + timedelta(days=1), + operation_names=["DropCampaignDetails"], + is_fully_imported=True, + ) + self.twitch_campaign.allow_channels.add(self.twitch_channel) + + self.twitch_drop: TimeBasedDrop = TimeBasedDrop.objects.create( + twitch_id="smoke-drop-1", + name="Smoke Drop", + campaign=self.twitch_campaign, + required_minutes_watched=15, + start_at=now - timedelta(days=1), + end_at=now + timedelta(days=1), + ) + self.twitch_benefit: DropBenefit = DropBenefit.objects.create( + twitch_id="smoke-benefit-1", + name="Smoke Benefit", + image_asset_url="https://example.com/smoke-benefit.png", + ) + self.twitch_drop.benefits.add(self.twitch_benefit) + + self.twitch_reward_campaign: RewardCampaign = RewardCampaign.objects.create( + twitch_id="smoke-reward-campaign-1", + name="Smoke Reward Campaign", + brand="Smoke Brand", + starts_at=now - timedelta(days=1), + ends_at=now + timedelta(days=2), + status="ACTIVE", + summary="Smoke reward summary", + external_url="https://example.com/smoke-reward", + is_sitewide=False, + game=self.twitch_game, + ) + + self.badge_set: ChatBadgeSet = ChatBadgeSet.objects.create( + set_id="smoke-badge-set", + ) + ChatBadge.objects.create( + badge_set=self.badge_set, + badge_id="1", + image_url_1x="https://example.com/badge-1x.png", + image_url_2x="https://example.com/badge-2x.png", + image_url_4x="https://example.com/badge-4x.png", + title="Smoke Badge", + description="Smoke badge description", + ) + + # Kick fixtures + self.kick_org: KickOrganization = KickOrganization.objects.create( + kick_id="smoke-kick-org-1", + name="Smoke Kick Organization", + ) + self.kick_category: KickCategory = KickCategory.objects.create( + kick_id=9101, + name="Smoke Kick Category", + slug="smoke-kick-category", + image_url="https://example.com/smoke-kick-category.png", + ) + self.kick_campaign: KickDropCampaign = KickDropCampaign.objects.create( + kick_id="smoke-kick-campaign-1", + name="Smoke Kick Campaign", + status="active", + starts_at=now - timedelta(days=1), + ends_at=now + timedelta(days=1), + organization=self.kick_org, + category=self.kick_category, + rule_id=1, + rule_name="Watch to redeem", + is_fully_imported=True, + ) + kick_user: KickUser = KickUser.objects.create( + kick_id=990001, + username="smokekickuser", + ) + kick_channel: KickChannel = KickChannel.objects.create( + kick_id=990002, + slug="smokekickchannel", + user=kick_user, + ) + self.kick_campaign.channels.add(kick_channel) + KickReward.objects.create( + kick_id="smoke-kick-reward-1", + name="Smoke Kick Reward", + image_url="drops/reward-image/smoke-kick-reward.png", + required_units=20, + campaign=self.kick_campaign, + category=self.kick_category, + organization=self.kick_org, + ) + + # CHZZK fixtures + self.chzzk_campaign: ChzzkCampaign = ChzzkCampaign.objects.create( + campaign_no=9901, + title="Smoke CHZZK Campaign", + description="Smoke CHZZK description", + category_type="game", + category_id="1", + category_value="SmokeGame", + service_id="chzzk", + state="ACTIVE", + start_date=now - timedelta(days=1), + end_date=now + timedelta(days=1), + has_ios_based_reward=False, + drops_campaign_not_started=False, + source_api="unit-test", + raw_json_v1={"ok": True}, + ) + self.chzzk_campaign.rewards.create( # pyright: ignore[reportAttributeAccessIssue] + reward_no=991, + title="Smoke CHZZK Reward", + reward_type="ITEM", + campaign_reward_type="Standard", + condition_type="watch", + condition_for_minutes=10, + ios_based_reward=False, + code_remaining_count=100, + ) + + # Core dataset download fixture + self.dataset_dir: Path = settings.DATA_DIR / "datasets" + self.dataset_dir.mkdir(parents=True, exist_ok=True) + self.dataset_name = "smoke-dataset.zst" + (self.dataset_dir / self.dataset_name).write_bytes(b"smoke") + + def tearDown(self) -> None: + """Clean up any files created for testing.""" + dataset_path: Path = self.dataset_dir / self.dataset_name + if dataset_path.exists(): + dataset_path.unlink() + + def test_all_site_endpoints_return_success(self) -> None: + """Test that all named site endpoints return a successful response with representative data.""" + endpoints: list[tuple[str, dict[str, str | int], int]] = [ + # Top-level config endpoints + ("sitemap", {}, 200), + ("sitemap-static", {}, 200), + ("sitemap-twitch-channels", {}, 200), + ("sitemap-twitch-drops", {}, 200), + ("sitemap-twitch-others", {}, 200), + ("sitemap-kick", {}, 200), + ("sitemap-youtube", {}, 200), + # Core endpoints + ("core:dashboard", {}, 200), + ("core:search", {}, 200), + ("core:debug", {}, 200), + ("core:dataset_backups", {}, 200), + ("core:dataset_backup_download", {"relative_path": self.dataset_name}, 200), + ("core:docs_rss", {}, 200), + ("core:campaign_feed", {}, 200), + ("core:game_feed", {}, 200), + ("core:game_campaign_feed", {"twitch_id": self.twitch_game.twitch_id}, 200), + ("core:organization_feed", {}, 200), + ("core:reward_campaign_feed", {}, 200), + ("core:campaign_feed_atom", {}, 200), + ("core:game_feed_atom", {}, 200), + ( + "core:game_campaign_feed_atom", + {"twitch_id": self.twitch_game.twitch_id}, + 200, + ), + ("core:organization_feed_atom", {}, 200), + ("core:reward_campaign_feed_atom", {}, 200), + ("core:campaign_feed_discord", {}, 200), + ("core:game_feed_discord", {}, 200), + ( + "core:game_campaign_feed_discord", + {"twitch_id": self.twitch_game.twitch_id}, + 200, + ), + ("core:organization_feed_discord", {}, 200), + ("core:reward_campaign_feed_discord", {}, 200), + # Twitch endpoints + ("twitch:dashboard", {}, 200), + ("twitch:badge_list", {}, 200), + ("twitch:badge_set_detail", {"set_id": self.badge_set.set_id}, 200), + ("twitch:campaign_list", {}, 200), + ( + "twitch:campaign_detail", + {"twitch_id": self.twitch_campaign.twitch_id}, + 200, + ), + ("twitch:channel_list", {}, 200), + ( + "twitch:channel_detail", + {"twitch_id": self.twitch_channel.twitch_id}, + 200, + ), + ("twitch:emote_gallery", {}, 200), + ("twitch:games_grid", {}, 200), + ("twitch:games_list", {}, 200), + ("twitch:game_detail", {"twitch_id": self.twitch_game.twitch_id}, 200), + ("twitch:org_list", {}, 200), + ( + "twitch:organization_detail", + {"twitch_id": self.twitch_org.twitch_id}, + 200, + ), + ("twitch:reward_campaign_list", {}, 200), + ( + "twitch:reward_campaign_detail", + {"twitch_id": self.twitch_reward_campaign.twitch_id}, + 200, + ), + ("twitch:export_campaigns_csv", {}, 200), + ("twitch:export_campaigns_json", {}, 200), + ("twitch:export_games_csv", {}, 200), + ("twitch:export_games_json", {}, 200), + ("twitch:export_organizations_csv", {}, 200), + ("twitch:export_organizations_json", {}, 200), + # Kick endpoints + ("kick:dashboard", {}, 200), + ("kick:campaign_list", {}, 200), + ("kick:campaign_detail", {"kick_id": self.kick_campaign.kick_id}, 200), + ("kick:game_list", {}, 200), + ("kick:game_detail", {"kick_id": self.kick_category.kick_id}, 200), + ("kick:category_list", {}, 200), + ("kick:category_detail", {"kick_id": self.kick_category.kick_id}, 200), + ("kick:organization_list", {}, 200), + ("kick:organization_detail", {"kick_id": self.kick_org.kick_id}, 200), + ("kick:campaign_feed", {}, 200), + ("kick:game_feed", {}, 200), + ("kick:game_campaign_feed", {"kick_id": self.kick_category.kick_id}, 200), + ("kick:category_feed", {}, 200), + ( + "kick:category_campaign_feed", + {"kick_id": self.kick_category.kick_id}, + 200, + ), + ("kick:organization_feed", {}, 200), + ("kick:campaign_feed_atom", {}, 200), + ("kick:game_feed_atom", {}, 200), + ( + "kick:game_campaign_feed_atom", + {"kick_id": self.kick_category.kick_id}, + 200, + ), + ("kick:category_feed_atom", {}, 200), + ( + "kick:category_campaign_feed_atom", + {"kick_id": self.kick_category.kick_id}, + 200, + ), + ("kick:organization_feed_atom", {}, 200), + ("kick:campaign_feed_discord", {}, 200), + ("kick:game_feed_discord", {}, 200), + ( + "kick:game_campaign_feed_discord", + {"kick_id": self.kick_category.kick_id}, + 200, + ), + ("kick:category_feed_discord", {}, 200), + ( + "kick:category_campaign_feed_discord", + {"kick_id": self.kick_category.kick_id}, + 200, + ), + ("kick:organization_feed_discord", {}, 200), + # CHZZK endpoints + ("chzzk:dashboard", {}, 200), + ("chzzk:campaign_list", {}, 200), + ( + "chzzk:campaign_detail", + {"campaign_no": self.chzzk_campaign.campaign_no}, + 200, + ), + ("chzzk:campaign_feed", {}, 200), + ("chzzk:campaign_feed_atom", {}, 200), + ("chzzk:campaign_feed_discord", {}, 200), + # YouTube endpoint + ("youtube:index", {}, 200), + ] + + for route_name, kwargs, expected_status in endpoints: + response: _MonkeyPatchedWSGIResponse = self.client.get( + reverse(route_name, kwargs=kwargs), + ) + assert response.status_code == expected_status, ( + f"{route_name} returned {response.status_code}, expected {expected_status}" + ) + response.close() diff --git a/kick/models.py b/kick/models.py index d58a379..450434f 100644 --- a/kick/models.py +++ b/kick/models.py @@ -361,7 +361,24 @@ class KickDropCampaign(auto_prefetch.Model): If both a base reward and a "(Con)" variant exist, prefer the base reward name. """ rewards_by_name: dict[str, KickReward] = {} - for reward in self.rewards.all().order_by("required_units", "name", "kick_id"): # pyright: ignore[reportAttributeAccessIssue] + prefetched_rewards: list[KickReward] | None = getattr( + self, + "_prefetched_objects_cache", + {}, + ).get("rewards") + if prefetched_rewards is not None: + rewards_iterable = sorted( + prefetched_rewards, + key=lambda reward: (reward.required_units, reward.name, reward.kick_id), + ) + else: + rewards_iterable = self.rewards.all().order_by( # pyright: ignore[reportAttributeAccessIssue] + "required_units", + "name", + "kick_id", + ) + + for reward in rewards_iterable: key: str = self._normalized_reward_name(reward.name) existing: KickReward | None = rewards_by_name.get(key) if existing is None: diff --git a/kick/tests/test_kick.py b/kick/tests/test_kick.py index 357f604..84428c2 100644 --- a/kick/tests/test_kick.py +++ b/kick/tests/test_kick.py @@ -12,8 +12,10 @@ from unittest.mock import patch import httpx import pytest from django.core.management import call_command +from django.db import connection from django.test import Client from django.test import TestCase +from django.test.utils import CaptureQueriesContext from django.urls import reverse from django.utils import timezone from pydantic import ValidationError @@ -414,6 +416,37 @@ class KickDropCampaignMergedRewardsTest(TestCase): assert len(merged) == 1 assert merged[0].name == "9th Anniversary Cake & Confetti" + def test_uses_prefetched_rewards_without_extra_queries(self) -> None: + """When rewards are prefetched, merged_rewards should not hit the database again.""" + campaign: KickDropCampaign = self._make_campaign() + KickReward.objects.create( + kick_id="reward-prefetch-a", + name="Alpha Reward", + image_url="drops/reward-image/alpha.png", + required_units=10, + campaign=campaign, + category=campaign.category, + organization=campaign.organization, + ) + KickReward.objects.create( + kick_id="reward-prefetch-b", + name="Alpha Reward (Con)", + image_url="drops/reward-image/alpha-con.png", + required_units=10, + campaign=campaign, + category=campaign.category, + organization=campaign.organization, + ) + + campaign = KickDropCampaign.objects.prefetch_related("rewards").get( + pk=campaign.pk, + ) + with self.assertNumQueries(0): + merged: list[KickReward] = campaign.merged_rewards + + assert len(merged) == 1 + assert merged[0].name == "Alpha Reward" + # MARK: Management command tests class ImportKickDropsCommandTest(TestCase): @@ -546,6 +579,92 @@ class KickDashboardViewTest(TestCase): ) assert campaign.name in response.content.decode() + def test_dashboard_query_count_stays_flat_with_more_campaigns(self) -> None: + """Dashboard SELECT query count should stay flat as active campaign count grows.""" + + def _create_active_campaign(index: int) -> KickDropCampaign: + org: KickOrganization = KickOrganization.objects.create( + kick_id=f"org-qc-{index}", + name=f"Org QC {index}", + ) + cat: KickCategory = KickCategory.objects.create( + kick_id=10000 + index, + name=f"Cat QC {index}", + slug=f"cat-qc-{index}", + ) + campaign: KickDropCampaign = KickDropCampaign.objects.create( + kick_id=f"camp-qc-{index}", + name=f"Campaign QC {index}", + status="active", + starts_at=dt(2020, 1, 1, tzinfo=UTC), + ends_at=dt(2099, 12, 31, tzinfo=UTC), + organization=org, + category=cat, + rule_id=1, + rule_name="Watch to redeem", + is_fully_imported=True, + ) + + user: KickUser = KickUser.objects.create( + kick_id=3000000 + index, + username=f"qcuser{index}", + ) + channel: KickChannel = KickChannel.objects.create( + kick_id=2000000 + index, + slug=f"qc-channel-{index}", + user=user, + ) + campaign.channels.add(channel) + + KickReward.objects.create( + kick_id=f"reward-qc-{index}-a", + name="Alpha Reward", + image_url="drops/reward-image/alpha.png", + required_units=30, + campaign=campaign, + category=cat, + organization=org, + ) + KickReward.objects.create( + kick_id=f"reward-qc-{index}-b", + name="Alpha Reward (Con)", + image_url="drops/reward-image/alpha-con.png", + required_units=30, + campaign=campaign, + category=cat, + organization=org, + ) + + return campaign + + def _capture_dashboard_select_count() -> int: + with CaptureQueriesContext(connection) as queries: + response: _MonkeyPatchedWSGIResponse = self.client.get( + reverse("kick:dashboard"), + ) + assert response.status_code == 200 + + select_queries: list[str] = [ + query_info["sql"] + for query_info in queries.captured_queries + if query_info["sql"].lstrip().upper().startswith("SELECT") + ] + return len(select_queries) + + _create_active_campaign(1) + baseline_select_count: int = _capture_dashboard_select_count() + + for i in range(2, 12): + _create_active_campaign(i) + + scaled_select_count: int = _capture_dashboard_select_count() + + assert scaled_select_count <= baseline_select_count + 2, ( + "Kick dashboard SELECT query count grew with campaign volume; " + f"possible N+1 regression. baseline={baseline_select_count}, " + f"scaled={scaled_select_count}" + ) + class KickCampaignListViewTest(TestCase): """Tests for the kick campaign list view.""" @@ -811,6 +930,95 @@ class KickOrganizationDetailViewTest(TestCase): ) assert response.status_code == 404 + def test_organization_detail_query_count_stays_flat_with_more_campaigns( + self, + ) -> None: + """Organization detail SELECT query count should stay flat as campaign count grows.""" + org: KickOrganization = KickOrganization.objects.create( + kick_id="org-orgdet-qc", + name="Orgdet Query Count", + ) + + def _create_org_campaign(index: int) -> None: + cat: KickCategory = KickCategory.objects.create( + kick_id=17000 + index, + name=f"Orgdet QC Cat {index}", + slug=f"orgdet-qc-cat-{index}", + ) + campaign: KickDropCampaign = KickDropCampaign.objects.create( + kick_id=f"camp-orgdet-qc-{index}", + name=f"Orgdet QC Campaign {index}", + status="active", + starts_at=dt(2020, 1, 1, tzinfo=UTC), + ends_at=dt(2099, 12, 31, tzinfo=UTC), + organization=org, + category=cat, + rule_id=1, + rule_name="Watch to redeem", + is_fully_imported=True, + ) + + user: KickUser = KickUser.objects.create( + kick_id=3700000 + index, + username=f"orgdetqcuser{index}", + ) + channel: KickChannel = KickChannel.objects.create( + kick_id=2700000 + index, + slug=f"orgdet-qc-channel-{index}", + user=user, + ) + campaign.channels.add(channel) + + KickReward.objects.create( + kick_id=f"reward-orgdet-qc-{index}-a", + name="Org Reward", + image_url="drops/reward-image/org.png", + required_units=30, + campaign=campaign, + category=cat, + organization=org, + ) + KickReward.objects.create( + kick_id=f"reward-orgdet-qc-{index}-b", + name="Org Reward (Con)", + image_url="drops/reward-image/org-con.png", + required_units=30, + campaign=campaign, + category=cat, + organization=org, + ) + + def _capture_org_detail_select_count() -> int: + with CaptureQueriesContext(connection) as queries: + response: _MonkeyPatchedWSGIResponse = self.client.get( + reverse( + "kick:organization_detail", + kwargs={"kick_id": org.kick_id}, + ), + ) + assert response.status_code == 200 + + select_queries: list[str] = [ + query_info["sql"] + for query_info in queries.captured_queries + if query_info["sql"].lstrip().upper().startswith("SELECT") + ] + return len(select_queries) + + _create_org_campaign(1) + baseline_select_count: int = _capture_org_detail_select_count() + + for i in range(2, 12): + _create_org_campaign(i) + + scaled_select_count: int = _capture_org_detail_select_count() + + assert scaled_select_count <= baseline_select_count + 2, ( + "Organization detail SELECT query count grew with campaign volume; " + f"possible N+1 regression. baseline={baseline_select_count}, " + f"scaled={scaled_select_count}" + ) + class KickFeedsTest(TestCase): """Tests for Kick RSS/Atom/Discord feed endpoints.""" @@ -942,6 +1150,109 @@ class KickFeedsTest(TestCase): assert not str(discord_timestamp(None)) +class KickEndpointCoverageTest(TestCase): + """Endpoint smoke coverage for all Kick routes in kick.urls.""" + + def setUp(self) -> None: + """Create shared fixtures used by detail and feed endpoints.""" + self.org: KickOrganization = KickOrganization.objects.create( + kick_id="org-endpoint-1", + name="Endpoint Org", + logo_url="https://example.com/org-endpoint.png", + ) + self.category: KickCategory = KickCategory.objects.create( + kick_id=9123, + name="Endpoint Category", + slug="endpoint-category", + image_url="https://example.com/endpoint-category.png", + ) + self.campaign: KickDropCampaign = KickDropCampaign.objects.create( + kick_id="camp-endpoint-1", + name="Endpoint Campaign", + status="active", + starts_at=timezone.now() - timedelta(days=1), + ends_at=timezone.now() + timedelta(days=1), + organization=self.org, + category=self.category, + connect_url="https://example.com/connect", + url="https://example.com/campaign", + rule_id=1, + rule_name="Watch to redeem", + is_fully_imported=True, + ) + + user: KickUser = KickUser.objects.create( + kick_id=5551001, + username="endpointuser", + ) + channel: KickChannel = KickChannel.objects.create( + kick_id=5551002, + slug="endpointchannel", + user=user, + ) + self.campaign.channels.add(channel) + + KickReward.objects.create( + kick_id="reward-endpoint-1", + name="Endpoint Reward", + image_url="drops/reward-image/endpoint.png", + required_units=20, + campaign=self.campaign, + category=self.category, + organization=self.org, + ) + + def test_all_kick_html_endpoints_return_success(self) -> None: + """All Kick HTML endpoints should render successfully with populated fixtures.""" + html_routes: list[tuple[str, dict[str, str | int]]] = [ + ("kick:dashboard", {}), + ("kick:campaign_list", {}), + ("kick:campaign_detail", {"kick_id": self.campaign.kick_id}), + ("kick:game_list", {}), + ("kick:game_detail", {"kick_id": self.category.kick_id}), + ("kick:category_list", {}), + ("kick:category_detail", {"kick_id": self.category.kick_id}), + ("kick:organization_list", {}), + ("kick:organization_detail", {"kick_id": self.org.kick_id}), + ] + + for route_name, kwargs in html_routes: + response: _MonkeyPatchedWSGIResponse = self.client.get( + reverse(route_name, kwargs=kwargs), + ) + assert response.status_code == 200, route_name + + def test_all_kick_feed_endpoints_return_success(self) -> None: + """All Kick RSS/Atom/Discord feed endpoints should return XML responses.""" + feed_routes: list[tuple[str, dict[str, int]]] = [ + ("kick:campaign_feed", {}), + ("kick:game_feed", {}), + ("kick:game_campaign_feed", {"kick_id": self.category.kick_id}), + ("kick:category_feed", {}), + ("kick:category_campaign_feed", {"kick_id": self.category.kick_id}), + ("kick:organization_feed", {}), + ("kick:campaign_feed_atom", {}), + ("kick:game_feed_atom", {}), + ("kick:game_campaign_feed_atom", {"kick_id": self.category.kick_id}), + ("kick:category_feed_atom", {}), + ("kick:category_campaign_feed_atom", {"kick_id": self.category.kick_id}), + ("kick:organization_feed_atom", {}), + ("kick:campaign_feed_discord", {}), + ("kick:game_feed_discord", {}), + ("kick:game_campaign_feed_discord", {"kick_id": self.category.kick_id}), + ("kick:category_feed_discord", {}), + ("kick:category_campaign_feed_discord", {"kick_id": self.category.kick_id}), + ("kick:organization_feed_discord", {}), + ] + + for route_name, kwargs in feed_routes: + response: _MonkeyPatchedWSGIResponse = self.client.get( + reverse(route_name, kwargs=kwargs), + ) + assert response.status_code == 200, route_name + assert response["Content-Type"] == "application/xml; charset=utf-8" + + class KickDropCampaignFullyImportedTest(TestCase): """Tests for KickDropCampaign.is_fully_imported field and filtering.""" diff --git a/kick/views.py b/kick/views.py index b8cc126..7d89560 100644 --- a/kick/views.py +++ b/kick/views.py @@ -532,7 +532,7 @@ def organization_detail_view(request: HttpRequest, kick_id: str) -> HttpResponse KickDropCampaign.objects .filter(organization=org) .select_related("category") - .prefetch_related("rewards") + .prefetch_related("rewards", "channels__user") .order_by("-starts_at"), ) diff --git a/templates/twitch/dashboard.html b/templates/twitch/dashboard.html index 0689a2d..3d7aa55 100644 --- a/templates/twitch/dashboard.html +++ b/templates/twitch/dashboard.html @@ -68,7 +68,7 @@ flex-shrink: 0">
- {% picture campaign_data.campaign.image_best_url|default:campaign_data.campaign.image_url alt="Image for "|add:campaign_data.campaign.name width=120 %} + {% picture campaign_data.image_url alt="Image for "|add:campaign_data.campaign.name width=120 %}

{{ campaign_data.campaign.clean_name }}

 diff --git a/twitch/models.py b/twitch/models.py index 1ed2a57..bd5e9a1 100644 --- a/twitch/models.py +++ b/twitch/models.py @@ -1,5 +1,7 @@ import logging +from collections import OrderedDict from typing import TYPE_CHECKING +from typing import Any import auto_prefetch from django.conf import settings @@ -508,6 +510,93 @@ class DropCampaign(auto_prefetch.Model): def __str__(self) -> str: return self.name + @classmethod + def active_for_dashboard( + cls, + now: datetime.datetime, + ) -> models.QuerySet[DropCampaign]: + """Return active campaigns with relations needed by the dashboard. + + Args: + now: Current timestamp used to evaluate active campaigns. + + Returns: + QuerySet of active campaigns ordered by newest start date. + """ + return ( + cls.objects + .filter(start_at__lte=now, end_at__gte=now) + .only( + "twitch_id", + "name", + "image_url", + "image_file", + "start_at", + "end_at", + "allow_is_enabled", + "game", + "game__twitch_id", + "game__display_name", + "game__slug", + "game__box_art", + "game__box_art_file", + ) + .select_related("game") + .prefetch_related( + models.Prefetch( + "game__owners", + queryset=Organization.objects.only("twitch_id", "name"), + ), + models.Prefetch( + "allow_channels", + queryset=Channel.objects.only( + "twitch_id", + "name", + "display_name", + ).order_by("display_name"), + to_attr="channels_ordered", + ), + ) + .order_by("-start_at") + ) + + @staticmethod + def grouped_by_game( + campaigns: models.QuerySet[DropCampaign], + ) -> OrderedDict[str, dict[str, Any]]: + """Group campaigns by game for dashboard rendering. + + The grouping keeps insertion order and avoids duplicate per-game cards when + games have multiple owners. + + Args: + campaigns: Campaign queryset from active_for_dashboard(). + + Returns: + Ordered mapping keyed by game twitch_id. + """ + campaigns_by_game: OrderedDict[str, dict[str, Any]] = OrderedDict() + + for campaign in campaigns: + game: Game = campaign.game + game_id: str = game.twitch_id + + if game_id not in campaigns_by_game: + campaigns_by_game[game_id] = { + "name": game.display_name, + "box_art": game.box_art_best_url, + "owners": list(game.owners.all()), + "campaigns": [], + } + + campaigns_by_game[game_id]["campaigns"].append({ + "campaign": campaign, + "image_url": campaign.listing_image_url, + "allowed_channels": getattr(campaign, "channels_ordered", []), + }) + + return campaigns_by_game + @property def is_active(self) -> bool: """Check if the campaign is currently active.""" @@ -526,19 +615,21 @@ class DropCampaign(auto_prefetch.Model): "Skull & Bones - Closed Beta" -> "Closed Beta" (& is replaced with "and") """ - if not self.game or not self.game.display_name: + self_game: Game | None = self.game + + if not self_game or not self_game.display_name: return self.name - game_variations = [self.game.display_name] - if "&" in self.game.display_name: - game_variations.append(self.game.display_name.replace("&", "and")) - if "and" in self.game.display_name: - game_variations.append(self.game.display_name.replace("and", "&")) + game_variations: list[str] = [self_game.display_name] + if "&" in self_game.display_name: + game_variations.append(self_game.display_name.replace("&", "and")) + if "and" in self_game.display_name: + game_variations.append(self_game.display_name.replace("and", "&")) for game_name in game_variations: # Check for different separators after the game name for separator in [" - ", " | ", " "]: - prefix_to_check = game_name + separator + prefix_to_check: str = game_name + separator name: str = self.name if name.startswith(prefix_to_check): @@ -573,6 +664,20 @@ class DropCampaign(auto_prefetch.Model): return "" + @property + def listing_image_url(self) -> str: + """Return a campaign image URL optimized for list views. + + This intentionally avoids traversing drops/benefits to prevent N+1 queries + in list pages that render many campaigns. + """ + try: + if self.image_file and getattr(self.image_file, "url", None): + return self.image_file.url + except (AttributeError, OSError, ValueError) as exc: + logger.debug("Failed to resolve DropCampaign.image_file url: %s", exc) + return self.image_url or "" + @property def duration_iso(self) -> str: """Return the campaign duration in ISO 8601 format (e.g., 'P3DT4H30M'). @@ -1006,6 +1111,32 @@ class RewardCampaign(auto_prefetch.Model): """Return a string representation of the reward campaign.""" return f"{self.brand}: {self.name}" if self.brand else self.name + @classmethod + def active_for_dashboard( + cls, + now: datetime.datetime, + ) -> models.QuerySet[RewardCampaign]: + """Return active reward campaigns with only dashboard-needed fields.""" + return ( + cls.objects + .filter(starts_at__lte=now, ends_at__gte=now) + .only( + "twitch_id", + "name", + "brand", + "summary", + "external_url", + "starts_at", + "ends_at", + "is_sitewide", + "game", + "game__twitch_id", + "game__display_name", + ) + .select_related("game") + .order_by("-starts_at") + ) + @property def is_active(self) -> bool: """Check if the reward campaign is currently active.""" diff --git a/twitch/tests/test_views.py b/twitch/tests/test_views.py index 03f67d6..be951b2 100644 --- a/twitch/tests/test_views.py +++ b/twitch/tests/test_views.py @@ -9,8 +9,10 @@ from typing import Literal import pytest from django.core.handlers.wsgi import WSGIRequest from django.core.paginator import Paginator +from django.db import connection from django.db.models import Max from django.test import RequestFactory +from django.test.utils import CaptureQueriesContext from django.urls import reverse from django.utils import timezone @@ -34,6 +36,7 @@ from twitch.views import _truncate_description if TYPE_CHECKING: from django.core.handlers.wsgi import WSGIRequest + from django.db.models import QuerySet from django.test import Client from django.test.client import _MonkeyPatchedWSGIResponse from django.test.utils import ContextList @@ -537,6 +540,225 @@ class TestChannelListView: assert game.twitch_id in context["campaigns_by_game"] assert len(context["campaigns_by_game"][game.twitch_id]["campaigns"]) == 1 + @pytest.mark.django_db + def test_dashboard_queries_use_indexes(self) -> None: + """Dashboard source queries should use indexes for active-window filtering.""" + now: datetime.datetime = timezone.now() + + org: Organization = Organization.objects.create( + twitch_id="org_index_test", + name="Org Index Test", + ) + game: Game = Game.objects.create( + twitch_id="game_index_test", + name="Game Index Test", + display_name="Game Index Test", + ) + game.owners.add(org) + + # Add enough rows so the query planner has a reason to pick indexes. + campaigns: list[DropCampaign] = [] + for i in range(250): + campaigns.extend(( + DropCampaign( + twitch_id=f"inactive_old_{i}", + name=f"Inactive old {i}", + game=game, + operation_names=["DropCampaignDetails"], + start_at=now - timedelta(days=60), + end_at=now - timedelta(days=30), + ), + DropCampaign( + twitch_id=f"inactive_future_{i}", + name=f"Inactive future {i}", + game=game, + operation_names=["DropCampaignDetails"], + start_at=now + timedelta(days=30), + end_at=now + timedelta(days=60), + ), + )) + campaigns.append( + DropCampaign( + twitch_id="active_for_dashboard_index_test", + name="Active campaign", + game=game, + operation_names=["DropCampaignDetails"], + start_at=now - timedelta(hours=1), + end_at=now + timedelta(hours=1), + ), + ) + DropCampaign.objects.bulk_create(campaigns) + + reward_campaigns: list[RewardCampaign] = [] + for i in range(250): + reward_campaigns.extend(( + RewardCampaign( + twitch_id=f"reward_inactive_old_{i}", + name=f"Reward inactive old {i}", + game=game, + starts_at=now - timedelta(days=60), + ends_at=now - timedelta(days=30), + ), + RewardCampaign( + twitch_id=f"reward_inactive_future_{i}", + name=f"Reward inactive future {i}", + game=game, + starts_at=now + timedelta(days=30), + ends_at=now + timedelta(days=60), + ), + )) + reward_campaigns.append( + RewardCampaign( + twitch_id="reward_active_for_dashboard_index_test", + name="Active reward campaign", + game=game, + starts_at=now - timedelta(hours=1), + ends_at=now + timedelta(hours=1), + ), + ) + RewardCampaign.objects.bulk_create(reward_campaigns) + + active_campaigns_qs: QuerySet[DropCampaign] = DropCampaign.active_for_dashboard( + now, + ) + active_reward_campaigns_qs: QuerySet[RewardCampaign] = ( + RewardCampaign.objects + .filter(starts_at__lte=now, ends_at__gte=now) + .select_related("game") + .order_by("-starts_at") + ) + + campaigns_plan: str = active_campaigns_qs.explain() + reward_plan: str = active_reward_campaigns_qs.explain() + + if connection.vendor == "sqlite": + campaigns_uses_index: bool = "USING INDEX" in campaigns_plan.upper() + rewards_uses_index: bool = "USING INDEX" in reward_plan.upper() + elif connection.vendor == "postgresql": + campaigns_uses_index = ( + "INDEX SCAN" in campaigns_plan.upper() + or "BITMAP INDEX SCAN" in campaigns_plan.upper() + ) + rewards_uses_index = ( + "INDEX SCAN" in reward_plan.upper() + or "BITMAP INDEX SCAN" in reward_plan.upper() + ) + else: + pytest.skip( + f"Unsupported DB vendor for index-plan assertion: {connection.vendor}", + ) + + assert campaigns_uses_index, campaigns_plan + assert rewards_uses_index, reward_plan + + @pytest.mark.django_db + def test_dashboard_query_count_stays_flat_with_more_data( + self, + client: Client, + ) -> None: + """Dashboard should avoid N+1 queries as campaign volume grows.""" + now: datetime.datetime = timezone.now() + + org: Organization = Organization.objects.create( + twitch_id="org_query_count", + name="Org Query Count", + ) + game: Game = Game.objects.create( + twitch_id="game_query_count", + name="game_query_count", + display_name="Game Query Count", + ) + game.owners.add(org) + + def _capture_dashboard_select_count() -> int: + with CaptureQueriesContext(connection) as queries: + response: _MonkeyPatchedWSGIResponse = client.get( + reverse("twitch:dashboard"), + ) + assert response.status_code == 200 + + select_queries: list[str] = [ + query_info["sql"] + for query_info in queries.captured_queries + if query_info["sql"].lstrip().upper().startswith("SELECT") + ] + return len(select_queries) + + # Baseline: one active drop campaign and one active reward campaign. + base_campaign: DropCampaign = DropCampaign.objects.create( + twitch_id="baseline_campaign", + name="Baseline campaign", + game=game, + operation_names=["DropCampaignDetails"], + start_at=now - timedelta(hours=1), + end_at=now + timedelta(hours=1), + ) + base_channel: Channel = Channel.objects.create( + twitch_id="baseline_channel", + name="baselinechannel", + display_name="BaselineChannel", + ) + base_campaign.allow_channels.add(base_channel) + + RewardCampaign.objects.create( + twitch_id="baseline_reward_campaign", + name="Baseline reward campaign", + game=game, + starts_at=now - timedelta(hours=1), + ends_at=now + timedelta(hours=1), + summary="Baseline summary", + external_url="https://example.com/reward/baseline", + ) + + baseline_select_count: int = _capture_dashboard_select_count() + + # Scale up active dashboard data substantially. + extra_campaigns: list[DropCampaign] = [ + DropCampaign( + twitch_id=f"scaled_campaign_{i}", + name=f"Scaled campaign {i}", + game=game, + operation_names=["DropCampaignDetails"], + start_at=now - timedelta(hours=2), + end_at=now + timedelta(hours=2), + ) + for i in range(12) + ] + DropCampaign.objects.bulk_create(extra_campaigns) + + for i, campaign in enumerate( + DropCampaign.objects.filter( + twitch_id__startswith="scaled_campaign_", + ).order_by("twitch_id"), + ): + channel: Channel = Channel.objects.create( + twitch_id=f"scaled_channel_{i}", + name=f"scaledchannel{i}", + display_name=f"ScaledChannel{i}", + ) + campaign.allow_channels.add(channel) + + extra_rewards: list[RewardCampaign] = [ + RewardCampaign( + twitch_id=f"scaled_reward_{i}", + name=f"Scaled reward {i}", + game=game, + starts_at=now - timedelta(hours=2), + ends_at=now + timedelta(hours=2), + summary=f"Scaled summary {i}", + external_url=f"https://example.com/reward/{i}", + ) + for i in range(12) + ] + RewardCampaign.objects.bulk_create(extra_rewards) + + scaled_select_count: int = _capture_dashboard_select_count() + + assert scaled_select_count <= baseline_select_count + 2, ( + "Dashboard SELECT query count grew with data volume; possible N+1 regression. " + f"baseline={baseline_select_count}, scaled={scaled_select_count}" + ) + @pytest.mark.django_db def test_debug_view(self, client: Client) -> None: """Test debug view returns 200 and has games_without_owner in context.""" diff --git a/twitch/views.py b/twitch/views.py index 974f829..0fb8a74 100644 --- a/twitch/views.py +++ b/twitch/views.py @@ -1071,48 +1071,14 @@ def dashboard(request: HttpRequest) -> HttpResponse: HttpResponse: The rendered dashboard template. """ now: datetime.datetime = timezone.now() - active_campaigns: QuerySet[DropCampaign] = ( - DropCampaign.objects - .filter(start_at__lte=now, end_at__gte=now) - .select_related("game") - .prefetch_related("game__owners") - .prefetch_related( - Prefetch( - "allow_channels", - queryset=Channel.objects.order_by("display_name"), - to_attr="channels_ordered", - ), - ) - .order_by("-start_at") + active_campaigns: QuerySet[DropCampaign] = DropCampaign.active_for_dashboard(now) + campaigns_by_game: OrderedDict[str, dict[str, Any]] = DropCampaign.grouped_by_game( + active_campaigns, ) - # Preserve insertion order (newest campaigns first). - # Group by game so games with multiple owners don't render duplicate campaign cards. - campaigns_by_game: OrderedDict[str, dict[str, Any]] = OrderedDict() - - for campaign in active_campaigns: - game: Game = campaign.game - game_id: str = game.twitch_id - - if game_id not in campaigns_by_game: - campaigns_by_game[game_id] = { - "name": game.display_name, - "box_art": game.box_art_best_url, - "owners": list(game.owners.all()), - "campaigns": [], - } - - campaigns_by_game[game_id]["campaigns"].append({ - "campaign": campaign, - "allowed_channels": getattr(campaign, "channels_ordered", []), - }) - # Get active reward campaigns (Quest rewards) active_reward_campaigns: QuerySet[RewardCampaign] = ( - RewardCampaign.objects - .filter(starts_at__lte=now, ends_at__gte=now) - .select_related("game") - .order_by("-starts_at") + RewardCampaign.active_for_dashboard(now) ) # WebSite schema with SearchAction for sitelinks search box From b7e10e766e10612034839640ca7fc38d4f4324b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Hells=C3=A9n?= Date: Sat, 11 Apr 2026 00:44:16 +0200 Subject: [PATCH 3/6] Improve performance and add type hints --- chzzk/tasks.py | 2 +- chzzk/tests/test_management_commands.py | 2 +- chzzk/views.py | 3 +- config/settings.py | 2 +- conftest.py | 25 ++ core/base_url.py | 2 +- core/tasks.py | 2 +- core/tests/test_sitemaps.py | 3 +- core/views.py | 2 - kick/feeds.py | 16 +- kick/management/commands/import_kick_drops.py | 224 +++++++++---- kick/tests/test_kick.py | 25 +- pyproject.toml | 1 + templates/twitch/dashboard.html | 16 +- twitch/feeds.py | 26 +- twitch/management/commands/backup_db.py | 14 +- .../commands/better_import_drops.py | 116 +++++-- .../management/commands/download_box_art.py | 8 +- .../commands/download_campaign_images.py | 8 +- .../management/commands/import_chat_badges.py | 27 +- twitch/models.py | 81 ++++- twitch/tests/test_views.py | 308 +++++++++++++++++- twitch/views.py | 10 +- 23 files changed, 745 insertions(+), 178 deletions(-) create mode 100644 conftest.py diff --git a/chzzk/tasks.py b/chzzk/tasks.py index 5707beb..55f929c 100644 --- a/chzzk/tasks.py +++ b/chzzk/tasks.py @@ -5,7 +5,7 @@ import logging from celery import shared_task from django.core.management import call_command -logger = logging.getLogger("ttvdrops.tasks") +logger: logging.Logger = logging.getLogger("ttvdrops.tasks") @shared_task(bind=True, queue="imports", max_retries=3, default_retry_delay=60) diff --git a/chzzk/tests/test_management_commands.py b/chzzk/tests/test_management_commands.py index ec41d22..dd10844 100644 --- a/chzzk/tests/test_management_commands.py +++ b/chzzk/tests/test_management_commands.py @@ -136,7 +136,7 @@ class ImportChzzkCampaignRangeCommandTest(TestCase): stdout = StringIO() stderr = StringIO() - def side_effect(command: str, *args: str, **kwargs: object) -> None: + def side_effect(command: str, *args: str, **kwargs: StringIO) -> None: if "4" in args: msg = "Campaign 4 not found" raise CommandError(msg) diff --git a/chzzk/views.py b/chzzk/views.py index 16cd547..d3c155c 100644 --- a/chzzk/views.py +++ b/chzzk/views.py @@ -1,7 +1,6 @@ from typing import TYPE_CHECKING from django.db.models import Q -from django.db.models.query import QuerySet from django.shortcuts import get_object_or_404 from django.shortcuts import render from django.urls import reverse @@ -16,9 +15,9 @@ from twitch.feeds import TTVDropsBaseFeed if TYPE_CHECKING: import datetime + from django.db.models.query import QuerySet from django.http import HttpResponse from django.http.request import HttpRequest - from pytest_django.asserts import QuerySet def dashboard_view(request: HttpRequest) -> HttpResponse: diff --git a/config/settings.py b/config/settings.py index c65859b..85a4757 100644 --- a/config/settings.py +++ b/config/settings.py @@ -224,7 +224,7 @@ DATABASES: dict[str, dict[str, Any]] = configure_databases( base_dir=BASE_DIR, ) -if DEBUG: +if DEBUG or TESTING: INSTALLED_APPS.append("zeal") MIDDLEWARE.append("zeal.middleware.zeal_middleware") diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..883c39d --- /dev/null +++ b/conftest.py @@ -0,0 +1,25 @@ +from typing import TYPE_CHECKING + +import pytest +from zeal import zeal_context + +if TYPE_CHECKING: + from collections.abc import Generator + + +@pytest.fixture(autouse=True) +def use_zeal(request: pytest.FixtureRequest) -> Generator[None]: + """Enable Zeal N+1 detection context for each pytest test. + + Use @pytest.mark.no_zeal for tests that intentionally exercise import paths + where Zeal's strict get() heuristics are too noisy. + + Yields: + None: Control back to pytest for test execution. + """ + if request.node.get_closest_marker("no_zeal") is not None: + yield + return + + with zeal_context(): + yield diff --git a/core/base_url.py b/core/base_url.py index 7eb763d..524532d 100644 --- a/core/base_url.py +++ b/core/base_url.py @@ -69,7 +69,7 @@ class _TTVDropsSite: domain: str -def get_current_site(request: object) -> _TTVDropsSite: +def get_current_site(request: HttpRequest | None) -> _TTVDropsSite: """Return a site-like object with domain derived from BASE_URL.""" base_url: str = _get_base_url() parts: SplitResult = urlsplit(base_url) diff --git a/core/tasks.py b/core/tasks.py index 5b20599..cf7f6f0 100644 --- a/core/tasks.py +++ b/core/tasks.py @@ -5,7 +5,7 @@ import logging from celery import shared_task from django.core.management import call_command -logger = logging.getLogger("ttvdrops.tasks") +logger: logging.Logger = logging.getLogger("ttvdrops.tasks") @shared_task(bind=True, queue="default", max_retries=3, default_retry_delay=300) diff --git a/core/tests/test_sitemaps.py b/core/tests/test_sitemaps.py index c833d26..7821279 100644 --- a/core/tests/test_sitemaps.py +++ b/core/tests/test_sitemaps.py @@ -5,6 +5,7 @@ from django.urls import reverse if TYPE_CHECKING: from django.test.client import Client + from pytest_django.fixtures import SettingsWrapper def _extract_locs(xml_bytes: bytes) -> list[str]: @@ -15,7 +16,7 @@ def _extract_locs(xml_bytes: bytes) -> list[str]: def test_sitemap_static_contains_expected_links( client: Client, - settings: object, + settings: SettingsWrapper, ) -> None: """Ensure the static sitemap contains the main site links across apps. diff --git a/core/views.py b/core/views.py index 6d26d1f..b03d991 100644 --- a/core/views.py +++ b/core/views.py @@ -15,11 +15,9 @@ from django.db.models import Max from django.db.models import OuterRef from django.db.models import Prefetch from django.db.models import Q -from django.db.models import QuerySet from django.db.models.functions import Trim from django.http import FileResponse from django.http import Http404 -from django.http import HttpRequest from django.http import HttpResponse from django.shortcuts import render from django.template.defaultfilters import filesizeformat diff --git a/kick/feeds.py b/kick/feeds.py index 6d3685a..36f55b9 100644 --- a/kick/feeds.py +++ b/kick/feeds.py @@ -206,8 +206,8 @@ class KickOrganizationFeed(TTVDropsBaseFeed): def __call__( self, request: HttpRequest, - *args: object, - **kwargs: object, + *args: str | int, + **kwargs: str | int, ) -> HttpResponse: """Capture optional ?limit query parameter. @@ -283,8 +283,8 @@ class KickCategoryFeed(TTVDropsBaseFeed): def __call__( self, request: HttpRequest, - *args: object, - **kwargs: object, + *args: str | int, + **kwargs: str | int, ) -> HttpResponse: """Capture optional ?limit query parameter. @@ -372,8 +372,8 @@ class KickCampaignFeed(TTVDropsBaseFeed): def __call__( self, request: HttpRequest, - *args: object, - **kwargs: object, + *args: str | int, + **kwargs: str | int, ) -> HttpResponse: """Capture optional ?limit query parameter. @@ -481,8 +481,8 @@ class KickCategoryCampaignFeed(TTVDropsBaseFeed): def __call__( self, request: HttpRequest, - *args: object, - **kwargs: object, + *args: str | int, + **kwargs: str | int, ) -> HttpResponse: """Capture optional ?limit query parameter. diff --git a/kick/management/commands/import_kick_drops.py b/kick/management/commands/import_kick_drops.py index 04face1..4116350 100644 --- a/kick/management/commands/import_kick_drops.py +++ b/kick/management/commands/import_kick_drops.py @@ -1,4 +1,7 @@ +from __future__ import annotations + import logging +from datetime import datetime from typing import TYPE_CHECKING import httpx @@ -14,6 +17,8 @@ from kick.models import KickUser from kick.schemas import KickDropsResponseSchema if TYPE_CHECKING: + from collections.abc import Mapping + from django.core.management.base import CommandParser from kick.schemas import KickCategorySchema @@ -23,6 +28,26 @@ if TYPE_CHECKING: logger: logging.Logger = logging.getLogger("ttvdrops") +type KickImportModel = ( + KickOrganization + | KickCategory + | KickDropCampaign + | KickUser + | KickChannel + | KickReward +) +type KickFieldValue = ( + str + | bool + | int + | datetime + | KickOrganization + | KickCategory + | KickDropCampaign + | KickUser + | None +) + KICK_DROPS_API_URL = "https://web.kick.com/api/v1/drops/campaigns" # Kick's public API requires a browser-like User-Agent. @@ -48,7 +73,26 @@ class Command(BaseCommand): help="API endpoint to fetch (default: %(default)s).", ) - def handle(self, *args: object, **options: object) -> None: # noqa: ARG002 + @staticmethod + def _save_if_changed( + obj: KickImportModel, + defaults: Mapping[str, KickFieldValue], + ) -> None: + """Persist only changed fields to avoid unnecessary updates.""" + changed_fields: list[str] = [] + for field, new_value in defaults.items(): + if getattr(obj, field, None) != new_value: + setattr(obj, field, new_value) + changed_fields.append(field) + + if changed_fields: + obj.save(update_fields=changed_fields) + + def handle( + self, + *_args: str, + **options: str | bool | int | None, + ) -> None: """Main entry point for the command.""" url: str = str(options["url"]) self.stdout.write(f"Fetching Kick drops from {url} ...") @@ -99,54 +143,75 @@ class Command(BaseCommand): self.style.SUCCESS(f"Imported {imported}/{len(campaigns)} campaign(s)."), ) - def _import_campaign(self, data: KickDropCampaignSchema) -> None: + def _import_campaign(self, data: KickDropCampaignSchema) -> None: # noqa: PLR0914, PLR0915 """Import a single campaign and all its related objects.""" - # Organisation + # Organization org_data: KickOrganizationSchema = data.organization - org, created = KickOrganization.objects.update_or_create( + org_defaults: dict[str, str | bool] = { + "name": org_data.name, + "logo_url": org_data.logo_url, + "url": org_data.url, + "restricted": org_data.restricted, + } + org: KickOrganization | None = KickOrganization.objects.filter( kick_id=org_data.id, - defaults={ - "name": org_data.name, - "logo_url": org_data.logo_url, - "url": org_data.url, - "restricted": org_data.restricted, - }, - ) + ).first() + created: bool = org is None + if org is None: + org = KickOrganization.objects.create(kick_id=org_data.id, **org_defaults) + else: + self._save_if_changed(org, org_defaults) if created: logger.info("Created new organization: %s", org.kick_id) # Category cat_data: KickCategorySchema = data.category - category, created = KickCategory.objects.update_or_create( + category_defaults: dict[str, KickFieldValue] = { + "name": cat_data.name, + "slug": cat_data.slug, + "image_url": cat_data.image_url, + } + category: KickCategory | None = KickCategory.objects.filter( kick_id=cat_data.id, - defaults={ - "name": cat_data.name, - "slug": cat_data.slug, - "image_url": cat_data.image_url, - }, - ) + ).first() + created = category is None + if category is None: + category = KickCategory.objects.create( + kick_id=cat_data.id, + **category_defaults, + ) + else: + self._save_if_changed(category, category_defaults) if created: logger.info("Created new category: %s", category.kick_id) # Campaign - campaign, created = KickDropCampaign.objects.update_or_create( + campaign_defaults: dict[str, KickFieldValue] = { + "name": data.name, + "status": data.status, + "starts_at": data.starts_at, + "ends_at": data.ends_at, + "connect_url": data.connect_url, + "url": data.url, + "rule_id": data.rule.id, + "rule_name": data.rule.name, + "organization": org, + "category": category, + "created_at": data.created_at, + "api_updated_at": data.updated_at, + "is_fully_imported": True, + } + campaign: KickDropCampaign | None = KickDropCampaign.objects.filter( kick_id=data.id, - defaults={ - "name": data.name, - "status": data.status, - "starts_at": data.starts_at, - "ends_at": data.ends_at, - "connect_url": data.connect_url, - "url": data.url, - "rule_id": data.rule.id, - "rule_name": data.rule.name, - "organization": org, - "category": category, - "created_at": data.created_at, - "api_updated_at": data.updated_at, - "is_fully_imported": True, - }, - ) + ).first() + created = campaign is None + if campaign is None: + campaign = KickDropCampaign.objects.create( + kick_id=data.id, + **campaign_defaults, + ) + else: + self._save_if_changed(campaign, campaign_defaults) if created: logger.info("Created new campaign: %s", campaign.kick_id) @@ -154,25 +219,38 @@ class Command(BaseCommand): channel_objs: list[KickChannel] = [] for ch_data in data.channels: user_data: KickUserSchema = ch_data.user - user, created = KickUser.objects.update_or_create( + user_defaults: dict[str, KickFieldValue] = { + "username": user_data.username, + "profile_picture": user_data.profile_picture, + } + user: KickUser | None = KickUser.objects.filter( kick_id=user_data.id, - defaults={ - "username": user_data.username, - "profile_picture": user_data.profile_picture, - }, - ) + ).first() + created = user is None + if user is None: + user = KickUser.objects.create(kick_id=user_data.id, **user_defaults) + else: + self._save_if_changed(user, user_defaults) if created: logger.info("Created new user: %s", user.kick_id) - channel, created = KickChannel.objects.update_or_create( + channel_defaults: dict[str, KickFieldValue] = { + "slug": ch_data.slug, + "description": ch_data.description, + "banner_picture_url": ch_data.banner_picture_url, + "user": user, + } + channel: KickChannel | None = KickChannel.objects.filter( kick_id=ch_data.id, - defaults={ - "slug": ch_data.slug, - "description": ch_data.description, - "banner_picture_url": ch_data.banner_picture_url, - "user": user, - }, - ) + ).first() + created = channel is None + if channel is None: + channel = KickChannel.objects.create( + kick_id=ch_data.id, + **channel_defaults, + ) + else: + self._save_if_changed(channel, channel_defaults) if created: logger.info("Created new channel: %s", channel.kick_id) @@ -184,36 +262,46 @@ class Command(BaseCommand): # Resolve reward's category (may differ from campaign category) reward_category: KickCategory = category if reward_data.category_id != cat_data.id: - reward_category, created = KickCategory.objects.get_or_create( + reward_category = KickCategory.objects.filter( kick_id=reward_data.category_id, - defaults={"name": "", "slug": "", "image_url": ""}, + ).first() or KickCategory.objects.create( + kick_id=reward_data.category_id, + name="", + slug="", + image_url="", ) + created = not reward_category.name and not reward_category.slug if created: logger.info("Created new category: %s", reward_category.kick_id) # Resolve reward's organization (may differ from campaign org) reward_org: KickOrganization = org if reward_data.organization_id != org_data.id: - reward_org, created = KickOrganization.objects.get_or_create( + reward_org = KickOrganization.objects.filter( kick_id=reward_data.organization_id, - defaults={ - "name": "", - "logo_url": "", - "url": "", - "restricted": False, - }, + ).first() or KickOrganization.objects.create( + kick_id=reward_data.organization_id, + name="", + logo_url="", + url="", + restricted=False, ) + created = not reward_org.name and not reward_org.url if created: logger.info("Created new organization: %s", reward_org.kick_id) - KickReward.objects.update_or_create( + reward_defaults: dict[str, KickFieldValue] = { + "name": reward_data.name, + "image_url": reward_data.image_url, + "required_units": reward_data.required_units, + "campaign": campaign, + "category": reward_category, + "organization": reward_org, + } + reward: KickReward | None = KickReward.objects.filter( kick_id=reward_data.id, - defaults={ - "name": reward_data.name, - "image_url": reward_data.image_url, - "required_units": reward_data.required_units, - "campaign": campaign, - "category": reward_category, - "organization": reward_org, - }, - ) + ).first() + if reward is None: + KickReward.objects.create(kick_id=reward_data.id, **reward_defaults) + else: + self._save_if_changed(reward, reward_defaults) diff --git a/kick/tests/test_kick.py b/kick/tests/test_kick.py index 84428c2..7fb1c43 100644 --- a/kick/tests/test_kick.py +++ b/kick/tests/test_kick.py @@ -669,20 +669,25 @@ class KickDashboardViewTest(TestCase): class KickCampaignListViewTest(TestCase): """Tests for the kick campaign list view.""" + @classmethod + def setUpTestData(cls) -> None: + """Set up shared test data for campaign list view tests.""" + cls.org: KickOrganization = KickOrganization.objects.create( + kick_id="org-list", + name="List Org", + ) + cls.cat: KickCategory = KickCategory.objects.create( + kick_id=300, + name="List Cat", + slug="list-cat", + ) + def _make_campaign( self, kick_id: str, name: str, status: str = "active", ) -> KickDropCampaign: - org, _ = KickOrganization.objects.get_or_create( - kick_id="org-list", - defaults={"name": "List Org"}, - ) - cat, _ = KickCategory.objects.get_or_create( - kick_id=300, - defaults={"name": "List Cat", "slug": "list-cat"}, - ) # Set dates so the active/expired filter works correctly if status == "active": starts_at = dt(2020, 1, 1, tzinfo=UTC) @@ -696,8 +701,8 @@ class KickCampaignListViewTest(TestCase): status=status, starts_at=starts_at, ends_at=ends_at, - organization=org, - category=cat, + organization=self.org, + category=self.cat, rule_id=1, rule_name="Watch to redeem", is_fully_imported=True, diff --git a/pyproject.toml b/pyproject.toml index 9fee185..74d15fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ dev = [ DJANGO_SETTINGS_MODULE = "config.settings" python_files = ["test_*.py", "*_test.py"] addopts = "--tb=short -n auto --cov" +markers = ["no_zeal: run test without zeal_context N+1 checks"] filterwarnings = [ "ignore:Parsing dates involving a day of month without a year specified is ambiguous:DeprecationWarning", ] diff --git a/templates/twitch/dashboard.html b/templates/twitch/dashboard.html index 3d7aa55..5e6f7cd 100644 --- a/templates/twitch/dashboard.html +++ b/templates/twitch/dashboard.html @@ -69,7 +69,7 @@
{% picture campaign_data.image_url alt="Image for "|add:campaign_data.campaign.name width=120 %} -

{{ campaign_data.campaign.clean_name }}

+

{{ campaign_data.clean_name }}