diff --git a/chzzk/tasks.py b/chzzk/tasks.py index 5707beb..55f929c 100644 --- a/chzzk/tasks.py +++ b/chzzk/tasks.py @@ -5,7 +5,7 @@ import logging from celery import shared_task from django.core.management import call_command -logger = logging.getLogger("ttvdrops.tasks") +logger: logging.Logger = logging.getLogger("ttvdrops.tasks") @shared_task(bind=True, queue="imports", max_retries=3, default_retry_delay=60) diff --git a/chzzk/tests/test_management_commands.py b/chzzk/tests/test_management_commands.py index ec41d22..dd10844 100644 --- a/chzzk/tests/test_management_commands.py +++ b/chzzk/tests/test_management_commands.py @@ -136,7 +136,7 @@ class ImportChzzkCampaignRangeCommandTest(TestCase): stdout = StringIO() stderr = StringIO() - def side_effect(command: str, *args: str, **kwargs: object) -> None: + def side_effect(command: str, *args: str, **kwargs: StringIO) -> None: if "4" in args: msg = "Campaign 4 not found" raise CommandError(msg) diff --git a/chzzk/tests/test_views.py b/chzzk/tests/test_views.py index 38e84be..1af3cdd 100644 --- a/chzzk/tests/test_views.py +++ b/chzzk/tests/test_views.py @@ -1,7 +1,9 @@ from datetime import timedelta from typing import TYPE_CHECKING +from django.db import connection from django.test import TestCase +from django.test.utils import CaptureQueriesContext from django.urls import reverse from django.utils import timezone @@ -16,6 +18,75 @@ if TYPE_CHECKING: class ChzzkDashboardViewTests(TestCase): """Test cases for the dashboard view of the chzzk app.""" + def test_dashboard_view_no_n_plus_one_on_rewards(self) -> None: + """Test that the dashboard view does not trigger an N+1 query for rewards.""" + now = timezone.now() + base_kwargs = { + "category_type": "game", + "category_id": "1", + "category_value": "TestGame", + "service_id": "chzzk", + "state": "ACTIVE", + "start_date": now - timedelta(days=1), + "end_date": now + timedelta(days=1), + "has_ios_based_reward": False, + "drops_campaign_not_started": False, + "source_api": "unit-test", + } + reward_kwargs = { + "reward_type": "ITEM", + "campaign_reward_type": "Standard", + "condition_type": "watch", + "ios_based_reward": False, + "code_remaining_count": 100, + } + + campaign1 = ChzzkCampaign.objects.create( + campaign_no=9001, + title="C1", + **base_kwargs, + ) + campaign1.rewards.create( # pyright: ignore[reportAttributeAccessIssue] + reward_no=901, + title="R1", + condition_for_minutes=10, + **reward_kwargs, + ) # pyright: ignore[reportAttributeAccessIssue] + campaign2 = ChzzkCampaign.objects.create( + campaign_no=9002, + title="C2", + **base_kwargs, + ) + campaign2.rewards.create( # pyright: ignore[reportAttributeAccessIssue] + reward_no=902, + title="R2", + condition_for_minutes=20, + **reward_kwargs, + ) # pyright: ignore[reportAttributeAccessIssue] + + with CaptureQueriesContext(connection) as one_campaign_ctx: + self.client.get(reverse("chzzk:dashboard")) + query_count_two = len(one_campaign_ctx) + + campaign3 = ChzzkCampaign.objects.create( + campaign_no=9003, + title="C3", + **base_kwargs, + ) + campaign3.rewards.create( # pyright: ignore[reportAttributeAccessIssue] + reward_no=903, + title="R3", + condition_for_minutes=30, + **reward_kwargs, + ) # pyright: ignore[reportAttributeAccessIssue] + + with CaptureQueriesContext(connection) as three_campaign_ctx: + self.client.get(reverse("chzzk:dashboard")) + query_count_three = len(three_campaign_ctx) + + # With prefetch_related, adding more campaigns should not add extra queries per campaign. + assert query_count_two == query_count_three + def test_dashboard_view_excludes_testing_state_campaigns(self) -> None: """Test that the dashboard view excludes campaigns in the TESTING state.""" now: datetime = timezone.now() diff --git a/chzzk/views.py b/chzzk/views.py index 6146c31..d3c155c 100644 --- a/chzzk/views.py +++ b/chzzk/views.py @@ -1,7 +1,6 @@ from typing import TYPE_CHECKING from django.db.models import Q -from django.db.models.query import QuerySet from django.shortcuts import get_object_or_404 from django.shortcuts import render from django.urls import reverse @@ -16,9 +15,9 @@ from twitch.feeds import TTVDropsBaseFeed if TYPE_CHECKING: import datetime + from django.db.models.query import QuerySet from django.http import HttpResponse from django.http.request import HttpRequest - from pytest_django.asserts import QuerySet def dashboard_view(request: HttpRequest) -> HttpResponse: @@ -34,6 +33,7 @@ def dashboard_view(request: HttpRequest) -> HttpResponse: models.ChzzkCampaign.objects .filter(end_date__gte=timezone.now()) .exclude(state="TESTING") + .prefetch_related("rewards") .order_by("-start_date") ) return render( diff --git a/config/settings.py b/config/settings.py index 7fb2c0f..85a4757 100644 --- a/config/settings.py +++ b/config/settings.py @@ -224,6 +224,10 @@ DATABASES: dict[str, dict[str, Any]] = configure_databases( base_dir=BASE_DIR, ) +if DEBUG or TESTING: + INSTALLED_APPS.append("zeal") + MIDDLEWARE.append("zeal.middleware.zeal_middleware") + if not TESTING: INSTALLED_APPS = [*INSTALLED_APPS, "debug_toolbar", "silk"] MIDDLEWARE = [ diff --git a/config/tests/test_site_endpoint_smoke.py b/config/tests/test_site_endpoint_smoke.py new file mode 100644 index 0000000..90537c0 --- /dev/null +++ b/config/tests/test_site_endpoint_smoke.py @@ -0,0 +1,349 @@ +from __future__ import annotations + +from datetime import timedelta +from pathlib import Path +from typing import TYPE_CHECKING + +from django.conf import settings +from django.test import TestCase +from django.urls import reverse +from django.utils import timezone + +from chzzk.models import ChzzkCampaign +from kick.models import KickCategory +from kick.models import KickChannel +from kick.models import KickDropCampaign +from kick.models import KickOrganization +from kick.models import KickReward +from kick.models import KickUser +from twitch.models import Channel +from twitch.models import ChatBadge +from twitch.models import ChatBadgeSet +from twitch.models import DropBenefit +from twitch.models import DropCampaign +from twitch.models import Game +from twitch.models import Organization +from twitch.models import RewardCampaign +from twitch.models import TimeBasedDrop + +if TYPE_CHECKING: + from datetime import datetime + from pathlib import Path + + from django.test.client import _MonkeyPatchedWSGIResponse + + +class SiteEndpointSmokeTest(TestCase): + """Smoke-test all named site endpoints with realistic fixture data.""" + + def setUp(self) -> None: + """Set up representative Twitch, Kick, and CHZZK data for endpoint smoke tests.""" + now: datetime = timezone.now() + + # Twitch fixtures + self.twitch_org: Organization = Organization.objects.create( + twitch_id="smoke-org-1", + name="Smoke Organization", + ) + self.twitch_game: Game = Game.objects.create( + twitch_id="smoke-game-1", + slug="smoke-game", + name="Smoke Game", + display_name="Smoke Game", + box_art="https://example.com/smoke-game.png", + ) + self.twitch_game.owners.add(self.twitch_org) + + self.twitch_channel: Channel = Channel.objects.create( + twitch_id="smoke-channel-1", + name="smokechannel", + display_name="SmokeChannel", + ) + + self.twitch_campaign: DropCampaign = DropCampaign.objects.create( + twitch_id="smoke-campaign-1", + name="Smoke Campaign", + description="Smoke campaign description", + game=self.twitch_game, + image_url="https://example.com/smoke-campaign.png", + start_at=now - timedelta(days=1), + end_at=now + timedelta(days=1), + operation_names=["DropCampaignDetails"], + is_fully_imported=True, + ) + self.twitch_campaign.allow_channels.add(self.twitch_channel) + + self.twitch_drop: TimeBasedDrop = TimeBasedDrop.objects.create( + twitch_id="smoke-drop-1", + name="Smoke Drop", + campaign=self.twitch_campaign, + required_minutes_watched=15, + start_at=now - timedelta(days=1), + end_at=now + timedelta(days=1), + ) + self.twitch_benefit: DropBenefit = DropBenefit.objects.create( + twitch_id="smoke-benefit-1", + name="Smoke Benefit", + image_asset_url="https://example.com/smoke-benefit.png", + ) + self.twitch_drop.benefits.add(self.twitch_benefit) + + self.twitch_reward_campaign: RewardCampaign = RewardCampaign.objects.create( + twitch_id="smoke-reward-campaign-1", + name="Smoke Reward Campaign", + brand="Smoke Brand", + starts_at=now - timedelta(days=1), + ends_at=now + timedelta(days=2), + status="ACTIVE", + summary="Smoke reward summary", + external_url="https://example.com/smoke-reward", + is_sitewide=False, + game=self.twitch_game, + ) + + self.badge_set: ChatBadgeSet = ChatBadgeSet.objects.create( + set_id="smoke-badge-set", + ) + ChatBadge.objects.create( + badge_set=self.badge_set, + badge_id="1", + image_url_1x="https://example.com/badge-1x.png", + image_url_2x="https://example.com/badge-2x.png", + image_url_4x="https://example.com/badge-4x.png", + title="Smoke Badge", + description="Smoke badge description", + ) + + # Kick fixtures + self.kick_org: KickOrganization = KickOrganization.objects.create( + kick_id="smoke-kick-org-1", + name="Smoke Kick Organization", + ) + self.kick_category: KickCategory = KickCategory.objects.create( + kick_id=9101, + name="Smoke Kick Category", + slug="smoke-kick-category", + image_url="https://example.com/smoke-kick-category.png", + ) + self.kick_campaign: KickDropCampaign = KickDropCampaign.objects.create( + kick_id="smoke-kick-campaign-1", + name="Smoke Kick Campaign", + status="active", + starts_at=now - timedelta(days=1), + ends_at=now + timedelta(days=1), + organization=self.kick_org, + category=self.kick_category, + rule_id=1, + rule_name="Watch to redeem", + is_fully_imported=True, + ) + kick_user: KickUser = KickUser.objects.create( + kick_id=990001, + username="smokekickuser", + ) + kick_channel: KickChannel = KickChannel.objects.create( + kick_id=990002, + slug="smokekickchannel", + user=kick_user, + ) + self.kick_campaign.channels.add(kick_channel) + KickReward.objects.create( + kick_id="smoke-kick-reward-1", + name="Smoke Kick Reward", + image_url="drops/reward-image/smoke-kick-reward.png", + required_units=20, + campaign=self.kick_campaign, + category=self.kick_category, + organization=self.kick_org, + ) + + # CHZZK fixtures + self.chzzk_campaign: ChzzkCampaign = ChzzkCampaign.objects.create( + campaign_no=9901, + title="Smoke CHZZK Campaign", + description="Smoke CHZZK description", + category_type="game", + category_id="1", + category_value="SmokeGame", + service_id="chzzk", + state="ACTIVE", + start_date=now - timedelta(days=1), + end_date=now + timedelta(days=1), + has_ios_based_reward=False, + drops_campaign_not_started=False, + source_api="unit-test", + raw_json_v1={"ok": True}, + ) + self.chzzk_campaign.rewards.create( # pyright: ignore[reportAttributeAccessIssue] + reward_no=991, + title="Smoke CHZZK Reward", + reward_type="ITEM", + campaign_reward_type="Standard", + condition_type="watch", + condition_for_minutes=10, + ios_based_reward=False, + code_remaining_count=100, + ) + + # Core dataset download fixture + self.dataset_dir: Path = settings.DATA_DIR / "datasets" + self.dataset_dir.mkdir(parents=True, exist_ok=True) + self.dataset_name = "smoke-dataset.zst" + (self.dataset_dir / self.dataset_name).write_bytes(b"smoke") + + def tearDown(self) -> None: + """Clean up any files created for testing.""" + dataset_path: Path = self.dataset_dir / self.dataset_name + if dataset_path.exists(): + dataset_path.unlink() + + def test_all_site_endpoints_return_success(self) -> None: + """Test that all named site endpoints return a successful response with representative data.""" + endpoints: list[tuple[str, dict[str, str | int], int]] = [ + # Top-level config endpoints + ("sitemap", {}, 200), + ("sitemap-static", {}, 200), + ("sitemap-twitch-channels", {}, 200), + ("sitemap-twitch-drops", {}, 200), + ("sitemap-twitch-others", {}, 200), + ("sitemap-kick", {}, 200), + ("sitemap-youtube", {}, 200), + # Core endpoints + ("core:dashboard", {}, 200), + ("core:search", {}, 200), + ("core:debug", {}, 200), + ("core:dataset_backups", {}, 200), + ("core:dataset_backup_download", {"relative_path": self.dataset_name}, 200), + ("core:docs_rss", {}, 200), + ("core:campaign_feed", {}, 200), + ("core:game_feed", {}, 200), + ("core:game_campaign_feed", {"twitch_id": self.twitch_game.twitch_id}, 200), + ("core:organization_feed", {}, 200), + ("core:reward_campaign_feed", {}, 200), + ("core:campaign_feed_atom", {}, 200), + ("core:game_feed_atom", {}, 200), + ( + "core:game_campaign_feed_atom", + {"twitch_id": self.twitch_game.twitch_id}, + 200, + ), + ("core:organization_feed_atom", {}, 200), + ("core:reward_campaign_feed_atom", {}, 200), + ("core:campaign_feed_discord", {}, 200), + ("core:game_feed_discord", {}, 200), + ( + "core:game_campaign_feed_discord", + {"twitch_id": self.twitch_game.twitch_id}, + 200, + ), + ("core:organization_feed_discord", {}, 200), + ("core:reward_campaign_feed_discord", {}, 200), + # Twitch endpoints + ("twitch:dashboard", {}, 200), + ("twitch:badge_list", {}, 200), + ("twitch:badge_set_detail", {"set_id": self.badge_set.set_id}, 200), + ("twitch:campaign_list", {}, 200), + ( + "twitch:campaign_detail", + {"twitch_id": self.twitch_campaign.twitch_id}, + 200, + ), + ("twitch:channel_list", {}, 200), + ( + "twitch:channel_detail", + {"twitch_id": self.twitch_channel.twitch_id}, + 200, + ), + ("twitch:emote_gallery", {}, 200), + ("twitch:games_grid", {}, 200), + ("twitch:games_list", {}, 200), + ("twitch:game_detail", {"twitch_id": self.twitch_game.twitch_id}, 200), + ("twitch:org_list", {}, 200), + ( + "twitch:organization_detail", + {"twitch_id": self.twitch_org.twitch_id}, + 200, + ), + ("twitch:reward_campaign_list", {}, 200), + ( + "twitch:reward_campaign_detail", + {"twitch_id": self.twitch_reward_campaign.twitch_id}, + 200, + ), + ("twitch:export_campaigns_csv", {}, 200), + ("twitch:export_campaigns_json", {}, 200), + ("twitch:export_games_csv", {}, 200), + ("twitch:export_games_json", {}, 200), + ("twitch:export_organizations_csv", {}, 200), + ("twitch:export_organizations_json", {}, 200), + # Kick endpoints + ("kick:dashboard", {}, 200), + ("kick:campaign_list", {}, 200), + ("kick:campaign_detail", {"kick_id": self.kick_campaign.kick_id}, 200), + ("kick:game_list", {}, 200), + ("kick:game_detail", {"kick_id": self.kick_category.kick_id}, 200), + ("kick:category_list", {}, 200), + ("kick:category_detail", {"kick_id": self.kick_category.kick_id}, 200), + ("kick:organization_list", {}, 200), + ("kick:organization_detail", {"kick_id": self.kick_org.kick_id}, 200), + ("kick:campaign_feed", {}, 200), + ("kick:game_feed", {}, 200), + ("kick:game_campaign_feed", {"kick_id": self.kick_category.kick_id}, 200), + ("kick:category_feed", {}, 200), + ( + "kick:category_campaign_feed", + {"kick_id": self.kick_category.kick_id}, + 200, + ), + ("kick:organization_feed", {}, 200), + ("kick:campaign_feed_atom", {}, 200), + ("kick:game_feed_atom", {}, 200), + ( + "kick:game_campaign_feed_atom", + {"kick_id": self.kick_category.kick_id}, + 200, + ), + ("kick:category_feed_atom", {}, 200), + ( + "kick:category_campaign_feed_atom", + {"kick_id": self.kick_category.kick_id}, + 200, + ), + ("kick:organization_feed_atom", {}, 200), + ("kick:campaign_feed_discord", {}, 200), + ("kick:game_feed_discord", {}, 200), + ( + "kick:game_campaign_feed_discord", + {"kick_id": self.kick_category.kick_id}, + 200, + ), + ("kick:category_feed_discord", {}, 200), + ( + "kick:category_campaign_feed_discord", + {"kick_id": self.kick_category.kick_id}, + 200, + ), + ("kick:organization_feed_discord", {}, 200), + # CHZZK endpoints + ("chzzk:dashboard", {}, 200), + ("chzzk:campaign_list", {}, 200), + ( + "chzzk:campaign_detail", + {"campaign_no": self.chzzk_campaign.campaign_no}, + 200, + ), + ("chzzk:campaign_feed", {}, 200), + ("chzzk:campaign_feed_atom", {}, 200), + ("chzzk:campaign_feed_discord", {}, 200), + # YouTube endpoint + ("youtube:index", {}, 200), + ] + + for route_name, kwargs, expected_status in endpoints: + response: _MonkeyPatchedWSGIResponse = self.client.get( + reverse(route_name, kwargs=kwargs), + ) + assert response.status_code == expected_status, ( + f"{route_name} returned {response.status_code}, expected {expected_status}" + ) + response.close() diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..883c39d --- /dev/null +++ b/conftest.py @@ -0,0 +1,25 @@ +from typing import TYPE_CHECKING + +import pytest +from zeal import zeal_context + +if TYPE_CHECKING: + from collections.abc import Generator + + +@pytest.fixture(autouse=True) +def use_zeal(request: pytest.FixtureRequest) -> Generator[None]: + """Enable Zeal N+1 detection context for each pytest test. + + Use @pytest.mark.no_zeal for tests that intentionally exercise import paths + where Zeal's strict get() heuristics are too noisy. + + Yields: + None: Control back to pytest for test execution. + """ + if request.node.get_closest_marker("no_zeal") is not None: + yield + return + + with zeal_context(): + yield diff --git a/core/base_url.py b/core/base_url.py index 7eb763d..524532d 100644 --- a/core/base_url.py +++ b/core/base_url.py @@ -69,7 +69,7 @@ class _TTVDropsSite: domain: str -def get_current_site(request: object) -> _TTVDropsSite: +def get_current_site(request: HttpRequest | None) -> _TTVDropsSite: """Return a site-like object with domain derived from BASE_URL.""" base_url: str = _get_base_url() parts: SplitResult = urlsplit(base_url) diff --git a/core/tasks.py b/core/tasks.py index 5b20599..cf7f6f0 100644 --- a/core/tasks.py +++ b/core/tasks.py @@ -5,7 +5,7 @@ import logging from celery import shared_task from django.core.management import call_command -logger = logging.getLogger("ttvdrops.tasks") +logger: logging.Logger = logging.getLogger("ttvdrops.tasks") @shared_task(bind=True, queue="default", max_retries=3, default_retry_delay=300) diff --git a/core/tests/test_sitemaps.py b/core/tests/test_sitemaps.py index c833d26..7821279 100644 --- a/core/tests/test_sitemaps.py +++ b/core/tests/test_sitemaps.py @@ -5,6 +5,7 @@ from django.urls import reverse if TYPE_CHECKING: from django.test.client import Client + from pytest_django.fixtures import SettingsWrapper def _extract_locs(xml_bytes: bytes) -> list[str]: @@ -15,7 +16,7 @@ def _extract_locs(xml_bytes: bytes) -> list[str]: def test_sitemap_static_contains_expected_links( client: Client, - settings: object, + settings: SettingsWrapper, ) -> None: """Ensure the static sitemap contains the main site links across apps. diff --git a/core/views.py b/core/views.py index 6d26d1f..b03d991 100644 --- a/core/views.py +++ b/core/views.py @@ -15,11 +15,9 @@ from django.db.models import Max from django.db.models import OuterRef from django.db.models import Prefetch from django.db.models import Q -from django.db.models import QuerySet from django.db.models.functions import Trim from django.http import FileResponse from django.http import Http404 -from django.http import HttpRequest from django.http import HttpResponse from django.shortcuts import render from django.template.defaultfilters import filesizeformat diff --git a/kick/feeds.py b/kick/feeds.py index 6d3685a..36f55b9 100644 --- a/kick/feeds.py +++ b/kick/feeds.py @@ -206,8 +206,8 @@ class KickOrganizationFeed(TTVDropsBaseFeed): def __call__( self, request: HttpRequest, - *args: object, - **kwargs: object, + *args: str | int, + **kwargs: str | int, ) -> HttpResponse: """Capture optional ?limit query parameter. @@ -283,8 +283,8 @@ class KickCategoryFeed(TTVDropsBaseFeed): def __call__( self, request: HttpRequest, - *args: object, - **kwargs: object, + *args: str | int, + **kwargs: str | int, ) -> HttpResponse: """Capture optional ?limit query parameter. @@ -372,8 +372,8 @@ class KickCampaignFeed(TTVDropsBaseFeed): def __call__( self, request: HttpRequest, - *args: object, - **kwargs: object, + *args: str | int, + **kwargs: str | int, ) -> HttpResponse: """Capture optional ?limit query parameter. @@ -481,8 +481,8 @@ class KickCategoryCampaignFeed(TTVDropsBaseFeed): def __call__( self, request: HttpRequest, - *args: object, - **kwargs: object, + *args: str | int, + **kwargs: str | int, ) -> HttpResponse: """Capture optional ?limit query parameter. diff --git a/kick/management/commands/import_kick_drops.py b/kick/management/commands/import_kick_drops.py index 04face1..4116350 100644 --- a/kick/management/commands/import_kick_drops.py +++ b/kick/management/commands/import_kick_drops.py @@ -1,4 +1,7 @@ +from __future__ import annotations + import logging +from datetime import datetime from typing import TYPE_CHECKING import httpx @@ -14,6 +17,8 @@ from kick.models import KickUser from kick.schemas import KickDropsResponseSchema if TYPE_CHECKING: + from collections.abc import Mapping + from django.core.management.base import CommandParser from kick.schemas import KickCategorySchema @@ -23,6 +28,26 @@ if TYPE_CHECKING: logger: logging.Logger = logging.getLogger("ttvdrops") +type KickImportModel = ( + KickOrganization + | KickCategory + | KickDropCampaign + | KickUser + | KickChannel + | KickReward +) +type KickFieldValue = ( + str + | bool + | int + | datetime + | KickOrganization + | KickCategory + | KickDropCampaign + | KickUser + | None +) + KICK_DROPS_API_URL = "https://web.kick.com/api/v1/drops/campaigns" # Kick's public API requires a browser-like User-Agent. @@ -48,7 +73,26 @@ class Command(BaseCommand): help="API endpoint to fetch (default: %(default)s).", ) - def handle(self, *args: object, **options: object) -> None: # noqa: ARG002 + @staticmethod + def _save_if_changed( + obj: KickImportModel, + defaults: Mapping[str, KickFieldValue], + ) -> None: + """Persist only changed fields to avoid unnecessary updates.""" + changed_fields: list[str] = [] + for field, new_value in defaults.items(): + if getattr(obj, field, None) != new_value: + setattr(obj, field, new_value) + changed_fields.append(field) + + if changed_fields: + obj.save(update_fields=changed_fields) + + def handle( + self, + *_args: str, + **options: str | bool | int | None, + ) -> None: """Main entry point for the command.""" url: str = str(options["url"]) self.stdout.write(f"Fetching Kick drops from {url} ...") @@ -99,54 +143,75 @@ class Command(BaseCommand): self.style.SUCCESS(f"Imported {imported}/{len(campaigns)} campaign(s)."), ) - def _import_campaign(self, data: KickDropCampaignSchema) -> None: + def _import_campaign(self, data: KickDropCampaignSchema) -> None: # noqa: PLR0914, PLR0915 """Import a single campaign and all its related objects.""" - # Organisation + # Organization org_data: KickOrganizationSchema = data.organization - org, created = KickOrganization.objects.update_or_create( + org_defaults: dict[str, str | bool] = { + "name": org_data.name, + "logo_url": org_data.logo_url, + "url": org_data.url, + "restricted": org_data.restricted, + } + org: KickOrganization | None = KickOrganization.objects.filter( kick_id=org_data.id, - defaults={ - "name": org_data.name, - "logo_url": org_data.logo_url, - "url": org_data.url, - "restricted": org_data.restricted, - }, - ) + ).first() + created: bool = org is None + if org is None: + org = KickOrganization.objects.create(kick_id=org_data.id, **org_defaults) + else: + self._save_if_changed(org, org_defaults) if created: logger.info("Created new organization: %s", org.kick_id) # Category cat_data: KickCategorySchema = data.category - category, created = KickCategory.objects.update_or_create( + category_defaults: dict[str, KickFieldValue] = { + "name": cat_data.name, + "slug": cat_data.slug, + "image_url": cat_data.image_url, + } + category: KickCategory | None = KickCategory.objects.filter( kick_id=cat_data.id, - defaults={ - "name": cat_data.name, - "slug": cat_data.slug, - "image_url": cat_data.image_url, - }, - ) + ).first() + created = category is None + if category is None: + category = KickCategory.objects.create( + kick_id=cat_data.id, + **category_defaults, + ) + else: + self._save_if_changed(category, category_defaults) if created: logger.info("Created new category: %s", category.kick_id) # Campaign - campaign, created = KickDropCampaign.objects.update_or_create( + campaign_defaults: dict[str, KickFieldValue] = { + "name": data.name, + "status": data.status, + "starts_at": data.starts_at, + "ends_at": data.ends_at, + "connect_url": data.connect_url, + "url": data.url, + "rule_id": data.rule.id, + "rule_name": data.rule.name, + "organization": org, + "category": category, + "created_at": data.created_at, + "api_updated_at": data.updated_at, + "is_fully_imported": True, + } + campaign: KickDropCampaign | None = KickDropCampaign.objects.filter( kick_id=data.id, - defaults={ - "name": data.name, - "status": data.status, - "starts_at": data.starts_at, - "ends_at": data.ends_at, - "connect_url": data.connect_url, - "url": data.url, - "rule_id": data.rule.id, - "rule_name": data.rule.name, - "organization": org, - "category": category, - "created_at": data.created_at, - "api_updated_at": data.updated_at, - "is_fully_imported": True, - }, - ) + ).first() + created = campaign is None + if campaign is None: + campaign = KickDropCampaign.objects.create( + kick_id=data.id, + **campaign_defaults, + ) + else: + self._save_if_changed(campaign, campaign_defaults) if created: logger.info("Created new campaign: %s", campaign.kick_id) @@ -154,25 +219,38 @@ class Command(BaseCommand): channel_objs: list[KickChannel] = [] for ch_data in data.channels: user_data: KickUserSchema = ch_data.user - user, created = KickUser.objects.update_or_create( + user_defaults: dict[str, KickFieldValue] = { + "username": user_data.username, + "profile_picture": user_data.profile_picture, + } + user: KickUser | None = KickUser.objects.filter( kick_id=user_data.id, - defaults={ - "username": user_data.username, - "profile_picture": user_data.profile_picture, - }, - ) + ).first() + created = user is None + if user is None: + user = KickUser.objects.create(kick_id=user_data.id, **user_defaults) + else: + self._save_if_changed(user, user_defaults) if created: logger.info("Created new user: %s", user.kick_id) - channel, created = KickChannel.objects.update_or_create( + channel_defaults: dict[str, KickFieldValue] = { + "slug": ch_data.slug, + "description": ch_data.description, + "banner_picture_url": ch_data.banner_picture_url, + "user": user, + } + channel: KickChannel | None = KickChannel.objects.filter( kick_id=ch_data.id, - defaults={ - "slug": ch_data.slug, - "description": ch_data.description, - "banner_picture_url": ch_data.banner_picture_url, - "user": user, - }, - ) + ).first() + created = channel is None + if channel is None: + channel = KickChannel.objects.create( + kick_id=ch_data.id, + **channel_defaults, + ) + else: + self._save_if_changed(channel, channel_defaults) if created: logger.info("Created new channel: %s", channel.kick_id) @@ -184,36 +262,46 @@ class Command(BaseCommand): # Resolve reward's category (may differ from campaign category) reward_category: KickCategory = category if reward_data.category_id != cat_data.id: - reward_category, created = KickCategory.objects.get_or_create( + reward_category = KickCategory.objects.filter( kick_id=reward_data.category_id, - defaults={"name": "", "slug": "", "image_url": ""}, + ).first() or KickCategory.objects.create( + kick_id=reward_data.category_id, + name="", + slug="", + image_url="", ) + created = not reward_category.name and not reward_category.slug if created: logger.info("Created new category: %s", reward_category.kick_id) # Resolve reward's organization (may differ from campaign org) reward_org: KickOrganization = org if reward_data.organization_id != org_data.id: - reward_org, created = KickOrganization.objects.get_or_create( + reward_org = KickOrganization.objects.filter( kick_id=reward_data.organization_id, - defaults={ - "name": "", - "logo_url": "", - "url": "", - "restricted": False, - }, + ).first() or KickOrganization.objects.create( + kick_id=reward_data.organization_id, + name="", + logo_url="", + url="", + restricted=False, ) + created = not reward_org.name and not reward_org.url if created: logger.info("Created new organization: %s", reward_org.kick_id) - KickReward.objects.update_or_create( + reward_defaults: dict[str, KickFieldValue] = { + "name": reward_data.name, + "image_url": reward_data.image_url, + "required_units": reward_data.required_units, + "campaign": campaign, + "category": reward_category, + "organization": reward_org, + } + reward: KickReward | None = KickReward.objects.filter( kick_id=reward_data.id, - defaults={ - "name": reward_data.name, - "image_url": reward_data.image_url, - "required_units": reward_data.required_units, - "campaign": campaign, - "category": reward_category, - "organization": reward_org, - }, - ) + ).first() + if reward is None: + KickReward.objects.create(kick_id=reward_data.id, **reward_defaults) + else: + self._save_if_changed(reward, reward_defaults) diff --git a/kick/models.py b/kick/models.py index 1c2b4c0..450434f 100644 --- a/kick/models.py +++ b/kick/models.py @@ -292,10 +292,19 @@ class KickDropCampaign(auto_prefetch.Model): def image_url(self) -> str: """Return the image URL for the campaign.""" # Image from first drop - if self.rewards.exists(): # pyright: ignore[reportAttributeAccessIssue] - first_reward: KickReward | None = self.rewards.first() # pyright: ignore[reportAttributeAccessIssue] - if first_reward and first_reward.image_url: - return first_reward.full_image_url + rewards_prefetched: list[KickReward] | None = getattr( + self, + "rewards_ordered", + None, + ) + if rewards_prefetched is not None: + first_reward: KickReward | None = ( + rewards_prefetched[0] if rewards_prefetched else None + ) + else: + first_reward = self.rewards.first() # pyright: ignore[reportAttributeAccessIssue] + if first_reward and first_reward.image_url: + return first_reward.full_image_url if self.category and self.category.image_url: return self.category.image_url @@ -352,7 +361,24 @@ class KickDropCampaign(auto_prefetch.Model): If both a base reward and a "(Con)" variant exist, prefer the base reward name. """ rewards_by_name: dict[str, KickReward] = {} - for reward in self.rewards.all().order_by("required_units", "name", "kick_id"): # pyright: ignore[reportAttributeAccessIssue] + prefetched_rewards: list[KickReward] | None = getattr( + self, + "_prefetched_objects_cache", + {}, + ).get("rewards") + if prefetched_rewards is not None: + rewards_iterable = sorted( + prefetched_rewards, + key=lambda reward: (reward.required_units, reward.name, reward.kick_id), + ) + else: + rewards_iterable = self.rewards.all().order_by( # pyright: ignore[reportAttributeAccessIssue] + "required_units", + "name", + "kick_id", + ) + + for reward in rewards_iterable: key: str = self._normalized_reward_name(reward.name) existing: KickReward | None = rewards_by_name.get(key) if existing is None: diff --git a/kick/tests/test_kick.py b/kick/tests/test_kick.py index 357f604..7fb1c43 100644 --- a/kick/tests/test_kick.py +++ b/kick/tests/test_kick.py @@ -12,8 +12,10 @@ from unittest.mock import patch import httpx import pytest from django.core.management import call_command +from django.db import connection from django.test import Client from django.test import TestCase +from django.test.utils import CaptureQueriesContext from django.urls import reverse from django.utils import timezone from pydantic import ValidationError @@ -414,6 +416,37 @@ class KickDropCampaignMergedRewardsTest(TestCase): assert len(merged) == 1 assert merged[0].name == "9th Anniversary Cake & Confetti" + def test_uses_prefetched_rewards_without_extra_queries(self) -> None: + """When rewards are prefetched, merged_rewards should not hit the database again.""" + campaign: KickDropCampaign = self._make_campaign() + KickReward.objects.create( + kick_id="reward-prefetch-a", + name="Alpha Reward", + image_url="drops/reward-image/alpha.png", + required_units=10, + campaign=campaign, + category=campaign.category, + organization=campaign.organization, + ) + KickReward.objects.create( + kick_id="reward-prefetch-b", + name="Alpha Reward (Con)", + image_url="drops/reward-image/alpha-con.png", + required_units=10, + campaign=campaign, + category=campaign.category, + organization=campaign.organization, + ) + + campaign = KickDropCampaign.objects.prefetch_related("rewards").get( + pk=campaign.pk, + ) + with self.assertNumQueries(0): + merged: list[KickReward] = campaign.merged_rewards + + assert len(merged) == 1 + assert merged[0].name == "Alpha Reward" + # MARK: Management command tests class ImportKickDropsCommandTest(TestCase): @@ -546,24 +579,115 @@ class KickDashboardViewTest(TestCase): ) assert campaign.name in response.content.decode() + def test_dashboard_query_count_stays_flat_with_more_campaigns(self) -> None: + """Dashboard SELECT query count should stay flat as active campaign count grows.""" + + def _create_active_campaign(index: int) -> KickDropCampaign: + org: KickOrganization = KickOrganization.objects.create( + kick_id=f"org-qc-{index}", + name=f"Org QC {index}", + ) + cat: KickCategory = KickCategory.objects.create( + kick_id=10000 + index, + name=f"Cat QC {index}", + slug=f"cat-qc-{index}", + ) + campaign: KickDropCampaign = KickDropCampaign.objects.create( + kick_id=f"camp-qc-{index}", + name=f"Campaign QC {index}", + status="active", + starts_at=dt(2020, 1, 1, tzinfo=UTC), + ends_at=dt(2099, 12, 31, tzinfo=UTC), + organization=org, + category=cat, + rule_id=1, + rule_name="Watch to redeem", + is_fully_imported=True, + ) + + user: KickUser = KickUser.objects.create( + kick_id=3000000 + index, + username=f"qcuser{index}", + ) + channel: KickChannel = KickChannel.objects.create( + kick_id=2000000 + index, + slug=f"qc-channel-{index}", + user=user, + ) + campaign.channels.add(channel) + + KickReward.objects.create( + kick_id=f"reward-qc-{index}-a", + name="Alpha Reward", + image_url="drops/reward-image/alpha.png", + required_units=30, + campaign=campaign, + category=cat, + organization=org, + ) + KickReward.objects.create( + kick_id=f"reward-qc-{index}-b", + name="Alpha Reward (Con)", + image_url="drops/reward-image/alpha-con.png", + required_units=30, + campaign=campaign, + category=cat, + organization=org, + ) + + return campaign + + def _capture_dashboard_select_count() -> int: + with CaptureQueriesContext(connection) as queries: + response: _MonkeyPatchedWSGIResponse = self.client.get( + reverse("kick:dashboard"), + ) + assert response.status_code == 200 + + select_queries: list[str] = [ + query_info["sql"] + for query_info in queries.captured_queries + if query_info["sql"].lstrip().upper().startswith("SELECT") + ] + return len(select_queries) + + _create_active_campaign(1) + baseline_select_count: int = _capture_dashboard_select_count() + + for i in range(2, 12): + _create_active_campaign(i) + + scaled_select_count: int = _capture_dashboard_select_count() + + assert scaled_select_count <= baseline_select_count + 2, ( + "Kick dashboard SELECT query count grew with campaign volume; " + f"possible N+1 regression. baseline={baseline_select_count}, " + f"scaled={scaled_select_count}" + ) + class KickCampaignListViewTest(TestCase): """Tests for the kick campaign list view.""" + @classmethod + def setUpTestData(cls) -> None: + """Set up shared test data for campaign list view tests.""" + cls.org: KickOrganization = KickOrganization.objects.create( + kick_id="org-list", + name="List Org", + ) + cls.cat: KickCategory = KickCategory.objects.create( + kick_id=300, + name="List Cat", + slug="list-cat", + ) + def _make_campaign( self, kick_id: str, name: str, status: str = "active", ) -> KickDropCampaign: - org, _ = KickOrganization.objects.get_or_create( - kick_id="org-list", - defaults={"name": "List Org"}, - ) - cat, _ = KickCategory.objects.get_or_create( - kick_id=300, - defaults={"name": "List Cat", "slug": "list-cat"}, - ) # Set dates so the active/expired filter works correctly if status == "active": starts_at = dt(2020, 1, 1, tzinfo=UTC) @@ -577,8 +701,8 @@ class KickCampaignListViewTest(TestCase): status=status, starts_at=starts_at, ends_at=ends_at, - organization=org, - category=cat, + organization=self.org, + category=self.cat, rule_id=1, rule_name="Watch to redeem", is_fully_imported=True, @@ -811,6 +935,95 @@ class KickOrganizationDetailViewTest(TestCase): ) assert response.status_code == 404 + def test_organization_detail_query_count_stays_flat_with_more_campaigns( + self, + ) -> None: + """Organization detail SELECT query count should stay flat as campaign count grows.""" + org: KickOrganization = KickOrganization.objects.create( + kick_id="org-orgdet-qc", + name="Orgdet Query Count", + ) + + def _create_org_campaign(index: int) -> None: + cat: KickCategory = KickCategory.objects.create( + kick_id=17000 + index, + name=f"Orgdet QC Cat {index}", + slug=f"orgdet-qc-cat-{index}", + ) + campaign: KickDropCampaign = KickDropCampaign.objects.create( + kick_id=f"camp-orgdet-qc-{index}", + name=f"Orgdet QC Campaign {index}", + status="active", + starts_at=dt(2020, 1, 1, tzinfo=UTC), + ends_at=dt(2099, 12, 31, tzinfo=UTC), + organization=org, + category=cat, + rule_id=1, + rule_name="Watch to redeem", + is_fully_imported=True, + ) + + user: KickUser = KickUser.objects.create( + kick_id=3700000 + index, + username=f"orgdetqcuser{index}", + ) + channel: KickChannel = KickChannel.objects.create( + kick_id=2700000 + index, + slug=f"orgdet-qc-channel-{index}", + user=user, + ) + campaign.channels.add(channel) + + KickReward.objects.create( + kick_id=f"reward-orgdet-qc-{index}-a", + name="Org Reward", + image_url="drops/reward-image/org.png", + required_units=30, + campaign=campaign, + category=cat, + organization=org, + ) + KickReward.objects.create( + kick_id=f"reward-orgdet-qc-{index}-b", + name="Org Reward (Con)", + image_url="drops/reward-image/org-con.png", + required_units=30, + campaign=campaign, + category=cat, + organization=org, + ) + + def _capture_org_detail_select_count() -> int: + with CaptureQueriesContext(connection) as queries: + response: _MonkeyPatchedWSGIResponse = self.client.get( + reverse( + "kick:organization_detail", + kwargs={"kick_id": org.kick_id}, + ), + ) + assert response.status_code == 200 + + select_queries: list[str] = [ + query_info["sql"] + for query_info in queries.captured_queries + if query_info["sql"].lstrip().upper().startswith("SELECT") + ] + return len(select_queries) + + _create_org_campaign(1) + baseline_select_count: int = _capture_org_detail_select_count() + + for i in range(2, 12): + _create_org_campaign(i) + + scaled_select_count: int = _capture_org_detail_select_count() + + assert scaled_select_count <= baseline_select_count + 2, ( + "Organization detail SELECT query count grew with campaign volume; " + f"possible N+1 regression. baseline={baseline_select_count}, " + f"scaled={scaled_select_count}" + ) + class KickFeedsTest(TestCase): """Tests for Kick RSS/Atom/Discord feed endpoints.""" @@ -942,6 +1155,109 @@ class KickFeedsTest(TestCase): assert not str(discord_timestamp(None)) +class KickEndpointCoverageTest(TestCase): + """Endpoint smoke coverage for all Kick routes in kick.urls.""" + + def setUp(self) -> None: + """Create shared fixtures used by detail and feed endpoints.""" + self.org: KickOrganization = KickOrganization.objects.create( + kick_id="org-endpoint-1", + name="Endpoint Org", + logo_url="https://example.com/org-endpoint.png", + ) + self.category: KickCategory = KickCategory.objects.create( + kick_id=9123, + name="Endpoint Category", + slug="endpoint-category", + image_url="https://example.com/endpoint-category.png", + ) + self.campaign: KickDropCampaign = KickDropCampaign.objects.create( + kick_id="camp-endpoint-1", + name="Endpoint Campaign", + status="active", + starts_at=timezone.now() - timedelta(days=1), + ends_at=timezone.now() + timedelta(days=1), + organization=self.org, + category=self.category, + connect_url="https://example.com/connect", + url="https://example.com/campaign", + rule_id=1, + rule_name="Watch to redeem", + is_fully_imported=True, + ) + + user: KickUser = KickUser.objects.create( + kick_id=5551001, + username="endpointuser", + ) + channel: KickChannel = KickChannel.objects.create( + kick_id=5551002, + slug="endpointchannel", + user=user, + ) + self.campaign.channels.add(channel) + + KickReward.objects.create( + kick_id="reward-endpoint-1", + name="Endpoint Reward", + image_url="drops/reward-image/endpoint.png", + required_units=20, + campaign=self.campaign, + category=self.category, + organization=self.org, + ) + + def test_all_kick_html_endpoints_return_success(self) -> None: + """All Kick HTML endpoints should render successfully with populated fixtures.""" + html_routes: list[tuple[str, dict[str, str | int]]] = [ + ("kick:dashboard", {}), + ("kick:campaign_list", {}), + ("kick:campaign_detail", {"kick_id": self.campaign.kick_id}), + ("kick:game_list", {}), + ("kick:game_detail", {"kick_id": self.category.kick_id}), + ("kick:category_list", {}), + ("kick:category_detail", {"kick_id": self.category.kick_id}), + ("kick:organization_list", {}), + ("kick:organization_detail", {"kick_id": self.org.kick_id}), + ] + + for route_name, kwargs in html_routes: + response: _MonkeyPatchedWSGIResponse = self.client.get( + reverse(route_name, kwargs=kwargs), + ) + assert response.status_code == 200, route_name + + def test_all_kick_feed_endpoints_return_success(self) -> None: + """All Kick RSS/Atom/Discord feed endpoints should return XML responses.""" + feed_routes: list[tuple[str, dict[str, int]]] = [ + ("kick:campaign_feed", {}), + ("kick:game_feed", {}), + ("kick:game_campaign_feed", {"kick_id": self.category.kick_id}), + ("kick:category_feed", {}), + ("kick:category_campaign_feed", {"kick_id": self.category.kick_id}), + ("kick:organization_feed", {}), + ("kick:campaign_feed_atom", {}), + ("kick:game_feed_atom", {}), + ("kick:game_campaign_feed_atom", {"kick_id": self.category.kick_id}), + ("kick:category_feed_atom", {}), + ("kick:category_campaign_feed_atom", {"kick_id": self.category.kick_id}), + ("kick:organization_feed_atom", {}), + ("kick:campaign_feed_discord", {}), + ("kick:game_feed_discord", {}), + ("kick:game_campaign_feed_discord", {"kick_id": self.category.kick_id}), + ("kick:category_feed_discord", {}), + ("kick:category_campaign_feed_discord", {"kick_id": self.category.kick_id}), + ("kick:organization_feed_discord", {}), + ] + + for route_name, kwargs in feed_routes: + response: _MonkeyPatchedWSGIResponse = self.client.get( + reverse(route_name, kwargs=kwargs), + ) + assert response.status_code == 200, route_name + assert response["Content-Type"] == "application/xml; charset=utf-8" + + class KickDropCampaignFullyImportedTest(TestCase): """Tests for KickDropCampaign.is_fully_imported field and filtering.""" diff --git a/kick/views.py b/kick/views.py index b8cc126..7d89560 100644 --- a/kick/views.py +++ b/kick/views.py @@ -532,7 +532,7 @@ def organization_detail_view(request: HttpRequest, kick_id: str) -> HttpResponse KickDropCampaign.objects .filter(organization=org) .select_related("category") - .prefetch_related("rewards") + .prefetch_related("rewards", "channels__user") .order_by("-starts_at"), ) diff --git a/pyproject.toml b/pyproject.toml index 836228a..74d15fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ dependencies = [ "setproctitle", "sitemap-parser", "tqdm", + "django-zeal>=2.1.0", ] @@ -51,6 +52,7 @@ dev = [ DJANGO_SETTINGS_MODULE = "config.settings" python_files = ["test_*.py", "*_test.py"] addopts = "--tb=short -n auto --cov" +markers = ["no_zeal: run test without zeal_context N+1 checks"] filterwarnings = [ "ignore:Parsing dates involving a day of month without a year specified is ambiguous:DeprecationWarning", ] diff --git a/templates/twitch/badge_list.html b/templates/twitch/badge_list.html index a91b3cc..ea15c09 100644 --- a/templates/twitch/badge_list.html +++ b/templates/twitch/badge_list.html @@ -4,11 +4,11 @@ Chat Badges {% endblock title %} {% block content %} -

{{ badge_sets.count }} Twitch Chat Badges

+

{{ badge_data|length }} Twitch Chat Badges

Twitch > Badges
- {% if badge_sets %} + {% if badge_data %} {% for data in badge_data %}

{{ data.set.set_id }} diff --git a/templates/twitch/dashboard.html b/templates/twitch/dashboard.html index 0689a2d..5e6f7cd 100644 --- a/templates/twitch/dashboard.html +++ b/templates/twitch/dashboard.html @@ -68,8 +68,8 @@ flex-shrink: 0">
- {% picture campaign_data.campaign.image_best_url|default:campaign_data.campaign.image_url alt="Image for "|add:campaign_data.campaign.name width=120 %} -

{{ campaign_data.campaign.clean_name }}

+ {% picture campaign_data.image_url alt="Image for "|add:campaign_data.campaign.name width=120 %} +

{{ campaign_data.clean_name }}