From b7e10e766e10612034839640ca7fc38d4f4324b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Hells=C3=A9n?= Date: Sat, 11 Apr 2026 00:44:16 +0200 Subject: [PATCH] Improve performance and add type hints --- chzzk/tasks.py | 2 +- chzzk/tests/test_management_commands.py | 2 +- chzzk/views.py | 3 +- config/settings.py | 2 +- conftest.py | 25 ++ core/base_url.py | 2 +- core/tasks.py | 2 +- core/tests/test_sitemaps.py | 3 +- core/views.py | 2 - kick/feeds.py | 16 +- kick/management/commands/import_kick_drops.py | 224 +++++++++---- kick/tests/test_kick.py | 25 +- pyproject.toml | 1 + templates/twitch/dashboard.html | 16 +- twitch/feeds.py | 26 +- twitch/management/commands/backup_db.py | 14 +- .../commands/better_import_drops.py | 116 +++++-- .../management/commands/download_box_art.py | 8 +- .../commands/download_campaign_images.py | 8 +- .../management/commands/import_chat_badges.py | 27 +- twitch/models.py | 81 ++++- twitch/tests/test_views.py | 308 +++++++++++++++++- twitch/views.py | 10 +- 23 files changed, 745 insertions(+), 178 deletions(-) create mode 100644 conftest.py diff --git a/chzzk/tasks.py b/chzzk/tasks.py index 5707beb..55f929c 100644 --- a/chzzk/tasks.py +++ b/chzzk/tasks.py @@ -5,7 +5,7 @@ import logging from celery import shared_task from django.core.management import call_command -logger = logging.getLogger("ttvdrops.tasks") +logger: logging.Logger = logging.getLogger("ttvdrops.tasks") @shared_task(bind=True, queue="imports", max_retries=3, default_retry_delay=60) diff --git a/chzzk/tests/test_management_commands.py b/chzzk/tests/test_management_commands.py index ec41d22..dd10844 100644 --- a/chzzk/tests/test_management_commands.py +++ b/chzzk/tests/test_management_commands.py @@ -136,7 +136,7 @@ class ImportChzzkCampaignRangeCommandTest(TestCase): stdout = StringIO() stderr = StringIO() - def side_effect(command: str, *args: str, **kwargs: object) -> None: + def side_effect(command: str, *args: str, **kwargs: StringIO) -> None: if "4" in args: msg = "Campaign 4 not found" raise CommandError(msg) diff --git a/chzzk/views.py b/chzzk/views.py index 16cd547..d3c155c 100644 --- a/chzzk/views.py +++ b/chzzk/views.py @@ -1,7 +1,6 @@ from typing import TYPE_CHECKING from django.db.models import Q -from django.db.models.query import QuerySet from django.shortcuts import get_object_or_404 from django.shortcuts import render from django.urls import reverse @@ -16,9 +15,9 @@ from twitch.feeds import TTVDropsBaseFeed if TYPE_CHECKING: import datetime + from django.db.models.query import QuerySet from django.http import HttpResponse from django.http.request import HttpRequest - from pytest_django.asserts import QuerySet def dashboard_view(request: HttpRequest) -> HttpResponse: diff --git a/config/settings.py b/config/settings.py index c65859b..85a4757 100644 --- a/config/settings.py +++ b/config/settings.py @@ -224,7 +224,7 @@ DATABASES: dict[str, dict[str, Any]] = configure_databases( base_dir=BASE_DIR, ) -if DEBUG: +if DEBUG or TESTING: INSTALLED_APPS.append("zeal") MIDDLEWARE.append("zeal.middleware.zeal_middleware") diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..883c39d --- /dev/null +++ b/conftest.py @@ -0,0 +1,25 @@ +from typing import TYPE_CHECKING + +import pytest +from zeal import zeal_context + +if TYPE_CHECKING: + from collections.abc import Generator + + +@pytest.fixture(autouse=True) +def use_zeal(request: pytest.FixtureRequest) -> Generator[None]: + """Enable Zeal N+1 detection context for each pytest test. + + Use @pytest.mark.no_zeal for tests that intentionally exercise import paths + where Zeal's strict get() heuristics are too noisy. + + Yields: + None: Control back to pytest for test execution. + """ + if request.node.get_closest_marker("no_zeal") is not None: + yield + return + + with zeal_context(): + yield diff --git a/core/base_url.py b/core/base_url.py index 7eb763d..524532d 100644 --- a/core/base_url.py +++ b/core/base_url.py @@ -69,7 +69,7 @@ class _TTVDropsSite: domain: str -def get_current_site(request: object) -> _TTVDropsSite: +def get_current_site(request: HttpRequest | None) -> _TTVDropsSite: """Return a site-like object with domain derived from BASE_URL.""" base_url: str = _get_base_url() parts: SplitResult = urlsplit(base_url) diff --git a/core/tasks.py b/core/tasks.py index 5b20599..cf7f6f0 100644 --- a/core/tasks.py +++ b/core/tasks.py @@ -5,7 +5,7 @@ import logging from celery import shared_task from django.core.management import call_command -logger = logging.getLogger("ttvdrops.tasks") +logger: logging.Logger = logging.getLogger("ttvdrops.tasks") @shared_task(bind=True, queue="default", max_retries=3, default_retry_delay=300) diff --git a/core/tests/test_sitemaps.py b/core/tests/test_sitemaps.py index c833d26..7821279 100644 --- a/core/tests/test_sitemaps.py +++ b/core/tests/test_sitemaps.py @@ -5,6 +5,7 @@ from django.urls import reverse if TYPE_CHECKING: from django.test.client import Client + from pytest_django.fixtures import SettingsWrapper def _extract_locs(xml_bytes: bytes) -> list[str]: @@ -15,7 +16,7 @@ def _extract_locs(xml_bytes: bytes) -> list[str]: def test_sitemap_static_contains_expected_links( client: Client, - settings: object, + settings: SettingsWrapper, ) -> None: """Ensure the static sitemap contains the main site links across apps. diff --git a/core/views.py b/core/views.py index 6d26d1f..b03d991 100644 --- a/core/views.py +++ b/core/views.py @@ -15,11 +15,9 @@ from django.db.models import Max from django.db.models import OuterRef from django.db.models import Prefetch from django.db.models import Q -from django.db.models import QuerySet from django.db.models.functions import Trim from django.http import FileResponse from django.http import Http404 -from django.http import HttpRequest from django.http import HttpResponse from django.shortcuts import render from django.template.defaultfilters import filesizeformat diff --git a/kick/feeds.py b/kick/feeds.py index 6d3685a..36f55b9 100644 --- a/kick/feeds.py +++ b/kick/feeds.py @@ -206,8 +206,8 @@ class KickOrganizationFeed(TTVDropsBaseFeed): def __call__( self, request: HttpRequest, - *args: object, - **kwargs: object, + *args: str | int, + **kwargs: str | int, ) -> HttpResponse: """Capture optional ?limit query parameter. @@ -283,8 +283,8 @@ class KickCategoryFeed(TTVDropsBaseFeed): def __call__( self, request: HttpRequest, - *args: object, - **kwargs: object, + *args: str | int, + **kwargs: str | int, ) -> HttpResponse: """Capture optional ?limit query parameter. @@ -372,8 +372,8 @@ class KickCampaignFeed(TTVDropsBaseFeed): def __call__( self, request: HttpRequest, - *args: object, - **kwargs: object, + *args: str | int, + **kwargs: str | int, ) -> HttpResponse: """Capture optional ?limit query parameter. @@ -481,8 +481,8 @@ class KickCategoryCampaignFeed(TTVDropsBaseFeed): def __call__( self, request: HttpRequest, - *args: object, - **kwargs: object, + *args: str | int, + **kwargs: str | int, ) -> HttpResponse: """Capture optional ?limit query parameter. diff --git a/kick/management/commands/import_kick_drops.py b/kick/management/commands/import_kick_drops.py index 04face1..4116350 100644 --- a/kick/management/commands/import_kick_drops.py +++ b/kick/management/commands/import_kick_drops.py @@ -1,4 +1,7 @@ +from __future__ import annotations + import logging +from datetime import datetime from typing import TYPE_CHECKING import httpx @@ -14,6 +17,8 @@ from kick.models import KickUser from kick.schemas import KickDropsResponseSchema if TYPE_CHECKING: + from collections.abc import Mapping + from django.core.management.base import CommandParser from kick.schemas import KickCategorySchema @@ -23,6 +28,26 @@ if TYPE_CHECKING: logger: logging.Logger = logging.getLogger("ttvdrops") +type KickImportModel = ( + KickOrganization + | KickCategory + | KickDropCampaign + | KickUser + | KickChannel + | KickReward +) +type KickFieldValue = ( + str + | bool + | int + | datetime + | KickOrganization + | KickCategory + | KickDropCampaign + | KickUser + | None +) + KICK_DROPS_API_URL = "https://web.kick.com/api/v1/drops/campaigns" # Kick's public API requires a browser-like User-Agent. @@ -48,7 +73,26 @@ class Command(BaseCommand): help="API endpoint to fetch (default: %(default)s).", ) - def handle(self, *args: object, **options: object) -> None: # noqa: ARG002 + @staticmethod + def _save_if_changed( + obj: KickImportModel, + defaults: Mapping[str, KickFieldValue], + ) -> None: + """Persist only changed fields to avoid unnecessary updates.""" + changed_fields: list[str] = [] + for field, new_value in defaults.items(): + if getattr(obj, field, None) != new_value: + setattr(obj, field, new_value) + changed_fields.append(field) + + if changed_fields: + obj.save(update_fields=changed_fields) + + def handle( + self, + *_args: str, + **options: str | bool | int | None, + ) -> None: """Main entry point for the command.""" url: str = str(options["url"]) self.stdout.write(f"Fetching Kick drops from {url} ...") @@ -99,54 +143,75 @@ class Command(BaseCommand): self.style.SUCCESS(f"Imported {imported}/{len(campaigns)} campaign(s)."), ) - def _import_campaign(self, data: KickDropCampaignSchema) -> None: + def _import_campaign(self, data: KickDropCampaignSchema) -> None: # noqa: PLR0914, PLR0915 """Import a single campaign and all its related objects.""" - # Organisation + # Organization org_data: KickOrganizationSchema = data.organization - org, created = KickOrganization.objects.update_or_create( + org_defaults: dict[str, str | bool] = { + "name": org_data.name, + "logo_url": org_data.logo_url, + "url": org_data.url, + "restricted": org_data.restricted, + } + org: KickOrganization | None = KickOrganization.objects.filter( kick_id=org_data.id, - defaults={ - "name": org_data.name, - "logo_url": org_data.logo_url, - "url": org_data.url, - "restricted": org_data.restricted, - }, - ) + ).first() + created: bool = org is None + if org is None: + org = KickOrganization.objects.create(kick_id=org_data.id, **org_defaults) + else: + self._save_if_changed(org, org_defaults) if created: logger.info("Created new organization: %s", org.kick_id) # Category cat_data: KickCategorySchema = data.category - category, created = KickCategory.objects.update_or_create( + category_defaults: dict[str, KickFieldValue] = { + "name": cat_data.name, + "slug": cat_data.slug, + "image_url": cat_data.image_url, + } + category: KickCategory | None = KickCategory.objects.filter( kick_id=cat_data.id, - defaults={ - "name": cat_data.name, - "slug": cat_data.slug, - "image_url": cat_data.image_url, - }, - ) + ).first() + created = category is None + if category is None: + category = KickCategory.objects.create( + kick_id=cat_data.id, + **category_defaults, + ) + else: + self._save_if_changed(category, category_defaults) if created: logger.info("Created new category: %s", category.kick_id) # Campaign - campaign, created = KickDropCampaign.objects.update_or_create( + campaign_defaults: dict[str, KickFieldValue] = { + "name": data.name, + "status": data.status, + "starts_at": data.starts_at, + "ends_at": data.ends_at, + "connect_url": data.connect_url, + "url": data.url, + "rule_id": data.rule.id, + "rule_name": data.rule.name, + "organization": org, + "category": category, + "created_at": data.created_at, + "api_updated_at": data.updated_at, + "is_fully_imported": True, + } + campaign: KickDropCampaign | None = KickDropCampaign.objects.filter( kick_id=data.id, - defaults={ - "name": data.name, - "status": data.status, - "starts_at": data.starts_at, - "ends_at": data.ends_at, - "connect_url": data.connect_url, - "url": data.url, - "rule_id": data.rule.id, - "rule_name": data.rule.name, - "organization": org, - "category": category, - "created_at": data.created_at, - "api_updated_at": data.updated_at, - "is_fully_imported": True, - }, - ) + ).first() + created = campaign is None + if campaign is None: + campaign = KickDropCampaign.objects.create( + kick_id=data.id, + **campaign_defaults, + ) + else: + self._save_if_changed(campaign, campaign_defaults) if created: logger.info("Created new campaign: %s", campaign.kick_id) @@ -154,25 +219,38 @@ class Command(BaseCommand): channel_objs: list[KickChannel] = [] for ch_data in data.channels: user_data: KickUserSchema = ch_data.user - user, created = KickUser.objects.update_or_create( + user_defaults: dict[str, KickFieldValue] = { + "username": user_data.username, + "profile_picture": user_data.profile_picture, + } + user: KickUser | None = KickUser.objects.filter( kick_id=user_data.id, - defaults={ - "username": user_data.username, - "profile_picture": user_data.profile_picture, - }, - ) + ).first() + created = user is None + if user is None: + user = KickUser.objects.create(kick_id=user_data.id, **user_defaults) + else: + self._save_if_changed(user, user_defaults) if created: logger.info("Created new user: %s", user.kick_id) - channel, created = KickChannel.objects.update_or_create( + channel_defaults: dict[str, KickFieldValue] = { + "slug": ch_data.slug, + "description": ch_data.description, + "banner_picture_url": ch_data.banner_picture_url, + "user": user, + } + channel: KickChannel | None = KickChannel.objects.filter( kick_id=ch_data.id, - defaults={ - "slug": ch_data.slug, - "description": ch_data.description, - "banner_picture_url": ch_data.banner_picture_url, - "user": user, - }, - ) + ).first() + created = channel is None + if channel is None: + channel = KickChannel.objects.create( + kick_id=ch_data.id, + **channel_defaults, + ) + else: + self._save_if_changed(channel, channel_defaults) if created: logger.info("Created new channel: %s", channel.kick_id) @@ -184,36 +262,46 @@ class Command(BaseCommand): # Resolve reward's category (may differ from campaign category) reward_category: KickCategory = category if reward_data.category_id != cat_data.id: - reward_category, created = KickCategory.objects.get_or_create( + reward_category = KickCategory.objects.filter( kick_id=reward_data.category_id, - defaults={"name": "", "slug": "", "image_url": ""}, + ).first() or KickCategory.objects.create( + kick_id=reward_data.category_id, + name="", + slug="", + image_url="", ) + created = not reward_category.name and not reward_category.slug if created: logger.info("Created new category: %s", reward_category.kick_id) # Resolve reward's organization (may differ from campaign org) reward_org: KickOrganization = org if reward_data.organization_id != org_data.id: - reward_org, created = KickOrganization.objects.get_or_create( + reward_org = KickOrganization.objects.filter( kick_id=reward_data.organization_id, - defaults={ - "name": "", - "logo_url": "", - "url": "", - "restricted": False, - }, + ).first() or KickOrganization.objects.create( + kick_id=reward_data.organization_id, + name="", + logo_url="", + url="", + restricted=False, ) + created = not reward_org.name and not reward_org.url if created: logger.info("Created new organization: %s", reward_org.kick_id) - KickReward.objects.update_or_create( + reward_defaults: dict[str, KickFieldValue] = { + "name": reward_data.name, + "image_url": reward_data.image_url, + "required_units": reward_data.required_units, + "campaign": campaign, + "category": reward_category, + "organization": reward_org, + } + reward: KickReward | None = KickReward.objects.filter( kick_id=reward_data.id, - defaults={ - "name": reward_data.name, - "image_url": reward_data.image_url, - "required_units": reward_data.required_units, - "campaign": campaign, - "category": reward_category, - "organization": reward_org, - }, - ) + ).first() + if reward is None: + KickReward.objects.create(kick_id=reward_data.id, **reward_defaults) + else: + self._save_if_changed(reward, reward_defaults) diff --git a/kick/tests/test_kick.py b/kick/tests/test_kick.py index 84428c2..7fb1c43 100644 --- a/kick/tests/test_kick.py +++ b/kick/tests/test_kick.py @@ -669,20 +669,25 @@ class KickDashboardViewTest(TestCase): class KickCampaignListViewTest(TestCase): """Tests for the kick campaign list view.""" + @classmethod + def setUpTestData(cls) -> None: + """Set up shared test data for campaign list view tests.""" + cls.org: KickOrganization = KickOrganization.objects.create( + kick_id="org-list", + name="List Org", + ) + cls.cat: KickCategory = KickCategory.objects.create( + kick_id=300, + name="List Cat", + slug="list-cat", + ) + def _make_campaign( self, kick_id: str, name: str, status: str = "active", ) -> KickDropCampaign: - org, _ = KickOrganization.objects.get_or_create( - kick_id="org-list", - defaults={"name": "List Org"}, - ) - cat, _ = KickCategory.objects.get_or_create( - kick_id=300, - defaults={"name": "List Cat", "slug": "list-cat"}, - ) # Set dates so the active/expired filter works correctly if status == "active": starts_at = dt(2020, 1, 1, tzinfo=UTC) @@ -696,8 +701,8 @@ class KickCampaignListViewTest(TestCase): status=status, starts_at=starts_at, ends_at=ends_at, - organization=org, - category=cat, + organization=self.org, + category=self.cat, rule_id=1, rule_name="Watch to redeem", is_fully_imported=True, diff --git a/pyproject.toml b/pyproject.toml index 9fee185..74d15fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ dev = [ DJANGO_SETTINGS_MODULE = "config.settings" python_files = ["test_*.py", "*_test.py"] addopts = "--tb=short -n auto --cov" +markers = ["no_zeal: run test without zeal_context N+1 checks"] filterwarnings = [ "ignore:Parsing dates involving a day of month without a year specified is ambiguous:DeprecationWarning", ] diff --git a/templates/twitch/dashboard.html b/templates/twitch/dashboard.html index 3d7aa55..5e6f7cd 100644 --- a/templates/twitch/dashboard.html +++ b/templates/twitch/dashboard.html @@ -69,7 +69,7 @@
{% picture campaign_data.image_url alt="Image for "|add:campaign_data.campaign.name width=120 %} -

{{ campaign_data.campaign.clean_name }}

+

{{ campaign_data.clean_name }}