diff --git a/core/views.py b/core/views.py index d5d2dc3..4c3000c 100644 --- a/core/views.py +++ b/core/views.py @@ -352,7 +352,9 @@ def sitemap_twitch_drops_view(request: HttpRequest) -> HttpResponse: base_url: str = _build_base_url(request) sitemap_urls: list[dict[str, str]] = [] - campaigns: QuerySet[DropCampaign] = DropCampaign.objects.all() + campaigns: QuerySet[DropCampaign] = DropCampaign.objects.filter( + is_fully_imported=True, + ) for campaign in campaigns: resource_url: str = reverse("twitch:campaign_detail", args=[campaign.twitch_id]) full_url: str = f"{base_url}{resource_url}" @@ -442,7 +444,9 @@ def sitemap_kick_view(request: HttpRequest) -> HttpResponse: base_url: str = _build_base_url(request) sitemap_urls: list[dict[str, str]] = [] - kick_campaigns: QuerySet[KickDropCampaign] = KickDropCampaign.objects.all() + kick_campaigns: QuerySet[KickDropCampaign] = KickDropCampaign.objects.filter( + is_fully_imported=True, + ) for campaign in kick_campaigns: resource_url: str = reverse("kick:campaign_detail", args=[campaign.kick_id]) full_url: str = f"{base_url}{resource_url}" diff --git a/kick/management/commands/import_kick_drops.py b/kick/management/commands/import_kick_drops.py index ab3ff68..04face1 100644 --- a/kick/management/commands/import_kick_drops.py +++ b/kick/management/commands/import_kick_drops.py @@ -144,6 +144,7 @@ class Command(BaseCommand): "category": category, "created_at": data.created_at, "api_updated_at": data.updated_at, + "is_fully_imported": True, }, ) if created: diff --git a/kick/migrations/0002_kickdropcampaign_is_fully_imported.py b/kick/migrations/0002_kickdropcampaign_is_fully_imported.py new file mode 100644 index 0000000..f847cbb --- /dev/null +++ b/kick/migrations/0002_kickdropcampaign_is_fully_imported.py @@ -0,0 +1,23 @@ +# Generated by Django 6.0.3 on 2026-03-19 19:20 + +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + """Add is_fully_imported field to KickDropCampaign to track import status.""" + + dependencies = [ + ("kick", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="kickdropcampaign", + name="is_fully_imported", + field=models.BooleanField( + default=False, + help_text="True if all images and formats are imported and ready for display.", + ), + ), + ] diff --git a/kick/models.py b/kick/models.py index 7f6ff01..1c2b4c0 100644 --- a/kick/models.py +++ b/kick/models.py @@ -260,6 +260,10 @@ class KickDropCampaign(auto_prefetch.Model): ) added_at = models.DateTimeField(auto_now_add=True, editable=False) updated_at = models.DateTimeField(auto_now=True, editable=False) + is_fully_imported = models.BooleanField( + default=False, + help_text="True if all images and formats are imported and ready for display.", + ) class Meta(auto_prefetch.Model.Meta): ordering = ["-starts_at"] diff --git a/kick/tests/test_kick.py b/kick/tests/test_kick.py index 32e1adf..357f604 100644 --- a/kick/tests/test_kick.py +++ b/kick/tests/test_kick.py @@ -1,3 +1,4 @@ +import datetime import re from datetime import UTC from datetime import datetime as dt @@ -11,6 +12,7 @@ from unittest.mock import patch import httpx import pytest from django.core.management import call_command +from django.test import Client from django.test import TestCase from django.urls import reverse from django.utils import timezone @@ -28,6 +30,7 @@ from kick.schemas import KickDropsResponseSchema if TYPE_CHECKING: from django.test.client import _MonkeyPatchedWSGIResponse + from pytest_django.asserts import QuerySet from kick.schemas import KickDropCampaignSchema from kick.schemas import KickRewardSchema @@ -525,6 +528,7 @@ class KickDashboardViewTest(TestCase): category=cat, rule_id=1, rule_name="Watch to redeem", + is_fully_imported=True, ) def test_dashboard_returns_200(self) -> None: @@ -577,6 +581,7 @@ class KickCampaignListViewTest(TestCase): category=cat, rule_id=1, rule_name="Watch to redeem", + is_fully_imported=True, ) def test_campaign_list_returns_200(self) -> None: @@ -935,3 +940,105 @@ class KickFeedsTest(TestCase): assert result.startswith("<t:") assert result.endswith(":R>") assert not str(discord_timestamp(None)) + + +class KickDropCampaignFullyImportedTest(TestCase): + """Tests for KickDropCampaign.is_fully_imported field and filtering.""" + + def setUp(self) -> None: + """Create common org and category fixtures for campaign import tests.""" + self.org: KickOrganization = KickOrganization.objects.create( + kick_id="org-fi", + name="Org", + ) + self.cat: KickCategory = KickCategory.objects.create( + kick_id=1, + name="Cat", + slug="cat", + ) + + def test_campaign_not_fully_imported_by_default(self) -> None: + """By default, a newly created campaign should have is_fully_imported set to False.""" + campaign: KickDropCampaign = KickDropCampaign.objects.create( + kick_id="camp-fi-1", + name="Not Imported", + organization=self.org, + category=self.cat, + rule_id=1, + rule_name="Rule", + ) + assert campaign.is_fully_imported is False + + def test_campaign_fully_imported_flag(self) -> None: + """When creating a campaign with is_fully_imported=True, the flag should be set correctly.""" + campaign: KickDropCampaign = KickDropCampaign.objects.create( + kick_id="camp-fi-2", + name="Imported", + organization=self.org, + category=self.cat, + rule_id=1, + rule_name="Rule", + is_fully_imported=True, + ) + assert campaign.is_fully_imported is True + + def test_queryset_filters_only_fully_imported(self) -> None: + """Filtering campaigns by is_fully_imported should return only those with the flag set to True.""" + KickDropCampaign.objects.create( + kick_id="camp-fi-3", + name="Not Imported", + organization=self.org, + category=self.cat, + rule_id=1, + rule_name="Rule", + ) + imported: KickDropCampaign = KickDropCampaign.objects.create( + kick_id="camp-fi-4", + name="Imported", + organization=self.org, + category=self.cat, + rule_id=1, + rule_name="Rule", + is_fully_imported=True, + ) + qs: QuerySet[KickDropCampaign, KickDropCampaign] = ( + KickDropCampaign.objects.filter(is_fully_imported=True) + ) + assert list(qs) == [imported] + + def test_dashboard_view_only_shows_fully_imported(self) -> None: + """Dashboard view should only show fully imported and active campaigns.""" + now: dt = timezone.now() + + # Not imported, but active + KickDropCampaign.objects.create( + kick_id="camp-fi-5", + name="Not Imported", + organization=self.org, + category=self.cat, + rule_id=1, + rule_name="Rule", + starts_at=now - datetime.timedelta(days=1), + ends_at=now + datetime.timedelta(days=1), + ) + + # Imported and active + imported: KickDropCampaign = KickDropCampaign.objects.create( + kick_id="camp-fi-6", + name="Imported", + organization=self.org, + category=self.cat, + rule_id=1, + rule_name="Rule", + is_fully_imported=True, + starts_at=now - datetime.timedelta(days=1), + ends_at=now + datetime.timedelta(days=1), + ) + + client = Client() + response: _MonkeyPatchedWSGIResponse = client.get(reverse("kick:dashboard")) + assert response.status_code == 200 + campaigns: QuerySet[KickDropCampaign, KickDropCampaign] = response.context[ + "active_campaigns" + ] + assert list(campaigns) == [imported] diff --git a/kick/views.py b/kick/views.py index 474806e..be7df1c 100644 --- a/kick/views.py +++ b/kick/views.py @@ -155,7 +155,7 @@ def dashboard(request: HttpRequest) -> HttpResponse: now: datetime.datetime = timezone.now() active_campaigns: QuerySet[KickDropCampaign] = ( KickDropCampaign.objects - .filter(starts_at__lte=now, ends_at__gte=now) + .filter(starts_at__lte=now, ends_at__gte=now, is_fully_imported=True) .select_related("organization", "category") .prefetch_related("channels__user", "rewards") .order_by("-starts_at") @@ -193,6 +193,7 @@ def campaign_list_view(request: HttpRequest) -> HttpResponse: queryset: QuerySet[KickDropCampaign] = ( KickDropCampaign.objects + .filter(is_fully_imported=True) .select_related("organization", "category") .prefetch_related("rewards") .order_by("-starts_at") diff --git a/twitch/management/commands/better_import_drops.py b/twitch/management/commands/better_import_drops.py index 0ec3771..b72a314 100644 --- a/twitch/management/commands/better_import_drops.py +++ b/twitch/management/commands/better_import_drops.py @@ -1,6 +1,7 @@ import json import os import sys +from collections.abc import Mapping from datetime import UTC from datetime import datetime from pathlib import Path @@ -15,6 +16,7 @@ from colorama import Fore from colorama import Style from colorama import init as colorama_init from django.core.files.base import ContentFile +from django.core.management import call_command from django.core.management.base import BaseCommand from django.core.management.base import CommandError from pydantic import ValidationError @@ -35,6 +37,8 @@ from twitch.utils import normalize_twitch_box_art_url from twitch.utils import parse_date if TYPE_CHECKING: + from collections.abc import Mapping + from django.core.management.base import CommandParser from django.db.models import Model from json_repair import JSONReturnType @@ -545,7 +549,7 @@ class Command(BaseCommand): return valid_responses, broken_dir - def _save_if_changed(self, obj: Model, defaults: dict[str, object]) -> bool: + def _save_if_changed(self, obj: Model, defaults: Mapping[str, object]) -> bool: """Save the model instance only when data actually changed. This prevents unnecessary updates and avoids touching fields like @@ -618,7 +622,7 @@ class Command(BaseCommand): if campaign_org_obj: owner_orgs.add(campaign_org_obj) - defaults: dict[str, str] = { + defaults: dict[str, object] = { "display_name": game_data.display_name or (game_data.name or ""), "name": game_data.name or "", "slug": game_data.slug or "", @@ -681,7 +685,7 @@ class Command(BaseCommand): # Use name as display_name fallback if displayName is None display_name: str = channel_info.display_name or channel_info.name - defaults: dict[str, str] = { + defaults: dict[str, object] = { "name": channel_info.name, "display_name": display_name, } @@ -698,7 +702,7 @@ class Command(BaseCommand): return channel_obj - def process_responses( + def process_responses( # noqa: PLR0915 self, responses: list[dict[str, Any]], file_path: Path, @@ -778,7 +782,7 @@ class Command(BaseCommand): raise ValueError(msg) continue - defaults: dict[str, str | datetime | Game | bool] = { + defaults: dict[str, object] = { "name": drop_campaign.name, "description": drop_campaign.description, "image_url": drop_campaign.image_url, @@ -800,6 +804,16 @@ class Command(BaseCommand): f"{Fore.GREEN}✓{Style.RESET_ALL} Created new campaign: {drop_campaign.name}", ) + # Always run additional commands after import + call_command("download_box_art") + call_command("download_campaign_images") + call_command("convert_images_to_modern_formats") + + # After all downloads and processing, mark as fully imported + if campaign_obj: + campaign_obj.is_fully_imported = True + campaign_obj.save(update_fields=["is_fully_imported"]) + action: Literal["Imported new", "Updated"] = ( "Imported new" if created else "Updated" ) @@ -1026,7 +1040,7 @@ class Command(BaseCommand): f"{Fore.YELLOW}→{Style.RESET_ALL} Game not found for reward campaign: {game_id}", ) - defaults: dict[str, str | datetime | Game | bool | None] = { + defaults: dict[str, object] = { "name": reward_campaign.name, "brand": reward_campaign.brand, "starts_at": starts_at_dt, @@ -1084,6 +1098,7 @@ class Command(BaseCommand): else: msg: str = f"Path does not exist: {input_path}" raise CommandError(msg) + except KeyboardInterrupt: tqdm.write(self.style.WARNING("\n\nInterrupted by user!")) tqdm.write(self.style.WARNING("Shutting down gracefully...")) diff --git a/twitch/migrations/0015_dropcampaign_is_fully_imported.py b/twitch/migrations/0015_dropcampaign_is_fully_imported.py new file mode 100644 index 0000000..1e1ce1c --- /dev/null +++ b/twitch/migrations/0015_dropcampaign_is_fully_imported.py @@ -0,0 +1,23 @@ +# Generated by Django 6.0.3 on 2026-03-19 19:20 + +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + """Add a new field `is_fully_imported` to the `DropCampaign` model to track whether all related images and formats have been imported and are ready for display. This field will help us filter out campaigns that are not yet fully imported in our views and APIs, ensuring that only complete campaigns are shown to users.""" + + dependencies = [ + ("twitch", "0014_dropcampaign_image_mime_type_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="dropcampaign", + name="is_fully_imported", + field=models.BooleanField( + default=False, + help_text="True if all images and formats are imported and ready for display.", + ), + ), + ] diff --git a/twitch/migrations/0016_mark_all_drops_fully_imported.py b/twitch/migrations/0016_mark_all_drops_fully_imported.py new file mode 100644 index 0000000..9fd5630 --- /dev/null +++ b/twitch/migrations/0016_mark_all_drops_fully_imported.py @@ -0,0 +1,25 @@ +from django.db import migrations + + +def mark_all_drops_fully_imported(apps, schema_editor) -> None: # noqa: ANN001, ARG001 + """Marks all existing DropCampaigns as fully imported. + + This was needed to ensure that the Twitch API view only returns campaigns that are ready for display. + """ + DropCampaign = apps.get_model("twitch", "DropCampaign") + DropCampaign.objects.all().update(is_fully_imported=True) + + +class Migration(migrations.Migration): + """Marks all existing DropCampaigns as fully imported. + + This was needed to ensure that the Twitch API view only returns campaigns that are ready for display. + """ + + dependencies = [ + ("twitch", "0015_dropcampaign_is_fully_imported"), + ] + + operations = [ + migrations.RunPython(mark_all_drops_fully_imported), + ] diff --git a/twitch/models.py b/twitch/models.py index 0c2030e..c148423 100644 --- a/twitch/models.py +++ b/twitch/models.py @@ -429,6 +429,10 @@ class DropCampaign(auto_prefetch.Model): auto_now=True, help_text="Timestamp when this campaign record was last updated.", ) + is_fully_imported = models.BooleanField( + default=False, + help_text="True if all images and formats are imported and ready for display.", + ) class Meta(auto_prefetch.Model.Meta): ordering = ["-start_at"] diff --git a/twitch/tests/test_views.py b/twitch/tests/test_views.py index c88e555..159a8a4 100644 --- a/twitch/tests/test_views.py +++ b/twitch/tests/test_views.py @@ -566,6 +566,7 @@ class TestChannelListView: start_at=now - timedelta(days=10), end_at=now + timedelta(days=10), operation_names=["DropCampaignDetails"], + is_fully_imported=True, ) for i in range(150) ] @@ -609,6 +610,7 @@ class TestChannelListView: start_at=now - timedelta(days=5), end_at=now + timedelta(days=5), operation_names=["DropCampaignDetails"], + is_fully_imported=True, ) # Create upcoming campaign @@ -619,6 +621,7 @@ class TestChannelListView: start_at=now + timedelta(days=5), end_at=now + timedelta(days=10), operation_names=["DropCampaignDetails"], + is_fully_imported=True, ) # Create expired campaign @@ -629,6 +632,7 @@ class TestChannelListView: start_at=now - timedelta(days=10), end_at=now - timedelta(days=5), operation_names=["DropCampaignDetails"], + is_fully_imported=True, ) # Test active filter @@ -648,7 +652,7 @@ class TestChannelListView: name="Game", display_name="Game", ) - now: datetime.datetime = timezone.now() + now = timezone.now() # Create active campaign DropCampaign.objects.create( @@ -658,16 +662,18 @@ class TestChannelListView: start_at=now - timedelta(days=5), end_at=now + timedelta(days=5), operation_names=["DropCampaignDetails"], + is_fully_imported=True, ) # Create upcoming campaign - _upcoming_campaign: DropCampaign = DropCampaign.objects.create( + DropCampaign.objects.create( twitch_id="upcoming", name="Upcoming Campaign", game=game, - start_at=now + timedelta(days=5), + start_at=now + timedelta(days=1), end_at=now + timedelta(days=10), operation_names=["DropCampaignDetails"], + is_fully_imported=True, ) # Create expired campaign @@ -678,6 +684,7 @@ class TestChannelListView: start_at=now - timedelta(days=10), end_at=now - timedelta(days=5), operation_names=["DropCampaignDetails"], + is_fully_imported=True, ) # Test upcoming filter @@ -707,6 +714,7 @@ class TestChannelListView: start_at=now - timedelta(days=5), end_at=now + timedelta(days=5), operation_names=["DropCampaignDetails"], + is_fully_imported=True, ) # Create upcoming campaign @@ -717,6 +725,7 @@ class TestChannelListView: start_at=now + timedelta(days=5), end_at=now + timedelta(days=10), operation_names=["DropCampaignDetails"], + is_fully_imported=True, ) # Create expired campaign @@ -727,6 +736,7 @@ class TestChannelListView: start_at=now - timedelta(days=10), end_at=now - timedelta(days=5), operation_names=["DropCampaignDetails"], + is_fully_imported=True, ) # Test expired filter @@ -761,6 +771,7 @@ class TestChannelListView: start_at=now - timedelta(days=5), end_at=now + timedelta(days=5), operation_names=["DropCampaignDetails"], + is_fully_imported=True, ) DropCampaign.objects.create( twitch_id="c2", @@ -769,6 +780,7 @@ class TestChannelListView: start_at=now - timedelta(days=5), end_at=now + timedelta(days=5), operation_names=["DropCampaignDetails"], + is_fully_imported=True, ) # Create campaign for game 2 @@ -779,6 +791,7 @@ class TestChannelListView: start_at=now - timedelta(days=5), end_at=now + timedelta(days=5), operation_names=["DropCampaignDetails"], + is_fully_imported=True, ) # Test filtering by game1 @@ -819,6 +832,7 @@ class TestChannelListView: start_at=now - timedelta(days=5), end_at=now + timedelta(days=5), operation_names=["DropCampaignDetails"], + is_fully_imported=True, ) for i in range(150) ] @@ -1337,6 +1351,7 @@ class TestSitemapView: operation_names=["DropCampaignDetails"], start_at=now - datetime.timedelta(days=1), end_at=now + datetime.timedelta(days=1), + is_fully_imported=True, ) inactive_campaign: DropCampaign = DropCampaign.objects.create( twitch_id="camp2", @@ -1346,6 +1361,7 @@ class TestSitemapView: operation_names=["DropCampaignDetails"], start_at=now - datetime.timedelta(days=10), end_at=now - datetime.timedelta(days=5), + is_fully_imported=True, ) kick_org: KickOrganization = KickOrganization.objects.create( @@ -1364,6 +1380,7 @@ class TestSitemapView: category=kick_cat, starts_at=now - datetime.timedelta(days=1), ends_at=now + datetime.timedelta(days=1), + is_fully_imported=True, ) kick_inactive: KickDropCampaign = KickDropCampaign.objects.create( kick_id="kcamp2", @@ -1372,6 +1389,7 @@ class TestSitemapView: category=kick_cat, starts_at=now - datetime.timedelta(days=10), ends_at=now - datetime.timedelta(days=5), + is_fully_imported=True, ) badge: ChatBadgeSet = ChatBadgeSet.objects.create(set_id="badge1") @@ -1707,6 +1725,7 @@ class TestSEOPaginationLinks: description="Desc", game=game, operation_names=["DropCampaignDetails"], + is_fully_imported=True, ) response = client.get(reverse("twitch:campaign_list")) @@ -1731,6 +1750,7 @@ class TestSEOPaginationLinks: description="Desc", game=game, operation_names=["DropCampaignDetails"], + is_fully_imported=True, ) response = client.get(reverse("twitch:campaign_list")) diff --git a/twitch/views.py b/twitch/views.py index 2b4df36..595bd42 100644 --- a/twitch/views.py +++ b/twitch/views.py @@ -430,7 +430,9 @@ def drop_campaign_list_view(request: HttpRequest) -> HttpResponse: # noqa: PLR0 game_filter: str | None = request.GET.get("game") status_filter: str | None = request.GET.get("status") per_page: int = 100 - queryset: QuerySet[DropCampaign] = DropCampaign.objects.all() + queryset: QuerySet[DropCampaign] = DropCampaign.objects.filter( + is_fully_imported=True, + ) if game_filter: queryset = queryset.filter(game__twitch_id=game_filter)