From 28dbab57c4e9f0f198aa8b644fa2866c50b9e3be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Hells=C3=A9n?= Date: Tue, 5 Aug 2025 02:07:02 +0200 Subject: [PATCH] Improve importer --- templates/twitch/dashboard.html | 56 ++++++----- twitch/management/commands/import_drops.py | 112 ++++++++++++++------- twitch/migrations/0005_game_box_art.py | 18 ++++ twitch/models.py | 1 + 4 files changed, 129 insertions(+), 58 deletions(-) create mode 100644 twitch/migrations/0005_game_box_art.py diff --git a/templates/twitch/dashboard.html b/templates/twitch/dashboard.html index fbebae1..2f672a7 100644 --- a/templates/twitch/dashboard.html +++ b/templates/twitch/dashboard.html @@ -4,29 +4,39 @@ Dashboard {% endblock title %} {% block content %} - {% if campaigns_by_org_game %} - {% for org_id, org_data in campaigns_by_org_game.items %} - {% for game_id, game_data in org_data.games.items %} -

- {{ game_data.name }} -

- {% for campaign in game_data.campaigns %} -

{{ campaign.clean_name }}

-

-

- - {{ campaign.name }} - -

- Ends in {{ campaign.end_at|timeuntil }} -

+
+ {% if campaigns_by_org_game %} + {% for org_id, org_data in campaigns_by_org_game.items %} + {% for game_id, game_data in org_data.games.items %} +
+
+

+ {{ game_data.name }} +

+
+
+ {% for campaign in game_data.campaigns %} +
+ + Image for {{ campaign.name }} + +

{{ campaign.clean_name }}

+ +
+ {% endfor %} +
+
{% endfor %} {% endfor %} - {% endfor %} - {% else %} - No active campaigns at the moment. - {% endif %} + {% else %} +

No active campaigns at the moment.

+ {% endif %} +
{% endblock content %} diff --git a/twitch/management/commands/import_drops.py b/twitch/management/commands/import_drops.py index e01c7fb..1510eb3 100644 --- a/twitch/management/commands/import_drops.py +++ b/twitch/management/commands/import_drops.py @@ -79,9 +79,9 @@ class Command(BaseCommand): self.stdout.write(self.style.ERROR(f"Error processing {json_file}: {e}")) except (orjson.JSONDecodeError, json.JSONDecodeError): broken_json_dir: Path = processed_path / "broken_json" - broken_json_dir.mkdir(exist_ok=True) + broken_json_dir.mkdir(parents=True, exist_ok=True) self.stdout.write(self.style.WARNING(f"Invalid JSON in '{json_file}'. Moving to '{broken_json_dir}'.")) - shutil.move(str(json_file), str(broken_json_dir)) + self.move_file(json_file, broken_json_dir / json_file.name) except (ValueError, TypeError, AttributeError, KeyError, IndexError): self.stdout.write(self.style.ERROR(f"Data error processing {json_file}")) self.stdout.write(self.style.ERROR(traceback.format_exc())) @@ -95,47 +95,88 @@ class Command(BaseCommand): Args: file_path: Path to the JSON file. processed_path: Subdirectory to move processed files to. - - Raises: - CommandError: If the file isn't a JSON file or has invalid JSON structure. """ data = orjson.loads(file_path.read_text(encoding="utf-8")) broken_dir: Path = processed_path / "broken" + broken_dir.mkdir(parents=True, exist_ok=True) - # Remove shit - if not isinstance(data, list): - try: - token = data["data"]["streamPlaybackAccessToken"] - if token["__typename"] == "PlaybackAccessToken" and len(data["data"]) == 1: - shutil.move(str(file_path), str(broken_dir)) - self.stdout.write(f"Moved {file_path} to {broken_dir}. This file only contains PlaybackAccessToken data.") - return + probably_shit: list[str] = [ + "ChannelPointsContext", + "DropCurrentSessionContext", + "DropsHighlightService_AvailableDrops", + "DropsPage_ClaimDropRewards", + "Inventory", + "PlaybackAccessToken", + "streamPlaybackAccessToken", + "VideoPlayerStreamInfoOverlayChannel", + "OnsiteNotifications_DeleteNotification", + "DirectoryPage_Game", + ] - if data["extensions"]["operationName"] == "PlaybackAccessToken" and ("data" not in data or len(data["data"]) <= 1): - shutil.move(str(file_path), str(broken_dir)) - self.stdout.write(f"Moved {file_path} to {broken_dir}. This file only contains PlaybackAccessToken data.") - return - except KeyError: + for keyword in probably_shit: + if f'"operationName": "{keyword}"' in file_path.read_text(): + target_dir: Path = broken_dir / keyword + target_dir.mkdir(parents=True, exist_ok=True) + + self.stdout.write(msg=f"Trying to move {file_path!s} to {target_dir / file_path.name!s}") + + self.move_file(file_path, target_dir / file_path.name) + self.stdout.write(f"Moved {file_path} to {target_dir} (matched '{keyword}')") return - # Move DropsHighlightService_AvailableDrops to its own dir - # TODO(TheLovinator): Check if we should import this # noqa: TD003 - if isinstance(data, list): - for item in data: - drop_campaign_data = item["data"]["user"]["dropCampaign"] - self._import_drop_campaign_with_retry(drop_campaign_data) + for _item in data: + self.import_drop_campaign(_item) else: - if "data" not in data or "user" not in data["data"] or "dropCampaign" not in data["data"]["user"]: - msg = "Invalid JSON structure: Missing data.user.dropCampaign" - raise CommandError(msg) + self.import_drop_campaign(data) + self.move_file(file_path, processed_path) + + def move_file(self, file_path: Path, processed_path: Path) -> None: + """Move file and check if already exists.""" + try: + shutil.move(str(file_path), str(processed_path)) + except FileExistsError: + # Rename the file if contents is different than the existing one + with file_path.open("rb") as f1, (processed_path / file_path.name).open("rb") as f2: + if f1.read() != f2.read(): + new_name: Path = processed_path / f"{file_path.stem}_duplicate{file_path.suffix}" + shutil.move(str(file_path), str(new_name)) + self.stdout.write(f"Moved {file_path!s} to {new_name!s} (content differs)") + else: + self.stdout.write(f"{file_path!s} already exists in {processed_path!s}, removing original file.") + file_path.unlink() + except (PermissionError, OSError, shutil.Error) as e: + self.stdout.write(self.style.ERROR(f"Error moving {file_path!s} to {processed_path!s}: {e}")) + traceback.print_exc() + + def import_drop_campaign(self, data: dict[str, Any]) -> None: + """Find the key with all the data. + + Args: + data: The JSON data. + + Raises: + CommandError: If the JSON structure is invalid. + """ + if "data" not in data: + msg = "Invalid JSON structure: Missing top-level 'data'" + raise CommandError(msg) + + if "user" in data["data"] and "dropCampaign" in data["data"]["user"]: drop_campaign_data = data["data"]["user"]["dropCampaign"] - self._import_drop_campaign_with_retry(drop_campaign_data) + self.import_to_db(drop_campaign_data) - shutil.move(str(file_path), str(processed_path)) + elif "currentUser" in data["data"] and "dropCampaigns" in data["data"]["currentUser"]: + campaigns = data["data"]["currentUser"]["dropCampaigns"] + for drop_campaign_data in campaigns: + self.import_to_db(drop_campaign_data) - def _import_drop_campaign_with_retry(self, campaign_data: dict[str, Any]) -> None: + else: + msg = "Invalid JSON structure: Missing either data.user.dropCampaign or data.currentUser.dropCampaigns" + raise CommandError(msg) + + def import_to_db(self, campaign_data: dict[str, Any]) -> None: """Import drop campaign data into the database with retry logic for SQLite locks. Args: @@ -204,14 +245,14 @@ class Command(BaseCommand): drop_campaign, created = DropCampaign.objects.update_or_create( id=campaign_data["id"], defaults={ - "name": campaign_data["name"], - "description": campaign_data["description"].replace("\\n", "\n"), + "name": campaign_data.get("name", ""), + "description": campaign_data.get("description", "").replace("\\n", "\n"), "details_url": campaign_data.get("detailsURL", ""), "account_link_url": campaign_data.get("accountLinkURL", ""), "image_url": campaign_data.get("imageURL", ""), - "start_at": campaign_data["startAt"], - "end_at": campaign_data["endAt"], - "is_account_connected": campaign_data["self"]["isAccountConnected"], + "start_at": campaign_data.get("startAt"), + "end_at": campaign_data.get("endAt"), + "is_account_connected": campaign_data.get("self", {}).get("isAccountConnected", False), "game": game, "owner": organization, }, @@ -253,6 +294,7 @@ class Command(BaseCommand): defaults={ "slug": game_data.get("slug", ""), "display_name": game_data["displayName"], + "box_art": game_data.get("boxArtURL", ""), }, ) if created: diff --git a/twitch/migrations/0005_game_box_art.py b/twitch/migrations/0005_game_box_art.py new file mode 100644 index 0000000..f5a5fe6 --- /dev/null +++ b/twitch/migrations/0005_game_box_art.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.4 on 2025-08-04 03:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('twitch', '0004_alter_dropbenefit_distribution_type'), + ] + + operations = [ + migrations.AddField( + model_name='game', + name='box_art', + field=models.URLField(blank=True, default='', max_length=500), + ), + ] diff --git a/twitch/models.py b/twitch/models.py index 407d865..27ee938 100644 --- a/twitch/models.py +++ b/twitch/models.py @@ -14,6 +14,7 @@ class Game(models.Model): id = models.TextField(primary_key=True) slug = models.TextField(blank=True, default="", db_index=True) display_name = models.TextField(db_index=True) + box_art = models.URLField(max_length=500, blank=True, default="") class Meta: indexes: ClassVar[list] = [