Improve importer
This commit is contained in:
parent
3d447e08ff
commit
28dbab57c4
4 changed files with 131 additions and 60 deletions
|
|
@ -4,29 +4,39 @@
|
|||
Dashboard
|
||||
{% endblock title %}
|
||||
{% block content %}
|
||||
<main>
|
||||
{% if campaigns_by_org_game %}
|
||||
{% for org_id, org_data in campaigns_by_org_game.items %}
|
||||
{% for game_id, game_data in org_data.games.items %}
|
||||
<h1>
|
||||
<section>
|
||||
<header>
|
||||
<h2>
|
||||
<a href="{% url 'twitch:game_detail' game_id %}">{{ game_data.name }}</a>
|
||||
</h1>
|
||||
</h2>
|
||||
</header>
|
||||
<div style="display: flex; flex-wrap: wrap; gap: 1.5rem;">
|
||||
{% for campaign in game_data.campaigns %}
|
||||
<h2>{{ campaign.clean_name }}</h2>
|
||||
<p>
|
||||
<h4>
|
||||
<article style="flex: 0 1 300px;">
|
||||
<a href="{% url 'twitch:campaign_detail' campaign.id %}">
|
||||
<img height="160"
|
||||
<img src="{{ campaign.image_url }}"
|
||||
alt="Image for {{ campaign.name }}"
|
||||
width="160"
|
||||
src="{{ campaign.image_url }}"
|
||||
alt="{{ campaign.name }}">
|
||||
height="160"
|
||||
style="border-radius: 0.5rem">
|
||||
</a>
|
||||
</h4>
|
||||
<span title="{{ campaign.end_at|date:'c' }}">Ends in {{ campaign.end_at|timeuntil }}</span>
|
||||
</p>
|
||||
<h3>{{ campaign.clean_name }}</h3>
|
||||
<time datetime="{{ campaign.end_at|date:'c' }}"
|
||||
title="{{ campaign.end_at|date:'DATETIME_FORMAT' }}">
|
||||
Ends in {{ campaign.end_at|timeuntil }}
|
||||
</time>
|
||||
</article>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</section>
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
No active campaigns at the moment.
|
||||
<p>No active campaigns at the moment.</p>
|
||||
{% endif %}
|
||||
</main>
|
||||
{% endblock content %}
|
||||
|
|
|
|||
|
|
@ -79,9 +79,9 @@ class Command(BaseCommand):
|
|||
self.stdout.write(self.style.ERROR(f"Error processing {json_file}: {e}"))
|
||||
except (orjson.JSONDecodeError, json.JSONDecodeError):
|
||||
broken_json_dir: Path = processed_path / "broken_json"
|
||||
broken_json_dir.mkdir(exist_ok=True)
|
||||
broken_json_dir.mkdir(parents=True, exist_ok=True)
|
||||
self.stdout.write(self.style.WARNING(f"Invalid JSON in '{json_file}'. Moving to '{broken_json_dir}'."))
|
||||
shutil.move(str(json_file), str(broken_json_dir))
|
||||
self.move_file(json_file, broken_json_dir / json_file.name)
|
||||
except (ValueError, TypeError, AttributeError, KeyError, IndexError):
|
||||
self.stdout.write(self.style.ERROR(f"Data error processing {json_file}"))
|
||||
self.stdout.write(self.style.ERROR(traceback.format_exc()))
|
||||
|
|
@ -95,47 +95,88 @@ class Command(BaseCommand):
|
|||
Args:
|
||||
file_path: Path to the JSON file.
|
||||
processed_path: Subdirectory to move processed files to.
|
||||
|
||||
Raises:
|
||||
CommandError: If the file isn't a JSON file or has invalid JSON structure.
|
||||
"""
|
||||
data = orjson.loads(file_path.read_text(encoding="utf-8"))
|
||||
broken_dir: Path = processed_path / "broken"
|
||||
broken_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Remove shit
|
||||
if not isinstance(data, list):
|
||||
try:
|
||||
token = data["data"]["streamPlaybackAccessToken"]
|
||||
if token["__typename"] == "PlaybackAccessToken" and len(data["data"]) == 1:
|
||||
shutil.move(str(file_path), str(broken_dir))
|
||||
self.stdout.write(f"Moved {file_path} to {broken_dir}. This file only contains PlaybackAccessToken data.")
|
||||
return
|
||||
probably_shit: list[str] = [
|
||||
"ChannelPointsContext",
|
||||
"DropCurrentSessionContext",
|
||||
"DropsHighlightService_AvailableDrops",
|
||||
"DropsPage_ClaimDropRewards",
|
||||
"Inventory",
|
||||
"PlaybackAccessToken",
|
||||
"streamPlaybackAccessToken",
|
||||
"VideoPlayerStreamInfoOverlayChannel",
|
||||
"OnsiteNotifications_DeleteNotification",
|
||||
"DirectoryPage_Game",
|
||||
]
|
||||
|
||||
if data["extensions"]["operationName"] == "PlaybackAccessToken" and ("data" not in data or len(data["data"]) <= 1):
|
||||
shutil.move(str(file_path), str(broken_dir))
|
||||
self.stdout.write(f"Moved {file_path} to {broken_dir}. This file only contains PlaybackAccessToken data.")
|
||||
return
|
||||
except KeyError:
|
||||
return
|
||||
for keyword in probably_shit:
|
||||
if f'"operationName": "{keyword}"' in file_path.read_text():
|
||||
target_dir: Path = broken_dir / keyword
|
||||
target_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Move DropsHighlightService_AvailableDrops to its own dir
|
||||
# TODO(TheLovinator): Check if we should import this # noqa: TD003
|
||||
self.stdout.write(msg=f"Trying to move {file_path!s} to {target_dir / file_path.name!s}")
|
||||
|
||||
self.move_file(file_path, target_dir / file_path.name)
|
||||
self.stdout.write(f"Moved {file_path} to {target_dir} (matched '{keyword}')")
|
||||
return
|
||||
|
||||
if isinstance(data, list):
|
||||
for item in data:
|
||||
drop_campaign_data = item["data"]["user"]["dropCampaign"]
|
||||
self._import_drop_campaign_with_retry(drop_campaign_data)
|
||||
for _item in data:
|
||||
self.import_drop_campaign(_item)
|
||||
else:
|
||||
if "data" not in data or "user" not in data["data"] or "dropCampaign" not in data["data"]["user"]:
|
||||
msg = "Invalid JSON structure: Missing data.user.dropCampaign"
|
||||
self.import_drop_campaign(data)
|
||||
|
||||
self.move_file(file_path, processed_path)
|
||||
|
||||
def move_file(self, file_path: Path, processed_path: Path) -> None:
|
||||
"""Move file and check if already exists."""
|
||||
try:
|
||||
shutil.move(str(file_path), str(processed_path))
|
||||
except FileExistsError:
|
||||
# Rename the file if contents is different than the existing one
|
||||
with file_path.open("rb") as f1, (processed_path / file_path.name).open("rb") as f2:
|
||||
if f1.read() != f2.read():
|
||||
new_name: Path = processed_path / f"{file_path.stem}_duplicate{file_path.suffix}"
|
||||
shutil.move(str(file_path), str(new_name))
|
||||
self.stdout.write(f"Moved {file_path!s} to {new_name!s} (content differs)")
|
||||
else:
|
||||
self.stdout.write(f"{file_path!s} already exists in {processed_path!s}, removing original file.")
|
||||
file_path.unlink()
|
||||
except (PermissionError, OSError, shutil.Error) as e:
|
||||
self.stdout.write(self.style.ERROR(f"Error moving {file_path!s} to {processed_path!s}: {e}"))
|
||||
traceback.print_exc()
|
||||
|
||||
def import_drop_campaign(self, data: dict[str, Any]) -> None:
|
||||
"""Find the key with all the data.
|
||||
|
||||
Args:
|
||||
data: The JSON data.
|
||||
|
||||
Raises:
|
||||
CommandError: If the JSON structure is invalid.
|
||||
"""
|
||||
if "data" not in data:
|
||||
msg = "Invalid JSON structure: Missing top-level 'data'"
|
||||
raise CommandError(msg)
|
||||
|
||||
if "user" in data["data"] and "dropCampaign" in data["data"]["user"]:
|
||||
drop_campaign_data = data["data"]["user"]["dropCampaign"]
|
||||
self._import_drop_campaign_with_retry(drop_campaign_data)
|
||||
self.import_to_db(drop_campaign_data)
|
||||
|
||||
shutil.move(str(file_path), str(processed_path))
|
||||
elif "currentUser" in data["data"] and "dropCampaigns" in data["data"]["currentUser"]:
|
||||
campaigns = data["data"]["currentUser"]["dropCampaigns"]
|
||||
for drop_campaign_data in campaigns:
|
||||
self.import_to_db(drop_campaign_data)
|
||||
|
||||
def _import_drop_campaign_with_retry(self, campaign_data: dict[str, Any]) -> None:
|
||||
else:
|
||||
msg = "Invalid JSON structure: Missing either data.user.dropCampaign or data.currentUser.dropCampaigns"
|
||||
raise CommandError(msg)
|
||||
|
||||
def import_to_db(self, campaign_data: dict[str, Any]) -> None:
|
||||
"""Import drop campaign data into the database with retry logic for SQLite locks.
|
||||
|
||||
Args:
|
||||
|
|
@ -204,14 +245,14 @@ class Command(BaseCommand):
|
|||
drop_campaign, created = DropCampaign.objects.update_or_create(
|
||||
id=campaign_data["id"],
|
||||
defaults={
|
||||
"name": campaign_data["name"],
|
||||
"description": campaign_data["description"].replace("\\n", "\n"),
|
||||
"name": campaign_data.get("name", ""),
|
||||
"description": campaign_data.get("description", "").replace("\\n", "\n"),
|
||||
"details_url": campaign_data.get("detailsURL", ""),
|
||||
"account_link_url": campaign_data.get("accountLinkURL", ""),
|
||||
"image_url": campaign_data.get("imageURL", ""),
|
||||
"start_at": campaign_data["startAt"],
|
||||
"end_at": campaign_data["endAt"],
|
||||
"is_account_connected": campaign_data["self"]["isAccountConnected"],
|
||||
"start_at": campaign_data.get("startAt"),
|
||||
"end_at": campaign_data.get("endAt"),
|
||||
"is_account_connected": campaign_data.get("self", {}).get("isAccountConnected", False),
|
||||
"game": game,
|
||||
"owner": organization,
|
||||
},
|
||||
|
|
@ -253,6 +294,7 @@ class Command(BaseCommand):
|
|||
defaults={
|
||||
"slug": game_data.get("slug", ""),
|
||||
"display_name": game_data["displayName"],
|
||||
"box_art": game_data.get("boxArtURL", ""),
|
||||
},
|
||||
)
|
||||
if created:
|
||||
|
|
|
|||
18
twitch/migrations/0005_game_box_art.py
Normal file
18
twitch/migrations/0005_game_box_art.py
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
# Generated by Django 5.2.4 on 2025-08-04 03:19
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('twitch', '0004_alter_dropbenefit_distribution_type'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='game',
|
||||
name='box_art',
|
||||
field=models.URLField(blank=True, default='', max_length=500),
|
||||
),
|
||||
]
|
||||
|
|
@ -14,6 +14,7 @@ class Game(models.Model):
|
|||
id = models.TextField(primary_key=True)
|
||||
slug = models.TextField(blank=True, default="", db_index=True)
|
||||
display_name = models.TextField(db_index=True)
|
||||
box_art = models.URLField(max_length=500, blank=True, default="")
|
||||
|
||||
class Meta:
|
||||
indexes: ClassVar[list] = [
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue