From b97118cffd98661ccfc6a4d78b49b63076beccfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Hells=C3=A9n?= Date: Sat, 13 Sep 2025 06:37:35 +0200 Subject: [PATCH] Cache images instead of serve from Twitch --- config/urls.py | 9 +- templates/twitch/campaign_detail.html | 8 +- templates/twitch/campaign_list.html | 8 +- templates/twitch/channel_detail.html | 12 +-- templates/twitch/dashboard.html | 2 +- templates/twitch/debug.html | 4 +- templates/twitch/game_detail.html | 16 +-- templates/twitch/games_grid.html | 4 +- twitch/management/commands/backfill_images.py | 69 +++++++++++++ twitch/management/commands/import_drops.py | 22 +++++ .../migrations/0016_add_local_image_fields.py | 29 ++++++ ...7_alter_dropbenefit_image_file_and_more.py | 28 ++++++ twitch/models.py | 57 +++++++++++ twitch/utils/__init__.py | 3 + twitch/utils/images.py | 97 +++++++++++++++++++ twitch/views.py | 2 +- 16 files changed, 340 insertions(+), 30 deletions(-) create mode 100644 twitch/management/commands/backfill_images.py create mode 100644 twitch/migrations/0016_add_local_image_fields.py create mode 100644 twitch/migrations/0017_alter_dropbenefit_image_file_and_more.py create mode 100644 twitch/utils/__init__.py create mode 100644 twitch/utils/images.py diff --git a/config/urls.py b/config/urls.py index 71cc63c..df2b84d 100644 --- a/config/urls.py +++ b/config/urls.py @@ -3,13 +3,14 @@ from __future__ import annotations from typing import TYPE_CHECKING from django.conf import settings +from django.conf.urls.static import static from django.contrib import admin from django.urls import include, path if TYPE_CHECKING: - from django.urls.resolvers import URLResolver + from django.urls.resolvers import URLPattern, URLResolver -urlpatterns: list[URLResolver] = [ +urlpatterns: list[URLResolver] | list[URLPattern | URLResolver] = [ # type: ignore[assignment] path(route="admin/", view=admin.site.urls), path(route="accounts/", view=include("accounts.urls", namespace="accounts")), path(route="", view=include("twitch.urls", namespace="twitch")), @@ -24,3 +25,7 @@ if not settings.TESTING: *debug_toolbar_urls(), path("__reload__/", include("django_browser_reload.urls")), ] + +# Serve media in development +if settings.DEBUG: + urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) diff --git a/templates/twitch/campaign_detail.html b/templates/twitch/campaign_detail.html index b1b6123..dbfd1d2 100644 --- a/templates/twitch/campaign_detail.html +++ b/templates/twitch/campaign_detail.html @@ -18,11 +18,11 @@

{% endif %} - {% if campaign.image_url %} + {% if campaign.image_best_url or campaign.image_url %} {{ campaign.name }} {% endif %} @@ -117,12 +117,12 @@ {% for benefit in drop.drop.benefits.all %} - {% if benefit.image_asset_url %} + {% if benefit.image_best_url or benefit.image_asset_url %} {{ benefit.name }} {% else %}
- {% if game_group.grouper.box_art_base_url %} - Box art for {{ game_group.grouper.display_name }} - {% if campaign.image_url %} + {% if campaign.image_best_url or campaign.image_url %} Campaign artwork for {{ campaign.name }} {% for benefit in campaign.sorted_benefits %} - {% if benefit.image_asset_url %} - {{ benefit.name }} {% for benefit in campaign.sorted_benefits %} - {% if benefit.image_asset_url %} - {{ benefit.name }} {% for benefit in campaign.sorted_benefits %} - {% if benefit.image_asset_url %} - {{ benefit.name }}
- Image for {{ campaign.name }} {{ c.name }} (Game: {{ c.game.display_name }}) - - URL: {{ c.image_url|default:'(empty)' }} + - URL: {{ c.image_best_url|default:c.image_url|default:'(empty)' }} {% endfor %} @@ -53,7 +53,7 @@ {% else %} (Game: Not linked) {% endif %} - - URL: {{ b.image_asset_url|default:'(empty)' }} + - URL: {{ b.image_best_url|default:b.image_asset_url|default:'(empty)' }} {% endwith %} {% endfor %} diff --git a/templates/twitch/game_detail.html b/templates/twitch/game_detail.html index bc07e9a..364d8aa 100644 --- a/templates/twitch/game_detail.html +++ b/templates/twitch/game_detail.html @@ -9,11 +9,11 @@ {% if game.display_name != game.name and game.name %}({{ game.name }}){% endif %} - {% if game.box_art_url %} + {% if game.box_art_best_url %} {{ game.name }} {% endif %} @@ -58,8 +58,8 @@ {% comment %}Show unique benefits sorted alphabetically{% endcomment %} {% for benefit in campaign.sorted_benefits %} - {% if benefit.image_asset_url %} - {{ benefit.name }} {% for benefit in campaign.sorted_benefits %} - {% if benefit.image_asset_url %} - {{ benefit.name }} - {% if benefit.image_asset_url %} - {{ benefit.name }}
- {% if item.game.box_art_base_url %} - Box art for {{ item.game.display_name }} None: # type: ignore[override] + """Add CLI arguments for the management command.""" + parser.add_argument("--limit", type=int, default=0, help="Limit number of objects per model to process (0 = no limit)") + + def handle(self, **options: object) -> None: + """Execute the backfill process using provided options.""" + limit: int = int(options.get("limit", 0)) # type: ignore[arg-type] + + def maybe_limit(qs: QuerySet) -> QuerySet: + """Apply slicing if --limit is provided. + + Returns: + Queryset possibly sliced by the limit. + """ + return qs[:limit] if limit > 0 else qs + + processed = 0 + + for game in maybe_limit(Game.objects.filter(box_art_file__isnull=True).exclude(box_art="")): + rel = cache_remote_image(game.box_art, "games/box_art") + if rel: + game.box_art_file.name = rel + game.save(update_fields=["box_art_file"]) # type: ignore[list-item] + processed += 1 + + self.stdout.write(self.style.SUCCESS(f"Processed game box art: {game.id}")) + + for campaign in maybe_limit(DropCampaign.objects.filter(image_file__isnull=True).exclude(image_url="")): + rel = cache_remote_image(campaign.image_url, "campaigns/images") + if rel: + campaign.image_file.name = rel + campaign.save(update_fields=["image_file"]) # type: ignore[list-item] + processed += 1 + + self.stdout.write(self.style.SUCCESS(f"Processed campaign image: {campaign.id}")) + + for benefit in maybe_limit(DropBenefit.objects.filter(image_file__isnull=True).exclude(image_asset_url="")): + rel = cache_remote_image(benefit.image_asset_url, "benefits/images") + if rel: + benefit.image_file.name = rel + benefit.save(update_fields=["image_file"]) # type: ignore[list-item] + processed += 1 + + self.stdout.write(self.style.SUCCESS(f"Processed benefit image: {benefit.id}")) + + self.stdout.write(self.style.SUCCESS(f"Backfill complete. Updated {processed} images.")) diff --git a/twitch/management/commands/import_drops.py b/twitch/management/commands/import_drops.py index d86cacd..9c48815 100644 --- a/twitch/management/commands/import_drops.py +++ b/twitch/management/commands/import_drops.py @@ -13,6 +13,7 @@ from django.db import transaction from django.utils import timezone from twitch.models import Channel, DropBenefit, DropBenefitEdge, DropCampaign, Game, Organization, TimeBasedDrop +from twitch.utils.images import cache_remote_image if TYPE_CHECKING: from datetime import datetime @@ -472,6 +473,13 @@ class Command(BaseCommand): defaults=benefit_defaults, ) + # Cache benefit image if available and not already cached + if (not benefit.image_file) and benefit.image_asset_url: + rel_path: str | None = cache_remote_image(benefit.image_asset_url, "benefits/images") + if rel_path: + benefit.image_file.name = rel_path + benefit.save(update_fields=["image_file"]) + DropBenefitEdge.objects.update_or_create( drop=time_based_drop, benefit=benefit, @@ -590,6 +598,13 @@ class Command(BaseCommand): if created: self.stdout.write(self.style.SUCCESS(f"Created new drop campaign: {drop_campaign.name} (ID: {drop_campaign.id})")) + + # Cache campaign image if available and not already cached + if (not drop_campaign.image_file) and drop_campaign.image_url: + rel_path: str | None = cache_remote_image(drop_campaign.image_url, "campaigns/images") + if rel_path: + drop_campaign.image_file.name = rel_path + drop_campaign.save(update_fields=["image_file"]) # type: ignore[list-item] return drop_campaign def owner_update_or_create(self, campaign_data: dict[str, Any]) -> Organization | None: @@ -648,4 +663,11 @@ class Command(BaseCommand): ) if created: self.stdout.write(self.style.SUCCESS(f"Created new game: {game.display_name} (ID: {game.id})")) + + # Cache game box art if available and not already cached + if (not game.box_art_file) and game.box_art: + rel_path: str | None = cache_remote_image(game.box_art, "games/box_art") + if rel_path: + game.box_art_file.name = rel_path + game.save(update_fields=["box_art_file"]) return game diff --git a/twitch/migrations/0016_add_local_image_fields.py b/twitch/migrations/0016_add_local_image_fields.py new file mode 100644 index 0000000..a596af9 --- /dev/null +++ b/twitch/migrations/0016_add_local_image_fields.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from django.db import migrations, models + + +class Migration(migrations.Migration): + """Add local image FileFields to models for caching Twitch images.""" + + dependencies = [ + ("twitch", "0015_alter_dropbenefitedge_benefit_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="game", + name="box_art_file", + field=models.FileField(blank=True, null=True, upload_to="games/box_art/"), + ), + migrations.AddField( + model_name="dropcampaign", + name="image_file", + field=models.FileField(blank=True, null=True, upload_to="campaigns/images/"), + ), + migrations.AddField( + model_name="dropbenefit", + name="image_file", + field=models.FileField(blank=True, null=True, upload_to="benefits/images/"), + ), + ] diff --git a/twitch/migrations/0017_alter_dropbenefit_image_file_and_more.py b/twitch/migrations/0017_alter_dropbenefit_image_file_and_more.py new file mode 100644 index 0000000..7981713 --- /dev/null +++ b/twitch/migrations/0017_alter_dropbenefit_image_file_and_more.py @@ -0,0 +1,28 @@ +# Generated by Django 5.2.6 on 2025-09-13 00:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('twitch', '0016_add_local_image_fields'), + ] + + operations = [ + migrations.AlterField( + model_name='dropbenefit', + name='image_file', + field=models.FileField(blank=True, help_text='Locally cached benefit image served from this site.', null=True, upload_to='benefits/images/'), + ), + migrations.AlterField( + model_name='dropcampaign', + name='image_file', + field=models.FileField(blank=True, help_text='Locally cached campaign image served from this site.', null=True, upload_to='campaigns/images/'), + ), + migrations.AlterField( + model_name='game', + name='box_art_file', + field=models.FileField(blank=True, help_text='Locally cached box art image served from this site.', null=True, upload_to='games/box_art/'), + ), + ] diff --git a/twitch/models.py b/twitch/models.py index 83d24a0..caeee43 100644 --- a/twitch/models.py +++ b/twitch/models.py @@ -95,6 +95,13 @@ class Game(auto_prefetch.Model): default="", verbose_name="Box art URL", ) + # Locally cached image file for the game's box art + box_art_file = models.FileField( + upload_to="games/box_art/", + blank=True, + null=True, + help_text="Locally cached box art image served from this site.", + ) # PostgreSQL full-text search field search_vector = SearchVectorField(null=True, blank=True) @@ -162,6 +169,22 @@ class Game(auto_prefetch.Model): ) return urlunsplit((parts.scheme, parts.netloc, path, "", "")) + @property + def box_art_best_url(self) -> str: + """Return the best available URL for the game's box art. + + Preference order: + 1) Local cached file (MEDIA) + 2) Remote Twitch base URL + 3) Empty string + """ + try: + if self.box_art_file and getattr(self.box_art_file, "url", None): + return self.box_art_file.url + except (AttributeError, OSError, ValueError) as exc: # storage might not be configured in some contexts + logger.debug("Failed to resolve Game.box_art_file url: %s", exc) + return self.box_art_base_url + @property def get_game_name(self) -> str: """Return the best available name for the game.""" @@ -260,6 +283,13 @@ class DropCampaign(auto_prefetch.Model): default="", help_text="URL to an image representing the campaign.", ) + # Locally cached campaign image + image_file = models.FileField( + upload_to="campaigns/images/", + blank=True, + null=True, + help_text="Locally cached campaign image served from this site.", + ) start_at = models.DateTimeField( db_index=True, null=True, @@ -368,6 +398,16 @@ class DropCampaign(auto_prefetch.Model): return self.name + @property + def image_best_url(self) -> str: + """Return the best available URL for the campaign image (local first).""" + try: + if self.image_file and getattr(self.image_file, "url", None): + return self.image_file.url + except (AttributeError, OSError, ValueError) as exc: + logger.debug("Failed to resolve DropCampaign.image_file url: %s", exc) + return self.image_url or "" + class DropBenefit(auto_prefetch.Model): """Represents a benefit that can be earned from a drop.""" @@ -390,6 +430,13 @@ class DropBenefit(auto_prefetch.Model): default="", help_text="URL to the benefit's image asset.", ) + # Locally cached benefit image + image_file = models.FileField( + upload_to="benefits/images/", + blank=True, + null=True, + help_text="Locally cached benefit image served from this site.", + ) created_at = models.DateTimeField( null=True, db_index=True, @@ -443,6 +490,16 @@ class DropBenefit(auto_prefetch.Model): """Return a string representation of the drop benefit.""" return self.name + @property + def image_best_url(self) -> str: + """Return the best available URL for the benefit image (local first).""" + try: + if self.image_file and getattr(self.image_file, "url", None): + return self.image_file.url + except (AttributeError, OSError, ValueError) as exc: + logger.debug("Failed to resolve DropBenefit.image_file url: %s", exc) + return self.image_asset_url or "" + class TimeBasedDrop(auto_prefetch.Model): """Represents a time-based drop in a drop campaign.""" diff --git a/twitch/utils/__init__.py b/twitch/utils/__init__.py new file mode 100644 index 0000000..efaab6d --- /dev/null +++ b/twitch/utils/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +# Utility package for twitch app diff --git a/twitch/utils/images.py b/twitch/utils/images.py new file mode 100644 index 0000000..8df1204 --- /dev/null +++ b/twitch/utils/images.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import hashlib +import logging +import mimetypes +import re +from pathlib import Path +from urllib.parse import urlparse +from urllib.request import Request, urlopen + +from django.conf import settings + +logger: logging.Logger = logging.getLogger(__name__) + + +def _sanitize_filename(name: str) -> str: + """Return a filesystem-safe filename.""" + name = re.sub(r"[^A-Za-z0-9._-]", "_", name) + return name[:150] or "file" + + +def _guess_extension(url: str, content_type: str | None) -> str: + """Guess a file extension from URL or content-type. + + Args: + url: Source URL. + content_type: Optional content type from HTTP response. + + Returns: + File extension including dot, like ".png". + """ + parsed = urlparse(url) + ext = Path(parsed.path).suffix.lower() + if ext in {".jpg", ".jpeg", ".png", ".gif", ".webp"}: + return ext + if content_type: + guessed = mimetypes.guess_extension(content_type.split(";")[0].strip()) + if guessed: + return guessed + return ".bin" + + +def cache_remote_image(url: str, subdir: str, *, timeout: float = 10.0) -> str | None: + """Download a remote image and save it under MEDIA_ROOT, returning storage path. + + The file name is the SHA256 of the content to de-duplicate downloads. + + Args: + url: Remote image URL. + subdir: Sub-directory under MEDIA_ROOT to store the file. + timeout: Network timeout in seconds. + + Returns: + Relative storage path (under MEDIA_ROOT) suitable for assigning to FileField.name, + or None if the operation failed. + """ + url = (url or "").strip() + if not url or not url.startswith(("http://", "https://")): + return None + + try: + # Enforce allowed schemes at runtime too + parsed = urlparse(url) + if parsed.scheme not in {"http", "https"}: + return None + req = Request(url, headers={"User-Agent": "TTVDrops/1.0"}) # noqa: S310 + # nosec: B310 - urlopen allowed because scheme is validated (http/https only) + with urlopen(req, timeout=timeout) as resp: # noqa: S310 + content: bytes = resp.read() + content_type = resp.headers.get("Content-Type") + except OSError as exc: + logger.debug("Failed to download image %s: %s", url, exc) + return None + + if not content: + return None + + sha = hashlib.sha256(content).hexdigest() + ext = _guess_extension(url, content_type) + # Shard into two-level directories by hash for scalability + shard1, shard2 = sha[:2], sha[2:4] + media_subdir = Path(subdir) / shard1 / shard2 + target_dir: Path = Path(settings.MEDIA_ROOT) / media_subdir + target_dir.mkdir(parents=True, exist_ok=True) + + filename = f"{sha}{ext}" + storage_rel_path = str(media_subdir / _sanitize_filename(filename)).replace("\\", "/") + storage_abs_path = Path(settings.MEDIA_ROOT) / storage_rel_path + + if not storage_abs_path.exists(): + try: + storage_abs_path.write_bytes(content) + except OSError as exc: + logger.debug("Failed to write image %s: %s", storage_abs_path, exc) + return None + + return storage_rel_path diff --git a/twitch/views.py b/twitch/views.py index 5ecfad5..abadb83 100644 --- a/twitch/views.py +++ b/twitch/views.py @@ -550,7 +550,7 @@ def dashboard(request: HttpRequest) -> HttpResponse: if game_id not in campaigns_by_org_game[org_id]["games"]: campaigns_by_org_game[org_id]["games"][game_id] = { "name": game_name, - "box_art": campaign.game.box_art_base_url, + "box_art": campaign.game.box_art_best_url, "campaigns": [], }