From 6f6116c3c784c281ae205cc36e107c022075dbd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Helle=C5=9Ben?= Date: Mon, 16 Mar 2026 15:27:33 +0100 Subject: [PATCH] Move Twitch stuff to /twitch/ --- config/settings.py | 1 + config/tests/test_urls.py | 4 +- config/urls.py | 10 +- core/__init__.py | 0 core/admin.py | 1 + core/apps.py | 7 + core/migrations/__init__.py | 0 core/models.py | 1 + core/tests/__init__.py | 0 core/urls.py | 96 +++ core/views.py | 810 +++++++++++++++++++ templates/base.html | 36 +- templates/twitch/campaign_detail.html | 12 +- templates/twitch/campaign_list.html | 12 +- templates/twitch/dashboard.html | 12 +- templates/twitch/dataset_backups.html | 2 +- templates/twitch/game_detail.html | 12 +- templates/twitch/games_grid.html | 12 +- templates/twitch/games_list.html | 12 +- templates/twitch/org_list.html | 6 +- templates/twitch/organization_detail.html | 6 +- templates/twitch/reward_campaign_detail.html | 12 +- templates/twitch/reward_campaign_list.html | 12 +- twitch/feeds.py | 32 +- twitch/tests/test_backup.py | 20 +- twitch/tests/test_badge_views.py | 6 +- twitch/tests/test_exports.py | 16 +- twitch/tests/test_feeds.py | 168 ++-- twitch/tests/test_views.py | 18 +- twitch/urls.py | 104 +-- twitch/views.py | 694 ---------------- 31 files changed, 1150 insertions(+), 984 deletions(-) create mode 100644 core/__init__.py create mode 100644 core/admin.py create mode 100644 core/apps.py create mode 100644 core/migrations/__init__.py create mode 100644 core/models.py create mode 100644 core/tests/__init__.py create mode 100644 core/urls.py create mode 100644 core/views.py diff --git a/config/settings.py b/config/settings.py index ca45e20..25f2043 100644 --- a/config/settings.py +++ b/config/settings.py @@ -140,6 +140,7 @@ INSTALLED_APPS: list[str] = [ "django.contrib.postgres", "twitch.apps.TwitchConfig", "kick.apps.KickConfig", + "core.apps.CoreConfig", ] MIDDLEWARE: list[str] = [ diff --git a/config/tests/test_urls.py b/config/tests/test_urls.py index 8e04206..04fd386 100644 --- a/config/tests/test_urls.py +++ b/config/tests/test_urls.py @@ -34,8 +34,10 @@ def _reload_urls_with(**overrides) -> ModuleType: def test_top_level_named_routes_available() -> None: """Top-level routes defined in `config.urls` are reversible.""" assert reverse("sitemap") == "/sitemap.xml" + # ensure the included `twitch` namespace is present - assert reverse("twitch:dashboard") == "/" + msg: str = f"Expected 'twitch:dashboard' to reverse to '/twitch/', got {reverse('twitch:dashboard')}" + assert reverse("twitch:dashboard") == "/twitch/", msg def test_debug_tools_not_present_while_testing() -> None: diff --git a/config/urls.py b/config/urls.py index 0a9ea65..bf0c81c 100644 --- a/config/urls.py +++ b/config/urls.py @@ -5,15 +5,19 @@ from django.conf.urls.static import static from django.urls import include from django.urls import path -from twitch import views as twitch_views +from core import views as core_views if TYPE_CHECKING: from django.urls.resolvers import URLPattern from django.urls.resolvers import URLResolver urlpatterns: list[URLPattern | URLResolver] = [ - path(route="sitemap.xml", view=twitch_views.sitemap_view, name="sitemap"), - path(route="", view=include("twitch.urls", namespace="twitch")), + path(route="sitemap.xml", view=core_views.sitemap_view, name="sitemap"), + # Core app + path(route="", view=include("core.urls", namespace="core")), + # Twitch app + path(route="twitch/", view=include("twitch.urls", namespace="twitch")), + # Kick app path(route="kick/", view=include("kick.urls", namespace="kick")), ] diff --git a/core/__init__.py b/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/core/admin.py b/core/admin.py new file mode 100644 index 0000000..846f6b4 --- /dev/null +++ b/core/admin.py @@ -0,0 +1 @@ +# Register your models here. diff --git a/core/apps.py b/core/apps.py new file mode 100644 index 0000000..0568ae1 --- /dev/null +++ b/core/apps.py @@ -0,0 +1,7 @@ +from django.apps import AppConfig + + +class CoreConfig(AppConfig): + """Core application configuration.""" + + name = "core" diff --git a/core/migrations/__init__.py b/core/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/core/models.py b/core/models.py new file mode 100644 index 0000000..6b20219 --- /dev/null +++ b/core/models.py @@ -0,0 +1 @@ +# Create your models here. diff --git a/core/tests/__init__.py b/core/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/core/urls.py b/core/urls.py new file mode 100644 index 0000000..6966557 --- /dev/null +++ b/core/urls.py @@ -0,0 +1,96 @@ +from typing import TYPE_CHECKING + +from django.urls import path + +from core import views +from twitch.feeds import DropCampaignFeed +from twitch.feeds import GameFeed + +if TYPE_CHECKING: + from django.urls.resolvers import URLPattern + from django.urls.resolvers import URLResolver + +app_name = "core" + + +urlpatterns: list[URLPattern | URLResolver] = [ + # / + path("", views.dashboard, name="dashboard"), + # /search/ + path("search/", views.search_view, name="search"), + # /debug/ + path("debug/", views.debug_view, name="debug"), + # /datasets/ + path("datasets/", views.dataset_backups_view, name="dataset_backups"), + # /datasets/download// + path( + "datasets/download//", + views.dataset_backup_download_view, + name="dataset_backup_download", + ), + # /docs/rss/ + path("docs/rss/", views.docs_rss_view, name="docs_rss"), + # RSS feeds + # /rss/campaigns/ - all active campaigns + path("rss/campaigns/", DropCampaignFeed(), name="campaign_feed"), + # /rss/games/ - newly added games + path("rss/games/", GameFeed(), name="game_feed"), + # /rss/games//campaigns/ - active campaigns for a specific game + path( + "rss/games//campaigns/", + views.GameCampaignFeed(), + name="game_campaign_feed", + ), + # /rss/organizations/ - newly added organizations + path( + "rss/organizations/", + views.OrganizationRSSFeed(), + name="organization_feed", + ), + # /rss/reward-campaigns/ - all active reward campaigns + path( + "rss/reward-campaigns/", + views.RewardCampaignFeed(), + name="reward_campaign_feed", + ), + # Atom feeds (added alongside RSS to preserve backward compatibility) + path("atom/campaigns/", views.DropCampaignAtomFeed(), name="campaign_feed_atom"), + path("atom/games/", views.GameAtomFeed(), name="game_feed_atom"), + path( + "atom/games//campaigns/", + views.GameCampaignAtomFeed(), + name="game_campaign_feed_atom", + ), + path( + "atom/organizations/", + views.OrganizationAtomFeed(), + name="organization_feed_atom", + ), + path( + "atom/reward-campaigns/", + views.RewardCampaignAtomFeed(), + name="reward_campaign_feed_atom", + ), + # Discord feeds (Atom feeds with Discord relative timestamps) + path( + "discord/campaigns/", + views.DropCampaignDiscordFeed(), + name="campaign_feed_discord", + ), + path("discord/games/", views.GameDiscordFeed(), name="game_feed_discord"), + path( + "discord/games//campaigns/", + views.GameCampaignDiscordFeed(), + name="game_campaign_feed_discord", + ), + path( + "discord/organizations/", + views.OrganizationDiscordFeed(), + name="organization_feed_discord", + ), + path( + "discord/reward-campaigns/", + views.RewardCampaignDiscordFeed(), + name="reward_campaign_feed_discord", + ), +] diff --git a/core/views.py b/core/views.py new file mode 100644 index 0000000..05465b8 --- /dev/null +++ b/core/views.py @@ -0,0 +1,810 @@ +import datetime +import json +import logging +import operator +from copy import copy +from typing import TYPE_CHECKING +from typing import Any + +from django.conf import settings +from django.db import connection +from django.db.models import Count +from django.db.models import Exists +from django.db.models import F +from django.db.models import OuterRef +from django.db.models import Q +from django.db.models.functions import Trim +from django.db.models.query import QuerySet +from django.http import FileResponse +from django.http import Http404 +from django.http import HttpResponse +from django.shortcuts import render +from django.template.defaultfilters import filesizeformat +from django.urls import reverse +from django.utils import timezone + +from twitch.feeds import DropCampaignAtomFeed +from twitch.feeds import DropCampaignDiscordFeed +from twitch.feeds import DropCampaignFeed +from twitch.feeds import GameAtomFeed +from twitch.feeds import GameCampaignAtomFeed +from twitch.feeds import GameCampaignDiscordFeed +from twitch.feeds import GameCampaignFeed +from twitch.feeds import GameDiscordFeed +from twitch.feeds import GameFeed +from twitch.feeds import OrganizationAtomFeed +from twitch.feeds import OrganizationDiscordFeed +from twitch.feeds import OrganizationRSSFeed +from twitch.feeds import RewardCampaignAtomFeed +from twitch.feeds import RewardCampaignDiscordFeed +from twitch.feeds import RewardCampaignFeed +from twitch.models import Channel +from twitch.models import ChatBadge +from twitch.models import ChatBadgeSet +from twitch.models import DropBenefit +from twitch.models import DropCampaign +from twitch.models import Game +from twitch.models import Organization +from twitch.models import RewardCampaign +from twitch.models import TimeBasedDrop + +if TYPE_CHECKING: + from collections.abc import Callable + from os import stat_result + from pathlib import Path + + from debug_toolbar.utils import QueryDict + from django.db.models import QuerySet + from django.http import HttpRequest + + +logger: logging.Logger = logging.getLogger("ttvdrops.views") + + +MIN_QUERY_LENGTH_FOR_FTS = 3 +MIN_SEARCH_RANK = 0.05 +DEFAULT_SITE_DESCRIPTION = "Archive of Twitch drops, campaigns, rewards, and more." + + +def _build_seo_context( # noqa: PLR0913, PLR0917 + page_title: str = "ttvdrops", + page_description: str | None = None, + page_image: str | None = None, + page_image_width: int | None = None, + page_image_height: int | None = None, + og_type: str = "website", + schema_data: dict[str, Any] | None = None, + breadcrumb_schema: dict[str, Any] | None = None, + pagination_info: list[dict[str, str]] | None = None, + published_date: str | None = None, + modified_date: str | None = None, + robots_directive: str = "index, follow", +) -> dict[str, Any]: + """Build SEO context for template rendering. + + Args: + page_title: Page title (shown in browser tab, og:title). + page_description: Page description (meta description, og:description). + page_image: Image URL for og:image meta tag. + page_image_width: Width of the image in pixels. + page_image_height: Height of the image in pixels. + og_type: OpenGraph type (e.g., "website", "article"). + schema_data: Dict representation of Schema.org JSON-LD data. + breadcrumb_schema: Breadcrumb schema dict for navigation hierarchy. + pagination_info: List of dicts with "rel" (prev|next|first|last) and "url". + published_date: ISO 8601 published date (e.g., "2025-01-01T00:00:00Z"). + modified_date: ISO 8601 modified date. + robots_directive: Robots meta content (e.g., "index, follow" or "noindex"). + + Returns: + Dict with SEO context variables to pass to render(). + """ + # TODO(TheLovinator): Instead of having so many parameters, # noqa: TD003 + # consider having a single "seo_info" parameter that + # can contain all of these optional fields. This would make + # it easier to extend in the future without changing the + # function signature. + + context: dict[str, Any] = { + "page_title": page_title, + "page_description": page_description or DEFAULT_SITE_DESCRIPTION, + "og_type": og_type, + "robots_directive": robots_directive, + } + if page_image: + context["page_image"] = page_image + if page_image_width and page_image_height: + context["page_image_width"] = page_image_width + context["page_image_height"] = page_image_height + if schema_data: + context["schema_data"] = json.dumps(schema_data) + if breadcrumb_schema: + context["breadcrumb_schema"] = json.dumps(breadcrumb_schema) + if pagination_info: + context["pagination_info"] = pagination_info + if published_date: + context["published_date"] = published_date + if modified_date: + context["modified_date"] = modified_date + return context + + +# MARK: /sitemap.xml +def sitemap_view(request: HttpRequest) -> HttpResponse: # noqa: PLR0915 + """Generate a dynamic XML sitemap for search engines. + + Args: + request: The HTTP request. + + Returns: + HttpResponse: XML sitemap. + """ + base_url: str = f"{request.scheme}://{request.get_host()}" + + # Start building sitemap XML + sitemap_urls: list[dict[str, str | dict[str, str]]] = [] + + # Static pages + sitemap_urls.extend([ + {"url": f"{base_url}/", "priority": "1.0", "changefreq": "daily"}, + {"url": f"{base_url}/campaigns/", "priority": "0.9", "changefreq": "daily"}, + { + "url": f"{base_url}/reward-campaigns/", + "priority": "0.9", + "changefreq": "daily", + }, + {"url": f"{base_url}/games/", "priority": "0.9", "changefreq": "weekly"}, + { + "url": f"{base_url}/organizations/", + "priority": "0.8", + "changefreq": "weekly", + }, + {"url": f"{base_url}/channels/", "priority": "0.8", "changefreq": "weekly"}, + {"url": f"{base_url}/badges/", "priority": "0.7", "changefreq": "monthly"}, + {"url": f"{base_url}/emotes/", "priority": "0.7", "changefreq": "monthly"}, + {"url": f"{base_url}/search/", "priority": "0.6", "changefreq": "monthly"}, + ]) + + # Dynamic detail pages - Games + games: QuerySet[Game] = Game.objects.all() + for game in games: + entry: dict[str, str | dict[str, str]] = { + "url": f"{base_url}{reverse('twitch:game_detail', args=[game.twitch_id])}", + "priority": "0.8", + "changefreq": "weekly", + } + if game.updated_at: + entry["lastmod"] = game.updated_at.isoformat() + sitemap_urls.append(entry) + + # Dynamic detail pages - Campaigns + campaigns: QuerySet[DropCampaign] = DropCampaign.objects.all() + for campaign in campaigns: + resource_url: str = reverse("twitch:campaign_detail", args=[campaign.twitch_id]) + full_url: str = f"{base_url}{resource_url}" + entry: dict[str, str | dict[str, str]] = { + "url": full_url, + "priority": "0.7", + "changefreq": "weekly", + } + if campaign.updated_at: + entry["lastmod"] = campaign.updated_at.isoformat() + sitemap_urls.append(entry) + + # Dynamic detail pages - Organizations + orgs: QuerySet[Organization] = Organization.objects.all() + for org in orgs: + resource_url = reverse("twitch:organization_detail", args=[org.twitch_id]) + full_url: str = f"{base_url}{resource_url}" + entry: dict[str, str | dict[str, str]] = { + "url": full_url, + "priority": "0.7", + "changefreq": "weekly", + } + if org.updated_at: + entry["lastmod"] = org.updated_at.isoformat() + sitemap_urls.append(entry) + + # Dynamic detail pages - Channels + channels: QuerySet[Channel] = Channel.objects.all() + for channel in channels: + resource_url = reverse("twitch:channel_detail", args=[channel.twitch_id]) + full_url: str = f"{base_url}{resource_url}" + entry: dict[str, str | dict[str, str]] = { + "url": full_url, + "priority": "0.6", + "changefreq": "weekly", + } + if channel.updated_at: + entry["lastmod"] = channel.updated_at.isoformat() + sitemap_urls.append(entry) + + # Dynamic detail pages - Badges + badge_sets: QuerySet[ChatBadgeSet] = ChatBadgeSet.objects.all() + for badge_set in badge_sets: + resource_url = reverse("twitch:badge_set_detail", args=[badge_set.set_id]) + full_url: str = f"{base_url}{resource_url}" + sitemap_urls.append({ + "url": full_url, + "priority": "0.5", + "changefreq": "monthly", + }) + + # Dynamic detail pages - Reward Campaigns + reward_campaigns: QuerySet[RewardCampaign] = RewardCampaign.objects.all() + for reward_campaign in reward_campaigns: + resource_url = reverse( + "twitch:reward_campaign_detail", + args=[ + reward_campaign.twitch_id, + ], + ) + full_url: str = f"{base_url}{resource_url}" + entry: dict[str, str | dict[str, str]] = { + "url": full_url, + "priority": "0.6", + "changefreq": "weekly", + } + if reward_campaign.updated_at: + entry["lastmod"] = reward_campaign.updated_at.isoformat() + sitemap_urls.append(entry) + + # Build XML + xml_content = '\n' + xml_content += '\n' + + for url_entry in sitemap_urls: + xml_content += " \n" + xml_content += f" {url_entry['url']}\n" + if url_entry.get("lastmod"): + xml_content += f" {url_entry['lastmod']}\n" + xml_content += ( + f" {url_entry.get('changefreq', 'monthly')}\n" + ) + xml_content += f" {url_entry.get('priority', '0.5')}\n" + xml_content += " \n" + + xml_content += "" + + return HttpResponse(xml_content, content_type="application/xml") + + +# MARK: /docs/rss/ +def docs_rss_view(request: HttpRequest) -> HttpResponse: + """View for /docs/rss that lists all available RSS feeds. + + Args: + request: The HTTP request object. + + Returns: + Rendered HTML response with list of RSS feeds. + """ + + def absolute(path: str) -> str: + try: + return request.build_absolute_uri(path) + except Exception: + logger.exception("Failed to build absolute URL for %s", path) + return path + + def _pretty_example(xml_str: str, max_items: int = 1) -> str: + try: + trimmed: str = xml_str.strip() + first_item: int = trimmed.find("", second_item) + if end_channel != -1: + trimmed = trimmed[:second_item] + trimmed[end_channel:] + formatted: str = trimmed.replace("><", ">\n<") + return "\n".join(line for line in formatted.splitlines() if line.strip()) + except Exception: + logger.exception("Failed to pretty-print RSS example") + return xml_str + + def render_feed(feed_view: Callable[..., HttpResponse], *args: object) -> str: + try: + limited_request: HttpRequest = copy(request) + # Add limit=1 to GET parameters + get_data: QueryDict = request.GET.copy() + get_data["limit"] = "1" + limited_request.GET = get_data # pyright: ignore[reportAttributeAccessIssue] + + response: HttpResponse = feed_view(limited_request, *args) + return _pretty_example(response.content.decode("utf-8")) + except Exception: + logger.exception( + "Failed to render %s for RSS docs", + feed_view.__class__.__name__, + ) + return "" + + show_atom: bool = bool(request.GET.get("show_atom")) + + feeds: list[dict[str, str]] = [ + { + "title": "All Organizations", + "description": "Latest organizations added to TTVDrops", + "url": absolute(reverse("core:organization_feed")), + "atom_url": absolute(reverse("core:organization_feed_atom")), + "discord_url": absolute(reverse("core:organization_feed_discord")), + "example_xml": render_feed(OrganizationRSSFeed()), + "example_xml_atom": render_feed(OrganizationAtomFeed()) + if show_atom + else "", + "example_xml_discord": render_feed(OrganizationDiscordFeed()) + if show_atom + else "", + }, + { + "title": "All Games", + "description": "Latest games added to TTVDrops", + "url": absolute(reverse("core:game_feed")), + "atom_url": absolute(reverse("core:game_feed_atom")), + "discord_url": absolute(reverse("core:game_feed_discord")), + "example_xml": render_feed(GameFeed()), + "example_xml_atom": render_feed(GameAtomFeed()) if show_atom else "", + "example_xml_discord": render_feed(GameDiscordFeed()) if show_atom else "", + }, + { + "title": "All Drop Campaigns", + "description": "Latest drop campaigns across all games", + "url": absolute(reverse("core:campaign_feed")), + "atom_url": absolute(reverse("core:campaign_feed_atom")), + "discord_url": absolute(reverse("core:campaign_feed_discord")), + "example_xml": render_feed(DropCampaignFeed()), + "example_xml_atom": render_feed(DropCampaignAtomFeed()) + if show_atom + else "", + "example_xml_discord": render_feed(DropCampaignDiscordFeed()) + if show_atom + else "", + }, + { + "title": "All Reward Campaigns", + "description": "Latest reward campaigns (Quest rewards) on Twitch", + "url": absolute(reverse("core:reward_campaign_feed")), + "atom_url": absolute(reverse("core:reward_campaign_feed_atom")), + "discord_url": absolute(reverse("core:reward_campaign_feed_discord")), + "example_xml": render_feed(RewardCampaignFeed()), + "example_xml_atom": render_feed(RewardCampaignAtomFeed()) + if show_atom + else "", + "example_xml_discord": render_feed(RewardCampaignDiscordFeed()) + if show_atom + else "", + }, + ] + + sample_game: Game | None = Game.objects.order_by("-added_at").first() + sample_org: Organization | None = Organization.objects.order_by("-added_at").first() + if sample_org is None and sample_game is not None: + sample_org = sample_game.owners.order_by("-pk").first() + + filtered_feeds: list[dict[str, str | bool]] = [ + { + "title": "Campaigns for a Single Game", + "description": "Latest drop campaigns for one game.", + "url": ( + absolute( + reverse("core:game_campaign_feed", args=[sample_game.twitch_id]), + ) + if sample_game + else absolute("/rss/games//campaigns/") + ), + "atom_url": ( + absolute( + reverse( + "core:game_campaign_feed_atom", + args=[sample_game.twitch_id], + ), + ) + if sample_game + else absolute("/atom/games//campaigns/") + ), + "discord_url": ( + absolute( + reverse( + "core:game_campaign_feed_discord", + args=[sample_game.twitch_id], + ), + ) + if sample_game + else absolute("/discord/games//campaigns/") + ), + "has_sample": bool(sample_game), + "example_xml": render_feed(GameCampaignFeed(), sample_game.twitch_id) + if sample_game + else "", + "example_xml_atom": ( + render_feed(GameCampaignAtomFeed(), sample_game.twitch_id) + if sample_game and show_atom + else "" + ), + "example_xml_discord": ( + render_feed(GameCampaignDiscordFeed(), sample_game.twitch_id) + if sample_game and show_atom + else "" + ), + }, + ] + + seo_context: dict[str, Any] = _build_seo_context( + page_title="Twitch RSS Feeds", + page_description="RSS feeds for Twitch drops.", + ) + return render( + request, + "twitch/docs_rss.html", + { + "feeds": feeds, + "filtered_feeds": filtered_feeds, + "sample_game": sample_game, + "sample_org": sample_org, + **seo_context, + }, + ) + + +# MARK: /debug/ +def debug_view(request: HttpRequest) -> HttpResponse: + """Debug view showing potentially broken or inconsistent data. + + Returns: + HttpResponse: Rendered debug template or redirect if unauthorized. + """ + now: datetime.datetime = timezone.now() + + # Games with no assigned owner organization + games_without_owner: QuerySet[Game] = Game.objects.filter( + owners__isnull=True, + ).order_by("display_name") + + # Campaigns with no images at all (no direct URL and no benefit image fallbacks) + broken_image_campaigns: QuerySet[DropCampaign] = ( + DropCampaign.objects + .filter( + Q(image_url__isnull=True) + | Q(image_url__exact="") + | ~Q(image_url__startswith="http"), + ) + .exclude( + Exists( + TimeBasedDrop.objects.filter(campaign=OuterRef("pk")).filter( + benefits__image_asset_url__startswith="http", + ), + ), + ) + .select_related("game") + ) + + # Benefits with missing images + broken_benefit_images: QuerySet[DropBenefit] = DropBenefit.objects.annotate( + trimmed_url=Trim("image_asset_url"), + ).filter( + Q(image_asset_url__isnull=True) + | Q(trimmed_url__exact="") + | ~Q(image_asset_url__startswith="http"), + ) + + # Time-based drops without any benefits + drops_without_benefits: QuerySet[TimeBasedDrop] = TimeBasedDrop.objects.filter( + benefits__isnull=True, + ).select_related("campaign__game") + + # Campaigns with invalid dates (start after end or missing either) + invalid_date_campaigns: QuerySet[DropCampaign] = DropCampaign.objects.filter( + Q(start_at__gt=F("end_at")) | Q(start_at__isnull=True) | Q(end_at__isnull=True), + ).select_related("game") + + # Duplicate campaign names per game. + # We retrieve the game's name for user-friendly display. + duplicate_name_campaigns: QuerySet[DropCampaign, dict[str, Any]] = ( + DropCampaign.objects + .values("game__display_name", "name", "game__twitch_id") + .annotate(name_count=Count("twitch_id")) + .filter(name_count__gt=1) + .order_by("game__display_name", "name") + ) + + # Active campaigns with no images at all + active_missing_image: QuerySet[DropCampaign] = ( + DropCampaign.objects + .filter(start_at__lte=now, end_at__gte=now) + .filter( + Q(image_url__isnull=True) + | Q(image_url__exact="") + | ~Q(image_url__startswith="http"), + ) + .exclude( + Exists( + TimeBasedDrop.objects.filter(campaign=OuterRef("pk")).filter( + benefits__image_asset_url__startswith="http", + ), + ), + ) + .select_related("game") + ) + + # Distinct GraphQL operation names used to fetch campaigns with counts + # Since operation_names is now a JSON list field, we need to flatten and count + operation_names_counter: dict[str, int] = {} + for campaign in DropCampaign.objects.only("operation_names"): + for op_name in campaign.operation_names: + if op_name and op_name.strip(): + operation_names_counter[op_name.strip()] = ( + operation_names_counter.get(op_name.strip(), 0) + 1 + ) + + operation_names_with_counts: list[dict[str, Any]] = [ + {"trimmed_op": op_name, "count": count} + for op_name, count in sorted(operation_names_counter.items()) + ] + + # Campaigns missing DropCampaignDetails operation name + # Need to handle SQLite separately since it doesn't support JSONField lookups + # Sqlite is used when testing + if connection.vendor == "sqlite": + all_campaigns: QuerySet[DropCampaign] = DropCampaign.objects.select_related( + "game", + ).order_by("game__display_name", "name") + campaigns_missing_dropcampaigndetails: list[DropCampaign] = [ + c + for c in all_campaigns + if c.operation_names is None + or "DropCampaignDetails" not in c.operation_names + ] + else: + campaigns_missing_dropcampaigndetails: list[DropCampaign] = list( + DropCampaign.objects + .filter( + Q(operation_names__isnull=True) + | ~Q(operation_names__contains=["DropCampaignDetails"]), + ) + .select_related("game") + .order_by("game__display_name", "name"), + ) + + context: dict[str, Any] = { + "now": now, + "games_without_owner": games_without_owner, + "broken_image_campaigns": broken_image_campaigns, + "broken_benefit_images": broken_benefit_images, + "drops_without_benefits": drops_without_benefits, + "invalid_date_campaigns": invalid_date_campaigns, + "duplicate_name_campaigns": duplicate_name_campaigns, + "active_missing_image": active_missing_image, + "operation_names_with_counts": operation_names_with_counts, + "campaigns_missing_dropcampaigndetails": campaigns_missing_dropcampaigndetails, + } + + seo_context: dict[str, Any] = _build_seo_context( + page_title="Debug", + page_description="Debug view showing potentially broken or inconsistent data.", + robots_directive="noindex, nofollow", + ) + context.update(seo_context) + + return render(request, "twitch/debug.html", context) + + +# MARK: /datasets/ +def dataset_backups_view(request: HttpRequest) -> HttpResponse: + """View to list database backup datasets on disk. + + Args: + request: The HTTP request. + + Returns: + HttpResponse: The rendered dataset backups page. + """ + # TODO(TheLovinator): Instead of only using sql we should also support other formats like parquet, csv, or json. # noqa: TD003 + # TODO(TheLovinator): Upload to s3 instead. # noqa: TD003 + # TODO(TheLovinator): https://developers.google.com/search/docs/appearance/structured-data/dataset#json-ld + datasets_root: Path = settings.DATA_DIR / "datasets" + search_dirs: list[Path] = [datasets_root] + seen_paths: set[str] = set() + datasets: list[dict[str, Any]] = [] + + for folder in search_dirs: + if not folder.exists() or not folder.is_dir(): + continue + + # Only include .zst files + for path in folder.glob("*.zst"): + if not path.is_file(): + continue + key = str(path.resolve()) + if key in seen_paths: + continue + seen_paths.add(key) + stat: stat_result = path.stat() + updated_at: datetime.datetime = datetime.datetime.fromtimestamp( + stat.st_mtime, + tz=timezone.get_current_timezone(), + ) + try: + display_path = str(path.relative_to(datasets_root)) + download_path: str | None = display_path + except ValueError: + display_path: str = path.name + download_path: str | None = None + datasets.append({ + "name": path.name, + "display_path": display_path, + "download_path": download_path, + "size": filesizeformat(stat.st_size), + "updated_at": updated_at, + }) + + datasets.sort(key=operator.itemgetter("updated_at"), reverse=True) + + seo_context: dict[str, Any] = _build_seo_context( + page_title="Twitch Dataset", + page_description="Database backups and datasets available for download.", + ) + context: dict[str, Any] = { + "datasets": datasets, + "data_dir": str(datasets_root), + "dataset_count": len(datasets), + **seo_context, + } + return render(request, "twitch/dataset_backups.html", context) + + +def dataset_backup_download_view( + request: HttpRequest, # noqa: ARG001 + relative_path: str, +) -> FileResponse: + """Download a dataset backup from the data directory. + + Args: + request: The HTTP request. + relative_path: The path relative to the data directory. + + Returns: + FileResponse: The file response for the requested dataset. + + Raises: + Http404: When the file is not found or is outside the data directory. + """ + # TODO(TheLovinator): Use s3 instead of local disk. # noqa: TD003 + + datasets_root: Path = settings.DATA_DIR / "datasets" + requested_path: Path = (datasets_root / relative_path).resolve() + data_root: Path = datasets_root.resolve() + + try: + requested_path.relative_to(data_root) + except ValueError as exc: + msg = "File not found" + raise Http404(msg) from exc + if not requested_path.exists() or not requested_path.is_file(): + msg = "File not found" + raise Http404(msg) + if not requested_path.name.endswith(".zst"): + msg = "File not found" + raise Http404(msg) + + return FileResponse( + requested_path.open("rb"), + as_attachment=True, + filename=requested_path.name, + ) + + +# MARK: /search/ +def search_view(request: HttpRequest) -> HttpResponse: + """Search view for all models. + + Args: + request: The HTTP request. + + Returns: + HttpResponse: The rendered search results. + """ + query: str = request.GET.get("q", "") + results: dict[str, QuerySet] = {} + + if query: + if len(query) < MIN_QUERY_LENGTH_FOR_FTS: + results["organizations"] = Organization.objects.filter( + name__istartswith=query, + ) + results["games"] = Game.objects.filter( + Q(name__istartswith=query) | Q(display_name__istartswith=query), + ) + + results["campaigns"] = DropCampaign.objects.filter( + Q(name__istartswith=query) | Q(description__icontains=query), + ).select_related("game") + + results["drops"] = TimeBasedDrop.objects.filter( + name__istartswith=query, + ).select_related("campaign") + + results["benefits"] = DropBenefit.objects.filter( + name__istartswith=query, + ).prefetch_related("drops__campaign") + + results["reward_campaigns"] = RewardCampaign.objects.filter( + Q(name__istartswith=query) + | Q(brand__istartswith=query) + | Q(summary__icontains=query), + ).select_related("game") + + results["badge_sets"] = ChatBadgeSet.objects.filter( + set_id__istartswith=query, + ) + + results["badges"] = ChatBadge.objects.filter( + Q(title__istartswith=query) | Q(description__icontains=query), + ).select_related("badge_set") + else: + results["organizations"] = Organization.objects.filter( + name__icontains=query, + ) + results["games"] = Game.objects.filter( + Q(name__icontains=query) | Q(display_name__icontains=query), + ) + + results["campaigns"] = DropCampaign.objects.filter( + Q(name__icontains=query) | Q(description__icontains=query), + ).select_related("game") + + results["drops"] = TimeBasedDrop.objects.filter( + name__icontains=query, + ).select_related("campaign") + + results["benefits"] = DropBenefit.objects.filter( + name__icontains=query, + ).prefetch_related("drops__campaign") + + results["reward_campaigns"] = RewardCampaign.objects.filter( + Q(name__icontains=query) + | Q(brand__icontains=query) + | Q(summary__icontains=query), + ).select_related("game") + + results["badge_sets"] = ChatBadgeSet.objects.filter(set_id__icontains=query) + results["badges"] = ChatBadge.objects.filter( + Q(title__icontains=query) | Q(description__icontains=query), + ).select_related("badge_set") + + total_results_count: int = sum(len(qs) for qs in results.values()) + + # TODO(TheLovinator): Make the description more informative by including counts of each result type, e.g. "Found 5 games, 3 campaigns, and 10 drops for 'rust'." # noqa: TD003 + if query: + page_title: str = f"Search Results for '{query}'"[:60] + page_description: str = f"Found {total_results_count} results for '{query}'." + else: + page_title = "Search" + page_description = "Search for drops, games, channels, and organizations." + + seo_context: dict[str, Any] = _build_seo_context( + page_title=page_title, + page_description=page_description, + ) + return render( + request, + "twitch/search_results.html", + {"query": query, "results": results, **seo_context}, + ) + + +# MARK: / +def dashboard(request: HttpRequest) -> HttpResponse: # noqa: ARG001 + """Dashboard view showing summary stats and latest campaigns. + + Args: + request: The HTTP request. + + Returns: + HttpResponse: The rendered dashboard page. + """ + # Return HTML to show that the view is working. + return HttpResponse( + "

Welcome to the Twitch Drops Dashboard

Use the navigation to explore campaigns, games, organizations, and more.

", + content_type="text/html", + ) diff --git a/templates/base.html b/templates/base.html index 71488ab..dcafd79 100644 --- a/templates/base.html +++ b/templates/base.html @@ -24,55 +24,55 @@ {% include "includes/meta_tags.html" %} - + + href="{% url 'core:campaign_feed' %}" /> + href="{% url 'core:campaign_feed_atom' %}" /> + href="{% url 'core:campaign_feed_discord' %}" /> + href="{% url 'core:game_feed' %}" /> + href="{% url 'core:game_feed_atom' %}" /> + href="{% url 'core:game_feed_discord' %}" /> + href="{% url 'core:organization_feed' %}" /> + href="{% url 'core:organization_feed_atom' %}" /> + href="{% url 'core:organization_feed_discord' %}" /> + href="{% url 'core:reward_campaign_feed' %}" /> + href="{% url 'core:reward_campaign_feed_atom' %}" /> + href="{% url 'core:reward_campaign_feed_discord' %}" />