From 3d10b210feef99b437102b89d4a8af5ea2588fde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Hells=C3=A9n?= Date: Wed, 12 Aug 2026 06:05:39 +0200 Subject: [PATCH] Fix broken Steam localized image URLs --- .vscode/settings.json | 4 + discord_rss_bot/custom_message.py | 5 + discord_rss_bot/extensions/steam.py | 147 +++++++++++++++++++++++++++- tests/test_custom_message.py | 26 +++++ tests/test_extensions.py | 128 ++++++++++++++++++++++++ 5 files changed, 309 insertions(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 5e7f664..fb0db71 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -6,13 +6,16 @@ "argvalues", "autoexport", "autoplay", + "autouse", "botuser", "DISCORDTIMESTAMPPLACEHOLDER", "domcontentloaded", "Genshins", "healthcheck", + "hentaigasm", "Hoyolab", "HTMX", + "jwplayer", "KHTML", "levelname", "Lovinator", @@ -27,6 +30,7 @@ "thead", "thelovinator", "ttvdrops", + "uncensor", "uvicorn", "youtu" ], diff --git a/discord_rss_bot/custom_message.py b/discord_rss_bot/custom_message.py index a9411fc..ca7d988 100644 --- a/discord_rss_bot/custom_message.py +++ b/discord_rss_bot/custom_message.py @@ -9,6 +9,7 @@ from bs4 import BeautifulSoup from bs4 import Tag from discord_rss_bot.extensions import run_extensions +from discord_rss_bot.extensions.steam import fix_steam_image_url from discord_rss_bot.html_format import format_entry_html_for_discord from discord_rss_bot.is_url_valid import is_url_valid @@ -201,6 +202,10 @@ def get_image_urls( logger.warning("Invalid URL: %s", src) continue + # Steam news feeds sometimes emit broken localized image names + # (e.g. ".../english.png"); rewrite them to the working URL. + src = fix_steam_image_url(src) + if src in seen_urls: continue diff --git a/discord_rss_bot/extensions/steam.py b/discord_rss_bot/extensions/steam.py index 5432665..dccf1c3 100644 --- a/discord_rss_bot/extensions/steam.py +++ b/discord_rss_bot/extensions/steam.py @@ -4,6 +4,10 @@ Detects feeds from ``store.steampowered.com`` and ``steamcommunity.com``, extracts the application ID, and exposes the game's capsule image URL as ``{{steam_thumbnail_url}}`` and the app ID as ``{{steam_app_id}}``. +It also rewrites broken Steam CDN image URLs: some news feeds reference +localized image names such as ``.../english.png`` that return a 404, and +the working URL omits the language segment (``.../.png``). + Usage: 1. Enable ``steam`` on the feed's Extensions page. 2. In embed settings, set Thumbnail URL to ``{{steam_thumbnail_url}}``. @@ -14,17 +18,22 @@ from __future__ import annotations import hashlib import json import logging +import re from pathlib import Path from typing import TYPE_CHECKING from typing import ClassVar +from typing import cast +from urllib.parse import ParseResult from urllib.parse import parse_qs from urllib.parse import urlparse +from urllib.parse import urlunsplit from reader import ReaderError from discord_rss_bot.extensions.base import FeedExtension from discord_rss_bot.settings import data_dir from discord_rss_bot.webhook import DiscordWebhook +from discord_rss_bot.webhook import JsonValue from discord_rss_bot.webhook import WebhookFile if TYPE_CHECKING: @@ -122,6 +131,133 @@ def extract_app_id(url: str) -> str | None: return _extract_app_id_from_query(parsed.query) +#: Steam language codes that appear as image filename suffixes +#: (e.g. ``.../aa83bcaaafe7cd4d11a841ec381ec28ccb039d41/english.png``). +#: These localized URLs 404; the working URL omits the language segment. +_STEAM_IMAGE_LOCALES: tuple[str, ...] = ( + "arabic", + "brazilian", + "bulgarian", + "czech", + "danish", + "dutch", + "english", + "finnish", + "french", + "german", + "greek", + "hungarian", + "indonesian", + "italian", + "japanese", + "koreana", + "latam", + "norwegian", + "polish", + "portuguese", + "romanian", + "russian", + "schinese", + "spanish", + "swedish", + "tchinese", + "thai", + "turkish", + "ukrainian", + "vietnamese", +) + +#: Image file extensions that may carry a Steam locale suffix. +_STEAM_IMAGE_EXTENSIONS: tuple[str, ...] = ("png", "jpg", "jpeg", "gif", "webp") + +_STEAM_IMAGE_PATH_RE: re.Pattern[str] = re.compile( + rf"^(.+)/({'|'.join(_STEAM_IMAGE_LOCALES)})\.({'|'.join(_STEAM_IMAGE_EXTENSIONS)})$", + re.IGNORECASE, +) + + +def fix_steam_image_url(url: str) -> str: + """Return *url* with Steam's broken localized image suffix removed. + + Some Steam news feeds reference localized images such as + ``...//english.png``, which return a 404. The working URL omits + the language segment: ``.../.png``. + + Args: + url: The image URL to inspect. + + Returns: + The fixed URL, or *url* unchanged when no fix applies. + """ + if not url: + return url + stripped_url: str = url.strip() + parsed: ParseResult = urlparse(stripped_url) + if parsed.scheme not in {"http", "https"} or not parsed.netloc.lower().endswith(".steamstatic.com"): + return url + match: re.Match[str] | None = _STEAM_IMAGE_PATH_RE.match(parsed.path) + if not match: + return url + fixed_path: str = f"{match.group(1)}.{match.group(3)}" + return urlunsplit((parsed.scheme, parsed.netloc, fixed_path, parsed.query, parsed.fragment)) + + +#: Matches http(s) URLs within larger text so they can be rewritten in place. +_STEAM_URL_IN_TEXT_RE: re.Pattern[str] = re.compile(r"""https?://[^\s<>"']+""", re.IGNORECASE) + +#: Punctuation that may trail a URL inside prose or a markdown link. +_STEAM_URL_TRAILING_CHARS: frozenset[str] = frozenset(")]},.;!?") + + +def _fix_steam_image_urls_in_text(text: str) -> str: + """Rewrite broken Steam image URLs that appear inside *text*. + + Handles both standalone URLs and URLs embedded in markdown or prose + (e.g. ``[image](https://.../english.png)``). + + Args: + text: The text to rewrite. + + Returns: + The text with matching Steam image URLs fixed. + """ + + def _replace(match: re.Match[str]) -> str: + raw_url: str = match.group(0) + trimmed_url: str = raw_url + while trimmed_url and trimmed_url[-1] in _STEAM_URL_TRAILING_CHARS: + trimmed_url = trimmed_url[:-1] + trailing: str = raw_url[len(trimmed_url) :] + fixed_url: str = fix_steam_image_url(trimmed_url) + if fixed_url == trimmed_url: + return raw_url + return f"{fixed_url}{trailing}" + + return _STEAM_URL_IN_TEXT_RE.sub(_replace, text) + + +def fix_steam_image_urls_in_payload(value: JsonValue) -> JsonValue: + """Recursively rewrite Steam image URLs throughout a webhook payload. + + Walks every string in a JSON-compatible payload (embeds, media gallery + components, description text, avatar URL, etc.) and applies + :func:`fix_steam_image_url` to each. + + Args: + value: A JSON-compatible webhook payload value. + + Returns: + A new payload with all matching Steam image URLs fixed. + """ + if isinstance(value, str): + return _fix_steam_image_urls_in_text(value) + if isinstance(value, list): + return [fix_steam_image_urls_in_payload(item) for item in value] + if isinstance(value, dict): + return {key: fix_steam_image_urls_in_payload(item) for key, item in value.items()} + return value + + def _try_read_icon_file(app_id: str) -> WebhookFile | None: """Read a Steam game icon from disk, returning a ``WebhookFile`` or ``None``. @@ -233,7 +369,9 @@ class SteamExtension(FeedExtension): ) -> DiscordWebhook: """Set the embed thumbnail to the Steam game's capsule image. - Also attaches a locally cached icon file when available. + Also rewrites broken Steam CDN image URLs (localized image names + like ``.../english.png``) anywhere in the payload, and attaches a + locally cached icon file when available. Only applies when ``show_steam_game_icon_in_thumbnail`` is enabled in the feed's embed settings. @@ -249,6 +387,13 @@ class SteamExtension(FeedExtension): if not is_steam_url(feed_url): return webhook + # Steam news feeds occasionally emit localized image names + # (e.g. ".../english.png") that 404. Rewrite them throughout the + # payload so embeds, media galleries, and text links resolve. + fixed_payload = cast("dict[str, JsonValue]", fix_steam_image_urls_in_payload(webhook.json)) + webhook.json.clear() + webhook.json.update(fixed_payload) + if not _steam_thumbnail_enabled(reader, entry.feed): return webhook diff --git a/tests/test_custom_message.py b/tests/test_custom_message.py index 51d7718..c5c82eb 100644 --- a/tests/test_custom_message.py +++ b/tests/test_custom_message.py @@ -271,6 +271,32 @@ def test_get_image_urls_respects_limit() -> None: assert images == ["https://example.com/one.jpg", "https://example.com/two.jpg"] +def test_get_image_urls_fixes_broken_steam_image_url() -> None: + """Broken Steam localized image URLs should be rewritten during extraction.""" + summary = ( + '

' + ) + + images: list[str] = get_image_urls(summary, None) + + assert images == [ + ("https://clan.akamai.steamstatic.com/images/3703047/aa83bcaaafe7cd4d11a841ec381ec28ccb039d41.png"), + ] + + +def test_get_first_image_fixes_broken_steam_image_url() -> None: + """get_first_image should return the fixed Steam image URL.""" + summary = ( + '

' + ) + + image: str = get_first_image(summary, None) + + assert image == ("https://clan.akamai.steamstatic.com/images/3703047/aa83bcaaafe7cd4d11a841ec381ec28ccb039d41.png") + + def test_get_first_image_returns_empty_when_images_have_no_src() -> None: summary = "

" content = '

missing source

' diff --git a/tests/test_extensions.py b/tests/test_extensions.py index 01d07ba..ede87b6 100644 --- a/tests/test_extensions.py +++ b/tests/test_extensions.py @@ -37,10 +37,13 @@ from discord_rss_bot.extensions.hoyolab import HoyolabExtension from discord_rss_bot.extensions.jwplayer_thumbnail import _SLUG_CACHE from discord_rss_bot.extensions.jwplayer_thumbnail import JWPlayerThumbnailExtension from discord_rss_bot.extensions.steam import SteamExtension +from discord_rss_bot.extensions.steam import fix_steam_image_url +from discord_rss_bot.extensions.steam import fix_steam_image_urls_in_payload from discord_rss_bot.extensions.storage import get_enabled_extensions_for_feed from discord_rss_bot.extensions.storage import set_enabled_extensions_for_feed from discord_rss_bot.extensions.wordpress import WordPressExtension from discord_rss_bot.extensions.youtube import YouTubeExtension +from discord_rss_bot.webhook import DiscordWebhook if TYPE_CHECKING: from collections.abc import Iterator @@ -701,6 +704,131 @@ def test_steam_extension_has_auto_enable_patterns() -> None: assert not SteamExtension.matches_feed_url("https://example.com/feed.xml") +def test_fix_steam_image_url_removes_locale_suffix() -> None: + """Broken Steam localized image URLs should be rewritten to the working URL.""" + cases: list[tuple[str, str]] = [ + ( + "https://clan.akamai.steamstatic.com/images/3703047/aa83bcaaafe7cd4d11a841ec381ec28ccb039d41/english.png", + "https://clan.akamai.steamstatic.com/images/3703047/aa83bcaaafe7cd4d11a841ec381ec28ccb039d41.png", + ), + ( + "https://clan.fastly.steamstatic.com/images/3703047/aa83bcaaafe7cd4d11a841ec381ec28ccb039d41/english.png", + "https://clan.fastly.steamstatic.com/images/3703047/aa83bcaaafe7cd4d11a841ec381ec28ccb039d41.png", + ), + ( + "https://clan.akamai.steamstatic.com/images/123/abc/german.jpg", + "https://clan.akamai.steamstatic.com/images/123/abc.jpg", + ), + ( + "https://community.akamai.steamstatic.com/images/123/abc/schinese.webp", + "https://community.akamai.steamstatic.com/images/123/abc.webp", + ), + ( + "https://clan.akamai.steamstatic.com/images/123/abc/english.png?foo=1", + "https://clan.akamai.steamstatic.com/images/123/abc.png?foo=1", + ), + ] + for broken, expected in cases: + assert fix_steam_image_url(broken) == expected + + +def test_fix_steam_image_url_leaves_other_urls_unchanged() -> None: + """Non-matching URLs should be returned unchanged.""" + unchanged: list[str] = [ + "", + "https://example.com/images/abc/english.png", + "https://clan.akamai.steamstatic.com/images/123/abc.png", + "https://clan.akamai.steamstatic.com/images/123/abc/capsule_sm_120.jpg", + "https://clan.akamai.steamstatic.com/images/123/abc/notalocale.png", + "attachment://steam-app-570.png", + ] + for url in unchanged: + assert fix_steam_image_url(url) == url + + +def test_steam_modify_webhook_fixes_broken_image_urls() -> None: + """modify_webhook should rewrite broken Steam localized image URLs.""" + reader = MagicMock() + reader.get_tag.return_value = '{"show_steam_game_icon_in_thumbnail": false}' + + entry = SimpleNamespace( + feed=SimpleNamespace(url="https://store.steampowered.com/feeds/news/app/570/"), + link="https://store.steampowered.com/news/app/570/", + ) + + webhook = DiscordWebhook(url="https://discord.com/api/webhooks/123/abc") + webhook.content = "See [image](https://clan.akamai.steamstatic.com/images/123/abc/english.png)" + webhook.json["embeds"] = [ + { + "image": {"url": "https://clan.akamai.steamstatic.com/images/123/abc/french.jpg"}, + "thumbnail": {"url": "https://clan.akamai.steamstatic.com/images/123/abc/english.png"}, + }, + ] + webhook.json["components"] = [ + { + "type": 12, + "items": [ + { + "media": {"url": "https://clan.fastly.steamstatic.com/images/123/abc/german.png"}, + "description": "desc", + }, + ], + }, + ] + + result: DiscordWebhook = SteamExtension().modify_webhook(webhook, entry, reader) # type: ignore[arg-type] + + assert result.json["content"] == ("See [image](https://clan.akamai.steamstatic.com/images/123/abc.png)") + embeds = result.json["embeds"] + assert isinstance(embeds, list) + assert isinstance(embeds[0], dict) + image = embeds[0]["image"] + assert isinstance(image, dict) + assert image["url"] == "https://clan.akamai.steamstatic.com/images/123/abc.jpg" + thumbnail = embeds[0]["thumbnail"] + assert isinstance(thumbnail, dict) + assert thumbnail["url"] == "https://clan.akamai.steamstatic.com/images/123/abc.png" + + components = result.json["components"] + assert isinstance(components, list) + assert isinstance(components[0], dict) + items = components[0]["items"] + assert isinstance(items, list) + assert isinstance(items[0], dict) + media = items[0]["media"] + assert isinstance(media, dict) + assert media["url"] == "https://clan.fastly.steamstatic.com/images/123/abc.png" + + +def test_fix_steam_image_urls_in_text() -> None: + """URLs embedded in prose or markdown should also be rewritten.""" + text = ( + "See https://clan.akamai.steamstatic.com/images/123/abc/english.png for details. " + "And [this](https://clan.akamai.steamstatic.com/images/123/abc/french.jpg) link." + ) + + fixed = fix_steam_image_urls_in_payload(text) + + assert fixed == ( + "See https://clan.akamai.steamstatic.com/images/123/abc.png for details. " + "And [this](https://clan.akamai.steamstatic.com/images/123/abc.jpg) link." + ) + + +def test_steam_modify_webhook_skips_non_steam_feed() -> None: + """modify_webhook should leave the payload unchanged for non-Steam feeds.""" + entry = SimpleNamespace( + feed=SimpleNamespace(url="https://example.com/feed.xml"), + link="", + ) + webhook = DiscordWebhook(url="https://discord.com/api/webhooks/123/abc") + webhook.content = "https://clan.akamai.steamstatic.com/images/123/abc/english.png" + + result: DiscordWebhook = SteamExtension().modify_webhook(webhook, entry, MagicMock()) # type: ignore[arg-type] + + assert result.json["content"] == "https://clan.akamai.steamstatic.com/images/123/abc/english.png" + + def test_youtube_extension_has_auto_enable_patterns() -> None: """The built-in YouTube extension should declare URL patterns.""" assert len(YouTubeExtension.auto_enable_url_patterns) > 0