Add support for extensions
Some checks failed
Test and build Docker image / docker (push) Failing after 3s

This commit is contained in:
Joakim Hellsén 2026-07-20 06:02:25 +02:00
commit 793f67db42
Signed by: Joakim Hellsén
SSH key fingerprint: SHA256:/9h/CsExpFp+PRhsfA0xznFx2CGfTT5R/kpuFfUgEQk
39 changed files with 3670 additions and 1309 deletions

View file

@ -1,7 +1,7 @@
FROM python:3.14-slim
COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
RUN useradd --create-home botuser && \
mkdir -p /home/botuser/discord-rss-bot/ /home/botuser/.local/share/discord_rss_bot/ && \
mkdir -p /home/botuser/discord-rss-bot/ /home/botuser/.local/share/discord_rss_bot/ /home/botuser/extensions/ && \
chown -R botuser:botuser /home/botuser/
USER botuser
WORKDIR /home/botuser/discord-rss-bot
@ -11,5 +11,9 @@ RUN --mount=type=cache,target=/root/.cache/uv \
COPY --chown=botuser:botuser discord_rss_bot/ /home/botuser/discord-rss-bot/discord_rss_bot/
EXPOSE 5000
VOLUME ["/home/botuser/.local/share/discord_rss_bot/"]
# Extensions directory — bind-mount your own plugins here.
ENV EXTENSIONS_DIR=/home/botuser/extensions
HEALTHCHECK --interval=10m --timeout=5s CMD ["uv", "run", "./discord_rss_bot/healthcheck.py"]
CMD ["uv", "run", "uvicorn", "discord_rss_bot.main:app", "--host=0.0.0.0", "--port=5000", "--proxy-headers", "--forwarded-allow-ips='*'", "--log-level", "debug"]

View file

@ -15,12 +15,10 @@ Discord: TheLovinator#9276
- Regex filters for RSS feeds.
- Blacklist/whitelist words in the title/description/author/etc.
- Set different update frequencies for each feed or use a global default.
- Gets extra information from APIs if available, currently for:
- [https://feeds.c3kay.de/](https://feeds.c3kay.de/)
- Genshin Impact News
- Honkai Impact 3rd News
- Honkai Starrail News
- Zenless Zone Zero News
- Extensions that pulls extra data from feeds.
- Built-in extensions for Steam, YouTube, Hoyolab, WordPress, and JWPlayer.
- Add your own by dropping a `.py` file in `extensions/` ([docs](discord_rss_bot/extensions/README.md)).
- Feel free to open a GitHub issue if you need one.
## Installation

View file

@ -1,55 +0,0 @@
from __future__ import annotations
import urllib.parse
from functools import lru_cache
from typing import TYPE_CHECKING
from discord_rss_bot.filter.evaluator import get_entry_filter_decision_from_reader
if TYPE_CHECKING:
from reader import Entry
from reader import Reader
@lru_cache
def encode_url(url_to_quote: str) -> str:
"""%-escape the URL so it can be used in a URL.
If we didn't do this, we couldn't go to feeds with a ? in the URL.
You can use this in templates with {{ url | encode_url }}.
Args:
url_to_quote: The url to encode.
Returns:
The encoded url.
"""
return urllib.parse.quote(string=url_to_quote) if url_to_quote else ""
def entry_is_whitelisted(entry_to_check: Entry, reader: Reader) -> bool:
"""Check if the entry is whitelisted.
Args:
entry_to_check: The feed to check.
reader: Custom Reader instance.
Returns:
bool: True if the feed is whitelisted, False otherwise.
"""
return get_entry_filter_decision_from_reader(reader, entry_to_check).whitelist_match is not None
def entry_is_blacklisted(entry_to_check: Entry, reader: Reader) -> bool:
"""Check if the entry is blacklisted.
Args:
entry_to_check: The feed to check.
reader: Custom Reader instance.
Returns:
bool: True if the feed is blacklisted, False otherwise.
"""
return get_entry_filter_decision_from_reader(reader, entry_to_check).blacklist_match is not None

View file

@ -1,16 +1,15 @@
from __future__ import annotations
import html
import json
import logging
import re
from dataclasses import dataclass
from typing import TYPE_CHECKING
from bs4 import BeautifulSoup
from bs4 import Tag
from markdownify import markdownify
from discord_rss_bot.extensions import run_extensions
from discord_rss_bot.html_format import format_entry_html_for_discord
from discord_rss_bot.is_url_valid import is_url_valid
if TYPE_CHECKING:
@ -23,8 +22,6 @@ if TYPE_CHECKING:
logger: logging.Logger = logging.getLogger(__name__)
DISCORD_TIMESTAMP_TAG_RE: re.Pattern[str] = re.compile(r"<t:\d+(?::[tTdDfFrRsS])?>")
# Discord webhook username: nickname rules, max 80 chars; no "clyde"/"discord" substrings.
DISCORD_WEBHOOK_USERNAME_MAX_LENGTH: int = 80
DISCORD_WEBHOOK_USERNAME_FORBIDDEN_CHARS: frozenset[str] = frozenset("@#:`")
@ -43,7 +40,7 @@ class CustomEmbed:
thumbnail_url: str = ""
footer_text: str = ""
footer_icon_url: str = ""
show_steam_game_icon_in_thumbnail: bool = False
show_steam_game_icon_in_thumbnail: bool = True
def try_to_replace(custom_message: str, template: str, replace_with: str) -> str:
@ -64,68 +61,6 @@ def try_to_replace(custom_message: str, template: str, replace_with: str) -> str
return custom_message
def _preserve_discord_timestamp_tags(text: str) -> tuple[str, dict[str, str]]:
"""Replace Discord timestamp tags with placeholders before markdown conversion.
Args:
text: The text to replace tags in.
Returns:
The text with Discord timestamp tags replaced by placeholders and a mapping of placeholders to original tags.
"""
replacements: dict[str, str] = {}
def replace_match(match: re.Match[str]) -> str:
placeholder: str = f"DISCORDTIMESTAMPPLACEHOLDER{len(replacements)}"
replacements[placeholder] = match.group(0)
return placeholder
return DISCORD_TIMESTAMP_TAG_RE.sub(replace_match, text), replacements
def _restore_discord_timestamp_tags(text: str, replacements: dict[str, str]) -> str:
"""Restore preserved Discord timestamp tags after markdown conversion.
Args:
text: The text to restore tags in.
replacements: A mapping of placeholders to original Discord timestamp tags.
Returns:
The text with placeholders replaced by the original Discord timestamp tags.
"""
for placeholder, original_value in replacements.items():
text = text.replace(placeholder, original_value)
return text
def format_entry_html_for_discord(text: str) -> str:
"""Convert entry HTML to Discord-friendly markdown while preserving Discord timestamp tags.
Args:
text: The HTML text to format.
Returns:
The formatted text with Discord timestamp tags preserved.
"""
if not text:
return ""
unescaped_text: str = html.unescape(text)
protected_text, replacements = _preserve_discord_timestamp_tags(unescaped_text)
formatted_text: str = markdownify(
html=protected_text,
strip=["img", "table", "td", "tr", "tbody", "thead"],
escape_misc=False,
heading_style="ATX",
)
if "[https://" in formatted_text or "[https://www." in formatted_text:
formatted_text = formatted_text.replace("[https://", "[")
formatted_text = formatted_text.replace("[https://www.", "[")
return _restore_discord_timestamp_tags(formatted_text, replacements)
def replace_tags_in_text_message(entry: Entry, reader: Reader) -> str:
"""Replace tags in custom_message.
@ -191,6 +126,12 @@ def replace_tags_in_text_message(entry: Entry, reader: Reader) -> str:
{"{{image_1}}": first_image},
]
# Compute extension variables (handled separately so they can use
# the already-computed values above without ordering issues).
extension_vars: dict[str, str] = run_extensions(entry, reader)
for var_name, var_value in extension_vars.items():
list_of_replacements.append({f"{{{{{var_name}}}}}": var_value})
for replacement in list_of_replacements:
for template, replace_with in replacement.items():
if not isinstance(replace_with, str):
@ -353,6 +294,12 @@ def replace_tags_in_embed(feed: Feed, entry: Entry, reader: Reader) -> CustomEmb
{"{{entry_updated}}": entry_updated or ""},
{"{{image_1}}": first_image or ""},
]
# Compute extension variables (handled separately so they can use
# the already-computed values above without ordering issues).
extension_vars: dict[str, str] = run_extensions(entry, reader)
for var_name, var_value in extension_vars.items():
list_of_replacements.append({f"{{{{{var_name}}}}}": var_value})
for replacement in list_of_replacements:
for template, replace_with in replacement.items():
_replace_embed_tags(embed, template, replace_with)
@ -547,25 +494,10 @@ def get_embed(reader: Reader, feed: Feed) -> CustomEmbed:
thumbnail_url="",
footer_text="",
footer_icon_url="",
show_steam_game_icon_in_thumbnail=False,
show_steam_game_icon_in_thumbnail=True,
)
def coerce_embed_bool(value: object) -> bool:
"""Normalize stored embed booleans from JSON or form-like values.
Returns:
The coerced boolean value.
"""
if isinstance(value, bool):
return value
if isinstance(value, int):
return value != 0
if isinstance(value, str):
return value.strip().lower() in {"1", "true", "on", "yes"}
return False
def get_embed_data(embed_data: dict[str, str | int | bool]) -> CustomEmbed:
"""Get embed data from embed_data.
@ -585,8 +517,9 @@ def get_embed_data(embed_data: dict[str, str | int | bool]) -> CustomEmbed:
thumbnail_url: str = str(embed_data.get("thumbnail_url", ""))
footer_text: str = str(embed_data.get("footer_text", ""))
footer_icon_url: str = str(embed_data.get("footer_icon_url", ""))
show_steam_game_icon_in_thumbnail: bool = coerce_embed_bool(
embed_data.get("show_steam_game_icon_in_thumbnail", False),
show_steam_game_icon_in_thumbnail: bool = embed_data.get( # pyright: ignore[reportAssignmentType]
"show_steam_game_icon_in_thumbnail",
True,
)
return CustomEmbed(

View file

@ -0,0 +1,157 @@
# Extension System
Extensions are Python plugins that extract data from RSS/Atom entries and/or mutate the Discord webhook payload before messages are sent.
**Two hooks per entry:**
1. **`process_entry()`** → return `{variable_name: string_value}` dict → available as `{{variable_name}}` in templates.
2. **`modify_webhook()`** → mutate the `DiscordWebhook` (files, embeds, author, etc.) after template replacement.
---
## TL;DR — Make an Extension
Drop a `.py` file in your `extensions/` dir (or `$EXTENSIONS_DIR`). Subclass `FeedExtension`:
```python
"""
Word count example — provides {{word_count}} per entry.
"""
from __future__ import annotations
import logging
from typing import ClassVar, TYPE_CHECKING
from discord_rss_bot.extensions.base import FeedExtension
if TYPE_CHECKING:
from reader import Entry, Reader
logger = logging.getLogger(__name__)
class WordCountExtension(FeedExtension):
name = "word_count"
description = "Counts words in each entry's content."
provides_variables: ClassVar[list[str]] = ["word_count"]
def process_entry(self, entry: Entry, reader: Reader) -> dict[str, str]:
text = " ".join(
c.value for c in (entry.content or []) if hasattr(c, "value") and c.value
) or (entry.summary or "")
return {"word_count": str(len(text.split())) if text else "0"}
```
Then restart the bot, enable it in the web UI's Extensions page, and use `{{word_count}}` in templates.
---
## API Reference
### Class Attributes
| Attribute | Type | Required | Purpose |
| -------------------------- | --------------------- | -------- | --------------------------------------------------------- |
| `name` | `ClassVar[str]` | **Yes** | Unique slug for logs, registry, enable/disable |
| `description` | `ClassVar[str]` | No | Shown in web UI |
| `provides_variables` | `ClassVar[list[str]]` | No | Template variables this extension produces |
| `auto_enable_url_patterns` | `ClassVar[list[str]]` | No | Regex patterns; matching feeds auto-enable this extension |
### `process_entry(self, entry: Entry, reader: Reader) -> dict[str, str]` (required)
Called for every entry. Return `{var: value}` pairs. Missing variables from `provides_variables` get filled with `""`.
- `entry.content` — list of `Content` objects (`.value`, `.type`)
- `entry.summary` — plain-text string
- `entry.feed.url`, `entry.link`, `entry.title`, `entry.id`, etc.
### `modify_webhook(self, webhook: DiscordWebhook, entry: Entry, reader: Reader) -> DiscordWebhook` (optional)
Called after template replacement. Return modified `webhook` or it unchanged.
```python
# Content
webhook.content = "..." # or None to clear
# Author
webhook.username = "Bot Name"
webhook.avatar_url = "https://..."
# Embeds
embed = DiscordEmbed()
embed.set_title("...")
embed.set_description("...")
embed.set_url("https://...")
embed.set_color(0x00FF00) # or hex str "00FF00"
embed.set_author(name="...", url="...", icon_url="...")
embed.set_thumbnail(url="...")
embed.set_image(url="...")
embed.set_footer(text="...", icon_url="...")
embed.add_embed_field(name="Key", value="Value", inline=True)
embed.set_timestamp(timestamp="1234567890")
webhook.add_embed(embed)
webhook.remove_embeds()
# Files
webhook.add_file(file=bytes_, filename="img.png")
```
### Utility Class Methods
- `matches_feed_url(cls, feed_url) → bool` — checks `feed_url` against `auto_enable_url_patterns`
- `get_enabled_variables(cls, registry, enabled_names) → list[str]` — sorted union of `provides_variables` for enabled extensions
---
## Discovery
Extensions are discovered from two sources (external overrides built-in):
1. **Built-in**`discord_rss_bot/extensions/*.py` (this dir)
2. **External**`EXTENSIONS_DIR` env var, default `extensions/` relative to CWD
Every `.py` (except `__init__`) is imported. Concrete `FeedExtension` subclasses with a non-empty `name` go into a global `registry: dict[str, type[FeedExtension]]`. Duplicate names log a warning; last one wins.
| Function | Description |
| ---------------------------------- | ------------------------------------------------ |
| `discover_plugins(*, force=False)` | Scan dirs, populate registry, return it |
| `get_registry()` | Return current registry (discover on first call) |
| `registry_clear()` | Clear registry (for tests) |
---
## Auto-Enable
Set `auto_enable_url_patterns` on your class:
```python
auto_enable_url_patterns: ClassVar[list[str]] = [
r"store\.steampowered\.com",
r"youtube\.com/feeds/videos\.xml",
]
```
Uses `re.search()` (matches anywhere in URL). Only applies when the feed's extensions tag was never explicitly saved by the user. Runs on feed creation and lazily on first process for old feeds.
---
## Per-Feed Storage
Enabled extensions stored as a JSON list under reader tag key `"extensions"`.
| Function | Description |
| --------------------------------------------------------------- | ---------------------------- |
| `get_enabled_extensions_for_feed(reader, feed_url) → list[str]` | Get enabled names for a feed |
| `set_enabled_extensions_for_feed(reader, feed_url, names)` | Persist enabled names |
Web UI: `GET /extensions?feed_url=...` renders checkboxes; `POST /extensions` saves.
---
## Built-in Extensions
| Extension | `name` | Variables | Auto-enable |
| ---------------------------------------------- | -------------------- | ----------------------------------------------------------------------------------------------------------- | ---------------------------------------------- |
| [youtube.py](youtube.py) | `youtube` | `youtube_video_id`, `youtube_embed_url` | `youtube.com/feeds/videos.xml` |
| [steam.py](steam.py) | `steam` | `steam_thumbnail_url`, `steam_app_id` | `store.steampowered.com`, `steamcommunity.com` |
| [hoyolab.py](hoyolab.py) | `hoyolab` | `hoyolab_subject`, `hoyolab_description`, `hoyolab_image`, `hoyolab_author` | `feeds.c3kay.de` |
| [jwplayer_thumbnail.py](jwplayer_thumbnail.py) | `jwplayer_thumbnail` | `jwplayer_thumbnail`, `jwplayer_file` | `hentaigasm.com`, `hgasm[0-9]*.com` |
| [wordpress.py](wordpress.py) | `wordpress` | `wp_content`, `wp_content_raw`, `wp_excerpt`, `wp_excerpt_raw`, `wp_jwplayer_thumbnail`, `wp_jwplayer_file` | _(none)_ |

View file

@ -0,0 +1,32 @@
"""Feed extension system: discover, configure, and run per-entry content extensions.
Extensions are Python classes that inherit from ``FeedExtension``,
auto-discovered from the ``EXTENSIONS_DIR`` directory (default: ``extensions/``).
Typical usage::
from discord_rss_bot.extensions import run_extensions
extra_vars: dict[str, str] = run_extensions(entry, reader)
# extra_vars == {"jwplayer_thumbnail": "https://...", ...}
"""
from __future__ import annotations
from discord_rss_bot.extensions.base import FeedExtension
from discord_rss_bot.extensions.discovery import discover_plugins
from discord_rss_bot.extensions.discovery import get_registry
from discord_rss_bot.extensions.discovery import registry_clear
from discord_rss_bot.extensions.runner import auto_enable_extensions_for_feed
from discord_rss_bot.extensions.runner import run_extensions
from discord_rss_bot.extensions.runner import run_modify_webhook
__all__ = [
"FeedExtension",
"auto_enable_extensions_for_feed",
"discover_plugins",
"get_registry",
"registry_clear",
"run_extensions",
"run_modify_webhook",
]

View file

@ -0,0 +1,127 @@
"""Abstract base class for feed extensions.
Every extension in the ``EXTENSIONS_DIR`` must inherit from
``FeedExtension`` and implement ``process_entry()``.
"""
from __future__ import annotations
import re
from abc import ABC
from abc import abstractmethod
from typing import TYPE_CHECKING
from typing import ClassVar
if TYPE_CHECKING:
from reader import Entry
from reader import Reader
from discord_rss_bot.webhook import DiscordWebhook
class FeedExtension(ABC):
"""Base class for per-feed content extensions.
Subclasses are auto-discovered from the ``EXTENSIONS_DIR`` directory.
Each instance processes one entry and returns template variables
that become available as ``{{variable_name}}`` in Discord messages
and embeds.
Class attributes set by subclasses:
``name``:
Unique identifier (used in logs and per-feed enable/disable config).
Should be a short slug, e.g. ``"jwplayer_thumbnail"``.
``description``:
Human-readable description shown in the web UI (optional).
"""
name: ClassVar[str] = ""
description: ClassVar[str] = ""
#: List of template variable names this extension provides.
#: Used by the web UI to show users what ``{{variable}}`` tags
#: are available when an extension is enabled.
provides_variables: ClassVar[list[str]] = []
#: List of regex patterns. When a feed URL matches any pattern,
#: this extension is automatically enabled for that feed.
auto_enable_url_patterns: ClassVar[list[str]] = []
@abstractmethod
def process_entry(self, entry: Entry, reader: Reader) -> dict[str, str]:
"""Return template variable pairs extracted from *entry*.
Args:
entry: The feed entry to process.
reader: The reader instance (useful for loading feed tags).
Returns:
A dict of ``{variable_name: value}`` pairs. Return an empty
dict if this entry does not need any extra variables.
"""
def modify_webhook(
self,
webhook: DiscordWebhook,
_entry: Entry,
_reader: Reader,
) -> DiscordWebhook:
"""Modify the Discord webhook before it is sent.
Override this method when your extension needs to change the
webhook payload itself (e.g. to add files, replace embeds, set
author information, etc.). By default it returns the webhook
unchanged.
Args:
webhook: The fully built webhook payload.
_entry: The feed entry being processed.
_reader: The reader instance.
Returns:
The (possibly modified) ``DiscordWebhook``. Return the
original *webhook* unchanged if no modifications are needed.
"""
return webhook
@classmethod
def get_enabled_variables(cls, registry: dict[str, type[FeedExtension]], enabled_names: list[str]) -> list[str]:
"""Return the union of ``provides_variables`` for all enabled extensions.
Args:
registry: The extension registry ``{name: class}``.
enabled_names: List of enabled extension names for a feed.
Returns:
Sorted list of unique variable names.
"""
seen: set[str] = set()
result: list[str] = []
for name in enabled_names:
ext_cls = registry.get(name)
if ext_cls is None:
continue
for var_name in ext_cls.provides_variables:
if var_name not in seen:
seen.add(var_name)
result.append(var_name)
result.sort()
return result
@classmethod
def matches_feed_url(cls, feed_url: str) -> bool:
"""Return ``True`` if *feed_url* matches any of the auto-enable patterns.
Args:
feed_url: The feed URL to test.
Returns:
``True`` if a match is found, ``False`` otherwise (including when
no patterns are defined).
"""
if not cls.auto_enable_url_patterns:
return False
return any(re.search(p, feed_url) for p in cls.auto_enable_url_patterns)

View file

@ -0,0 +1,191 @@
"""Plugin discovery — import ``.py`` files from ``EXTENSIONS_DIR``.
All ``FeedExtension`` subclasses found in those files are collected into
a global registry (``dict[str, type[FeedExtension]]``).
"""
from __future__ import annotations
import importlib.util
import logging
import os
import sys
from pathlib import Path
from typing import TYPE_CHECKING
if TYPE_CHECKING:
import types
from discord_rss_bot.extensions.base import FeedExtension
logger: logging.Logger = logging.getLogger(__name__)
#: Environment variable that points to the extensions directory.
ENV_EXTENSIONS_DIR: str = "EXTENSIONS_DIR"
#: Fallback if the env var is not set.
DEFAULT_EXTENSIONS_DIR: str = "extensions"
_registry: dict[str, type[FeedExtension]] = {}
_discovered: bool = False
def _get_extensions_dir() -> Path:
"""Return the path to the extensions directory.
Respects the ``EXTENSIONS_DIR`` environment variable. Falls back to
``extensions/`` relative to the current working directory.
"""
raw: str = os.getenv(ENV_EXTENSIONS_DIR, "").strip()
if raw:
return Path(raw).resolve()
return Path.cwd() / DEFAULT_EXTENSIONS_DIR
def _try_import_module(module_name: str, filepath: Path) -> types.ModuleType | None:
"""Create a module spec and execute it, returning the module or ``None``.
Args:
module_name: The name to give the imported module.
filepath: Absolute path to a ``.py`` file.
Returns:
The imported module, or ``None`` if the spec could not be created.
"""
spec = importlib.util.spec_from_file_location(module_name, filepath)
if spec is None or spec.loader is None:
logger.warning("Could not create module spec for %s", filepath)
return None
module = importlib.util.module_from_spec(spec)
# Make the parent package importable so relative imports in plugins work.
sys.modules[module_name] = module
spec.loader.exec_module(module)
return module
def _import_module_from_path(filepath: Path) -> types.ModuleType | None:
"""Import a single ``.py`` file as a module and return the module object.
Args:
filepath: Absolute path to a ``.py`` file.
Returns:
The imported module, or ``None`` on failure.
"""
module_name: str = f"_ext_plugin_{filepath.stem}"
# Avoid re-importing if already loaded.
if module_name in sys.modules:
return sys.modules[module_name]
try:
return _try_import_module(module_name, filepath)
except Exception:
logger.exception("Failed to import extension plugin from %s", filepath)
return None
def _collect_extensions_from_module(module: types.ModuleType) -> list[type[FeedExtension]]:
"""Return ``FeedExtension`` subclasses defined in *module*.
Only concrete (non-ABC) subclasses with a non-empty ``name`` are
returned.
Args:
module: An imported module object.
Returns:
List of extension classes found in the module.
"""
extensions: list[type[FeedExtension]] = []
for attr_name in dir(module):
obj = getattr(module, attr_name)
if (
isinstance(obj, type)
and issubclass(obj, FeedExtension)
and obj is not FeedExtension
and not getattr(obj, "__abstractmethods__", None)
):
name: str = getattr(obj, "name", "") or attr_name
extensions.append(obj)
logger.debug("Discovered extension %r (class %s)", name, attr_name)
return extensions
def _scan_directory(ext_dir: Path) -> None:
"""Scan a single directory and register any ``FeedExtension`` subclasses found."""
if not ext_dir.is_dir():
return
py_files: list[Path] = sorted(p for p in ext_dir.iterdir() if p.suffix == ".py" and p.stem != "__init__")
for filepath in py_files:
module = _import_module_from_path(filepath)
if module is None:
continue
classes = _collect_extensions_from_module(module)
for cls in classes:
cls_name: str = cls.name or cls.__name__
if cls_name in _registry:
logger.warning("Duplicate extension name %r — overwriting with %s", cls_name, filepath)
_registry[cls_name] = cls
def discover_plugins(*, force: bool = False) -> dict[str, type[FeedExtension]]:
"""Scan built-in and external extension directories and register.
All ``FeedExtension`` subclasses found in those directories are
collected into a global registry.
Order of discovery:
1. Built-in extensions shipped with the bot (``discord_rss_bot/extensions/``)
2. External user-provided plugins from ``EXTENSIONS_DIR``
External plugins can override built-in ones by using the same ``name``.
This is called automatically on first import. Call with
``force=True`` to re-scan (useful in tests).
Args:
force: If ``True``, re-scan even if already discovered.
Returns:
The global registry ``{name: class}``.
"""
global _discovered # ruff:ignore[global-statement]
if _discovered and not force:
return _registry
# 1. Scan the built-in package directory first.
built_in_dir: Path = Path(__file__).resolve().parent
logger.debug("Scanning built-in extensions directory: %s", built_in_dir)
_scan_directory(built_in_dir)
# 2. Scan the external user-provided extensions directory.
ext_dir: Path = _get_extensions_dir()
if ext_dir != built_in_dir:
logger.info("Scanning external extensions directory: %s", ext_dir)
_scan_directory(ext_dir)
else:
logger.debug("External extensions directory is the same as built-in — skipping duplicate scan")
_discovered = True
logger.info("Discovered %d extension plugin(s): %s", len(_registry), list(_registry.keys()))
return _registry
def get_registry() -> dict[str, type[FeedExtension]]:
"""Return the current extension registry, discovering plugins if needed.
Returns:
The global registry ``{name: class}``.
"""
if not _discovered:
return discover_plugins()
return _registry
def registry_clear() -> None:
"""Clear the registry (useful for testing)."""
global _discovered, _registry # ruff:ignore[global-statement]
_registry = {}
_discovered = False

View file

@ -0,0 +1,331 @@
"""Hoyolab feed extension: fetches full post data from the Hoyolab API.
Detects feeds from ``feeds.c3kay.de`` and replaces the Discord embed
with richer content fetched directly from Hoyolab (Genshin Impact,
Honkai Starrail, Honkai Impact 3rd, Zenless Zone Zero).
Usage:
1. Enable ``hoyolab`` on the feed's Extensions page.
2. The extension automatically decorates the embed with the post
title, author avatar, game color, images, and more.
"""
from __future__ import annotations
import contextlib
import json
import logging
import re
from typing import TYPE_CHECKING
from typing import ClassVar
from typing import cast
import requests
from discord_rss_bot.extensions.base import FeedExtension
from discord_rss_bot.webhook import DiscordEmbed
from discord_rss_bot.webhook import DiscordWebhook
if TYPE_CHECKING:
from reader import Entry
from reader import Reader
logger: logging.Logger = logging.getLogger(__name__)
type JsonValue = bool | int | float | str | list[JsonValue] | dict[str, JsonValue] | None
type JsonObject = dict[str, JsonValue]
def is_c3kay_feed_url(feed_url: str) -> bool:
"""Return whether a feed URL is from feeds.c3kay.de."""
return "feeds.c3kay.de" in (feed_url or "")
def extract_post_id(url: str) -> str | None:
"""Extract the Hoyolab post ID from a URL.
Args:
url: The URL to inspect (e.g. ``https://www.hoyolab.com/article/38588239``).
Returns:
The numeric post ID if found, otherwise ``None``.
"""
if not url:
return None
try:
match = re.search(r"/article/(\d+)", url)
if match:
return match.group(1)
except (ValueError, AttributeError, TypeError) as exc:
logger.warning("Error extracting post ID from URL %s: %s", url, exc)
return None
def _try_fetch_post(post_id: str) -> JsonObject | None:
"""Attempt to fetch a Hoyolab post, returning the payload or ``None``.
Does not catch exceptions the caller handles them.
Returns:
The post payload dict, or ``None`` on failure.
"""
url: str = f"https://bbs-api-os.hoyolab.com/community/post/wapi/getPostFull?post_id={post_id}"
response: requests.Response = requests.get(url, timeout=10)
if response.status_code == http_ok:
data = cast("JsonObject", response.json())
data_payload: JsonObject = cast("JsonObject", data.get("data", {}))
post_payload: JsonObject = cast("JsonObject", data_payload.get("post", {}))
if data.get("retcode") == 0 and post_payload:
return post_payload
logger.warning("Failed to fetch Hoyolab post %s: %s", post_id, response.text)
return None
def fetch_post(post_id: str) -> JsonObject | None:
"""Fetch post data from the Hoyolab API.
Args:
post_id: The numeric post ID.
Returns:
The post payload dict, or ``None`` on failure.
"""
if not post_id:
return None
try:
return _try_fetch_post(post_id)
except (requests.RequestException, ValueError):
logger.exception("Error fetching Hoyolab post %s", post_id)
return None
http_ok: int = 200
def _as_json_object(value: JsonValue) -> JsonObject:
return cast("JsonObject", value) if isinstance(value, dict) else {}
class HoyolabExtension(FeedExtension):
"""Fetches full post data from the Hoyolab API for c3kay.de feeds.
Replaces the Discord embed with richer content (title, author avatar,
game color, images, video, and more).
"""
name = "hoyolab"
description = (
"Detects c3kay.de / Hoyolab feeds and replaces the Discord embed "
"with full post data from the Hoyolab API "
"(Genshin Impact, Honkai Starrail, Honkai Impact 3rd, ZZZ)."
)
provides_variables: ClassVar[list[str]] = [
"hoyolab_subject",
"hoyolab_description",
"hoyolab_image",
"hoyolab_author",
]
auto_enable_url_patterns: ClassVar[list[str]] = [
r"feeds\.c3kay\.de",
]
def process_entry(self, entry: Entry, reader: Reader) -> dict[str, str]: # ruff:ignore[unused-method-argument]
"""Provide Hoyolab post data as template variables.
Args:
entry: The feed entry to process.
reader: The reader instance.
Returns:
Dict with ``hoyolab_subject``, ``hoyolab_description``,
``hoyolab_image`` and ``hoyolab_author`` if the data was
fetched successfully, otherwise empty dict.
"""
if not is_c3kay_feed_url(entry.feed.url):
return {}
post_id: str | None = extract_post_id(str(entry.link or ""))
if not post_id:
return {}
post_data: JsonObject | None = fetch_post(post_id)
if not post_data:
return {}
post: JsonObject = _as_json_object(post_data.get("post"))
subject: str = str(post.get("subject", ""))
description: str = str(post.get("desc", ""))
image_url: str = ""
image_list = post_data.get("image_list", [])
if isinstance(image_list, list) and image_list:
first_image = image_list[0]
if isinstance(first_image, dict):
image_url = str(first_image.get("url", ""))
author: str = ""
user: JsonObject = _as_json_object(post_data.get("user"))
if user:
author = str(user.get("nickname", ""))
return {
"hoyolab_subject": subject,
"hoyolab_description": description,
"hoyolab_image": image_url,
"hoyolab_author": author,
}
def modify_webhook(
self,
webhook: DiscordWebhook,
entry: Entry,
reader: Reader, # ruff:ignore[unused-method-argument]
) -> DiscordWebhook:
"""Replace the embed with richer Hoyolab content when applicable.
Args:
webhook: The fully built webhook payload.
entry: The feed entry being processed.
reader: The reader instance.
Returns:
The modified webhook with Hoyolab data.
"""
if not is_c3kay_feed_url(entry.feed.url):
return webhook
# Don't interfere with non-embed delivery modes.
if not webhook.json.get("embeds"):
return webhook
post_id: str | None = extract_post_id(str(entry.link or ""))
if not post_id:
return webhook
post_data: JsonObject | None = fetch_post(post_id)
if not post_data:
return webhook
discord_embed = self._build_embed_from_post(post_data, entry.link or entry.feed.url)
self._attach_video_if_present(webhook, entry, post_data)
self._apply_author_from_post(webhook, post_data)
self._apply_structured_content(webhook, post_data)
webhook.remove_embeds()
webhook.add_embed(discord_embed)
return webhook
def _build_embed_from_post(self, post_data: JsonObject, entry_link: str) -> DiscordEmbed:
"""Build a ``DiscordEmbed`` populated with post data.
Args:
post_data: Raw Hoyolab post payload.
entry_link: Fallback link URL.
Returns:
A populated embed.
"""
post: JsonObject = _as_json_object(post_data.get("post"))
subject: str = str(post.get("subject", ""))
content_raw: str = str(post.get("content", "{}"))
content_data: JsonObject = {}
with contextlib.suppress(json.JSONDecodeError, ValueError):
loaded = cast("JsonValue", json.loads(content_raw))
content_data = _as_json_object(loaded)
description: str = str(content_data.get("describe", ""))
if not description:
description = str(post.get("desc", ""))
discord_embed = DiscordEmbed()
discord_embed.set_title(subject)
discord_embed.set_url(entry_link)
# Image list
image_list_value: JsonValue = post_data.get("image_list", [])
image_list: list[JsonObject] = (
[cast("JsonObject", item) for item in image_list_value if isinstance(item, dict)]
if isinstance(image_list_value, list)
else []
)
if image_list:
discord_embed.set_image(url=str(image_list[0].get("url", "")))
# Game colour
game: JsonObject = _as_json_object(post_data.get("game"))
if game and game.get("color"):
discord_embed.set_color(str(game.get("color", "")).removeprefix("#"))
# Footer
classification: JsonObject = _as_json_object(post_data.get("classification"))
if classification and classification.get("name"):
discord_embed.set_footer(text=str(classification.get("name", "")))
# Event dates
event_start: str = str(post.get("event_start_date", ""))
if event_start and event_start != "0":
discord_embed.add_embed_field(name="Start", value=f"<t:{event_start}:R>")
event_end: str = str(post.get("event_end_date", ""))
if event_end and event_end != "0":
discord_embed.add_embed_field(name="End", value=f"<t:{event_end}:R>")
created_at: str = str(post.get("created_at", ""))
if created_at and created_at != "0":
discord_embed.set_timestamp(timestamp=created_at)
if description:
discord_embed.set_description(description)
return discord_embed
def _attach_video_if_present(self, webhook: DiscordWebhook, entry: Entry, post_data: JsonObject) -> None:
"""Download and attach a Hoyolab video to the webhook if present."""
video: JsonObject = _as_json_object(post_data.get("video"))
if not video or not video.get("url"):
return
video_url: str = str(video.get("url", ""))
with contextlib.suppress(requests.RequestException):
video_response: requests.Response = requests.get(video_url, stream=True, timeout=10)
if video_response.ok:
webhook.add_file(file=video_response.content, filename=f"{entry.id}.mp4")
def _apply_author_from_post(self, webhook: DiscordWebhook, post_data: JsonObject) -> None:
"""Set webhook author (username and avatar) from post user data."""
user: JsonObject = _as_json_object(post_data.get("user"))
author_name: str = str(user.get("nickname", ""))
avatar_url: str = str(user.get("avatar_url", ""))
if author_name:
webhook.avatar_url = avatar_url
webhook.username = author_name
def _apply_structured_content(self, webhook: DiscordWebhook, post_data: JsonObject) -> None:
"""Parse structured content for YouTube embeds and add them to the webhook."""
post: JsonObject = _as_json_object(post_data.get("post"))
structured_content: str = str(post.get("structured_content", ""))
if not structured_content:
return
with contextlib.suppress(json.JSONDecodeError, ValueError):
loaded_structured = cast("JsonValue", json.loads(structured_content))
structured_items: list[JsonObject] = (
[cast("JsonObject", item) for item in loaded_structured if isinstance(item, dict)]
if isinstance(loaded_structured, list)
else []
)
for item in structured_items:
self._apply_structured_item(webhook, item)
def _apply_structured_item(self, webhook: DiscordWebhook, item: JsonObject) -> None:
"""Apply a single structured content item (e.g. YouTube embed)."""
insert: JsonObject = _as_json_object(item.get("insert"))
if not insert:
return
video_url_str: str = str(insert.get("video", ""))
if not video_url_str:
return
video_id_match = re.search(r"embed/([a-zA-Z0-9_-]+)", video_url_str)
if not video_id_match:
return
webhook.content = f"https://www.youtube.com/watch?v={video_id_match.group(1)}"
webhook.remove_embeds()

View file

@ -0,0 +1,285 @@
"""Extension: extract JWPlayer thumbnail/video URLs from WordPress sites.
`JWPlayer <https://www.jwplayer.com/>`_ is a commercial video player
used by many sites to serve self-hosted or CDN-hosted video. The player
is embedded via a ``<script>`` block with a ``jwplayer()`` setup call
that contains ``image:`` (thumbnail) and ``file:`` (video URL) properties.
Feedparser strips ``<script>`` tags, so these URLs are invisible in the
parsed feed content. This extension recovers them by either scanning the
raw feed content directly or, for WordPress sites, fetching post HTML
from the REST API in a single batched request (cached by slug).
Created in response to `issue #432
<https://github.com/TheLovinator1/discord-rss-bot/issues/432>`_.
Currently auto-enabled for sites matching ``hentaigasm.com`` and
``hgasm[0-9]*.com``.
"""
from __future__ import annotations
import logging
import re
from typing import TYPE_CHECKING
from typing import ClassVar
from urllib.parse import quote
from urllib.parse import urlparse
import httpx2
from discord_rss_bot.extensions.base import FeedExtension
if TYPE_CHECKING:
from reader import Entry
from reader import Reader
from discord_rss_bot.feeds import JsonValue
logger: logging.Logger = logging.getLogger(__name__)
_IMAGE_PATTERN: re.Pattern[str] = re.compile(
r'image:\s*["\']([^"\']+)["\']',
re.IGNORECASE,
)
_FILE_PATTERN: re.Pattern[str] = re.compile(
r'file:\s*["\']([^"\']+\.\w+)["\']',
re.IGNORECASE,
)
#: Shared HTTP client — pooled connections, no TLS handshake per call.
_HTTP_CLIENT: httpx2.Client | None = None
#: Cache of ``{slug: post_data}`` — populated by the first entry
#: that needs it, then reused for all subsequent entries from the same site.
#: Each post_data dict contains ``content``, ``excerpt``, and ``title`` keys.
_SLUG_CACHE: dict[str, dict[str, dict[str, str]]] = {}
_HTTP_OK: int = 200
def _get_http() -> httpx2.Client:
"""Return the shared ``httpx2.Client``, creating it on first call.
Returns:
The shared ``httpx2.Client`` instance.
"""
global _HTTP_CLIENT # ruff:ignore[global-statement]
if _HTTP_CLIENT is None:
_HTTP_CLIENT = httpx2.Client(timeout=10)
return _HTTP_CLIENT
def _extract_slug(url: str) -> str | None:
"""Extract the last non-numeric path segment as a potential post slug.
Args:
url: The URL to extract the slug from.
Returns:
The extracted slug, or ``None`` if no slug could be extracted.
"""
if not url:
return None
try:
path: str = urlparse(url).path.rstrip("/")
segments: list[str] = [s for s in path.split("/") if s]
if not segments or segments[-1].isdigit():
return None
return segments[-1]
except ValueError:
return None
def _build_slug_cache(base_url: str) -> dict[str, dict[str, str]]:
"""Fetch all recent posts from the WordPress API and build the slug cache.
Fetches ``content``, ``excerpt``, and ``title`` for each post so that
callers can access full post data without additional API requests.
Args:
base_url: The site base URL (scheme + netloc).
Returns:
A dict mapping slugs to ``{content: ..., excerpt: ..., title: ...}``.
"""
client: httpx2.Client = _get_http()
fields: str = "slug,content,excerpt,title"
api_url: str = f"{base_url}/wp-json/wp/v2/posts?per_page=100&orderby=date&order=desc&_fields={fields}"
resp = client.get(api_url)
if resp.status_code != _HTTP_OK:
logger.warning("WordPress API returned %s for batch slug lookup", resp.status_code)
return {}
data = resp.json()
if not isinstance(data, list):
return {}
fresh: dict[str, dict[str, str]] = {}
for post in data:
if not isinstance(post, dict):
continue
ps = post.get("slug")
if not isinstance(ps, str):
continue
content_rendered: str = _extract_rendered(post.get("content"))
excerpt_rendered: str = _extract_rendered(post.get("excerpt"))
title_rendered: str = _extract_rendered(post.get("title"))
fresh[ps] = {
"content": content_rendered,
"excerpt": excerpt_rendered,
"title": title_rendered,
}
logger.info("Batch-fetched %d posts from %s", len(fresh), base_url)
return fresh
def _extract_rendered(obj: JsonValue) -> str:
"""Extract the ``rendered`` string from a WordPress API object.
Args:
obj: A WordPress API object (typically a dict with a ``rendered`` key).
Returns:
The rendered string, or ``""`` if the object is not a dict or
has no ``rendered`` key.
"""
if isinstance(obj, dict):
rendered = obj.get("rendered")
if isinstance(rendered, str):
return rendered
return ""
def _get_rendered_for_slug(slug: str, base_url: str) -> str | None:
"""Return the content.rendered HTML for *slug*, fetching on first miss.
The first call for a given *base_url* triggers a **single** batch
request (``?per_page=100``) that caches all recent posts. Every
subsequent call for the same site is a dict lookup zero network.
Args:
slug: The post slug to look up.
base_url: The site base URL (scheme + netloc).
Returns:
The rendered content HTML for the post, or ``None`` if not found.
"""
post_data: dict[str, str] | None = _get_post_data_for_slug(slug, base_url)
if post_data is None:
return None
return post_data.get("content") or None
def _get_post_data_for_slug(slug: str, base_url: str) -> dict[str, str] | None:
"""Return the full post data dict for *slug*, fetching on first miss.
The returned dict contains ``content``, ``excerpt``, and ``title`` keys.
The first call for a given *base_url* triggers a **single** batch
request (``?per_page=100``) that caches all recent posts. Every
subsequent call for the same site is a dict lookup zero network.
Args:
slug: The post slug to look up.
base_url: The site base URL (scheme + netloc).
Returns:
A dict with ``content``, ``excerpt``, ``title`` keys, or ``None``.
"""
site_cache: dict[str, dict[str, str]] | None = _SLUG_CACHE.get(base_url)
if site_cache is not None:
return site_cache.get(slug)
try:
fresh: dict[str, dict[str, str]] = _build_slug_cache(base_url)
except Exception:
logger.exception("Failed to batch-fetch WordPress posts from %s", base_url)
fresh = {}
_SLUG_CACHE[base_url] = fresh
return fresh.get(slug)
class JWPlayerThumbnailExtension(FeedExtension):
"""Extract the ``image`` URL from a JWPlayer ``setup()`` call.
Uses a single batch WordPress API request per site to fetch all
recent post content, then looks up entries by slug zero API
calls per entry.
"""
name = "jwplayer_thumbnail"
description = (
"Extracts JWPlayer thumbnail and video URLs from entry content "
"or the WordPress REST API (batched). "
"Available as {{jwplayer_thumbnail}} and {{jwplayer_file}}."
)
provides_variables: ClassVar[list[str]] = ["jwplayer_thumbnail", "jwplayer_file"]
auto_enable_url_patterns: ClassVar[list[str]] = [
r"hentaigasm\.com",
r"hgasm[0-9]*\.com",
]
def process_entry(self, entry: Entry, _reader: Reader) -> dict[str, str]:
"""Extract JWPlayer URLs from feed content or WordPress API.
Args:
entry: The feed entry to process.
Returns:
Dict with ``jwplayer_thumbnail`` and ``jwplayer_file``
if found, otherwise empty dict.
"""
result: dict[str, str] = {}
# Phase 1: search feed content (fast).
sources: list[str] = []
if entry.content:
sources.extend(item.value for item in entry.content if hasattr(item, "value") and item.value)
entry_summary: str | None = getattr(entry, "summary", None)
if entry_summary:
sources.append(entry_summary)
for source in sources:
self._search(source, result)
if "jwplayer_thumbnail" in result and "jwplayer_file" in result:
return result
# Phase 2: batch WordPress API lookup (one request per site).
if not result:
entry_link: str | None = getattr(entry, "link", None)
if entry_link:
slug: str | None = _extract_slug(entry_link)
if slug:
base_url: str | None = None
try:
parsed = urlparse(entry.feed.url)
base_url = f"{parsed.scheme}://{parsed.netloc}"
except ValueError:
return result
html: str | None = _get_rendered_for_slug(slug, base_url)
if html:
self._search(html, result)
return result
@staticmethod
def _search(html: str, result: dict[str, str]) -> None:
"""Search a single HTML source for JWPlayer patterns.
Args:
html: The HTML content to search.
result: The result dict to populate.
"""
if "jwplayer_thumbnail" not in result:
match = _IMAGE_PATTERN.search(html)
if match:
result["jwplayer_thumbnail"] = quote(match.group(1).strip(), safe=":/")
if "jwplayer_file" not in result:
match = _FILE_PATTERN.search(html)
if match:
result["jwplayer_file"] = quote(match.group(1).strip(), safe=":/")

View file

@ -0,0 +1,239 @@
"""Functions for running extensions against entries.
This module is separated from ``__init__.py`` to keep the package's
public API as pure re-exports (Ruff rule: non-empty-init-module).
"""
from __future__ import annotations
import logging
import re
from typing import TYPE_CHECKING
from discord_rss_bot.extensions.discovery import discover_plugins
from discord_rss_bot.extensions.discovery import get_registry
from discord_rss_bot.extensions.storage import EXTENSIONS_TAG
from discord_rss_bot.extensions.storage import get_enabled_extensions_for_feed
from discord_rss_bot.extensions.storage import set_enabled_extensions_for_feed
if TYPE_CHECKING:
from reader import Entry
from reader import Reader
from discord_rss_bot.extensions.base import FeedExtension
from discord_rss_bot.webhook import DiscordWebhook
logger: logging.Logger = logging.getLogger(__name__)
# Auto-discover plugins on first import so the registry is ready.
discover_plugins()
def auto_enable_extensions_for_feed(reader: Reader, feed_url: str) -> list[str]:
"""Enable extensions whose URL patterns match *feed_url*, if not already enabled.
Auto-enable only runs when the extensions tag has **never** been explicitly
set for this feed. Once the user visits the Extensions page and saves
(even with an empty list), auto-enable is skipped the user's choice
is respected.
This is called automatically when a feed is created, and lazily on first
processing for existing feeds.
Args:
reader: The reader instance.
feed_url: The feed URL to match against.
Returns:
The updated list of enabled extensions for this feed.
"""
# If the user has ever saved the Extensions page, respect their choice.
if reader.get_tag(feed_url, EXTENSIONS_TAG, None) is not None:
return get_enabled_extensions_for_feed(reader, feed_url)
registry: dict[str, type[FeedExtension]] = get_registry()
already_enabled: list[str] = get_enabled_extensions_for_feed(reader, feed_url)
to_add: list[str] = []
for name, cls in registry.items():
if name in already_enabled:
continue
if not getattr(cls, "auto_enable_url_patterns", None):
continue
for pattern in cls.auto_enable_url_patterns:
if re.search(pattern, feed_url):
logger.info(
"Auto-enabling extension %r for feed %s (matched pattern %r)",
name,
feed_url,
pattern,
)
to_add.append(name)
break
if not to_add:
return already_enabled
updated: list[str] = already_enabled + [n for n in to_add if n not in already_enabled]
set_enabled_extensions_for_feed(reader, feed_url, updated)
return updated
def _get_enabled_instances(entry: Entry, reader: Reader) -> list[FeedExtension]:
"""Return enabled extension instances for the given entry.
Auto-enables extensions whose URL patterns match the feed URL
(lazy initialisation for feeds that were created before the
extension system existed).
Args:
entry: The feed entry to process.
reader: The reader instance.
Returns:
List of ``FeedExtension`` instances enabled for this entry's feed.
Empty list if no extensions are enabled.
"""
feed_url: str = entry.feed.url
enabled: list[str] = auto_enable_extensions_for_feed(reader, feed_url)
if not enabled:
return []
registry: dict[str, type[FeedExtension]] = get_registry()
instances: list[FeedExtension] = []
for name in enabled:
cls = registry.get(name)
if cls is None:
logger.warning(
"Extension %r is enabled for feed %r but was not found in the registry",
name,
feed_url,
)
continue
instances.append(cls())
return instances
def _process_extension_instance(
instance: FeedExtension,
entry: Entry,
reader: Reader,
results: dict[str, str],
) -> bool:
"""Process a single extension and update *results* in place.
The ``try/except`` wrapping belongs in the caller so that this
helper stays focused on happy-path logic.
Args:
instance: The extension instance to process.
entry: The feed entry to process.
reader: The reader instance.
results: Accumulator dict to update with variable values.
Returns:
``True`` if the extension result was processed successfully,
``False`` if the result was not a ``dict`` (and was skipped).
"""
extra: dict[str, str] = instance.process_entry(entry, reader)
if not isinstance(extra, dict):
logger.warning(
"Extension %r returned %s, expected dict — skipping",
instance.name,
type(extra).__name__,
)
return False
for var_name, var_value in extra.items():
if not isinstance(var_value, str):
logger.warning(
"Extension %r returned non-string value for %r (%s) — coercing",
instance.name,
var_name,
type(var_value).__name__,
)
results[var_name] = str(var_value)
else:
results[var_name] = var_value
return True
def run_extensions(entry: Entry, reader: Reader) -> dict[str, str]:
"""Run all enabled extensions for the given entry.
For every enabled extension, all of its ``provides_variables`` are
guaranteed to appear in the result dict. Variables that the
extension did not produce (no match in the entry content) are set
to an empty string so that ``{{variable}}`` tags are always
replaced rather than left as literal text.
If an extension raises, its error is logged and processing
continues with the next one.
Args:
entry: The feed entry to process.
reader: The reader instance (used to load per-feed config).
Returns:
Flat dict of ``{variable_name: value}`` pairs. Always returns a
dict (possibly empty if no extensions are enabled).
"""
results: dict[str, str] = {}
for instance in _get_enabled_instances(entry, reader):
# Seed with empty strings so every declared variable is at
# least present (prevents literal ``{{var}}`` in output).
for var_name in getattr(type(instance), "provides_variables", []):
results.setdefault(var_name, "")
try:
_process_extension_instance(instance, entry, reader, results)
except Exception:
logger.exception(
"Extension %r failed while processing entry %s",
instance.name,
entry.id,
)
return results
def run_modify_webhook(
webhook: DiscordWebhook,
entry: Entry,
reader: Reader,
) -> DiscordWebhook:
"""Let enabled extensions modify the Discord webhook before sending.
Each enabled extension's ``modify_webhook()`` is called in order.
If an extension raises, its error is logged and the webhook is
passed to the next extension unchanged.
Args:
webhook: The fully built webhook payload.
entry: The feed entry being processed.
reader: The reader instance.
Returns:
The (possibly modified) webhook.
"""
current: DiscordWebhook = webhook
for instance in _get_enabled_instances(entry, reader):
try:
current = instance.modify_webhook(current, entry, reader)
if current is None:
logger.warning(
"Extension %r returned None from modify_webhook — skipping",
instance.name,
)
current = webhook
except Exception:
logger.exception(
"Extension %r modify_webhook failed for entry %s",
instance.name,
entry.id,
)
return current

View file

@ -0,0 +1,276 @@
"""Steam feed extension: extracts app metadata and provides thumbnail URLs.
Detects feeds from ``store.steampowered.com`` and ``steamcommunity.com``,
extracts the application ID, and exposes the game's capsule image URL
as ``{{steam_thumbnail_url}}`` and the app ID as ``{{steam_app_id}}``.
Usage:
1. Enable ``steam`` on the feed's Extensions page.
2. In embed settings, set Thumbnail URL to ``{{steam_thumbnail_url}}``.
"""
from __future__ import annotations
import hashlib
import json
import logging
from pathlib import Path
from typing import TYPE_CHECKING
from typing import ClassVar
from urllib.parse import parse_qs
from urllib.parse import urlparse
from reader import ReaderError
from discord_rss_bot.extensions.base import FeedExtension
from discord_rss_bot.settings import data_dir
from discord_rss_bot.webhook import DiscordWebhook
from discord_rss_bot.webhook import WebhookFile
if TYPE_CHECKING:
from reader import Entry
from reader import Feed
from reader import Reader
logger: logging.Logger = logging.getLogger(__name__)
_STEAM_NETLOCS: frozenset[str] = frozenset({"store.steampowered.com", "steamcommunity.com"})
def is_steam_url(url: str) -> bool:
"""Return whether *url* belongs to Steam."""
if not url:
return False
try:
parsed = urlparse(url)
except ValueError:
return False
return parsed.netloc.lower().removeprefix("www.") in _STEAM_NETLOCS
def _extract_store_app_id(segments: list[str]) -> str | None:
"""Extract app ID from ``store.steampowered.com`` path segments.
Returns:
The numeric app ID, or ``None`` if no match.
"""
for prefix in (
("feeds", "news", "app"),
("news", "app"),
("app",),
):
if len(segments) > len(prefix) and tuple(segments[: len(prefix)]) == prefix:
candidate: str = segments[len(prefix)]
if candidate.isdigit():
return candidate
return None
def _extract_community_app_id(segments: list[str]) -> str | None:
"""Extract app ID from ``steamcommunity.com`` path segments.
Returns:
The numeric app ID, or ``None`` if no match.
"""
if len(segments) > 1 and segments[0] in frozenset({"games", "app"}) and segments[1].isdigit():
return segments[1]
return None
def _extract_app_id_from_query(query: str) -> str | None:
"""Extract app ID from query parameters as a fallback.
Args:
query: The URL query string.
Returns:
The numeric app ID, or ``None`` if no match.
"""
for key in ("appid", "app_id", "appids"):
for value in parse_qs(query).get(key, []):
if value.strip().isdigit():
return value.strip()
return None
def extract_app_id(url: str) -> str | None:
"""Extract a Steam application ID from a URL.
Args:
url: The URL to inspect.
Returns:
The numeric app ID if found, otherwise ``None``.
"""
if not url:
return None
try:
parsed = urlparse(url)
except ValueError:
return None
netloc: str = parsed.netloc.lower().removeprefix("www.")
segments: list[str] = [s for s in parsed.path.split("/") if s]
if netloc == "store.steampowered.com":
return _extract_store_app_id(segments) or _extract_app_id_from_query(parsed.query)
if netloc == "steamcommunity.com":
return _extract_community_app_id(segments) or _extract_app_id_from_query(parsed.query)
# Query parameter fallback for any other netloc
return _extract_app_id_from_query(parsed.query)
def _try_read_icon_file(app_id: str) -> WebhookFile | None:
"""Read a Steam game icon from disk, returning a ``WebhookFile`` or ``None``.
Does not catch exceptions the caller handles them.
Returns:
A ``WebhookFile`` with the icon content, or ``None`` if no icon
was found on disk.
"""
# Check the old project-relative icons/ directory first,
# then fall back to the data directory.
old_icon: Path = Path(__file__).resolve().parent.parent.parent / "icons" / f"{app_id}.png"
icon_path = old_icon if old_icon.is_file() else Path(data_dir) / "steam_icons" / f"{app_id}.png"
if not icon_path.is_file():
return None
icon_bytes: bytes = icon_path.read_bytes()
if not icon_bytes:
return None
content_hash: str = hashlib.sha256(icon_bytes).hexdigest()[:12]
return WebhookFile(
filename=f"steam-app-{app_id}-{content_hash}.png",
content=icon_bytes,
)
def get_icon_file_for_app(app_id: str) -> WebhookFile | None:
"""Return a local Steam game icon file if one exists on disk."""
try:
return _try_read_icon_file(app_id)
except Exception:
logger.exception("Failed to read local Steam icon for app %s", app_id)
return None
def _steam_thumbnail_enabled(reader: Reader, feed: Entry | Feed) -> bool:
"""Return ``True`` if the "use Steam game icon" toggle is on for *feed*.
Reads the embed tag directly from the reader to avoid importing
``custom_message`` (which would create a circular dependency).
Defaults to ``True`` when the tag is missing or unparseable.
"""
try:
embed_tag_raw = reader.get_tag(feed, "embed", None)
except ReaderError:
return True
raw_val = True
if isinstance(embed_tag_raw, str) and embed_tag_raw.strip():
try:
embed_data = json.loads(embed_tag_raw)
except (json.JSONDecodeError, ValueError):
return True
if isinstance(embed_data, dict):
raw_val = embed_data.get("show_steam_game_icon_in_thumbnail", True)
elif isinstance(embed_tag_raw, dict):
raw_val = embed_tag_raw.get("show_steam_game_icon_in_thumbnail", True)
return raw_val # pyright: ignore[reportReturnType]
class SteamExtension(FeedExtension):
"""Provide ``{{steam_thumbnail_url}}`` and ``{{steam_app_id}}``.
Extracts the app ID from Steam store and community feeds and makes
the game's capsule image URL and app ID available as template variables.
"""
name = "steam"
description = (
"Extracts the Steam app ID from feed URLs and exposes the game's "
"capsule image as {{steam_thumbnail_url}}. "
"Available as {{steam_app_id}}."
)
provides_variables: ClassVar[list[str]] = ["steam_thumbnail_url", "steam_app_id"]
auto_enable_url_patterns: ClassVar[list[str]] = [
r"store\.steampowered\.com",
r"steamcommunity\.com",
]
def process_entry(self, entry: Entry, reader: Reader) -> dict[str, str]: # ruff:ignore[unused-method-argument]
"""Extract Steam app metadata from the entry.
Args:
entry: The feed entry to process.
reader: The reader instance.
Returns:
Dict with ``steam_thumbnail_url`` and ``steam_app_id`` if
this is a Steam feed, otherwise empty dict.
"""
feed_url: str = entry.feed.url or ""
if not is_steam_url(feed_url):
return {}
app_id: str | None = extract_app_id(feed_url) or extract_app_id(str(entry.link or ""))
if not app_id:
return {}
return {
"steam_app_id": app_id,
"steam_thumbnail_url": f"https://cdn.cloudflare.steamstatic.com/steam/apps/{app_id}/capsule_sm_120.jpg",
}
def modify_webhook(
self,
webhook: DiscordWebhook,
entry: Entry,
reader: Reader,
) -> DiscordWebhook:
"""Set the embed thumbnail to the Steam game's capsule image.
Also attaches a locally cached icon file when available.
Only applies when ``show_steam_game_icon_in_thumbnail`` is enabled
in the feed's embed settings.
Args:
webhook: The fully built webhook payload.
entry: The feed entry being processed.
reader: The reader instance.
Returns:
The (possibly modified) webhook.
"""
feed_url: str = entry.feed.url or ""
if not is_steam_url(feed_url):
return webhook
if not _steam_thumbnail_enabled(reader, entry.feed):
return webhook
app_id: str | None = extract_app_id(feed_url) or extract_app_id(str(entry.link or ""))
if not app_id:
return webhook
# Only modify embed thumbnail when there IS an embed (not in text mode).
embeds = webhook.json.get("embeds", [])
if not isinstance(embeds, list) or not embeds:
return webhook
# Use a local icon file if available (better quality), otherwise CDN.
icon_file: WebhookFile | None = get_icon_file_for_app(app_id)
if icon_file:
webhook.add_file(file=icon_file.content, filename=icon_file.filename)
thumbnail_url: str = f"attachment://{icon_file.filename}"
else:
thumbnail_url = f"https://cdn.cloudflare.steamstatic.com/steam/apps/{app_id}/capsule_sm_120.jpg"
embed = embeds[0]
if isinstance(embed, dict):
embed["thumbnail"] = {"url": thumbnail_url}
return webhook

View file

@ -0,0 +1,68 @@
"""Per-feed storage of enabled extension lists.
Enabled extensions for a feed are stored as a JSON list in the reader
tag ``"extensions"``. An empty list or missing tag means no extensions
are active for that feed.
"""
from __future__ import annotations
import json
import logging
from typing import TYPE_CHECKING
from typing import cast
if TYPE_CHECKING:
from reader import Reader
from reader.types import JSONType
logger: logging.Logger = logging.getLogger(__name__)
EXTENSIONS_TAG: str = "extensions"
def get_enabled_extensions_for_feed(reader: Reader, feed_url: str) -> list[str]:
"""Return the list of enabled extension names for the given feed.
Args:
reader: The reader instance.
feed_url: The feed URL.
Returns:
List of extension names enabled for this feed. Never ``None``.
"""
raw = reader.get_tag(feed_url, EXTENSIONS_TAG, None)
if raw is None:
return []
if isinstance(raw, list):
return [str(name) for name in raw]
if isinstance(raw, str):
if not raw.strip():
return []
try:
parsed = json.loads(raw)
if isinstance(parsed, list):
return [str(name) for name in parsed]
except (json.JSONDecodeError, ValueError):
logger.warning("Failed to parse extensions tag for feed %r: %r", feed_url, raw)
return []
logger.warning("Unexpected type for extensions tag on feed %r: %s (%s)", feed_url, type(raw).__name__, raw)
return []
def set_enabled_extensions_for_feed(
reader: Reader,
feed_url: str,
extension_names: list[str],
) -> None:
"""Persist a list of enabled extension names for a feed.
Args:
reader: The reader instance.
feed_url: The feed URL.
extension_names: Extension names to enable (empty = disable all).
"""
reader.set_tag(feed_url, EXTENSIONS_TAG, cast("JSONType", extension_names))

View file

@ -0,0 +1,116 @@
"""WordPress REST API extension: fetch post content with scripts intact.
Shares the batch cache with ``jwplayer_thumbnail`` so only one API call
is made per site regardless of which extension is enabled.
Provides ``{{wp_content}}`` (formatted) / ``{{wp_content_raw}}`` (raw HTML),
``{{wp_excerpt}}`` (formatted) / ``{{wp_excerpt_raw}}`` (raw HTML),
``{{wp_jwplayer_thumbnail}}`` and ``{{wp_jwplayer_file}}``.
"""
from __future__ import annotations
import logging
import re
from typing import TYPE_CHECKING
from typing import ClassVar
from urllib.parse import quote
from urllib.parse import urlparse
from discord_rss_bot.extensions.base import FeedExtension
from discord_rss_bot.extensions.jwplayer_thumbnail import _extract_slug
from discord_rss_bot.extensions.jwplayer_thumbnail import _get_post_data_for_slug
from discord_rss_bot.html_format import format_entry_html_for_discord
if TYPE_CHECKING:
from reader import Entry
from reader import Reader
logger: logging.Logger = logging.getLogger(__name__)
_IMAGE_PATTERN: re.Pattern[str] = re.compile(
r'image:\s*["\']([^"\']+)["\']',
re.IGNORECASE,
)
_FILE_PATTERN: re.Pattern[str] = re.compile(
r'file:\s*["\']([^"\']+\.\w+)["\']',
re.IGNORECASE,
)
class WordPressExtension(FeedExtension):
"""Fetch post content from the WordPress REST API.
Retrieves full post HTML so that ``<script>`` blocks survive.
Shares the batch cache with ``jwplayer_thumbnail``.
"""
name = "wordpress"
description = (
"Fetches post HTML from the WordPress REST API (batched). "
"Extracts JWPlayer URLs. "
"Available as {{wp_content}} (formatted) / {{wp_content_raw}} (raw HTML), "
"{{wp_excerpt}} (formatted) / {{wp_excerpt_raw}} (raw HTML), "
"{{wp_jwplayer_thumbnail}} and {{wp_jwplayer_file}}."
)
provides_variables: ClassVar[list[str]] = [
"wp_content",
"wp_content_raw",
"wp_excerpt",
"wp_excerpt_raw",
"wp_jwplayer_file",
"wp_jwplayer_thumbnail",
]
def process_entry(self, entry: Entry, _reader: Reader) -> dict[str, str]:
"""Return WordPress post data as template variables.
Uses the shared batch cache (single API call per site, not per entry).
Exposes the full post content and excerpt, plus any JWPlayer URLs
found within the content.
Args:
entry: The feed entry to process.
Returns:
Dict with ``wp_content`` (formatted) / ``wp_content_raw`` (raw), ``wp_excerpt``
(formatted) / ``wp_excerpt_raw`` (raw), ``wp_jwplayer_thumbnail``
and ``wp_jwplayer_file`` if found, otherwise empty dict.
"""
entry_link: str = entry.link or ""
slug: str | None = _extract_slug(entry_link)
if not slug:
return {}
try:
parsed = urlparse(entry.feed.url)
base_url: str = f"{parsed.scheme}://{parsed.netloc}"
except ValueError:
return {}
post_data: dict[str, str] | None = _get_post_data_for_slug(slug, base_url)
if not post_data:
return {}
result: dict[str, str] = {}
# Expose full content and excerpt from the WordPress REST API.
content_html: str = post_data.get("content", "")
if content_html:
result["wp_content"] = format_entry_html_for_discord(content_html)
result["wp_content_raw"] = content_html
excerpt_html: str = post_data.get("excerpt", "")
if excerpt_html:
result["wp_excerpt"] = format_entry_html_for_discord(excerpt_html)
result["wp_excerpt_raw"] = excerpt_html
# Extract JWPlayer URLs from the full content HTML.
image_match = _IMAGE_PATTERN.search(content_html)
if image_match:
result["wp_jwplayer_thumbnail"] = quote(image_match.group(1).strip(), safe=":/")
file_match = _FILE_PATTERN.search(content_html)
if file_match:
result["wp_jwplayer_file"] = quote(file_match.group(1).strip(), safe=":/")
return result

View file

@ -0,0 +1,110 @@
"""YouTube feed extension: extracts video IDs from YouTube video URLs.
Detects feeds from ``youtube.com/feeds/videos.xml`` and exposes the
video ID as ``{{youtube_video_id}}`` and the full embeddable URL as
``{{youtube_embed_url}}``.
Usage:
1. Enable ``youtube`` on the feed's Extensions page.
2. Use ``{{youtube_video_id}}`` or ``{{youtube_embed_url}}`` in
your message template or embed.
"""
from __future__ import annotations
import logging
import re
from typing import TYPE_CHECKING
from typing import ClassVar
from urllib.parse import parse_qs
from urllib.parse import urlparse
from discord_rss_bot.extensions.base import FeedExtension
if TYPE_CHECKING:
from reader import Entry
from reader import Reader
logger: logging.Logger = logging.getLogger(__name__)
# Various YouTube URL formats for video ID extraction.
_VIDEO_ID_PATTERNS: tuple[re.Pattern[str], ...] = (
re.compile(r"(?:v|vi|video)=([\w-]+)"),
re.compile(r"(?:youtu\.be|youtube\.com/embed)/([\w-]+)"),
re.compile(r"^([\w-]{11})$"),
)
def extract_youtube_video_id(url: str) -> str | None:
"""Extract a YouTube video ID from a URL.
Args:
url: The YouTube video URL.
Returns:
The video ID if found, otherwise ``None``.
"""
if not url:
return None
try:
parsed = urlparse(url)
except ValueError:
return None
# Try query parameter first
for value in parse_qs(parsed.query).get("v", []):
if value.strip():
return value.strip()
# Try path-based patterns
for pattern in _VIDEO_ID_PATTERNS:
match = pattern.search(url)
if match:
return match.group(1)
return None
def is_youtube_feed_url(feed_url: str) -> bool:
"""Return whether a feed URL is a YouTube video feed."""
return "youtube.com/feeds/videos.xml" in (feed_url or "")
class YouTubeExtension(FeedExtension):
"""Extract YouTube video IDs from entry links.
Makes ``{{youtube_video_id}}`` and ``{{youtube_embed_url}}``
available for YouTube channel feeds.
"""
name = "youtube"
description = (
"Extracts the YouTube video ID from entry links. Available as {{youtube_video_id}} and {{youtube_embed_url}}."
)
provides_variables: ClassVar[list[str]] = ["youtube_video_id", "youtube_embed_url"]
auto_enable_url_patterns: ClassVar[list[str]] = [
r"youtube\.com/feeds/videos\.xml",
]
def process_entry(self, entry: Entry, _reader: Reader) -> dict[str, str]:
"""Extract the video ID from the entry link.
Args:
entry: The feed entry to process.
Returns:
Dict with ``youtube_video_id`` and ``youtube_embed_url`` if
this is a YouTube feed, otherwise empty dict.
"""
if not is_youtube_feed_url(entry.feed.url):
return {}
entry_link: str = entry.link or ""
video_id: str | None = extract_youtube_video_id(entry_link)
if not video_id:
return {}
return {
"youtube_video_id": video_id,
"youtube_embed_url": f"https://www.youtube.com/watch?v={video_id}",
}

View file

@ -5,6 +5,7 @@ import concurrent.futures
import datetime
import functools
import hashlib
import html
import json
import logging
import os
@ -13,7 +14,6 @@ import re
import time
from collections.abc import Callable
from contextlib import suppress
from pathlib import Path
from typing import TYPE_CHECKING
from typing import Any
from typing import Literal
@ -54,11 +54,12 @@ from discord_rss_bot.custom_message import get_validated_message_avatar_url
from discord_rss_bot.custom_message import get_validated_message_username
from discord_rss_bot.custom_message import replace_tags_in_embed
from discord_rss_bot.custom_message import replace_tags_in_text_message
from discord_rss_bot.extensions import auto_enable_extensions_for_feed
from discord_rss_bot.extensions import run_modify_webhook
from discord_rss_bot.extensions.steam import extract_app_id as steam_extract_app_id
from discord_rss_bot.extensions.steam import get_icon_file_for_app as steam_get_icon_file
from discord_rss_bot.extensions.steam import is_steam_url
from discord_rss_bot.filter.evaluator import get_entry_filter_decision_from_reader
from discord_rss_bot.hoyolab_api import create_hoyolab_webhook
from discord_rss_bot.hoyolab_api import extract_post_id_from_hoyolab_url
from discord_rss_bot.hoyolab_api import fetch_hoyolab_post
from discord_rss_bot.hoyolab_api import is_c3kay_feed
from discord_rss_bot.is_url_valid import is_url_valid
from discord_rss_bot.settings import default_custom_embed
from discord_rss_bot.settings import default_custom_message
@ -92,7 +93,7 @@ class FeedUpdateError(HTTPException):
*,
status_code: int,
detail: str,
autodiscover_links: object | None = None,
autodiscover_links: JsonValue = None,
) -> None:
"""Initialize an initial-update error and preserve advertised feed links."""
super().__init__(status_code=status_code, detail=detail)
@ -142,6 +143,27 @@ MESSAGE_PAYLOAD_KEYS: tuple[str, ...] = (
)
def get_feed_display_name(feed: Feed) -> str:
"""Return a human-readable display name for a feed.
For YouTube feeds whose title is just "Videos", shows
``"{author}'s Videos"`` instead. Other feeds return the title
as-is.
"""
url: str = feed.url or ""
title: str = feed.title or ""
if "youtube.com/feeds/videos.xml" in url and (not title or title == "Videos"):
author: str = feed.authors_str or ""
if author:
return f"{author}'s Videos"
try:
return html.unescape(title) if title else url
except (TypeError, AttributeError):
return title or url
def extract_domain(url: str) -> str: # ruff:ignore[too-many-return-statements]
"""Extract the domain name from a URL.
@ -156,11 +178,10 @@ def extract_domain(url: str) -> str: # ruff:ignore[too-many-return-statements]
return "Other"
try: # ruff:ignore[too-many-statements-in-try-clause]
# Special handling for YouTube feeds
# Special handling for YouTube and Reddit feeds
if "youtube.com/feeds/videos.xml" in url:
return "YouTube"
# Special handling for Reddit feeds
if "reddit.com" in url and ".rss" in url:
return "Reddit"
@ -191,123 +212,6 @@ def extract_domain(url: str) -> str: # ruff:ignore[too-many-return-statements]
return "Other"
STEAM_STORE_APP_ID_PATH_PREFIXES: tuple[tuple[str, ...], ...] = (
("feeds", "news", "app"),
("news", "app"),
("app",),
)
def is_steam_feed_url(url: str) -> bool:
"""Return whether a feed URL belongs to Steam."""
if not url:
return False
try:
parsed_url: ParseResult = urlparse(url)
except ValueError:
return False
normalized_netloc: str = parsed_url.netloc.lower().removeprefix("www.")
return normalized_netloc in frozenset({"store.steampowered.com", "steamcommunity.com"})
def _extract_steam_app_id_from_path(normalized_netloc: str, path_segments: list[str]) -> str | None:
if normalized_netloc == "store.steampowered.com":
for prefix in STEAM_STORE_APP_ID_PATH_PREFIXES:
prefix_length: int = len(prefix)
if len(path_segments) > prefix_length and tuple(path_segments[:prefix_length]) == prefix:
app_id: str = path_segments[prefix_length]
if app_id.isdigit():
return app_id
if normalized_netloc == "steamcommunity.com" and len(path_segments) > 1:
app_type: str = path_segments[0]
app_id = path_segments[1]
if app_type in frozenset({"games", "app"}) and app_id.isdigit():
return app_id
return None
def _extract_steam_app_id_from_query(parsed_url: ParseResult) -> str | None:
query_params: dict[str, list[str]] = parse_qs(parsed_url.query)
for key in ("appid", "app_id", "appids"):
for value in query_params.get(key, []):
cleaned_value: str = value.strip()
if cleaned_value.isdigit():
return cleaned_value
return None
def extract_steam_app_id_from_url(url: str) -> str | None:
"""Extract a Steam application ID from a Steam URL when present.
Args:
url: The Steam URL to inspect.
Returns:
str | None: The application ID when present in the path or query string, otherwise None.
"""
if not url:
return None
try:
parsed_url: ParseResult = urlparse(url)
except ValueError:
return None
normalized_netloc: str = parsed_url.netloc.lower().removeprefix("www.")
path_segments: list[str] = [segment for segment in parsed_url.path.split("/") if segment]
return _extract_steam_app_id_from_path(normalized_netloc, path_segments) or _extract_steam_app_id_from_query(
parsed_url,
)
def get_local_steam_game_icon_file(app_id: str) -> WebhookFile | None:
"""Return a local Steam game icon file for Discord upload when available."""
icon_path: Path = Path(__file__).resolve().parent.parent / "icons" / f"{app_id}.png"
if not icon_path.is_file():
return None
try:
icon_bytes: bytes = icon_path.read_bytes()
except OSError:
logger.exception("Failed to read local Steam icon for app %s from %s", app_id, icon_path)
return None
if not icon_bytes:
logger.warning("Local Steam icon file is empty for app %s: %s", app_id, icon_path)
return None
content_hash: str = hashlib.sha256(icon_bytes).hexdigest()[:12]
return WebhookFile(
filename=f"steam-app-{app_id}-{content_hash}.png",
content=icon_bytes,
)
def get_steam_game_thumbnail(entry: Entry) -> tuple[str | None, WebhookFile | None]:
"""Return the preferred Steam thumbnail source for an entry.
Returns:
tuple[str | None, WebhookFile | None]: Thumbnail URL and optional uploaded file.
"""
feed_url: str = str(getattr(entry.feed, "url", "") or "")
if not is_steam_feed_url(feed_url):
return None, None
app_id: str | None = extract_steam_app_id_from_url(feed_url) or extract_steam_app_id_from_url(str(entry.link or ""))
if not app_id:
return None, None
local_icon_file: WebhookFile | None = get_local_steam_game_icon_file(app_id)
if local_icon_file:
return f"attachment://{local_icon_file.filename}", local_icon_file
return f"https://cdn.cloudflare.steamstatic.com/steam/apps/{app_id}/capsule_sm_120.jpg", None
def send_entry_to_discord(entry: Entry, reader: Reader) -> str | None:
"""Send a single entry to Discord.
@ -349,16 +253,12 @@ def get_entry_delivery_mode(reader: Reader, entry: Entry) -> DeliveryMode:
"""Resolve the effective delivery mode for an entry.
Priority order:
1. YouTube feeds are forced to text mode.
2. New `delivery_mode` tag when valid.
3. Legacy `should_send_embed` flag for backwards compatibility.
1. New `delivery_mode` tag when valid.
2. Legacy `should_send_embed` flag for backwards compatibility.
Returns:
DeliveryMode: The effective delivery mode for this entry.
"""
if is_youtube_feed(entry.feed.url):
return "text"
try:
delivery_mode_raw: str = str(reader.get_tag(entry.feed, "delivery_mode", "")).strip().lower()
except ReaderError:
@ -385,9 +285,6 @@ def get_feed_delivery_mode(reader: Reader, feed: Feed) -> DeliveryMode:
Returns:
DeliveryMode: The effective delivery mode for this feed.
"""
if is_youtube_feed(feed.url):
return "text"
try:
delivery_mode_raw: str = str(reader.get_tag(feed, "delivery_mode", "")).strip().lower()
except ReaderError:
@ -973,11 +870,22 @@ def edit_sent_webhook_message(
def apply_feed_webhook_identity(webhook: DiscordWebhook, entry: Entry, reader: Reader) -> DiscordWebhook:
"""Apply per-feed custom username and avatar when valid; ignore blank/invalid values.
Falls back to the feed's author or title when no custom username is set.
Returns:
The same webhook instance with optional identity overrides.
"""
username: str = get_validated_message_username(reader, entry.feed)
avatar_url: str = get_validated_message_avatar_url(reader, entry.feed)
feed: Feed = entry.feed
username: str = get_validated_message_username(reader, feed)
avatar_url: str = get_validated_message_avatar_url(reader, feed)
if not username:
# Fall back to feed author or title so the webhook name is
# identifiable rather than the generic Discord webhook default.
feed_author: str = feed.authors_str or feed.title or ""
if feed_author:
username = html.unescape(feed_author)[:80]
if username:
webhook.username = username
if avatar_url:
@ -999,22 +907,6 @@ def create_webhook_for_entry(
"""
delivery_mode: DeliveryMode = get_entry_delivery_mode(reader, entry)
if delivery_mode == "embed" and is_c3kay_feed(entry.feed.url):
entry_link: str | None = entry.link
if entry_link:
post_id: str | None = extract_post_id_from_hoyolab_url(entry_link)
if post_id:
post_data = fetch_hoyolab_post(post_id)
if post_data:
webhook = create_hoyolab_webhook(webhook_url, entry, post_data)
return apply_feed_webhook_identity(webhook, entry, reader), delivery_mode
logger.warning(
"Failed to create Hoyolab webhook for feed %s, falling back to regular processing",
entry.feed.url,
)
else:
logger.warning("No entry link found for feed %s, falling back to regular processing", entry.feed.url)
if delivery_mode == "embed":
webhook = create_embed_webhook(webhook_url, entry, reader=reader)
return apply_feed_webhook_identity(webhook, entry, reader), delivery_mode
@ -1873,7 +1765,7 @@ def create_components_v2_webhook(
)
def create_embed_webhook( # ruff:ignore[complex-structure, too-many-branches]
def create_embed_webhook( # ruff:ignore[complex-structure, too-many-branches, too-many-statements]
webhook_url: str,
entry: Entry,
reader: Reader,
@ -1895,10 +1787,6 @@ def create_embed_webhook( # ruff:ignore[complex-structure, too-many-branches]
custom_embed: CustomEmbed = replace_tags_in_embed(feed=feed, entry=entry, reader=reader)
media_gallery_image_limit: int = get_feed_media_gallery_image_limit(reader, feed)
webhook_text_length_limit: int = get_feed_webhook_text_length_limit(reader, feed)
steam_game_thumbnail_url: str | None = None
steam_game_thumbnail_file: WebhookFile | None = None
if custom_embed.show_steam_game_icon_in_thumbnail:
steam_game_thumbnail_url, steam_game_thumbnail_file = get_steam_game_thumbnail(entry)
if media_gallery_image_limit == 0:
custom_embed.image_url = ""
custom_embed.thumbnail_url = ""
@ -1951,9 +1839,22 @@ def create_embed_webhook( # ruff:ignore[complex-structure, too-many-branches]
icon_url=custom_embed.author_icon_url,
)
embed_thumbnail_url: str = steam_game_thumbnail_url or custom_embed.thumbnail_url
if embed_thumbnail_url:
discord_embed.set_thumbnail(url=embed_thumbnail_url)
if custom_embed.thumbnail_url:
discord_embed.set_thumbnail(url=custom_embed.thumbnail_url)
# Steam feed: override thumbnail with the game's capsule image when enabled.
steam_thumbnail_file: WebhookFile | None = None
if custom_embed.show_steam_game_icon_in_thumbnail and is_steam_url(feed.url or ""):
app_id: str | None = steam_extract_app_id(feed.url) or steam_extract_app_id(str(entry.link or ""))
if app_id:
icon_file: WebhookFile | None = steam_get_icon_file(app_id)
if icon_file:
steam_thumbnail_file = icon_file
discord_embed.set_thumbnail(url=f"attachment://{icon_file.filename}")
else:
discord_embed.set_thumbnail(
url=f"https://cdn.cloudflare.steamstatic.com/steam/apps/{app_id}/capsule_sm_120.jpg",
)
if custom_embed.image_url:
discord_embed.set_image(url=custom_embed.image_url)
@ -1967,8 +1868,8 @@ def create_embed_webhook( # ruff:ignore[complex-structure, too-many-branches]
if custom_embed.footer_icon_url and not custom_embed.footer_text:
discord_embed.set_footer(text="-", icon_url=custom_embed.footer_icon_url)
if steam_game_thumbnail_file:
webhook.add_file(file=steam_game_thumbnail_file.content, filename=steam_game_thumbnail_file.filename)
if steam_thumbnail_file:
webhook.add_file(file=steam_thumbnail_file.content, filename=steam_thumbnail_file.filename)
webhook.add_embed(discord_embed)
return webhook
@ -2098,6 +1999,9 @@ def execute_webhook(
logger.warning("Feed not found in reader, not sending entry to Discord: %s", entry_feed.url)
return
# Let enabled extensions modify the webhook before it is sent.
webhook = run_modify_webhook(webhook, entry, reader)
request_payload: JsonObject = get_webhook_request_payload(webhook)
payload: JsonObject = get_webhook_message_payload(webhook)
response: Response = send_webhook_message(webhook, request_payload)
@ -2116,31 +2020,6 @@ def execute_webhook(
upsert_sent_webhook_record(reader, entry, webhook_url, webhook, response, payload)
def is_youtube_feed(feed_url: str) -> bool:
"""Check if the feed is a YouTube feed.
Args:
feed_url: The feed URL to check.
Returns:
bool: True if the feed is a YouTube feed, False otherwise.
"""
return "youtube.com/feeds/videos.xml" in feed_url
def should_send_embed_check(reader: Reader, entry: Entry) -> bool:
"""Check if we should send an embed to Discord.
Args:
reader (Reader): The reader to use.
entry (Entry): The entry to check.
Returns:
bool: True if we should send an embed, False otherwise.
"""
return get_entry_delivery_mode(reader, entry) == "embed"
def truncate_webhook_message(
webhook_message: str,
*,
@ -2165,7 +2044,7 @@ def truncate_webhook_message(
return f"{webhook_message[:head_length]}...{webhook_message[-tail_length:]}"
def get_raw_autodiscover_links(reader: Reader, feed_url: str) -> object | None:
def get_raw_autodiscover_links(reader: Reader, feed_url: str):
"""Return advertised feed links stored after a failed initial update."""
try:
return reader.get_tag(feed_url, ".reader.autodiscover", None)
@ -2181,7 +2060,7 @@ def remove_invalid_new_feed(reader: Reader, feed_url: str) -> None:
logger.exception("Failed to remove invalid feed after initial update: %s", feed_url)
def create_feed(reader: Reader, feed_url: str, webhook_dropdown: str) -> None: # ruff:ignore[complex-structure, too-many-branches]
def create_feed(reader: Reader, feed_url: str, webhook_dropdown: str) -> None: # ruff:ignore[complex-structure, too-many-branches, too-many-statements]
"""Add a new feed, update it and mark every entry as read.
Args:
@ -2281,5 +2160,8 @@ def create_feed(reader: Reader, feed_url: str, webhook_dropdown: str) -> None:
# Set the default embed tag when creating the feed
reader.set_tag(clean_feed_url, "embed", json.dumps(default_custom_embed)) # pyright: ignore[reportArgumentType]
# Auto-enable extensions whose URL patterns match this feed URL.
auto_enable_extensions_for_feed(reader, clean_feed_url)
# Update the full-text search index so our new feed is searchable.
reader.update_search()

View file

@ -1,48 +0,0 @@
from __future__ import annotations
from typing import TYPE_CHECKING
from discord_rss_bot.filter.evaluator import find_filter_match
from discord_rss_bot.filter.evaluator import get_filter_values_from_reader
from discord_rss_bot.filter.evaluator import has_filter_values
if TYPE_CHECKING:
from reader import Entry
from reader import Feed
from reader import Reader
def feed_has_blacklist_tags(reader: Reader, feed: Feed) -> bool:
"""Return True if the feed has blacklist tags.
The following tags are checked:
- blacklist_author
- blacklist_content
- blacklist_summary
- blacklist_title
- regex_blacklist_author
- regex_blacklist_content
- regex_blacklist_summary
- regex_blacklist_title
Args:
reader: The reader.
feed: The feed to check.
Returns:
bool: If the feed has any of the tags.
"""
return has_filter_values(get_filter_values_from_reader(reader, feed, "blacklist"))
def entry_should_be_skipped(reader: Reader, entry: Entry) -> bool:
"""Return True if the entry is in the blacklist.
Args:
reader: The reader.
entry: The entry to check.
Returns:
bool: If the entry is in the blacklist.
"""
return bool(find_filter_match(entry, get_filter_values_from_reader(reader, entry.feed, "blacklist"), "blacklist"))

View file

@ -1,6 +1,8 @@
from __future__ import annotations
import urllib.parse
from dataclasses import dataclass
from functools import cache
from typing import TYPE_CHECKING
from discord_rss_bot.filter.utils import is_regex_match
@ -259,3 +261,46 @@ def get_entry_decision_key(entry: Entry) -> str:
str: A stable key based on feed URL and entry id.
"""
return f"{entry.feed.url}|{entry.id}"
# ── Convenience wrappers (formerly in separate modules) ──────────────
def feed_has_blacklist_tags(reader: Reader, feed: Feed) -> bool:
"""Return True if the feed has any blacklist tags."""
return has_filter_values(get_filter_values_from_reader(reader, feed, "blacklist"))
def entry_should_be_skipped(reader: Reader, entry: Entry) -> bool:
"""Return True if the entry matches a blacklist rule."""
return bool(find_filter_match(entry, get_filter_values_from_reader(reader, entry.feed, "blacklist"), "blacklist"))
def has_white_tags(reader: Reader, feed: Feed) -> bool:
"""Return True if the feed has any whitelist tags."""
return has_filter_values(get_filter_values_from_reader(reader, feed, "whitelist"))
def should_be_sent(reader: Reader, entry: Entry) -> bool:
"""Return True if the entry matches a whitelist rule."""
return bool(find_filter_match(entry, get_filter_values_from_reader(reader, entry.feed, "whitelist"), "whitelist"))
def entry_is_whitelisted(entry_to_check: Entry, reader: Reader) -> bool:
"""Return True if the entry is whitelisted."""
return get_entry_filter_decision_from_reader(reader, entry_to_check).whitelist_match is not None
def entry_is_blacklisted(entry_to_check: Entry, reader: Reader) -> bool:
"""Return True if the entry is blacklisted."""
return get_entry_filter_decision_from_reader(reader, entry_to_check).blacklist_match is not None
@cache
def encode_url(url_to_quote: str | None) -> str:
"""%-escape a URL so it can be used in a URL query parameter.
Returns:
str: The percent-encoded URL.
"""
return urllib.parse.quote(string=url_to_quote) if url_to_quote else ""

View file

@ -1,48 +0,0 @@
from __future__ import annotations
from typing import TYPE_CHECKING
from discord_rss_bot.filter.evaluator import find_filter_match
from discord_rss_bot.filter.evaluator import get_filter_values_from_reader
from discord_rss_bot.filter.evaluator import has_filter_values
if TYPE_CHECKING:
from reader import Entry
from reader import Feed
from reader import Reader
def has_white_tags(reader: Reader, feed: Feed) -> bool:
"""Return True if the feed has whitelist tags.
The following tags are checked:
- regex_whitelist_author
- regex_whitelist_content
- regex_whitelist_summary
- regex_whitelist_title
- whitelist_author
- whitelist_content
- whitelist_summary
- whitelist_title
Args:
reader: The reader.
feed: The feed to check.
Returns:
bool: If the feed has any of the tags.
"""
return has_filter_values(get_filter_values_from_reader(reader, feed, "whitelist"))
def should_be_sent(reader: Reader, entry: Entry) -> bool:
"""Return True if the entry is in the whitelist.
Args:
reader: The reader.
entry: The entry to check.
Returns:
bool: If the entry is in the whitelist.
"""
return bool(find_filter_match(entry, get_filter_values_from_reader(reader, entry.feed, "whitelist"), "whitelist"))

View file

@ -1,222 +0,0 @@
from __future__ import annotations
import contextlib
import json
import logging
import re
from typing import TYPE_CHECKING
from typing import cast
import requests
from discord_rss_bot.webhook import DiscordEmbed
from discord_rss_bot.webhook import DiscordWebhook
if TYPE_CHECKING:
from reader import Entry
logger: logging.Logger = logging.getLogger(__name__)
type JsonValue = bool | int | float | str | list[JsonValue] | dict[str, JsonValue] | None
type JsonObject = dict[str, JsonValue]
def is_c3kay_feed(feed_url: str) -> bool:
"""Check if the feed is from c3kay.de.
Args:
feed_url: The feed URL to check.
Returns:
bool: True if the feed is from c3kay.de, False otherwise.
"""
return "feeds.c3kay.de" in feed_url
def extract_post_id_from_hoyolab_url(url: str) -> str | None:
"""Extract the post ID from a Hoyolab URL.
Args:
url: The Hoyolab URL to extract the post ID from.
For example: https://www.hoyolab.com/article/38588239
Returns:
str | None: The post ID if found, None otherwise.
"""
try:
match: re.Match[str] | None = re.search(r"/article/(\d+)", url)
if match:
return match.group(1)
except (ValueError, AttributeError, TypeError) as e:
logger.warning("Error extracting post ID from Hoyolab URL %s: %s", url, e)
return None
def as_json_object(value: JsonValue) -> JsonObject:
"""Return value as a JSON object when possible.
Returns:
JsonObject: The input dict or an empty dict.
"""
return cast("JsonObject", value) if isinstance(value, dict) else {}
def fetch_hoyolab_post(post_id: str) -> JsonObject | None:
"""Fetch post data from the Hoyolab API.
Args:
post_id: The post ID to fetch.
Returns:
JsonObject | None: The post data if successful, None otherwise.
"""
if not post_id:
return None
http_ok = 200
try: # ruff:ignore[too-many-statements-in-try-clause]
url: str = f"https://bbs-api-os.hoyolab.com/community/post/wapi/getPostFull?post_id={post_id}"
response: requests.Response = requests.get(url, timeout=10)
if response.status_code == http_ok:
data = cast("JsonObject", response.json())
data_payload: JsonObject = as_json_object(data.get("data"))
post_payload: JsonObject = as_json_object(data_payload.get("post"))
if data.get("retcode") == 0 and post_payload:
return post_payload
logger.warning("Failed to fetch Hoyolab post %s: %s", post_id, response.text)
except (requests.RequestException, ValueError):
logger.exception("Error fetching Hoyolab post %s", post_id)
return None
def create_hoyolab_webhook(webhook_url: str, entry: Entry, post_data: JsonObject) -> DiscordWebhook: # ruff:ignore[complex-structure, too-many-branches, too-many-locals, too-many-statements]
"""Create a webhook with data from the Hoyolab API.
Args:
webhook_url: The webhook URL.
entry: The entry to send to Discord.
post_data: The post data from the Hoyolab API.
Returns:
DiscordWebhook: The webhook with the embed.
"""
entry_link: str = entry.link or entry.feed.url
webhook = DiscordWebhook(url=webhook_url, rate_limit_retry=True)
# Extract relevant data from the post
post: JsonObject = as_json_object(post_data.get("post"))
subject: str = str(post.get("subject", ""))
content: str = str(post.get("content", "{}"))
logger.debug("Post subject: %s", subject)
logger.debug("Post content: %s", content)
content_data: JsonObject = {}
with contextlib.suppress(json.JSONDecodeError, ValueError):
loaded_content = cast("JsonValue", json.loads(content))
content_data = as_json_object(loaded_content)
logger.debug("Content data: %s", content_data)
description: str = str(content_data.get("describe", ""))
if not description:
description = str(post.get("desc", ""))
# Create the embed
discord_embed = DiscordEmbed()
# Set title and description
discord_embed.set_title(subject)
discord_embed.set_url(entry_link)
# Get post.image_list
image_list_value: JsonValue = post_data.get("image_list", [])
image_list: list[JsonObject] = (
[cast("JsonObject", item) for item in image_list_value if isinstance(item, dict)]
if isinstance(image_list_value, list)
else []
)
if image_list:
image_url: str = str(image_list[0].get("url", ""))
image_height: int = int(image_list[0].get("height", "1080")) # pyright: ignore[reportArgumentType]
image_width: int = int(image_list[0].get("width", "1920")) # pyright: ignore[reportArgumentType]
logger.debug("Image URL: %s, Height: %s, Width: %s", image_url, image_height, image_width)
discord_embed.set_image(url=image_url, height=image_height, width=image_width)
video: JsonObject = as_json_object(post_data.get("video"))
if video and video.get("url"):
video_url: str = str(video.get("url", ""))
logger.debug("Video URL: %s", video_url)
with contextlib.suppress(requests.RequestException):
video_response: requests.Response = requests.get(video_url, stream=True, timeout=10)
if video_response.ok:
webhook.add_file(
file=video_response.content,
filename=f"{entry.id}.mp4",
)
game: JsonObject = as_json_object(post_data.get("game"))
if game and game.get("color"):
game_color = str(game.get("color", ""))
discord_embed.set_color(game_color.removeprefix("#"))
user: JsonObject = as_json_object(post_data.get("user"))
author_name: str = str(user.get("nickname", ""))
avatar_url: str = str(user.get("avatar_url", ""))
if author_name:
webhook.avatar_url = avatar_url
webhook.username = author_name
classification: JsonObject = as_json_object(post_data.get("classification"))
if classification and classification.get("name"):
footer = str(classification.get("name", ""))
discord_embed.set_footer(text=footer)
webhook.add_embed(discord_embed)
# Only show Youtube URL if available
structured_content: str = str(post.get("structured_content", ""))
if structured_content: # ruff:ignore[too-many-nested-blocks]
try: # ruff:ignore[too-many-statements-in-try-clause]
loaded_structured_content = cast("JsonValue", json.loads(structured_content))
structured_content_data: list[JsonObject] = (
[cast("JsonObject", item) for item in loaded_structured_content if isinstance(item, dict)]
if isinstance(loaded_structured_content, list)
else []
)
for item in structured_content_data:
insert: JsonObject = as_json_object(item.get("insert"))
if insert:
video_url: str = str(insert.get("video", ""))
if video_url:
video_id_match: re.Match[str] | None = re.search(r"embed/([a-zA-Z0-9_-]+)", video_url)
if video_id_match:
video_id: str = video_id_match.group(1)
logger.debug("Video ID: %s", video_id)
webhook.content = f"https://www.youtube.com/watch?v={video_id}"
webhook.remove_embeds()
except (json.JSONDecodeError, ValueError) as e:
logger.warning("Error parsing structured content: %s", e)
event_start_date: str = str(post.get("event_start_date", ""))
if event_start_date and event_start_date != "0":
discord_embed.add_embed_field(name="Start", value=f"<t:{event_start_date}:R>")
event_end_date: str = str(post.get("event_end_date", ""))
if event_end_date and event_end_date != "0":
discord_embed.add_embed_field(name="End", value=f"<t:{event_end_date}:R>")
created_at: str = str(post.get("created_at", ""))
if created_at and created_at != "0":
discord_embed.set_timestamp(timestamp=created_at)
return webhook

View file

@ -0,0 +1,77 @@
"""HTML formatting utilities for Discord message content.
Converts entry HTML to Discord-friendly markdown while preserving
Discord timestamp tags (``<t:12345:R>``).
"""
from __future__ import annotations
import html
import re
from markdownify import markdownify
DISCORD_TIMESTAMP_TAG_RE: re.Pattern[str] = re.compile(r"<t:\d+(?::[tTdDfFrRsS])?>")
_REDUNDANT_LINK_PREFIX_RE: re.Pattern[str] = re.compile(r"\[https://(www\.)?")
def _preserve_discord_timestamp_tags(text: str) -> tuple[str, dict[str, str]]:
"""Replace Discord timestamp tags with placeholders before markdown conversion.
Args:
text: The text to replace tags in.
Returns:
The text with Discord timestamp tags replaced by placeholders
and a mapping of placeholders to original tags.
"""
replacements: dict[str, str] = {}
def replace_match(match: re.Match[str]) -> str:
placeholder: str = f"DISCORDTIMESTAMPPLACEHOLDER{len(replacements)}"
replacements[placeholder] = match.group(0)
return placeholder
return DISCORD_TIMESTAMP_TAG_RE.sub(replace_match, text), replacements
def _restore_discord_timestamp_tags(text: str, replacements: dict[str, str]) -> str:
"""Restore preserved Discord timestamp tags after markdown conversion.
Args:
text: The text to restore tags in.
replacements: A mapping of placeholders to original Discord timestamp tags.
Returns:
The text with placeholders replaced by the original Discord timestamp tags.
"""
for placeholder, original_value in replacements.items():
text = text.replace(placeholder, original_value)
return text
def format_entry_html_for_discord(text: str) -> str:
"""Convert entry HTML to Discord-friendly markdown while preserving Discord timestamp tags.
Args:
text: The HTML text to format.
Returns:
The formatted text with Discord timestamp tags preserved.
"""
if not text:
return ""
unescaped_text: str = html.unescape(text)
protected_text, replacements = _preserve_discord_timestamp_tags(unescaped_text)
formatted_text: str = markdownify(
html=protected_text,
strip=["img", "table", "td", "tr", "tbody", "thead"],
escape_misc=False,
heading_style="ATX",
)
formatted_text = _REDUNDANT_LINK_PREFIX_RE.sub("[", formatted_text)
return _restore_discord_timestamp_tags(formatted_text, replacements)

View file

@ -52,8 +52,6 @@ from reader import opml
from starlette.responses import RedirectResponse
from starlette.responses import Response as StarletteResponse
from discord_rss_bot.custom_filters import entry_is_blacklisted
from discord_rss_bot.custom_filters import entry_is_whitelisted
from discord_rss_bot.custom_message import CustomEmbed
from discord_rss_bot.custom_message import get_custom_message
from discord_rss_bot.custom_message import get_embed
@ -62,6 +60,14 @@ from discord_rss_bot.custom_message import get_message_avatar_url
from discord_rss_bot.custom_message import get_message_username
from discord_rss_bot.custom_message import replace_tags_in_text_message
from discord_rss_bot.custom_message import save_embed
from discord_rss_bot.extensions import FeedExtension as FeedExtensionABC
from discord_rss_bot.extensions import get_registry as get_extension_registry
from discord_rss_bot.extensions import run_extensions
from discord_rss_bot.extensions.steam import is_steam_url as is_steam_feed_url
from discord_rss_bot.extensions.storage import get_enabled_extensions_for_feed as get_enabled_extensions
from discord_rss_bot.extensions.storage import set_enabled_extensions_for_feed as set_enabled_extensions
from discord_rss_bot.extensions.youtube import extract_youtube_video_id
from discord_rss_bot.extensions.youtube import is_youtube_feed_url
from discord_rss_bot.feeds import FeedUpdateError
from discord_rss_bot.feeds import JsonValue
from discord_rss_bot.feeds import SentWebhookRecord
@ -71,12 +77,12 @@ from discord_rss_bot.feeds import create_feed
from discord_rss_bot.feeds import extract_domain
from discord_rss_bot.feeds import feed_saves_sent_webhooks
from discord_rss_bot.feeds import get_feed_delivery_mode
from discord_rss_bot.feeds import get_feed_display_name
from discord_rss_bot.feeds import get_feed_media_gallery_image_limit
from discord_rss_bot.feeds import get_feed_webhook_text_length_limit
from discord_rss_bot.feeds import get_screenshot_layout
from discord_rss_bot.feeds import get_sent_webhook_records
from discord_rss_bot.feeds import is_chromium_installed
from discord_rss_bot.feeds import is_steam_feed_url
from discord_rss_bot.feeds import send_entry_to_discord
from discord_rss_bot.feeds import send_to_discord
from discord_rss_bot.feeds import update_feed_and_collect_modified_entries
@ -85,6 +91,8 @@ from discord_rss_bot.filter.evaluator import FILTER_FIELDS
from discord_rss_bot.filter.evaluator import EntryFilterDecision
from discord_rss_bot.filter.evaluator import FilterMatch
from discord_rss_bot.filter.evaluator import coerce_filter_values
from discord_rss_bot.filter.evaluator import entry_is_blacklisted
from discord_rss_bot.filter.evaluator import entry_is_whitelisted
from discord_rss_bot.filter.evaluator import evaluate_entry_filters
from discord_rss_bot.filter.evaluator import get_entry_decision_key
from discord_rss_bot.filter.evaluator import get_entry_fields
@ -97,6 +105,7 @@ from discord_rss_bot.search import create_search_context
from discord_rss_bot.settings import data_dir
from discord_rss_bot.settings import default_custom_embed
from discord_rss_bot.settings import default_custom_message
from discord_rss_bot.settings import extensions_dir
from discord_rss_bot.settings import get_reader
from discord_rss_bot.settings import make_app_reader
@ -283,6 +292,7 @@ templates: Jinja2Templates = Jinja2Templates(directory="discord_rss_bot/template
templates.env.filters["encode_url"] = lambda url: urllib.parse.quote(str(url)) if url else ""
templates.env.filters["discord_markdown"] = markdownify # pyright: ignore[reportArgumentType]
templates.env.filters["relative_time"] = relative_time
templates.env.filters["feed_display_name"] = get_feed_display_name
templates.env.globals["get_backup_path"] = get_backup_path # pyright: ignore[reportArgumentType]
templates.env.globals["has_webhooks"] = has_webhooks # pyright: ignore[reportArgumentType]
@ -493,7 +503,7 @@ def get_global_delivery_mode(reader: Reader) -> str:
def get_autodiscover_links(
reader: Reader,
feed_url: str,
stored_links: object | None = None,
stored_links: JsonValue = None,
) -> list[AutodiscoverLink]:
"""Return valid autodiscovered links stored for a failed feed update.
@ -1402,18 +1412,41 @@ async def get_custom(
"""
feed: Feed = reader.get_feed(urllib.parse.unquote(feed_url.strip()))
context: dict[str, Request | Feed | str | Entry] = {
# Collect extension variable names for the template.
ext_registry: dict[str, type] = get_extension_registry()
enabled_ext_names: list[str] = get_enabled_extensions(reader, feed.url)
extension_variables: list[str] = FeedExtensionABC.get_enabled_variables(ext_registry, enabled_ext_names)
# Try to resolve extension values from the first entry.
extension_values: dict[str, str] = {}
first_entry: Entry | None = None
for entry in reader.get_entries(feed=feed, limit=1):
first_entry = entry
break
if first_entry is not None:
try:
extension_values = run_extensions(first_entry, reader)
except Exception:
logger.exception("Failed to run extensions for preview")
# Compute first_image for template preview.
first_image: str = ""
if first_entry is not None:
content_str: str = first_entry.content[0].value if first_entry.content else ""
first_image = get_first_image(first_entry.summary, content_str)
context: dict[str, object] = {
"request": request,
"feed": feed,
"custom_message": get_custom_message(reader, feed),
"message_username": get_message_username(reader, feed),
"message_avatar_url": get_message_avatar_url(reader, feed),
"extension_variables": extension_variables,
"extension_values": extension_values,
"entry": first_entry,
"first_image": first_image,
}
# Get the first entry, this is used to show the user what the custom message will look like.
for entry in reader.get_entries(feed=feed, limit=1):
context["entry"] = entry
return templates.TemplateResponse(request=request, name="custom.html", context=context)
@ -1431,34 +1464,58 @@ async def get_embed_page(
reader: The Reader instance.
Returns:
HTMLResponse: The embed page.
HTMLResponse: The custom message page.
"""
feed: Feed = reader.get_feed(urllib.parse.unquote(feed_url.strip()))
# Get previous data, this is used when creating the form.
embed: CustomEmbed = get_embed(reader, feed)
context: dict[str, Request | Feed | str | Entry | CustomEmbed | bool] = {
# Collect extension variables so the template can list them.
ext_registry: dict[str, type] = get_extension_registry()
enabled_ext_names: list[str] = get_enabled_extensions(reader, feed.url)
extension_variables: list[str] = FeedExtensionABC.get_enabled_variables(ext_registry, enabled_ext_names)
# Try to resolve extension values from the first entry.
extension_values: dict[str, str] = {}
first_entry: Entry | None = None
for entry in reader.get_entries(feed=feed, limit=1):
first_entry = entry
break
if first_entry is not None:
try:
extension_values = run_extensions(first_entry, reader)
except Exception:
logger.exception("Failed to run extensions for preview")
# Compute first_image for template preview.
first_image: str = ""
if first_entry is not None:
content_str: str = first_entry.content[0].value if first_entry.content else ""
first_image = get_first_image(first_entry.summary, content_str)
context: dict[str, object] = {
"request": request,
"feed": feed,
"title": embed.title,
"description": embed.description,
"color": embed.color,
"image_url": embed.image_url,
"thumbnail_url": embed.thumbnail_url,
"author_name": embed.author_name,
"author_url": embed.author_url,
"author_icon_url": embed.author_icon_url,
"image_url": embed.image_url,
"thumbnail_url": embed.thumbnail_url,
"footer_text": embed.footer_text,
"footer_icon_url": embed.footer_icon_url,
"show_steam_game_icon_in_thumbnail": embed.show_steam_game_icon_in_thumbnail,
"is_steam_feed": is_steam_feed_url(feed.url or ""),
"extension_variables": extension_variables,
"extension_values": extension_values,
"entry": first_entry,
"first_image": first_image,
}
if custom_embed := get_embed(reader, feed):
context["custom_embed"] = custom_embed
for entry in reader.get_entries(feed=feed, limit=1):
# Append to context.
context["entry"] = entry
return templates.TemplateResponse(request=request, name="embed.html", context=context)
@ -1476,8 +1533,7 @@ async def post_embed( # ruff:ignore[complex-structure]
author_icon_url: Annotated[str, Form()] = "",
footer_text: Annotated[str, Form()] = "",
footer_icon_url: Annotated[str, Form()] = "",
*,
show_steam_game_icon_in_thumbnail: Annotated[bool, Form()] = False,
show_steam_game_icon_in_thumbnail: Annotated[str, Form()] = "",
) -> RedirectResponse:
"""Set the embed settings.
@ -1493,7 +1549,7 @@ async def post_embed( # ruff:ignore[complex-structure]
author_icon_url: The author icon url of the embed.
footer_text: The footer text of the embed.
footer_icon_url: The footer icon url of the embed.
show_steam_game_icon_in_thumbnail: Whether to use the Steam game icon as the embed thumbnail.
show_steam_game_icon_in_thumbnail: Whether to use Steam game icon as thumbnail.
reader: The Reader instance.
Returns:
@ -1524,8 +1580,11 @@ async def post_embed( # ruff:ignore[complex-structure]
custom_embed.footer_text = footer_text
if footer_icon_url != custom_embed.footer_icon_url:
custom_embed.footer_icon_url = footer_icon_url
if show_steam_game_icon_in_thumbnail != custom_embed.show_steam_game_icon_in_thumbnail:
custom_embed.show_steam_game_icon_in_thumbnail = show_steam_game_icon_in_thumbnail
# Handle the steam thumbnail toggle (checkbox: "true" when checked, "" when unchecked).
new_steam_icon_toggle: bool = show_steam_game_icon_in_thumbnail.strip().lower() in {"true", "on", "1"}
if new_steam_icon_toggle != custom_embed.show_steam_game_icon_in_thumbnail:
custom_embed.show_steam_game_icon_in_thumbnail = new_steam_icon_toggle
# Save the data.
save_embed(reader, feed, custom_embed)
@ -1534,6 +1593,60 @@ async def post_embed( # ruff:ignore[complex-structure]
return RedirectResponse(url=f"/feed?feed_url={urllib.parse.quote(clean_feed_url)}", status_code=303)
@app.get("/extensions", response_class=HTMLResponse)
async def get_extensions(
feed_url: str,
request: Request,
reader: Annotated[Reader, Depends(get_reader_dependency)],
):
"""Show the extensions configuration page for a feed.
Args:
feed_url: The feed to configure extensions for.
request: The request object.
reader: The Reader instance.
Returns:
HTMLResponse: The extensions configuration page.
"""
feed: Feed = reader.get_feed(urllib.parse.unquote(feed_url.strip()))
registry: dict[str, type] = get_extension_registry()
enabled: list[str] = get_enabled_extensions(reader, feed.url)
context: dict[str, object] = {
"request": request,
"feed": feed,
"discovered_extensions": registry,
"enabled_extensions": enabled,
"extensions_dir": extensions_dir,
}
return templates.TemplateResponse(request=request, name="extensions.html", context=context)
@app.post("/extensions")
async def post_extensions(
feed_url: Annotated[str, Form()],
reader: Annotated[Reader, Depends(get_reader_dependency)],
enabled_extensions: Annotated[list[str] | None, Form()] = None,
) -> RedirectResponse:
"""Save the enabled extensions for a feed.
Args:
feed_url: The feed URL.
reader: The Reader instance.
enabled_extensions: List of extension names to enable.
Returns:
RedirectResponse: Redirect to the feed page.
"""
if enabled_extensions is None:
enabled_extensions = []
clean_feed_url: str = feed_url.strip()
set_enabled_extensions(reader, clean_feed_url, enabled_extensions)
commit_state_change(reader, f"Update extensions for {clean_feed_url}")
return RedirectResponse(url=f"/feed?feed_url={urllib.parse.quote(clean_feed_url)}", status_code=303)
@app.post("/use_embed")
async def post_use_embed(
feed_url: Annotated[str, Form()],
@ -2078,7 +2191,6 @@ async def get_feed( # ruff:ignore[complex-structure, too-many-branches, too-man
"webhook_text_length_limit": get_feed_webhook_text_length_limit(reader, feed),
"max_webhook_text_length_limit": 4000,
"save_sent_webhooks": feed_saves_sent_webhooks(reader, feed),
"is_steam_feed": is_steam_feed_url(feed.url),
"chromium_installed": is_chromium_installed(),
}
return templates.TemplateResponse(request=request, name="feed.html", context=context)
@ -2146,7 +2258,6 @@ async def get_feed( # ruff:ignore[complex-structure, too-many-branches, too-man
"webhook_text_length_limit": get_feed_webhook_text_length_limit(reader, feed),
"max_webhook_text_length_limit": 4000,
"save_sent_webhooks": feed_saves_sent_webhooks(reader, feed),
"is_steam_feed": is_steam_feed_url(feed.url),
"chromium_installed": is_chromium_installed(),
}
return templates.TemplateResponse(request=request, name="feed.html", context=context)
@ -2229,10 +2340,9 @@ def create_html_for_feed( # ruff:ignore[complex-structure, too-many-locals]
)
# Check if this is a YouTube feed entry and the entry has a link
is_youtube_feed = "youtube.com/feeds/videos.xml" in entry.feed.url
video_embed_html = ""
if is_youtube_feed and entry.link:
if is_youtube_feed_url(entry.feed.url) and entry.link:
# Extract the video ID and create an embed if possible
video_id: str | None = extract_youtube_video_id(entry.link)
if video_id:
@ -3213,29 +3323,6 @@ def modify_webhook(
return RedirectResponse(url=redirect_url, status_code=303)
def extract_youtube_video_id(url: str) -> str | None:
"""Extract YouTube video ID from a YouTube video URL.
Args:
url: The YouTube video URL.
Returns:
The video ID if found, None otherwise.
"""
if not url:
return None
# Handle standard YouTube URLs (youtube.com/watch?v=VIDEO_ID)
if "youtube.com/watch" in url and "v=" in url:
return url.split("v=")[1].split("&", maxsplit=1)[0]
# Handle shortened YouTube URLs (youtu.be/VIDEO_ID)
if "youtu.be/" in url:
return url.split("youtu.be/")[1].split("?", maxsplit=1)[0]
return None
def resolve_final_feed_url(url: str) -> tuple[str, str | None]:
"""Resolve a feed URL by following redirects.

View file

@ -20,6 +20,10 @@ data_dir: str = os.getenv("DISCORD_RSS_BOT_DATA_DIR", "").strip() or user_data_d
ensure_exists=True,
)
#: Directory where user-supplied Python extension plugins live.
#: Override with the ``EXTENSIONS_DIR`` environment variable.
extensions_dir: str = os.getenv("EXTENSIONS_DIR", "").strip() or str(Path.cwd() / "extensions")
# TODO(TheLovinator): Add default things to the database and make the edible.
default_custom_message: JSONType | str = "{{entry_title}}\n{{entry_link}}"

View file

@ -202,7 +202,7 @@
{% raw %}
{{entry_text}}
{% endraw %}
</code> Same as entry_content if it exists, otherwise entry_summary
</code>{{ entry.content[0].value|discord_markdown if entry and entry.content else (entry.summary|discord_markdown if entry and entry.summary else '') }}
</li>
<li>
<code>
@ -217,18 +217,22 @@
{% raw %}
{{image_1}}
{% endraw %}
</code>First image in the entry if it exists
</li>
</ul>
<ul class="list-inline">
<li>Examples:</li>
<li>
<code>
{% raw %}
{{feed_title}}\n{{entry_content}}
{% endraw %}
</code>
</code>{{ first_image }}
</li>
{% if extension_variables %}
<br />
<li>Extension variables (from enabled extensions):</li>
{% for var_name in extension_variables %}
<li>
<code>{% raw %}{{{% endraw %}{{ var_name }}{% raw %}}}{% endraw %}</code>
{% if var_name in extension_values and extension_values[var_name] %}
{{ extension_values[var_name] }}
{% else %}
(empty)
{% endif %}
</li>
{% endfor %}
{% endif %}
</ul>
{% else %}
Something went wrong, there was no entry found. If this feed has entries and you still see this

View file

@ -195,7 +195,7 @@
{% raw %}
{{entry_text}}
{% endraw %}
</code> Same as entry_content if it exists, otherwise entry_summary
</code>{{ entry.content[0].value|discord_markdown if entry and entry.content else (entry.summary|discord_markdown if entry and entry.summary else '') }}
</li>
<li>
<code>
@ -210,13 +210,27 @@
{% raw %}
{{image_1}}
{% endraw %}
</code>First image in the entry if it exists
</li>
</ul>
{% else %}
Something went wrong, there was no entry found. If this feed has entries and you still see this
message, please contact the developer.
{% endif %}
</code>{{ first_image }}
</li>
{% if extension_variables %}
<br />
<li>Extension variables (from enabled extensions):</li>
{% for var_name in extension_variables %}
<li>
<code>{% raw %}{{{% endraw %}{{ var_name }}{% raw %}}}{% endraw %}</code>
{% if var_name in extension_values and extension_values[var_name] %}
{{ extension_values[var_name] }}
{% else %}
(empty)
{% endif %}
</li>
{% endfor %}
{% endif %}
</ul>
{% else %}
Something went wrong, there was no entry found. If this feed has entries and you still see this
message, please contact the developer.
{% endif %}
</div>
<div class="form-text">
<ul class="list-inline">
@ -251,17 +265,16 @@
<label for="thumbnail_url" class="col-sm-6 col-form-label">Thumbnail</label>
<input name="thumbnail_url" type="text" class="form-control bg-dark border-dark text-muted"
id="thumbnail_url" {% if thumbnail_url %} value="{{- thumbnail_url -}}" {% endif %} />
{% if is_steam_feed %}
<div class="form-check mt-2">
<input
class="form-check-input"
type="checkbox"
id="show_steam_game_icon_in_thumbnail"
name="show_steam_game_icon_in_thumbnail"
value="true"
{% if show_steam_game_icon_in_thumbnail %}checked{% endif %}
/>
<label class="form-check-label" for="show_steam_game_icon_in_thumbnail"> Use Steam game icon as thumbnail when available </label>
<input class="form-check-input" type="checkbox" name="show_steam_game_icon_in_thumbnail"
id="show_steam_game_icon_in_thumbnail" value="true"
{% if show_steam_game_icon_in_thumbnail %}checked{% endif %}>
<label class="form-check-label" for="show_steam_game_icon_in_thumbnail">
Use Steam game icon as thumbnail override
</label>
</div>
{% endif %}
<label for="footer_text" class="col-sm-6 col-form-label">Footer text</label>
<input name="footer_text" type="text" class="form-control bg-dark border-dark text-muted"
id="footer_text" {% if footer_text %} value="{{- footer_text -}}" {% endif %} />

View file

@ -0,0 +1,68 @@
{% extends "base.html" %}
{% block title %}
Extensions: {{ feed.title if feed.title else feed.url }} | discord-rss-bot
{% endblock title %}
{% block description %}
Enable or disable content extension plugins for {{ feed.title if feed.title else feed.url }}.
{% endblock description %}
{% block content %}
<div class="p-2 border border-dark">
<form action="/extensions" method="post">
<div class="row pb-2">
<div class="col-sm-12">
<div class="form-text">
<ul class="list-inline">
<li>Extensions extract additional variables from feed entries that you can use in your message template or embed.</li>
<li>Each extension creates a template variable like <code>{% raw %}{{ extension_variable_name }}{% endraw %}</code>.</li>
<li>Enable the ones you need and use their variables in the <a href="/custom?feed_url={{ feed.url|encode_url }}">message template</a> or <a href="/embed?feed_url={{ feed.url|encode_url }}">embed settings</a>.</li>
</ul>
</div>
{% if not discovered_extensions %}
<div class="alert alert-info mt-3" role="alert">
<p class="mb-0">
No extension plugins were found.
Place <code>.py</code> files containing <code>FeedExtension</code> subclasses
in the extensions directory
(<code>{{ extensions_dir }}</code>).
</p>
</div>
{% else %}
<div class="mt-3">
<p class="text-muted small mb-2">
Discovered extensions in <code>{{ extensions_dir }}</code>:
</p>
<div class="list-group list-group-flush">
{% for name, cls in discovered_extensions|dictsort %}
<div class="list-group-item bg-transparent border-secondary text-light px-0">
<div class="d-flex align-items-center gap-2">
<input class="form-check-input"
type="checkbox"
id="ext-{{ name }}"
name="enabled_extensions"
value="{{ name }}"
{% if name in enabled_extensions %}checked{% endif %} />
<label class="form-check-label flex-grow-1" for="ext-{{ name }}">
<code>{{ name }}</code>
{% if cls.description %}
<br />
<span class="text-muted small">{{ cls.description }}</span>
{% endif %}
</label>
</div>
</div>
{% endfor %}
</div>
</div>
{% endif %}
</div>
</div>
<!-- Add a hidden feed_url field to the form -->
<input type="hidden" name="feed_url" value="{{ feed.url }}" />
<!-- Submit button -->
<div class="d-md-flex mt-3">
<button class="btn btn-dark btn-sm"
{% if not discovered_extensions %}disabled{% endif %}>Save extensions</button>
</div>
</form>
</div>
{% endblock content %}

View file

@ -14,7 +14,7 @@
<div class="feed-page__content">
<h2 class="h3 mb-1">
<a class="text-muted text-decoration-none feed-page__wrap"
href="{{ feed.url }}">{{ feed.title }}</a>
href="{{ feed.url }}">{{ feed | feed_display_name }}</a>
</h2>
<p class="text-muted mb-0">{{ total_entries }} entries</p>
</div>
@ -30,18 +30,16 @@
Text
{% endif %}
</span>
{% if not "youtube.com/feeds/videos.xml" in feed.url %}
<span class="badge status-chip bg-secondary">
Images:
{% if media_gallery_image_limit == 0 %}
No images
{% elif media_gallery_image_limit == 1 %}
First image only
{% else %}
Up to {{ media_gallery_image_limit }} images
{% endif %}
</span>
{% endif %}
<span class="badge status-chip bg-secondary">
Images:
{% if media_gallery_image_limit == 0 %}
No images
{% elif media_gallery_image_limit == 1 %}
First image only
{% else %}
Up to {{ media_gallery_image_limit }} images
{% endif %}
</span>
{% if delivery_mode == "screenshot" %}
<span class="badge status-chip bg-secondary">
Screenshot layout:
@ -177,10 +175,8 @@
<br>
Embed: a Discord embed with title, description, and images.
<br>
{% if not "youtube.com/feeds/videos.xml" in feed.url %}
Screenshot: full-page screenshot of the entry link.
<br>
{% endif %}
Screenshot: full-page screenshot of the entry link.
<br>
Text: plain message.
</p>
<div class="d-flex" role="group" aria-label="Delivery mode">
@ -197,23 +193,21 @@
style="border-top-right-radius: 0;
border-bottom-right-radius: 0">Embed</span>
{% endif %}
{% if not "youtube.com/feeds/videos.xml" in feed.url %}
{% if chromium_installed %}
{% if delivery_mode != "screenshot" %}
<form action="/use_screenshot" method="post" class="d-inline">
<button class="btn btn-outline-light btn-sm rounded-0"
name="feed_url"
value="{{ feed.url }}">Screenshot</button>
</form>
{% else %}
<span class="btn btn-primary btn-sm disabled rounded-0">Screenshot</span>
{% endif %}
{% if chromium_installed %}
{% if delivery_mode != "screenshot" %}
<form action="/use_screenshot" method="post" class="d-inline">
<button class="btn btn-outline-light btn-sm rounded-0"
name="feed_url"
value="{{ feed.url }}">Screenshot</button>
</form>
{% else %}
<span class="btn btn-outline-light btn-sm rounded-0"
title="Playwright Chromium is not installed. Run 'uv run playwright install chromium' to enable."
style="opacity: 0.5;
cursor: not-allowed">Screenshot</span>
<span class="btn btn-primary btn-sm disabled rounded-0">Screenshot</span>
{% endif %}
{% else %}
<span class="btn btn-outline-light btn-sm rounded-0"
title="Playwright Chromium is not installed. Run 'uv run playwright install chromium' to enable."
style="opacity: 0.5;
cursor: not-allowed">Screenshot</span>
{% endif %}
{% if delivery_mode != "text" %}
<form action="/use_text" method="post" class="d-inline">
@ -229,14 +223,14 @@
border-bottom-left-radius: 0">Text</span>
{% endif %}
</div>
{% if not chromium_installed and not "youtube.com/feeds/videos.xml" in feed.url %}
{% if not chromium_installed %}
<div class="mt-2 small text-muted">
Screenshot mode requires Chromium for Playwright.
Run <code class="text-light">uv run playwright install chromium</code> to enable it.
</div>
{% endif %}
</section>
{% if delivery_mode == "screenshot" and chromium_installed and not "youtube.com/feeds/videos.xml" in feed.url %}
{% if delivery_mode == "screenshot" and chromium_installed %}
<hr class="border-secondary" />
<section class="mb-3">
<h4 class="h6 text-muted mb-2">Screenshot Layout</h4>
@ -259,54 +253,52 @@
</div>
</section>
{% endif %}
{% if not "youtube.com/feeds/videos.xml" in feed.url %}
<hr class="border-secondary" />
<section class="mb-3">
<div class="d-flex flex-wrap align-items-center justify-content-between gap-2 mb-2">
<h4 class="h6 text-muted mb-0">Image Delivery</h4>
<span class="badge {{ 'bg-info' if media_gallery_image_limit < max_media_gallery_items else 'bg-secondary' }}">
{% if media_gallery_image_limit == 0 %}
No images
{% elif media_gallery_image_limit == 1 %}
First image only
{% else %}
Up to {{ media_gallery_image_limit }} images
{% endif %}
</span>
</div>
<p id="imageDeliveryHelp" class="text-muted small mb-2">
How many images to include per entry. Only applies to embed mode.
</p>
<form action="/set_feed_media_gallery_image_limit"
method="post"
class="mb-0">
<input type="hidden" name="feed_url" value="{{ feed.url }}" />
<label class="form-label small text-muted mb-2" for="image_limit">
Images per entry:
<output name="image_limit_value" for="image_limit">{{ media_gallery_image_limit }}</output>
</label>
<div class="d-flex flex-wrap align-items-center gap-3">
<div class="flex-grow-1 image-limit-control">
<input id="image_limit"
class="form-range"
type="range"
name="image_limit"
min="0"
max="{{ max_media_gallery_items }}"
step="1"
value="{{ media_gallery_image_limit }}"
aria-describedby="imageDeliveryHelp"
oninput="this.form.elements.image_limit_value.value = this.value" />
<div class="d-flex justify-content-between text-muted small">
<span>0</span>
<span>{{ max_media_gallery_items }}</span>
</div>
<hr class="border-secondary" />
<section class="mb-3">
<div class="d-flex flex-wrap align-items-center justify-content-between gap-2 mb-2">
<h4 class="h6 text-muted mb-0">Image Delivery</h4>
<span class="badge {{ 'bg-info' if media_gallery_image_limit < max_media_gallery_items else 'bg-secondary' }}">
{% if media_gallery_image_limit == 0 %}
No images
{% elif media_gallery_image_limit == 1 %}
First image only
{% else %}
Up to {{ media_gallery_image_limit }} images
{% endif %}
</span>
</div>
<p id="imageDeliveryHelp" class="text-muted small mb-2">
How many images to include per entry. Only applies to embed mode.
</p>
<form action="/set_feed_media_gallery_image_limit"
method="post"
class="mb-0">
<input type="hidden" name="feed_url" value="{{ feed.url }}" />
<label class="form-label small text-muted mb-2" for="image_limit">
Images per entry:
<output name="image_limit_value" for="image_limit">{{ media_gallery_image_limit }}</output>
</label>
<div class="d-flex flex-wrap align-items-center gap-3">
<div class="flex-grow-1 image-limit-control">
<input id="image_limit"
class="form-range"
type="range"
name="image_limit"
min="0"
max="{{ max_media_gallery_items }}"
step="1"
value="{{ media_gallery_image_limit }}"
aria-describedby="imageDeliveryHelp"
oninput="this.form.elements.image_limit_value.value = this.value" />
<div class="d-flex justify-content-between text-muted small">
<span>0</span>
<span>{{ max_media_gallery_items }}</span>
</div>
<button class="btn btn-outline-light btn-sm" type="submit">Save image limit</button>
</div>
</form>
</section>
{% endif %}
<button class="btn btn-outline-light btn-sm" type="submit">Save image limit</button>
</div>
</form>
</section>
<hr class="border-secondary" />
<section>
<div class="d-flex flex-wrap align-items-center justify-content-between gap-2 mb-2">
@ -489,15 +481,12 @@
<div class="btn-group btn-group-sm" role="group">
<a class="btn {{ 'btn-primary' if delivery_mode == 'text' else 'btn-outline-light' }}"
href="/custom?feed_url={{ feed.url|encode_url }}">Customize message</a>
{% if not "youtube.com/feeds/videos.xml" in feed.url %}
<a class="btn {{ 'btn-primary' if delivery_mode == 'embed' else 'btn-outline-light' }}"
href="/embed?feed_url={{ feed.url|encode_url }}">Customize embed</a>
{% endif %}
<a class="btn {{ 'btn-primary' if delivery_mode == 'embed' else 'btn-outline-light' }}"
href="/embed?feed_url={{ feed.url|encode_url }}">Customize embed</a>
</div>
<a class="btn btn-sm btn-outline-light"
href="/extensions?feed_url={{ feed.url|encode_url }}">Extensions</a>
</div>
{% if is_steam_feed %}
<div class="form-text small mt-2">Steam feeds can enable thumbnails in embed settings.</div>
{% endif %}
</div>
</div>
</div>

View file

@ -73,11 +73,7 @@
{% for feed in domain_feeds %}
<li>
<a class="text-muted" href="/feed?feed_url={{ feed.url|encode_url }}">
{% if feed.title %}
{{ feed.title }}
{% else %}
{{ feed.url }}
{% endif %}
{{ feed | feed_display_name }}
</a>
{% if not feed.updates_enabled %}<span class="text-warning">Disabled</span>{% endif %}
{% if feed.last_exception %}<span class="text-danger">({{ feed.last_exception.value_str }})</span>{% endif %}
@ -116,18 +112,7 @@
{% for broken_feed in broken_feeds %}
<a class="text-muted"
href="/feed?feed_url={{ broken_feed.url|encode_url }}">
{# Display username@youtube for YouTube feeds #}
{% if "youtube.com/feeds/videos.xml" in broken_feed.url %}
{% if "user=" in broken_feed.url %}
{{ broken_feed.url.split("user=")[1] }}@youtube
{% elif "channel_id=" in broken_feed.url %}
{{ broken_feed.title if broken_feed.title else broken_feed.url.split("channel_id=")[1] }}@youtube
{% else %}
{{ broken_feed.url }}
{% endif %}
{% else %}
{{ broken_feed.url }}
{% endif %}
{{ broken_feed | feed_display_name }}
</a>
{% endfor %}
</ul>
@ -142,18 +127,7 @@
<li class="list-group-item bg-dark border-dark text-danger">
<div class="d-flex flex-wrap align-items-center gap-2">
<a class="text-muted" href="/feed?feed_url={{ feed.url|encode_url }}">
{# Display username@youtube for YouTube feeds #}
{% if "youtube.com/feeds/videos.xml" in feed.url %}
{% if "user=" in feed.url %}
{{ feed.url.split("user=")[1] }}@youtube
{% elif "channel_id=" in feed.url %}
{{ feed.title if feed.title else feed.url.split("channel_id=")[1] }}@youtube
{% else %}
{{ feed.url }}
{% endif %}
{% else %}
{{ feed.url }}
{% endif %}
{{ feed | feed_display_name }}
</a>
{% if webhooks %}
<form action="/attach_feed_webhook"

View file

@ -5,12 +5,19 @@ services:
ports:
- "5000:5000"
environment:
EXTENSIONS_DIR=/home/botuser/extensions
volumes:
# To use a host directory instead of a named volume:
# Linux: - /Docker/Bots/discord-rss-bot:/home/botuser/.local/share/discord_rss_bot/
# Windows: - C:\Docker\Bots\discord-rss-bot:/home/botuser/.local/share/discord_rss_bot/
- data:/home/botuser/.local/share/discord_rss_bot/
# Uncomment and point to a directory with your .py extension plugins:
# Linux: - /path/to/extensions:/home/botuser/extensions
# Windows: - C:\Extensions:/home/botuser/extensions
healthcheck:
test: [ "CMD", "uv", "run", "./discord_rss_bot/healthcheck.py" ]
interval: 1m

View file

@ -91,7 +91,7 @@ lint.ignore = [
]
[tool.ruff.lint.per-file-ignores]
"tests/*" = ["assert", "undocumented-public-function", "magic-value-comparison"]
"tests/*" = ["assert", "undocumented-public-function", "magic-value-comparison", "import-private-name"]
[tool.pytest.ini_options]
addopts = "-n 5 --dist loadfile -m \"not integration and not slow\""

View file

@ -11,9 +11,9 @@ from reader import Feed
from reader import Reader
from reader import make_reader
from discord_rss_bot.filter.blacklist import entry_should_be_skipped
from discord_rss_bot.filter.blacklist import feed_has_blacklist_tags
from discord_rss_bot.filter.evaluator import entry_should_be_skipped
from discord_rss_bot.filter.evaluator import evaluate_entry_filters
from discord_rss_bot.filter.evaluator import feed_has_blacklist_tags
from discord_rss_bot.filter.evaluator import get_entry_fields
from discord_rss_bot.filter.evaluator import get_filter_values_from_reader

View file

@ -5,9 +5,9 @@ import tempfile
from pathlib import Path
from typing import TYPE_CHECKING
from discord_rss_bot.custom_filters import encode_url
from discord_rss_bot.custom_filters import entry_is_blacklisted
from discord_rss_bot.custom_filters import entry_is_whitelisted
from discord_rss_bot.filter.evaluator import encode_url
from discord_rss_bot.filter.evaluator import entry_is_blacklisted
from discord_rss_bot.filter.evaluator import entry_is_whitelisted
from discord_rss_bot.settings import get_reader
if TYPE_CHECKING:

View file

@ -394,7 +394,6 @@ def test_save_embed_serializes_embed_and_writes_feed_tag() -> None:
thumbnail_url="https://example.com/thumb.png",
footer_text="Footer",
footer_icon_url="https://example.com/footer.png",
show_steam_game_icon_in_thumbnail=True,
)
save_embed(reader=reader, feed=feed, embed=embed)
@ -405,7 +404,6 @@ def test_save_embed_serializes_embed_and_writes_feed_tag() -> None:
parsed = typing.cast("dict[str, str]", __import__("json").loads(call_args[2]))
assert parsed["title"] == "Title"
assert parsed["footer_icon_url"] == "https://example.com/footer.png"
assert parsed["show_steam_game_icon_in_thumbnail"] is True
def test_get_embed_returns_default_embed_when_tag_is_empty() -> None:
@ -461,13 +459,11 @@ def test_get_embed_data_coerces_values_to_strings() -> None:
"thumbnail_url": 8,
"footer_text": 9,
"footer_icon_url": 10,
"show_steam_game_icon_in_thumbnail": "true",
},
)
assert embed.title == "1"
assert embed.footer_icon_url == "10"
assert embed.show_steam_game_icon_in_thumbnail is True
class TestNormalizeMessageUsername:

1137
tests/test_extensions.py Normal file
View file

@ -0,0 +1,1137 @@
"""Tests for the feed extension system.
Tests cover the ABC base class, plugin discovery, per-feed storage, the
``run_extensions`` integration function, and the built-in example plugin.
"""
from __future__ import annotations
import json
import os
import tempfile
from contextlib import contextmanager
from http.server import BaseHTTPRequestHandler
from http.server import ThreadingHTTPServer
from pathlib import Path
from threading import Thread
from types import SimpleNamespace
from typing import TYPE_CHECKING
from unittest.mock import MagicMock
import pytest
from reader import Reader as ReaderType
from reader import make_reader
from discord_rss_bot.custom_message import CustomEmbed
from discord_rss_bot.custom_message import get_embed
from discord_rss_bot.custom_message import replace_tags_in_embed
from discord_rss_bot.custom_message import replace_tags_in_text_message
from discord_rss_bot.custom_message import save_embed
from discord_rss_bot.extensions import FeedExtension
from discord_rss_bot.extensions import auto_enable_extensions_for_feed
from discord_rss_bot.extensions import discover_plugins
from discord_rss_bot.extensions import registry_clear
from discord_rss_bot.extensions import run_extensions
from discord_rss_bot.extensions.base import FeedExtension as FeedExtensionABC
from discord_rss_bot.extensions.hoyolab import HoyolabExtension
from discord_rss_bot.extensions.jwplayer_thumbnail import _SLUG_CACHE
from discord_rss_bot.extensions.jwplayer_thumbnail import JWPlayerThumbnailExtension
from discord_rss_bot.extensions.steam import SteamExtension
from discord_rss_bot.extensions.storage import get_enabled_extensions_for_feed
from discord_rss_bot.extensions.storage import set_enabled_extensions_for_feed
from discord_rss_bot.extensions.wordpress import WordPressExtension
from discord_rss_bot.extensions.youtube import YouTubeExtension
if TYPE_CHECKING:
from collections.abc import Iterator
from reader import Entry
from reader import Reader
from discord_rss_bot.feeds import JsonValue
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture(autouse=True)
def _clear_registry() -> Iterator[None]:
"""Clear the extension registry before and after each test.
Yields:
Control back to the test body, then cleans up.
"""
registry_clear()
yield
registry_clear()
@pytest.fixture
def temp_extensions_dir() -> Iterator[str]:
"""Create a temporary directory to act as the extensions directory.
Yields:
The path to the temporary extensions directory.
"""
tmpdir: str = tempfile.mkdtemp(prefix="ext-test-")
old_env: str | None = os.environ.pop("EXTENSIONS_DIR", None)
os.environ["EXTENSIONS_DIR"] = tmpdir
yield tmpdir
if old_env is not None:
os.environ["EXTENSIONS_DIR"] = old_env
else:
os.environ.pop("EXTENSIONS_DIR", None)
@pytest.fixture
def mock_reader() -> MagicMock:
"""A mock Reader with tag storage support.
Returns:
A MagicMock configured with set_tag/get_tag side effects.
"""
reader: MagicMock = MagicMock()
reader.tags = {} # type: ignore[valid-type]
def _resolve_feed_key(feed_or_url: str | SimpleNamespace) -> str:
"""Extract the feed URL string from either a string or SimpleNamespace.
Returns:
The URL as a string.
"""
if isinstance(feed_or_url, str):
return feed_or_url
url = getattr(feed_or_url, "url", None)
if url is not None:
return str(url)
return str(feed_or_url)
def set_tag(feed_or_url: str | SimpleNamespace, key: str, value: JsonValue) -> None:
feed_key: str = _resolve_feed_key(feed_or_url)
if feed_key not in reader.tags:
reader.tags[feed_key] = {}
reader.tags[feed_key][key] = value
def get_tag(feed_or_url: str | SimpleNamespace, key: str, default: JsonValue = None) -> JsonValue:
feed_key: str = _resolve_feed_key(feed_or_url)
feed_tags = reader.tags.get(feed_key, {})
return feed_tags.get(key, default)
reader.set_tag.side_effect = set_tag
reader.get_tag.side_effect = get_tag
return reader
@pytest.fixture
def mock_feed() -> SimpleNamespace:
"""A feed object with all attributes required by tag replacement.
Returns:
A SimpleNamespace with feed-like attributes.
"""
return SimpleNamespace(
added=None,
author="Feed Author",
authors_str="Feed Author",
last_exception=None,
last_updated=None,
link="https://example.com/feed",
subtitle="",
title="Example Feed",
updated=None,
updates_enabled=True,
url="https://example.com/feed.xml",
user_title="",
version="atom10",
)
@pytest.fixture
def mock_entry(mock_feed: SimpleNamespace) -> SimpleNamespace:
"""An entry object with all attributes required by tag replacement.
Returns:
A SimpleNamespace with entry-like attributes.
"""
return SimpleNamespace(
added=None,
author="Entry Author",
authors_str="Entry Author",
content=[SimpleNamespace(value="<p>Hello world</p>")],
feed=mock_feed,
feed_url=mock_feed.url,
id="entry-1",
important=False,
link="https://example.com/entry-1",
published=None,
read=False,
read_modified=None,
summary="",
title="Test Entry",
updated=None,
)
# ---------------------------------------------------------------------------
# Tests: base.py
# ---------------------------------------------------------------------------
def test_feed_extension_abc_cannot_be_instantiated() -> None:
"""FeedExtension should be abstract and not instantiable directly."""
with pytest.raises(TypeError):
FeedExtensionABC() # type: ignore[abstract]
def test_feed_extension_subclass_can_be_instantiated() -> None:
"""A concrete subclass with ``process_entry`` should work."""
class ConcreteExtension(FeedExtensionABC):
name = "test_concrete"
def process_entry(self, entry: Entry, reader: Reader) -> dict[str, str]: # ruff:ignore[unused-method-argument]
return {"hello": "world"}
instance = ConcreteExtension()
assert instance.name == "test_concrete"
def test_feed_extension_name_defaults_to_empty_string() -> None:
"""The ``name`` class variable defaults to ``""``."""
class UnnamedExtension(FeedExtensionABC):
def process_entry(self, entry: Entry, reader: Reader) -> dict[str, str]: # ruff:ignore[unused-method-argument]
return {}
assert not UnnamedExtension.name, "Expected name to be empty"
# ---------------------------------------------------------------------------
# Tests: discovery.py
# ---------------------------------------------------------------------------
BUILT_IN_EXTENSIONS: frozenset[str] = frozenset({"steam", "youtube", "hoyolab", "jwplayer_thumbnail", "wordpress"})
def _assert_only_built_in_extensions(registry: dict[str, type[FeedExtension]]) -> None:
"""Assert that *registry* contains exactly the built-in extensions."""
registered: set[str] = set(registry.keys())
unexpected: set[str] = registered - BUILT_IN_EXTENSIONS
missing: frozenset[str] = BUILT_IN_EXTENSIONS - registered
assert not unexpected, f"Registry contains unexpected extensions: {unexpected}"
assert not missing, f"Registry missing built-in extensions: {missing}"
def test_discover_plugins_empty_directory(temp_extensions_dir: str) -> None:
"""An empty external directory still has built-in extensions."""
registry: dict[str, type[FeedExtension]] = discover_plugins(force=True)
_assert_only_built_in_extensions(registry)
def test_discover_plugins_missing_directory() -> None:
"""If the external extensions directory doesn't exist, built-ins still load."""
old_env: str | None = os.environ.pop("EXTENSIONS_DIR", None)
try:
os.environ["EXTENSIONS_DIR"] = str(Path(tempfile.gettempdir()) / "nonexistent-extensions-dir-12345")
registry: dict[str, type[FeedExtension]] = discover_plugins(force=True)
_assert_only_built_in_extensions(registry)
finally:
if old_env is not None:
os.environ["EXTENSIONS_DIR"] = old_env
else:
os.environ.pop("EXTENSIONS_DIR", None)
def test_discover_plugins_imports_plugin(temp_extensions_dir: str) -> None:
"""A valid plugin file should be discovered and registered alongside built-ins."""
plugin_code: str = """
from discord_rss_bot.extensions.base import FeedExtension
class TestPlugin(FeedExtension):
name = "test_plugin"
description = "A test plugin."
def process_entry(self, entry, reader):
return {"test_var": "hello"}
"""
plugin_path: Path = Path(temp_extensions_dir) / "test_plugin.py"
plugin_path.write_text(plugin_code, encoding="utf-8")
registry: dict[str, type[FeedExtension]] = discover_plugins(force=True)
assert "test_plugin" in registry
assert registry["test_plugin"].name == "test_plugin"
assert registry["test_plugin"].description == "A test plugin."
# Built-in extensions should also be present.
for name in BUILT_IN_EXTENSIONS:
assert name in registry, f"Missing built-in extension {name}"
def test_discover_plugins_skips_broken_plugin(temp_extensions_dir: str) -> None:
"""A plugin that raises during import should be skipped, not crash."""
plugin_path: Path = Path(temp_extensions_dir) / "broken.py"
plugin_path.write_text("raise SyntaxError('bad syntax'", encoding="utf-8")
# Should not raise.
registry: dict[str, type[FeedExtension]] = discover_plugins(force=True)
assert "broken" not in registry
# Built-in extensions should still load.
_assert_only_built_in_extensions(registry)
def test_discover_plugins_skips_init_py(temp_extensions_dir: str) -> None:
"""__init__.py files in the extensions directory should be ignored."""
init_path: Path = Path(temp_extensions_dir) / "__init__.py"
init_path.write_text("# package init", encoding="utf-8")
registry: dict[str, type[FeedExtension]] = discover_plugins(force=True)
_assert_only_built_in_extensions(registry)
def test_discover_plugins_deduplicates_by_name(temp_extensions_dir: str) -> None:
"""If two plugins define the same name, the last one wins."""
plugin_a: str = """
from discord_rss_bot.extensions.base import FeedExtension
class PluginA(FeedExtension):
name = "dup_name"
def process_entry(self, entry, reader):
return {"from": "a"}
"""
plugin_b: str = """
from discord_rss_bot.extensions.base import FeedExtension
class PluginB(FeedExtension):
name = "dup_name"
def process_entry(self, entry, reader):
return {"from": "b"}
"""
(Path(temp_extensions_dir) / "a.py").write_text(plugin_a)
(Path(temp_extensions_dir) / "b.py").write_text(plugin_b)
registry: dict[str, type[FeedExtension]] = discover_plugins(force=True)
assert "dup_name" in registry
# The last one alphabetically (b.py) should win.
assert registry["dup_name"].__name__ == "PluginB"
# ---------------------------------------------------------------------------
# Tests: storage.py
# ---------------------------------------------------------------------------
def test_get_enabled_extensions_empty_when_no_tag(mock_reader: MagicMock, mock_feed: SimpleNamespace) -> None:
"""If no extensions tag is set, an empty list is returned."""
enabled: list[str] = get_enabled_extensions_for_feed(mock_reader, mock_feed.url)
assert enabled == []
def test_set_and_get_enabled_extensions(mock_reader: MagicMock, mock_feed: SimpleNamespace) -> None:
"""Setting enabled extensions and retrieving them should round-trip."""
names: list[str] = ["jwplayer_thumbnail", "encode_links"]
set_enabled_extensions_for_feed(mock_reader, mock_feed.url, names)
enabled: list[str] = get_enabled_extensions_for_feed(mock_reader, mock_feed.url)
assert enabled == names
def test_set_enabled_extensions_clears_list(mock_reader: MagicMock, mock_feed: SimpleNamespace) -> None:
"""Setting an empty list should clear the enabled extensions."""
set_enabled_extensions_for_feed(mock_reader, mock_feed.url, ["some_plugin"])
set_enabled_extensions_for_feed(mock_reader, mock_feed.url, [])
enabled: list[str] = get_enabled_extensions_for_feed(mock_reader, mock_feed.url)
assert enabled == []
def test_get_enabled_extensions_handles_string_tag(mock_reader: MagicMock, mock_feed: SimpleNamespace) -> None:
"""If the tag is stored as a JSON string, it should still be parsed."""
def json_string_tag(feed_url: str, key: str, default: JsonValue = None) -> str:
return json.dumps(["plugin_a", "plugin_b"])
mock_reader.get_tag.side_effect = json_string_tag
enabled: list[str] = get_enabled_extensions_for_feed(mock_reader, mock_feed.url)
assert enabled == ["plugin_a", "plugin_b"]
def test_get_enabled_extensions_handles_empty_string_tag(mock_reader: MagicMock, mock_feed: SimpleNamespace) -> None:
"""An empty string tag should return an empty list."""
def empty_string_tag(feed_url: str, key: str, default: JsonValue = None) -> str:
return ""
mock_reader.get_tag.side_effect = empty_string_tag
enabled: list[str] = get_enabled_extensions_for_feed(mock_reader, mock_feed.url)
assert enabled == []
# ---------------------------------------------------------------------------
# Tests: run_extensions (integration)
# ---------------------------------------------------------------------------
def test_run_extensions_empty_when_none_enabled(
mock_reader: MagicMock,
mock_entry: SimpleNamespace,
mock_feed: SimpleNamespace,
) -> None:
"""If no extensions are enabled for the feed, the result is empty."""
set_enabled_extensions_for_feed(mock_reader, mock_feed.url, [])
result: dict[str, str] = run_extensions(mock_entry, mock_reader) # type: ignore[arg-type]
assert result == {}
def test_run_extensions_with_missing_plugin(
mock_reader: MagicMock,
mock_entry: SimpleNamespace,
mock_feed: SimpleNamespace,
) -> None:
"""Enabled extension that doesn't exist in the registry is skipped."""
set_enabled_extensions_for_feed(mock_reader, mock_feed.url, ["nonexistent_plugin"])
result: dict[str, str] = run_extensions(mock_entry, mock_reader) # type: ignore[arg-type]
assert result == {}
def test_run_extensions_with_registered_plugin(
mock_reader: MagicMock,
mock_entry: SimpleNamespace,
mock_feed: SimpleNamespace,
temp_extensions_dir: str,
) -> None:
"""A registered and enabled extension should produce variables."""
# Register a plugin via discovery.
plugin_code: str = """
from discord_rss_bot.extensions.base import FeedExtension
class TestPlugin(FeedExtension):
name = "test_plugin"
def process_entry(self, entry, reader):
return {"custom_var": "hello_from_plugin"}
"""
(Path(temp_extensions_dir) / "my_plugin.py").write_text(plugin_code)
discover_plugins(force=True)
set_enabled_extensions_for_feed(mock_reader, mock_feed.url, ["test_plugin"])
result: dict[str, str] = run_extensions(mock_entry, mock_reader) # type: ignore[arg-type]
assert result == {"custom_var": "hello_from_plugin"}
def test_run_extensions_continues_after_plugin_error(
mock_reader: MagicMock,
mock_entry: SimpleNamespace,
mock_feed: SimpleNamespace,
temp_extensions_dir: str,
) -> None:
"""If one plugin raises, others should still run and produce results."""
# Register two plugins: one that raises and one that works.
good_code: str = """
from discord_rss_bot.extensions.base import FeedExtension
class GoodPlugin(FeedExtension):
name = "good_plugin"
def process_entry(self, entry, reader):
return {"good_var": "ok"}
"""
bad_code: str = """
from discord_rss_bot.extensions.base import FeedExtension
class BadPlugin(FeedExtension):
name = "bad_plugin"
def process_entry(self, entry, reader):
raise RuntimeError("This plugin failed!")
"""
(Path(temp_extensions_dir) / "good.py").write_text(good_code)
(Path(temp_extensions_dir) / "bad.py").write_text(bad_code)
discover_plugins(force=True)
set_enabled_extensions_for_feed(mock_reader, mock_feed.url, ["bad_plugin", "good_plugin"])
result: dict[str, str] = run_extensions(mock_entry, mock_reader) # type: ignore[arg-type]
assert result == {"good_var": "ok"}
# ---------------------------------------------------------------------------
# Tests: tag replacement integration (custom_message.py)
# ---------------------------------------------------------------------------
def test_replace_tags_in_embed_uses_extension_variables(
mock_reader: MagicMock,
mock_feed: SimpleNamespace,
mock_entry: SimpleNamespace,
temp_extensions_dir: str,
) -> None:
"""Extension variables should be available in embed tag replacement."""
# Register a test plugin.
plugin_code: str = """
from discord_rss_bot.extensions.base import FeedExtension
class EmbedVarPlugin(FeedExtension):
name = "embed_var"
def process_entry(self, entry, reader):
return {"custom_thumbnail": "https://example.com/thumb.jpg"}
"""
(Path(temp_extensions_dir) / "embed_var.py").write_text(plugin_code)
discover_plugins(force=True)
# Enable the plugin for this feed.
set_enabled_extensions_for_feed(mock_reader, mock_feed.url, ["embed_var"])
# Create an embed that uses the extension variable.
embed: CustomEmbed = get_embed(mock_reader, mock_feed) # type: ignore[arg-type]
embed.image_url = "{{custom_thumbnail}}"
save_embed(mock_reader, mock_feed, embed) # type: ignore[arg-type]
# Run replacement.
result: CustomEmbed = replace_tags_in_embed(mock_feed, mock_entry, mock_reader) # type: ignore[arg-type]
assert "https://example.com/thumb.jpg" in result.image_url
def test_replace_tags_in_text_message_uses_extension_variables(
mock_reader: MagicMock,
mock_feed: SimpleNamespace,
mock_entry: SimpleNamespace,
temp_extensions_dir: str,
) -> None:
"""Extension variables should be available in text message tag replacement."""
# Register a test plugin.
plugin_code: str = """
from discord_rss_bot.extensions.base import FeedExtension
class TextVarPlugin(FeedExtension):
name = "text_var"
def process_entry(self, entry, reader):
return {"custom_text": "hello_from_extension"}
"""
(Path(temp_extensions_dir) / "text_var.py").write_text(plugin_code)
discover_plugins(force=True)
# Enable the plugin for this feed.
set_enabled_extensions_for_feed(mock_reader, mock_feed.url, ["text_var"])
# Set a custom message that uses the extension variable.
mock_reader.set_tag(mock_feed.url, "custom_message", "{{custom_text}}")
# Run replacement.
result: str = replace_tags_in_text_message(mock_entry, mock_reader) # type: ignore[arg-type]
assert "hello_from_extension" in result
# ---------------------------------------------------------------------------
# Tests: JWPlayer example plugin
# ---------------------------------------------------------------------------
def test_jwplayer_thumbnail_extension_extracts_image() -> None:
"""The JWPlayer thumbnail extension should extract the image URL."""
ext = JWPlayerThumbnailExtension()
raw_html: str = """
<div id="player_01"></div>
<script type="text/javascript">
jwplayer("player_01").setup({
file: "https://example.com/video.mp4",
image: "https://example.com/thumbnail.jpg",
});
</script>
"""
entry: SimpleNamespace = SimpleNamespace(
id="test",
content=[SimpleNamespace(value=raw_html)],
summary="",
feed=SimpleNamespace(url="https://example.com/feed.xml"),
)
result: dict[str, str] = ext.process_entry(entry, MagicMock()) # type: ignore[arg-type]
assert result.get("jwplayer_thumbnail") == "https://example.com/thumbnail.jpg"
assert result.get("jwplayer_file") == "https://example.com/video.mp4"
def test_jwplayer_thumbnail_extension_returns_empty_without_content() -> None:
"""If the entry has no content, the extension should return an empty dict."""
ext = JWPlayerThumbnailExtension()
entry: SimpleNamespace = SimpleNamespace(
id="test",
content=[],
summary="",
link="https://example.com/video",
feed=SimpleNamespace(url="https://example.com/feed.xml"),
)
result: dict[str, str] = ext.process_entry(entry, MagicMock()) # type: ignore[arg-type]
assert result == {}
def test_jwplayer_thumbnail_extension_returns_empty_without_match() -> None:
"""If the HTML has no JWPlayer pattern, the extension should return empty."""
ext = JWPlayerThumbnailExtension()
raw_html: str = "<p>No player here.</p>"
entry: SimpleNamespace = SimpleNamespace(
id="test",
content=[SimpleNamespace(value=raw_html)],
summary="",
link="https://example.com/video",
feed=SimpleNamespace(url="https://example.com/feed.xml"),
)
result: dict[str, str] = ext.process_entry(entry, MagicMock()) # type: ignore[arg-type]
assert result == {}
def test_jwplayer_thumbnail_extension_matches_hentaigasm_format() -> None:
"""The extension should extract URLs from the actual hentaigasm.com feed format."""
ext = JWPlayerThumbnailExtension()
# This is the actual HTML structure from the main hentaigasm feed.
raw_html: str = """
<p style="text-align: center;"><strong>HENTAIGASM EXCLUSIVE!</strong></p>
<div style="width:620px; height:349px">
<script src="https://content.jwplatform.com/libraries/SAHhwvZq.js"></script>
<script>jwplayer.key="zTEbSn/eAplL0RLXT030FzOcek6qXmtrxju6Jg=="</script>
<div id="player_01"></div>
<script type="text/javascript">
.jwplayer("player_01").setup({
file: "https://hgasm2.com/Test Video 1 Subbed.mp4",
width: "620",
height: "349",
skin: "seven",
preload: "none",
autostart: "false",
image: "https://hgasm1.com/thumbnail/Test Video 1 Subbed.jpg",
advertising: {}
});
</script>
</div>
<a class="btn btn-pink" href="https://hgasm3.com/Test Video 1 Subbed.mp4" download>DOWNLOAD</a>
"""
entry: SimpleNamespace = SimpleNamespace(
id="test-hentai",
content=[SimpleNamespace(value=raw_html)],
summary="",
feed=SimpleNamespace(url="https://hentaigasm.com/feed/"),
)
result: dict[str, str] = ext.process_entry(entry, MagicMock()) # type: ignore[arg-type]
assert result.get("jwplayer_thumbnail") == "https://hgasm1.com/thumbnail/Test%20Video%201%20Subbed.jpg"
assert result.get("jwplayer_file") == "https://hgasm2.com/Test%20Video%201%20Subbed.mp4"
# ---------------------------------------------------------------------------
# Tests: auto_enable_url_patterns
# ---------------------------------------------------------------------------
def test_auto_enable_by_url_pattern_matches(
mock_reader: MagicMock,
temp_extensions_dir: str,
) -> None:
"""An extension with matching auto_enable_url_patterns should be auto-enabled.
Check that ``auto_enable_extensions_for_feed`` activates the plugin.
"""
# Register a test plugin with a URL pattern.
plugin_code: str = """
from discord_rss_bot.extensions.base import FeedExtension
class AutoPlugin(FeedExtension):
name = "auto_test"
auto_enable_url_patterns = [r"example\\.com/feeds"]
def process_entry(self, entry, reader):
return {"auto_var": "enabled"}
"""
plugin_path: Path = Path(temp_extensions_dir) / "auto_plugin.py"
plugin_path.write_text(plugin_code, encoding="utf-8")
discover_plugins(force=True)
mock_feed_url: str = "https://example.com/feeds/test.xml"
result: list[str] = auto_enable_extensions_for_feed(mock_reader, mock_feed_url)
assert "auto_test" in result
def test_auto_enable_by_url_pattern_does_not_match(
mock_reader: MagicMock,
temp_extensions_dir: str,
) -> None:
"""An extension whose pattern does not match should NOT be auto-enabled."""
# Register the same plugin as the matching test, so this test actually
# validates that a non-matching URL does not trigger auto-enable.
plugin_code: str = """
from discord_rss_bot.extensions.base import FeedExtension
class AutoPlugin(FeedExtension):
name = "auto_test"
auto_enable_url_patterns = [r"example\\.com/feeds"]
def process_entry(self, entry, reader):
return {"auto_var": "enabled"}
"""
plugin_path: Path = Path(temp_extensions_dir) / "auto_plugin.py"
plugin_path.write_text(plugin_code, encoding="utf-8")
discover_plugins(force=True)
mock_feed_url: str = "https://other-site.com/feed.xml"
result: list[str] = auto_enable_extensions_for_feed(mock_reader, mock_feed_url)
assert "auto_test" not in result
def test_auto_enable_does_not_duplicate_explicitly_enabled(
mock_reader: MagicMock,
) -> None:
"""If an extension is already explicitly enabled, auto-enable should not add it again."""
mock_feed_url: str = "https://example.com/feeds/test.xml"
# Explicitly enable the extension first.
set_enabled_extensions_for_feed(mock_reader, mock_feed_url, ["auto_test"])
result: list[str] = auto_enable_extensions_for_feed(mock_reader, mock_feed_url)
# Should contain auto_test only once.
assert result == ["auto_test"]
def test_matches_feed_url_classmethod() -> None:
"""The ``matches_feed_url`` classmethod should work correctly."""
assert FeedExtension.matches_feed_url("http://example.com") is False, "Base class has no patterns"
def test_steam_extension_has_auto_enable_patterns() -> None:
"""The built-in Steam extension should declare URL patterns."""
assert len(SteamExtension.auto_enable_url_patterns) > 0
assert SteamExtension.matches_feed_url("https://store.steampowered.com/feeds/news/app/570/")
assert SteamExtension.matches_feed_url("https://steamcommunity.com/games/570/")
assert not SteamExtension.matches_feed_url("https://example.com/feed.xml")
def test_youtube_extension_has_auto_enable_patterns() -> None:
"""The built-in YouTube extension should declare URL patterns."""
assert len(YouTubeExtension.auto_enable_url_patterns) > 0
assert YouTubeExtension.matches_feed_url("https://www.youtube.com/feeds/videos.xml?channel_id=123")
assert not YouTubeExtension.matches_feed_url("https://example.com/feed.xml")
def test_hoyolab_extension_has_auto_enable_patterns() -> None:
"""The built-in Hoyolab extension should declare URL patterns."""
assert len(HoyolabExtension.auto_enable_url_patterns) > 0
assert HoyolabExtension.matches_feed_url("https://feeds.c3kay.de/hoyolab.xml")
assert not HoyolabExtension.matches_feed_url("https://example.com/feed.xml")
# ---------------------------------------------------------------------------
# Integration test with a real WordPress comment RSS feed (the original issue)
# ---------------------------------------------------------------------------
_COMMENTS_FEED_XML: str = r"""<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0"
xmlns:content="http://purl.org/rss/1.0/modules/content/"
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:atom="http://www.w3.org/2005/Atom"
xmlns:sy="http://purl.org/rss/1.0/modules/syndication/">
<channel>
<title>Comments on: Test Article</title>
<link>https://example.com/test-article/</link>
<description></description>
<lastBuildDate>Sat, 18 Jul 2026 05:11:01 +0000</lastBuildDate>
<sy:updatePeriod>hourly</sy:updatePeriod>
<sy:updateFrequency>1</sy:updateFrequency>
<generator>https://wordpress.org/?v=5.8.2</generator>
<item>
<title>By: Anonymous</title>
<link>https://example.com/test-article/comment-page-1/#comment-820928</link>
<dc:creator><![CDATA[Anonymous]]></dc:creator>
<pubDate>Sat, 18 Jul 2026 05:11:01 +0000</pubDate>
<guid isPermaLink="false">https://example.com/?p=5633#comment-820928</guid>
<description><![CDATA[Am I the only one that saw the &quot;Bookmark us&quot; message?]]></description>
<content:encoded><![CDATA[<p>Am I the only one that saw &#8220;Bookmark us&#8221; message?</p>]]></content:encoded>
</item>
<item>
<title>By: Unknown</title>
<link>https://example.com/test-article/comment-page-1/#comment-820900</link>
<dc:creator><![CDATA[Unknown]]></dc:creator>
<pubDate>Fri, 17 Jul 2026 14:29:42 +0000</pubDate>
<guid isPermaLink="false">https://example.com/?p=5633#comment-820900</guid>
<description><![CDATA[Please uncensor the video]]></description>
<content:encoded><![CDATA[<p>Please uncensor the video</p>]]></content:encoded>
</item>
<item>
<title>By: mid</title>
<link>https://example.com/test-article/comment-page-1/#comment-820887</link>
<dc:creator><![CDATA[mid]]></dc:creator>
<pubDate>Fri, 17 Jul 2026 06:20:56 +0000</pubDate>
<guid isPermaLink="false">https://example.com/?p=5633#comment-820887</guid>
<description><![CDATA[that&#039;s not uncensored]]></description>
<content:encoded><![CDATA[<p>that&#8217;s not uncensored</p>]]></content:encoded>
</item>
</channel>
</rss>
"""
class _CommentsFeedHandler(BaseHTTPRequestHandler):
"""Serves the WordPress comment RSS XML on any request."""
def do_GET(self) -> None:
self.send_response(200)
self.send_header("Content-Type", "application/xml")
self.end_headers()
self.wfile.write(_COMMENTS_FEED_XML.encode("utf-8"))
def log_message(self, _format: str, *args: str | int) -> None:
pass
@contextmanager
def _serve_feed() -> Iterator[str]:
"""Start a local HTTP server serving the comments feed XML.
Yields:
The URL of the feed.
"""
with ThreadingHTTPServer(("127.0.0.1", 0), _CommentsFeedHandler) as server:
server_thread = Thread(target=server.serve_forever, daemon=True)
server_thread.start()
try:
yield f"http://127.0.0.1:{server.server_port}/feed.xml"
finally:
server.shutdown()
server_thread.join()
@pytest.mark.slow
def test_wordpress_comments_feed_pipeline() -> None:
"""End-to-end test using the WordPress comment RSS feed from the original issue.
Validates that:
1. The feed XML parses correctly via the reader library
2. Entry content and summary are populated as expected
3. The jwplayer_thumbnail extension gracefully returns empty (no script tags)
4. Tag replacement does not crash with comment feed data
"""
with (
tempfile.TemporaryDirectory() as tmpdir,
_serve_feed() as feed_url,
):
db_path: Path = Path(tmpdir) / "test.sqlite"
reader: ReaderType = make_reader(url=str(db_path))
try:
# Add and update the feed.
reader.add_feed(feed_url)
reader.update_feed(feed_url)
feed = reader.get_feed(feed_url)
entries: list[Entry] = list(reader.get_entries(feed=feed_url))
# The comments feed should have 3 entries.
assert len(entries) == 3, f"Expected 3 comments, got {len(entries)}"
for entry in entries:
# Each entry should have a title (comment author).
assert entry.title is not None
assert entry.title.startswith("By:")
# Each entry should have a link back to the comment.
assert entry.link is not None
assert "comment-page" in entry.link
# Summary should be plain text (from <description>),
# content should be HTML (from <content:encoded>).
assert entry.summary is not None
assert "<p>" not in entry.summary
if entry.content:
assert "<p>" in entry.content[0].value
# The jwplayer_thumbnail extension should return empty
# because this feed has no JWPlayer script blocks.
ext = JWPlayerThumbnailExtension()
result: dict[str, str] = ext.process_entry(entry, reader) # type: ignore[arg-type]
assert result == {}, f"JWPlayer extension should return empty for comment feed, got {result}"
# Tag replacement should not crash.
replaced_text: str = replace_tags_in_text_message(entry, reader)
assert isinstance(replaced_text, str)
# Embed replacement should not crash.
replaced_embed = replace_tags_in_embed(feed, entry, reader)
assert replaced_embed is not None
# Verify content:encoded was parsed into the content field.
first_entry = entries[0]
assert first_entry.content is not None
raw_content: str = first_entry.content[0].value
assert "Bookmark us" in raw_content or "bookmark" in raw_content.lower()
assert "<p>" in raw_content
finally:
reader.close()
# ---------------------------------------------------------------------------
# Main feed XML (hentaigasm-style, with JWPlayer video entries)
# ---------------------------------------------------------------------------
_MAIN_FEED_XML: str = """<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0"
xmlns:content="http://purl.org/rss/1.0/modules/content/"
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:atom="http://www.w3.org/2005/Atom"
xmlns:sy="http://purl.org/rss/1.0/modules/syndication/">
<channel>
<title>Hentaigasm - Test Feed</title>
<link>https://example.com</link>
<description></description>
<lastBuildDate>Sun, 19 Jul 2026 04:20:40 +0000</lastBuildDate>
<item>
<title>Test Video 1 Subbed</title>
<link>https://example.com/test-video-1/</link>
<dc:creator><![CDATA[admin]]></dc:creator>
<pubDate>Sun, 19 Jul 2026 04:00:49 +0000</pubDate>
<guid isPermaLink="false">https://example.com/?p=5501</guid>
<description><![CDATA[]]></description>
<content:encoded><![CDATA[
<p style="text-align: center;"><strong>HENTAIGASM EXCLUSIVE!</strong></p>
<div style="width:620px; height:349px">
<script src="https://content.jwplatform.com/libraries/SAHhwvZq.js"></script>
<script>jwplayer.key="zTEbSn/eAplL0RLXT030FzOcek6qXmtrxju6Jg=="</script>
<div id="player_01"></div>
<script type="text/javascript">
jwplayer("player_01").setup({
file: "https://cdn.example.com/Test Video 1 Subbed.mp4",
width: "620",
height: "349",
skin: "seven",
preload: "none",
autostart: "false",
image: "https://cdn.example.com/thumbnail/Test Video 1 Subbed.jpg",
advertising: {}
});
</script>
</div>
]]></content:encoded>
</item>
</channel>
</rss>
"""
class _MainFeedHandler(BaseHTTPRequestHandler):
"""Serves the main feed XML with JWPlayer video entries."""
def do_GET(self) -> None:
self.send_response(200)
self.send_header("Content-Type", "application/xml")
self.end_headers()
self.wfile.write(_MAIN_FEED_XML.encode("utf-8"))
def log_message(self, _format: str, *args: str | int) -> None:
pass
@contextmanager
def _serve_main_feed() -> Iterator[str]:
"""Start a local HTTP server serving the main feed XML with JWPlayer content.
Yields:
The URL of the feed.
"""
with ThreadingHTTPServer(("127.0.0.1", 0), _MainFeedHandler) as server:
server_thread = Thread(target=server.serve_forever, daemon=True)
server_thread.start()
try:
yield f"http://127.0.0.1:{server.server_port}/feed.xml"
finally:
server.shutdown()
server_thread.join()
@pytest.mark.slow
def test_hentaigasm_main_feed_jwplayer_extraction() -> None:
"""End-to-end test using a hentaigasm-style main feed with JWPlayer content.
This test confirms that feedparser strips ``<script>`` tags from
``<content:encoded>``, so the JWPlayer ``image:`` and ``file:``
properties are NOT available in ``entry.content``.
The extension falls back to fetching the HTML page at ``entry.link``,
but in the test environment no real server is available, so the
extension returns empty. A separate test validates the HTTP fetch
fallback with a local server.
"""
with (
tempfile.TemporaryDirectory() as tmpdir,
_serve_main_feed() as feed_url,
):
db_path: Path = Path(tmpdir) / "test.sqlite"
reader: ReaderType = make_reader(url=str(db_path))
try:
reader.add_feed(feed_url)
reader.update_feed(feed_url)
entries = list(reader.get_entries(feed=feed_url))
# Should have 1 video entry.
assert len(entries) == 1, f"Expected 1 entry, got {len(entries)}"
entry = entries[0]
assert entry.title is not None
# CONFIRMATION: feedparser strips <script> tags entirely.
# The content <p> and <div> survive, but all <script> blocks
# (including the jwplayer setup with image:/file:) are gone.
assert entry.content is not None
assert len(entry.content) > 0
raw_content: str = entry.content[0].value
assert "jwplayer" not in raw_content, (
f"BUG: script content survived parsing! Content: {raw_content[:300]!r}"
)
assert "image:" not in raw_content
assert "file:" not in raw_content
assert "player_01" in raw_content # non-script div survives
# Auto-enable is URL-based — the test feed URL (127.0.0.1) doesn't
# match "hentaigasm\.com", so we manually enable for this test.
set_enabled_extensions_for_feed(reader, entry.feed.url, ["jwplayer_thumbnail"])
enabled: list[str] = get_enabled_extensions_for_feed(reader, entry.feed.url)
assert "jwplayer_thumbnail" in enabled
finally:
reader.close()
@pytest.mark.slow
def test_jwplayer_thumbnail_wordpress_batch_fallback() -> None:
"""Verify the batch WordPress API fallback works.
Serves a fake WordPress REST API response (a list of posts) so the
extension extracts JWPlayer URLs from the cached batch data.
"""
_SLUG_CACHE.clear()
wp_json: str = json.dumps([
{
"slug": "test-slug",
"content": {
"rendered": (
'<p>Test</p><div id="player_01"></div>'
'<script>jwplayer("player_01").setup({'
'file: "https://cdn.example.com/video.mp4", '
'image: "https://cdn.example.com/thumb.jpg"'
"});</script>"
),
},
},
{
"slug": "other-post",
"content": {"rendered": "<p>No player here.</p>"},
},
])
class _APIHandler(BaseHTTPRequestHandler):
def do_GET(self) -> None:
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.end_headers()
self.wfile.write(wp_json.encode("utf-8"))
def log_message(self, _format: str, *args: str | int) -> None:
pass
with ThreadingHTTPServer(("127.0.0.1", 0), _APIHandler) as server:
server_thread = Thread(target=server.serve_forever, daemon=True)
server_thread.start()
port: int = server.server_port
try:
entry = SimpleNamespace(
id="test-wp-batch",
content=[],
summary="",
link=f"http://127.0.0.1:{port}/test-slug/",
feed=SimpleNamespace(url=f"http://127.0.0.1:{port}/feed/"),
)
ext = JWPlayerThumbnailExtension()
result: dict[str, str] = ext.process_entry(entry, MagicMock()) # type: ignore[arg-type]
assert result.get("jwplayer_thumbnail") == "https://cdn.example.com/thumb.jpg", (
f"Batch WP fallback should extract thumbnail, got {result}"
)
assert result.get("jwplayer_file") == "https://cdn.example.com/video.mp4", (
f"Batch WP fallback should extract file, got {result}"
)
# Second entry from the same site uses cache — no API call.
entry2 = SimpleNamespace(
id="other-post",
content=[],
summary="",
link=f"http://127.0.0.1:{port}/other-post/",
feed=SimpleNamespace(url=f"http://127.0.0.1:{port}/feed/"),
)
result2: dict[str, str] = ext.process_entry(entry2, MagicMock()) # type: ignore[arg-type]
assert result2 == {}, "Entry without player should return empty"
finally:
server.shutdown()
server_thread.join()
_SLUG_CACHE.clear()
# ---------------------------------------------------------------------------
# Tests: WordPress extension
# ---------------------------------------------------------------------------
def test_wordpress_extension_uses_shared_batch_cache() -> None:
"""The WordPress extension should use the shared batch cache."""
_SLUG_CACHE.clear()
content_html: str = (
"<p>Test</p>"
"<script>jwplayer().setup({file: 'https://cdn.example.com/v.mp4', image: 'https://cdn.example.com/thumb.jpg'});</script>"
)
# Pre-populate the shared cache with the new richer format.
_SLUG_CACHE["https://example.com"] = {
"test-slug": {
"content": content_html,
"excerpt": "<p>Test excerpt</p>",
"title": "Test Post",
},
}
ext = WordPressExtension()
entry = SimpleNamespace(
id="test",
link="https://example.com/test-slug/",
feed=SimpleNamespace(url="https://example.com/feed/"),
)
result = ext.process_entry(entry, MagicMock()) # type: ignore[arg-type]
assert result.get("wp_jwplayer_thumbnail") == "https://cdn.example.com/thumb.jpg"
assert result.get("wp_jwplayer_file") == "https://cdn.example.com/v.mp4"
assert result.get("wp_content_raw") == content_html
assert result.get("wp_content") is not None
assert "Test" in result.get("wp_content", "")
assert result.get("wp_excerpt_raw") == "<p>Test excerpt</p>"
assert result.get("wp_excerpt") is not None
assert "Test excerpt" in result.get("wp_excerpt", "")
# No spaces in URL values.
for key, val in result.items():
if key.startswith("wp_jwplayer"):
assert " " not in val, f"URL should be encoded, got spaces: {val}"
_SLUG_CACHE.clear()
def test_wordpress_extension_provides_correct_variables() -> None:
"""The WordPress extension should declare the correct variables."""
expected: set[str] = {
"wp_content",
"wp_content_raw",
"wp_excerpt",
"wp_excerpt_raw",
"wp_jwplayer_file",
"wp_jwplayer_thumbnail",
}
assert set(WordPressExtension.provides_variables) == expected

View file

@ -22,6 +22,8 @@ from reader import StorageError
from reader import make_reader
from discord_rss_bot import feeds
from discord_rss_bot.extensions.steam import extract_app_id
from discord_rss_bot.extensions.youtube import is_youtube_feed_url
from discord_rss_bot.feeds import JsonObject
from discord_rss_bot.feeds import capture_full_page_screenshot
from discord_rss_bot.feeds import create_feed
@ -31,13 +33,11 @@ from discord_rss_bot.feeds import extract_domain
from discord_rss_bot.feeds import get_entry_delivery_mode
from discord_rss_bot.feeds import get_screenshot_layout
from discord_rss_bot.feeds import get_webhook_url
from discord_rss_bot.feeds import is_youtube_feed
from discord_rss_bot.feeds import screenshot_filename_for_entry
from discord_rss_bot.feeds import send_discord_quest_notification
from discord_rss_bot.feeds import send_entry_to_discord
from discord_rss_bot.feeds import send_to_discord
from discord_rss_bot.feeds import set_entry_as_read
from discord_rss_bot.feeds import should_send_embed_check
from discord_rss_bot.feeds import truncate_webhook_message
@ -119,57 +119,15 @@ def test_truncate_webhook_message_long_message():
def test_is_youtube_feed():
"""Test the is_youtube_feed function."""
"""Test the is_youtube_feed_url function."""
# YouTube feed URLs
assert is_youtube_feed("https://www.youtube.com/feeds/videos.xml?channel_id=123456") is True
assert is_youtube_feed("https://www.youtube.com/feeds/videos.xml?user=username") is True
assert is_youtube_feed_url("https://www.youtube.com/feeds/videos.xml?channel_id=123456")
assert is_youtube_feed_url("https://www.youtube.com/feeds/videos.xml?user=username")
# Non-YouTube feed URLs
assert is_youtube_feed("https://www.example.com/feed.xml") is False
assert is_youtube_feed("https://www.youtube.com/watch?v=123456") is False
assert is_youtube_feed("https://www.reddit.com/r/Python/.rss") is False
@patch("discord_rss_bot.feeds.logger")
def test_should_send_embed_check_youtube_feeds(mock_logger: MagicMock) -> None:
"""Test should_send_embed_check returns False for YouTube feeds regardless of settings."""
# Create mocks
mock_reader = MagicMock()
mock_entry = MagicMock()
# Configure a YouTube feed
mock_entry.feed.url = "https://www.youtube.com/feeds/videos.xml?channel_id=123456"
# Set reader to return True for should_send_embed (would normally create an embed)
mock_reader.get_tag.return_value = True
# Result should be False, overriding the feed settings
result = should_send_embed_check(mock_reader, mock_entry)
assert result is False, "YouTube feeds should never use embeds"
# Function should not even call get_tag for YouTube feeds
mock_reader.get_tag.assert_not_called()
@patch("discord_rss_bot.feeds.logger")
def test_should_send_embed_check_normal_feeds(mock_logger: MagicMock) -> None:
"""Test should_send_embed_check returns feed settings for non-YouTube feeds."""
# Create mocks
mock_reader = MagicMock()
mock_entry = MagicMock()
# Configure a normal feed
mock_entry.feed.url = "https://www.example.com/feed.xml"
# Test with should_send_embed set to True
mock_reader.get_tag.return_value = True
result = should_send_embed_check(mock_reader, mock_entry)
assert result is True, "Normal feeds should use embeds when enabled"
# Test with should_send_embed set to False
mock_reader.get_tag.return_value = False
result = should_send_embed_check(mock_reader, mock_entry)
assert result is False, "Normal feeds should not use embeds when disabled"
assert not is_youtube_feed_url("https://www.example.com/feed.xml")
assert not is_youtube_feed_url("https://www.youtube.com/watch?v=123456")
assert not is_youtube_feed_url("https://www.reddit.com/r/Python/.rss")
def test_get_entry_delivery_mode_prefers_delivery_mode_tag() -> None:
@ -204,11 +162,7 @@ def test_get_entry_delivery_mode_falls_back_to_legacy_embed_flag() -> None:
@patch("discord_rss_bot.feeds.execute_webhook")
@patch("discord_rss_bot.feeds.create_text_webhook")
@patch("discord_rss_bot.feeds.create_hoyolab_webhook")
@patch("discord_rss_bot.feeds.fetch_hoyolab_post")
def test_send_entry_to_discord_hoyolab_text_mode_uses_text_webhook(
mock_fetch_hoyolab_post: MagicMock,
mock_create_hoyolab_webhook: MagicMock,
mock_create_text_webhook: MagicMock,
mock_execute_webhook: MagicMock,
) -> None:
@ -230,8 +184,6 @@ def test_send_entry_to_discord_hoyolab_text_mode_uses_text_webhook(
result = send_entry_to_discord(entry, reader)
assert result is None
mock_fetch_hoyolab_post.assert_not_called()
mock_create_hoyolab_webhook.assert_not_called()
mock_create_text_webhook.assert_called_once_with(
"https://discord.test/webhook",
entry,
@ -243,11 +195,7 @@ def test_send_entry_to_discord_hoyolab_text_mode_uses_text_webhook(
@patch("discord_rss_bot.feeds.execute_webhook")
@patch("discord_rss_bot.feeds.create_screenshot_webhook")
@patch("discord_rss_bot.feeds.create_hoyolab_webhook")
@patch("discord_rss_bot.feeds.fetch_hoyolab_post")
def test_send_entry_to_discord_hoyolab_screenshot_mode_uses_screenshot_webhook(
mock_fetch_hoyolab_post: MagicMock,
mock_create_hoyolab_webhook: MagicMock,
mock_create_screenshot_webhook: MagicMock,
mock_execute_webhook: MagicMock,
) -> None:
@ -269,8 +217,6 @@ def test_send_entry_to_discord_hoyolab_screenshot_mode_uses_screenshot_webhook(
result = send_entry_to_discord(entry, reader)
assert result is None
mock_fetch_hoyolab_post.assert_not_called()
mock_create_hoyolab_webhook.assert_not_called()
mock_create_screenshot_webhook.assert_called_once_with(
"https://discord.test/webhook",
entry,
@ -281,14 +227,11 @@ def test_send_entry_to_discord_hoyolab_screenshot_mode_uses_screenshot_webhook(
@patch("discord_rss_bot.feeds.execute_webhook")
@patch("discord_rss_bot.feeds.create_embed_webhook")
@patch("discord_rss_bot.feeds.create_hoyolab_webhook")
@patch("discord_rss_bot.feeds.fetch_hoyolab_post")
def test_send_entry_to_discord_hoyolab_embed_mode_uses_hoyolab_webhook(
mock_fetch_hoyolab_post: MagicMock,
mock_create_hoyolab_webhook: MagicMock,
def test_send_entry_to_discord_hoyolab_embed_mode_uses_embed_webhook(
mock_create_embed_webhook: MagicMock,
mock_execute_webhook: MagicMock,
) -> None:
"""Embed mode should use the standard embed pipeline, not a Hoyolab bypass."""
entry = MagicMock()
entry.id = "entry-3"
entry.feed.url = "https://feeds.c3kay.de/hoyolab.xml"
@ -301,21 +244,14 @@ def test_send_entry_to_discord_hoyolab_embed_mode_uses_hoyolab_webhook(
"delivery_mode": "embed",
}.get(key, default)
mock_fetch_hoyolab_post.return_value = {"post": {"subject": "News"}}
hoyolab_webhook = MagicMock()
mock_create_hoyolab_webhook.return_value = hoyolab_webhook
embed_webhook = MagicMock()
mock_create_embed_webhook.return_value = embed_webhook
result = send_entry_to_discord(entry, reader)
assert result is None
mock_fetch_hoyolab_post.assert_called_once_with("38588239")
mock_create_hoyolab_webhook.assert_called_once_with(
"https://discord.test/webhook",
entry,
{"post": {"subject": "News"}},
)
mock_create_embed_webhook.assert_not_called()
mock_execute_webhook.assert_called_once_with(hoyolab_webhook, entry, reader=reader)
mock_create_embed_webhook.assert_called_once()
mock_execute_webhook.assert_called_once_with(embed_webhook, entry, reader=reader)
def test_get_screenshot_layout_prefers_mobile_tag() -> None:
@ -391,7 +327,7 @@ def test_get_feed_media_gallery_image_limit_defaults_to_first_image() -> None:
],
)
def test_extract_steam_app_id_from_url(url: str, expected_app_id: str | None) -> None:
assert feeds.extract_steam_app_id_from_url(url) == expected_app_id
assert extract_app_id(url) == expected_app_id
@pytest.mark.parametrize(
@ -942,102 +878,6 @@ def test_create_embed_webhook_can_use_steam_game_icon_thumbnail(
entry.summary = ""
entry.content = []
entry.feed.url = "https://store.steampowered.com/feeds/news/app/570/?cc=us&l=english"
mock_replace_tags_in_embed.return_value = feeds.CustomEmbed(
description="Steam news",
thumbnail_url="https://example.com/custom-thumb.jpg",
show_steam_game_icon_in_thumbnail=True,
)
with patch("discord_rss_bot.feeds.Path.is_file", return_value=False):
webhook = feeds.create_embed_webhook("https://discord.com/api/webhooks/123/abc", entry, reader)
assert "components" not in webhook.json
embeds = webhook.json.get("embeds")
assert isinstance(embeds, list)
assert isinstance(embeds[0], dict)
assert embeds[0]["thumbnail"] == {
"url": "https://cdn.cloudflare.steamstatic.com/steam/apps/570/capsule_sm_120.jpg",
}
assert webhook.files == []
mock_fetch_ttvdrops_campaign_media_items.assert_not_called()
@patch("discord_rss_bot.feeds.fetch_ttvdrops_campaign_media_items", return_value=[])
@patch("discord_rss_bot.feeds.replace_tags_in_embed")
def test_create_embed_webhook_prefers_local_steam_game_icon_thumbnail(
mock_replace_tags_in_embed: MagicMock,
mock_fetch_ttvdrops_campaign_media_items: MagicMock,
) -> None:
local_icon_bytes = b"local-steam-icon"
reader = MagicMock()
reader.get_tag.side_effect = lambda resource, key, default=None: { # ruff:ignore[unused-lambda-argument]
"media_gallery_image_limit": 0,
"webhook_text_length_limit": 4000,
}.get(key, default)
entry = MagicMock()
entry.id = "entry-steam-local-1"
entry.title = "Dota 2 patch notes"
entry.link = "https://steamcommunity.com/games/570/announcements/detail/1234567890"
entry.summary = ""
entry.content = []
entry.feed.url = "https://store.steampowered.com/feeds/news/app/570/?cc=us&l=english"
mock_replace_tags_in_embed.return_value = feeds.CustomEmbed(
description="Steam news",
thumbnail_url="https://example.com/custom-thumb.jpg",
show_steam_game_icon_in_thumbnail=True,
)
with (
patch("discord_rss_bot.feeds.Path.is_file", return_value=True),
patch("discord_rss_bot.feeds.Path.read_bytes", return_value=local_icon_bytes),
):
webhook = feeds.create_embed_webhook("https://discord.com/api/webhooks/123/abc", entry, reader)
embeds = webhook.json.get("embeds")
assert isinstance(embeds, list)
assert isinstance(embeds[0], dict)
assert len(webhook.files) == 1
uploaded_icon = webhook.files[0]
assert uploaded_icon.content == local_icon_bytes
assert uploaded_icon.filename.startswith("steam-app-570-")
assert uploaded_icon.filename.endswith(".png")
assert embeds[0]["thumbnail"] == {"url": f"attachment://{uploaded_icon.filename}"}
mock_fetch_ttvdrops_campaign_media_items.assert_not_called()
@patch("discord_rss_bot.feeds.fetch_ttvdrops_campaign_media_items", return_value=[])
@patch("discord_rss_bot.feeds.replace_tags_in_embed")
def test_create_embed_webhook_does_not_inject_steam_thumbnail_when_app_id_is_missing(
mock_replace_tags_in_embed: MagicMock,
mock_fetch_ttvdrops_campaign_media_items: MagicMock,
) -> None:
reader = MagicMock()
reader.get_tag.side_effect = lambda resource, key, default=None: { # ruff:ignore[unused-lambda-argument]
"media_gallery_image_limit": 0,
"webhook_text_length_limit": 4000,
}.get(key, default)
entry = MagicMock()
entry.id = "entry-steam-2"
entry.title = "Steam group post"
entry.link = "https://steamcommunity.com/groups/example/announcements/detail/1234567890"
entry.summary = ""
entry.content = []
entry.feed.url = "https://steamcommunity.com/groups/example/rss/"
mock_replace_tags_in_embed.return_value = feeds.CustomEmbed(
description="Steam group news",
thumbnail_url="https://example.com/custom-thumb.jpg",
show_steam_game_icon_in_thumbnail=True,
)
webhook = feeds.create_embed_webhook("https://discord.com/api/webhooks/123/abc", entry, reader)
assert "components" not in webhook.json
embeds = webhook.json.get("embeds")
assert isinstance(embeds, list)
assert isinstance(embeds[0], dict)
assert "thumbnail" not in embeds[0]
mock_fetch_ttvdrops_campaign_media_items.assert_not_called()
@patch("discord_rss_bot.feeds.fetch_ttvdrops_campaign_media_items", return_value=[])
@ -1338,10 +1178,11 @@ def test_send_entry_to_discord_youtube_feed(
mock_entry.feed.url = "https://www.youtube.com/feeds/videos.xml?channel_id=123456"
mock_entry.feed_url = "https://www.youtube.com/feeds/videos.xml?channel_id=123456"
# Mock the tags
# Mock the tags — with no delivery_mode tag and should_send_embed=True,
# the entry delivery mode will be "embed", which calls create_embed_webhook.
mock_reader.get_tag.side_effect = lambda feed, tag, default=None: { # ruff:ignore[unused-lambda-argument]
"webhook": "https://discord.com/api/webhooks/123/abc",
"should_send_embed": True, # This should be ignored for YouTube feeds
"should_send_embed": True,
}.get(tag, default)
# Mock custom message
@ -1355,23 +1196,19 @@ def test_send_entry_to_discord_youtube_feed(
# Call the function
send_entry_to_discord(mock_entry, mock_reader)
# Assertions
mock_create_embed.assert_not_called()
mock_discord_webhook.assert_called_once()
# Check webhook was created with the right message
webhook_call_kwargs = mock_discord_webhook.call_args[1]
assert "content" in webhook_call_kwargs, "Webhook should have content"
assert webhook_call_kwargs["url"] == "https://discord.com/api/webhooks/123/abc"
# Assertions — YouTube feeds now follow standard delivery mode.
# With should_send_embed=True, embed mode is used.
mock_create_embed.assert_called_once()
mock_discord_webhook.assert_not_called()
# Verify execute_webhook was called
mock_execute_webhook.assert_called_once_with(mock_webhook, mock_entry, reader=mock_reader)
mock_execute_webhook.assert_called_once()
def test_extract_domain_youtube_feed() -> None:
"""Test extract_domain for YouTube feeds."""
url: str = "https://www.youtube.com/feeds/videos.xml?channel_id=123456"
assert extract_domain(url) == "YouTube", "YouTube feeds should return 'YouTube' as the domain."
assert extract_domain(url) == "YouTube"
def test_extract_domain_reddit_feed() -> None:

View file

@ -1,248 +0,0 @@
from __future__ import annotations
import json
import typing
from types import SimpleNamespace
from unittest.mock import MagicMock
from unittest.mock import patch
import pytest
import requests
from discord_rss_bot.hoyolab_api import create_hoyolab_webhook
from discord_rss_bot.hoyolab_api import extract_post_id_from_hoyolab_url
from discord_rss_bot.hoyolab_api import fetch_hoyolab_post
from discord_rss_bot.hoyolab_api import is_c3kay_feed
if typing.TYPE_CHECKING:
from reader import Entry
class TestExtractPostIdFromHoyolabUrl:
def test_extract_post_id_from_article_url(self) -> None:
"""Test extracting post ID from a direct article URL."""
test_cases: list[str] = [
"https://www.hoyolab.com/article/38588239",
"http://hoyolab.com/article/12345",
"https://www.hoyolab.com/article/987654321/comments",
]
expected_ids: list[str] = ["38588239", "12345", "987654321"]
for url, expected_id in zip(test_cases, expected_ids, strict=False):
assert extract_post_id_from_hoyolab_url(url) == expected_id
def test_url_without_post_id(self) -> None:
"""Test with a URL that doesn't have a post ID."""
test_cases: list[str] = [
"https://www.hoyolab.com/community",
]
for url in test_cases:
assert extract_post_id_from_hoyolab_url(url) is None
def test_edge_cases(self) -> None:
"""Test edge cases like None, empty string, and malformed URLs."""
test_cases: list[str | None] = [
None,
"",
"not_a_url",
"http:/", # Malformed URL
]
for url in test_cases:
assert extract_post_id_from_hoyolab_url(url) is None # type: ignore
def make_entry(link: str | None = "https://www.hoyolab.com/article/38588239") -> SimpleNamespace:
feed: SimpleNamespace = SimpleNamespace(url="https://feeds.c3kay.de/hoyolab.xml")
return SimpleNamespace(
id="entry-123",
link=link,
feed=feed,
)
class TestIsC3KayFeed:
def test_true_for_c3kay_feed(self) -> None:
assert is_c3kay_feed("https://feeds.c3kay.de/rss") is True
def test_false_for_non_c3kay_feed(self) -> None:
assert is_c3kay_feed("https://example.com/rss") is False
class TestFetchHoyolabPost:
@patch("discord_rss_bot.hoyolab_api.requests.get")
def test_returns_none_for_empty_post_id(self, mock_get: MagicMock) -> None:
assert fetch_hoyolab_post("") is None
mock_get.assert_not_called()
@patch("discord_rss_bot.hoyolab_api.requests.get")
def test_returns_post_data_for_success_response(self, mock_get: MagicMock) -> None:
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {
"retcode": 0,
"data": {
"post": {
"post_id": "38588239",
"subject": "Event",
},
},
}
mock_get.return_value = mock_response
result = fetch_hoyolab_post("38588239")
assert result == {"post_id": "38588239", "subject": "Event"}
assert mock_get.call_args.args[0].endswith("post_id=38588239")
@patch("discord_rss_bot.hoyolab_api.logger")
@patch("discord_rss_bot.hoyolab_api.requests.get")
def test_returns_none_and_logs_warning_for_non_success_payload(
self,
mock_get: MagicMock,
mock_logger: MagicMock,
) -> None:
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.text = "bad payload"
mock_response.json.return_value = {
"retcode": -1,
"data": {},
}
mock_get.return_value = mock_response
result = fetch_hoyolab_post("38588239")
assert result is None
mock_logger.warning.assert_called_once()
@patch("discord_rss_bot.hoyolab_api.logger")
@patch("discord_rss_bot.hoyolab_api.requests.get")
def test_returns_none_and_logs_exception_on_request_error(
self,
mock_get: MagicMock,
mock_logger: MagicMock,
) -> None:
mock_get.side_effect = requests.RequestException("network issue")
result = fetch_hoyolab_post("38588239")
assert result is None
mock_logger.exception.assert_called_once()
class TestCreateHoyolabWebhook:
@patch("discord_rss_bot.hoyolab_api.requests.get")
@patch("discord_rss_bot.hoyolab_api.DiscordEmbed")
@patch("discord_rss_bot.hoyolab_api.DiscordWebhook")
def test_builds_embed_webhook_with_full_post_data(
self,
mock_webhook_cls: MagicMock,
mock_embed_cls: MagicMock,
mock_requests_get: MagicMock,
) -> None:
webhook_instance = MagicMock()
embed_instance = MagicMock()
mock_webhook_cls.return_value = webhook_instance
mock_embed_cls.return_value = embed_instance
video_response = MagicMock()
video_response.ok = True
video_response.content = b"video-bytes"
mock_requests_get.return_value = video_response
post_data = {
"post": {
"subject": "Update 4.0",
"content": json.dumps({"describe": "Patch notes"}),
"desc": "fallback description",
"structured_content": json.dumps(
[{"insert": {"video": "https://www.youtube.com/embed/abc123_XY"}}],
),
"event_start_date": "1712000000",
"event_end_date": "1712600000",
"created_at": "1711000000",
},
"image_list": [{"url": "https://img.example.com/hero.jpg", "height": 1080, "width": 1920}],
"video": {"url": "https://cdn.example.com/video.mp4"},
"game": {"color": "#11AAFF"},
"user": {"nickname": "Paimon", "avatar_url": "https://img.example.com/avatar.jpg"},
"classification": {"name": "Official"},
}
entry = make_entry(link=None)
entry = typing.cast("Entry", entry)
webhook = create_hoyolab_webhook("https://discord.test/webhook", entry, post_data) # type: ignore
assert webhook is webhook_instance
mock_webhook_cls.assert_called_once_with(url="https://discord.test/webhook", rate_limit_retry=True)
embed_instance.set_title.assert_called_once_with("Update 4.0")
embed_instance.set_url.assert_called_once_with("https://feeds.c3kay.de/hoyolab.xml")
embed_instance.set_image.assert_called_once_with(
url="https://img.example.com/hero.jpg",
height=1080,
width=1920,
)
embed_instance.set_color.assert_called_once_with("11AAFF")
embed_instance.set_footer.assert_called_once_with(text="Official")
embed_instance.add_embed_field.assert_any_call(name="Start", value="<t:1712000000:R>")
embed_instance.add_embed_field.assert_any_call(name="End", value="<t:1712600000:R>")
embed_instance.set_timestamp.assert_called_once_with(timestamp="1711000000")
webhook_instance.add_file.assert_called_once_with(file=b"video-bytes", filename="entry-123.mp4")
webhook_instance.add_embed.assert_called_once_with(embed_instance)
assert webhook_instance.content == "https://www.youtube.com/watch?v=abc123_XY"
webhook_instance.remove_embeds.assert_called_once()
@patch("discord_rss_bot.hoyolab_api.requests.get")
@patch("discord_rss_bot.hoyolab_api.DiscordEmbed")
@patch("discord_rss_bot.hoyolab_api.DiscordWebhook")
def test_handles_invalid_structured_content_without_removing_embeds(
self,
mock_webhook_cls: MagicMock,
mock_embed_cls: MagicMock,
mock_requests_get: MagicMock,
) -> None:
webhook_instance = MagicMock()
embed_instance = MagicMock()
mock_webhook_cls.return_value = webhook_instance
mock_embed_cls.return_value = embed_instance
mock_requests_get.return_value = MagicMock(ok=False)
post_data = {
"post": {
"subject": "News",
"content": "{}",
"structured_content": "not-json",
},
}
entry = make_entry()
entry = typing.cast("Entry", entry)
webhook = create_hoyolab_webhook("https://discord.test/webhook", entry, post_data) # type: ignore
assert webhook is webhook_instance
webhook_instance.remove_embeds.assert_not_called()
def test_extract_post_id_with_querystring() -> None:
url = "https://www.hoyolab.com/article/38588239?utm_source=feed"
assert extract_post_id_from_hoyolab_url(url) == "38588239"
def test_extract_post_id_non_string_input_returns_none() -> None:
assert extract_post_id_from_hoyolab_url(None) is None # type: ignore[arg-type]
@pytest.mark.parametrize(
("url", "expected"),
[
("https://feeds.c3kay.de/rss", True),
("https://www.hoyolab.com/feed", False),
],
)
def test_is_c3kay_feed_parametrized(*, url: str, expected: bool) -> None:
assert is_c3kay_feed(url) is expected

View file

@ -32,6 +32,8 @@ if TYPE_CHECKING:
from reader import Entry
from reader import Reader
from discord_rss_bot.feeds import JsonValue
client: TestClient = TestClient(app)
webhook_name: str = "Hello, I am a webhook!"
webhook_url: str = "https://discord.com/api/webhooks/1234567890/abcdefghijklmnopqrstuvwxyz"
@ -318,7 +320,7 @@ def test_feed_page_shows_steam_thumbnail_hint_for_steam_feeds() -> None:
assert feed_url == self.feed.url
return self.feed
def get_tag(self, _resource: object, key: str, default: TestTagValue = None) -> TestTagValue:
def get_tag(self, _resource: str | DummyFeed, key: str, default: TestTagValue = None) -> TestTagValue:
return {
"webhooks": [],
"delivery_mode": "embed",
@ -2645,7 +2647,7 @@ def test_export_opml_with_stub_reader() -> None:
filename="reader-feeds-2026-07-18-12-00-00.opml",
)
def get_tag(self, _resource: tuple, _key: str, default: object = None) -> object:
def get_tag(self, _resource: tuple[()], _key: str, default: TestTagValue = None) -> TestTagValue:
return default
def get_feeds(self) -> list:
@ -2672,7 +2674,7 @@ class StubImportConfirmReader:
self.added_urls: list[str] = []
self.tags: dict[tuple[str, str], str] = {}
def get_tag(self, _resource: tuple, _key: str, default: object = None) -> object:
def get_tag(self, _resource: tuple[()], _key: str, default: TestTagValue = None) -> TestTagValue:
"""Stub get_tag.
Returns:
@ -2705,7 +2707,7 @@ class StubImportConfirmReaderWithExisting:
self.added_urls: list[str] = []
self.tags: dict[tuple[str, str], str] = {}
def get_tag(self, _resource: tuple, _key: str, default: object = None) -> object:
def get_tag(self, _resource: tuple[()], _key: str, default: TestTagValue = None) -> TestTagValue:
"""Stub get_tag.
Returns:
@ -2857,7 +2859,6 @@ def test_post_embed_saves_all_fields() -> None:
"thumbnail_url": "https://example.com/thumb.png",
"footer_text": "Footer Text",
"footer_icon_url": "https://example.com/footer.png",
"show_steam_game_icon_in_thumbnail": "true",
},
follow_redirects=False,
)
@ -2880,7 +2881,6 @@ def test_post_embed_saves_all_fields() -> None:
assert saved["thumbnail_url"] == "https://example.com/thumb.png"
assert saved["footer_text"] == "Footer Text"
assert saved["footer_icon_url"] == "https://example.com/footer.png"
assert saved["show_steam_game_icon_in_thumbnail"] is True
finally:
app.dependency_overrides = {}
@ -2958,7 +2958,6 @@ def test_post_embed_allows_clearing_all_fields() -> None:
"thumbnail_url": "https://old.example.com/thumb.png",
"footer_text": "Old Footer",
"footer_icon_url": "https://old.example.com/footer.png",
"show_steam_game_icon_in_thumbnail": True,
})
stub = _make_stub_reader_for_embed(stored_embed=existing)
app.dependency_overrides[get_reader_dependency] = lambda: stub
@ -2998,7 +2997,6 @@ def test_post_embed_allows_clearing_all_fields() -> None:
assert not saved["thumbnail_url"]
assert not saved["footer_text"]
assert not saved["footer_icon_url"]
assert saved["show_steam_game_icon_in_thumbnail"] is False
finally:
app.dependency_overrides = {}
@ -3016,7 +3014,6 @@ def test_post_embed_untouched_fields_retain_values() -> None:
"thumbnail_url": "",
"footer_text": "Old Footer",
"footer_icon_url": "",
"show_steam_game_icon_in_thumbnail": False,
})
stub = _make_stub_reader_for_embed(stored_embed=existing)
app.dependency_overrides[get_reader_dependency] = lambda: stub
@ -3056,7 +3053,6 @@ def test_post_embed_untouched_fields_retain_values() -> None:
assert saved["author_name"] == "Author"
assert saved["author_url"] == "https://a.example.com"
assert saved["footer_text"] == "Old Footer"
assert saved["show_steam_game_icon_in_thumbnail"] is False
finally:
app.dependency_overrides = {}
@ -3094,7 +3090,6 @@ def test_post_embed_saves_empty_description_when_no_prior_embed_exists() -> None
saved: dict[str, str] = json.loads(json_arg)
assert saved["title"] == "Just a Title"
assert not saved["description"], f"Expected empty description, got {saved['description']!r}"
assert saved["show_steam_game_icon_in_thumbnail"] is False
finally:
app.dependency_overrides = {}
@ -3412,7 +3407,7 @@ class _StubReaderForMass:
return self._webhook_url
return default
def set_tag(self, resource: str, key: str, value: object) -> None: # pyright: ignore[reportArgumentType]
def set_tag(self, resource: str, key: str, value: JsonValue) -> None:
"""Record tag set calls."""
self.set_tag_calls.append((resource, key, value))

View file

@ -39,7 +39,7 @@ class _AutodiscoverHandler(BaseHTTPRequestHandler):
b'type="application/rss+xml" title="Example"></head></html>',
)
def log_message(self, _format: str, *_args: object) -> None:
def log_message(self, _format: str, *_args: str | int) -> None:
"""Suppress HTTP request logging during tests."""

View file

@ -11,8 +11,8 @@ from reader import make_reader
from discord_rss_bot.filter.evaluator import evaluate_entry_filters
from discord_rss_bot.filter.evaluator import get_filter_values_from_reader
from discord_rss_bot.filter.whitelist import has_white_tags
from discord_rss_bot.filter.whitelist import should_be_sent
from discord_rss_bot.filter.evaluator import has_white_tags
from discord_rss_bot.filter.evaluator import should_be_sent
if TYPE_CHECKING:
from collections.abc import Iterable