TheLovinator/discord-rss-bot
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Compare commits

base: TheLovinator:master
Branches Tags
TheLovinator:master
..
compare: TheLovinator:2bc2bc008bed8b785c7c62078fcd432ed15d4144
Branches Tags
TheLovinator:master

No commits in common. "master" and "2bc2bc008bed8b785c7c62078fcd432ed15d4144" have entirely different histories.
master ... 2bc2bc008b

53 changed files with 926 additions and 6115 deletions
Download patch file Download diff file Expand all files Collapse all files

19
.env.example
View file

@ -1,19 +0,0 @@
# You can optionally store backups of your bot's configuration in a git repository.
# This allows you to track changes by subscribing to the repository or using a RSS feed.
# Local path for the backup git repository (e.g., /data/backup or /home/user/backups/discord-rss-bot)
# When set, the bot will initialize a git repo here and commit state.json after every configuration change
# GIT_BACKUP_PATH=
# Remote URL for pushing backup commits (e.g., git@github.com:username/private-config.git)
# Optional - only set if you want automatic pushes to a remote repository
# Leave empty to keep git history local only
# GIT_BACKUP_REMOTE=
# Sentry Configuration (Optional)
# Sentry DSN for error tracking and monitoring
# Leave empty to disable Sentry integration
# SENTRY_DSN=
# Testing Configuration
# Discord webhook URL used for testing (optional, only needed when running tests)
# TEST_WEBHOOK_URL=

100
.forgejo/workflows/build.yml
View file

@ -1,100 +0,0 @@
---
# Required setup for self-hosted runner:
# 1. Install dependencies:
# sudo pacman -S qemu-user-static qemu-user-static-binfmt docker docker-buildx
# 2. Add runner to docker group:
# sudo usermod -aG docker forgejo-runner
# 3. Restart runner service to apply group membership:
# sudo systemctl restart forgejo-runner
# 4. Install uv and ruff for the runner user
# 5. Login to GitHub Container Registry:
# echo "ghp_YOUR_TOKEN_HERE" | sudo -u forgejo-runner docker login ghcr.io -u TheLovinator1 --password-stdin
# 6. Configure sudoers for deployment (sudo EDITOR=nvim visudo):
# forgejo-runner ALL=(discord-rss) NOPASSWD: /usr/bin/git -C /home/discord-rss/discord-rss-bot pull
# forgejo-runner ALL=(discord-rss) NOPASSWD: /usr/bin/uv sync -U --directory /home/discord-rss/discord-rss-bot
# forgejo-runner ALL=(root) NOPASSWD: /bin/systemctl restart discord-rss-bot
name: Test and build Docker image
on:
push:
branches:
- master
pull_request:
workflow_dispatch:
schedule:
- cron: "0 0 1 * *"
jobs:
docker:
runs-on: self-hosted
steps:
# Download the latest commit from the master branch
- uses: actions/checkout@v6
# Verify local tools are available on the self-hosted runner
- name: Check local toolchain
run: |
python --version
uv --version
ruff --version
docker version
# Bootstrap a local Buildx builder for multi-arch builds
# (requires qemu-user-static and qemu-user-static-binfmt installed via pacman)
- name: Configure local buildx for multi-arch
run: |
docker buildx inspect local-multiarch-builder >/dev/null 2>&1 || \
docker buildx create --name local-multiarch-builder --driver docker-container
docker buildx use local-multiarch-builder
docker buildx inspect --bootstrap
- name: Lint Python code
run: ruff check --exit-non-zero-on-fix --verbose
- name: Check Python formatting
run: ruff format --check --verbose
- name: Lint Dockerfile
run: docker build --check .
- name: Install dependencies
run: uv sync --all-extras --all-groups
- name: Run tests
run: uv run pytest
- id: tags
name: Compute image tags
run: |
IMAGE="ghcr.io/thelovinator1/discord-rss-bot"
if [ "${FORGEJO_REF}" = "refs/heads/master" ]; then
echo "tags=${IMAGE}:latest,${IMAGE}:master" >> "$FORGEJO_OUTPUT"
else
SHORT_SHA="$(echo "$FORGEJO_SHA" | cut -c1-12)"
echo "tags=${IMAGE}:sha-${SHORT_SHA}" >> "$FORGEJO_OUTPUT"
fi
# Build (and optionally push) Docker image
- name: Build and push Docker image
env:
TAGS: ${{ steps.tags.outputs.tags }}
run: |
IFS=',' read -r -a tag_array <<< "$TAGS"
tag_args=()
for tag in "${tag_array[@]}"; do
tag_args+=( -t "$tag" )
done
if [ "${{ forge.event_name }}" = "pull_request" ]; then
docker buildx build --platform linux/amd64,linux/arm64 "${tag_args[@]}" --load .
else
docker buildx build --platform linux/amd64,linux/arm64 "${tag_args[@]}" --push .
fi
# Deploy to production server
- name: Deploy to Server
if: success() && forge.ref == 'refs/heads/master'
run: |
sudo -u discord-rss git -C /home/discord-rss/discord-rss-bot pull
sudo -u discord-rss uv sync -U --directory /home/discord-rss/discord-rss-bot
sudo systemctl restart discord-rss-bot

1
.gitattributes vendored
View file

@ -1 +0,0 @@
*.html linguist-language=jinja

4
.forgejo/renovate.json → .github/renovate.json vendored
View file

@ -1,8 +1,6 @@
{
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
"extends": [
"config:recommended"
],
"extends": ["config:recommended"],
"automerge": true,
"configMigration": true,
"dependencyDashboard": false,

64
.github/workflows/build.yml vendored Normal file
View file

@ -0,0 +1,64 @@
---
name: Test and build Docker image
on:
push:
pull_request:
workflow_dispatch:
schedule:
- cron: "0 6 * * *"
env:
TEST_WEBHOOK_URL: ${{ secrets.TEST_WEBHOOK_URL }}
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: 3.12
- uses: astral-sh/setup-uv@v5
with:
version: "latest"
- run: uv sync --all-extras --all-groups
- run: uv run pytest
ruff:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: astral-sh/ruff-action@v3
with:
version: "latest"
- run: ruff check --exit-non-zero-on-fix --verbose
- run: ruff format --check --verbose
build:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
if: github.event_name != 'pull_request'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
needs: [test, ruff]
steps:
- uses: actions/checkout@v4
- uses: docker/setup-qemu-action@v3
with:
platforms: all
- uses: docker/setup-buildx-action@v3
- uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- uses: docker/build-push-action@v6
with:
context: .
platforms: linux/amd64, linux/arm64
push: ${{ github.event_name != 'pull_request' }}
tags: |
ghcr.io/thelovinator1/discord-rss-bot:latest
ghcr.io/thelovinator1/discord-rss-bot:master

29
.gitignore vendored
View file

@ -92,7 +92,7 @@ ipython_config.py
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
# Pipfile.lock
Pipfile.lock
# UV
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
@ -105,12 +105,11 @@ uv.lock
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
# poetry.lock
# poetry.toml
poetry.lock
# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
# pdm.lock
#pdm.lock
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
# in version control.
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
@ -166,20 +165,7 @@ cython_debug/
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
# .idea/
# Abstra
# Abstra is an AI-powered process automation framework.
# Ignore directories containing user credentials, local state, and settings.
# Learn more at https://abstra.io/docs
.abstra/
# Visual Studio Code
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
# and can be added to the global gitignore or merged into this file. However, if you prefer,
# you could uncomment the following to ignore the entire vscode folder
# .vscode/
#.idea/
# Ruff stuff:
.ruff_cache/
@ -187,13 +173,6 @@ cython_debug/
# PyPI configuration file
.pypirc
# Cursor
# Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
# exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
# refer to https://docs.cursor.com/context/ignore-files
.cursorignore
.cursorindexingignore
# Database stuff
*.sqlite
*.sqlite-shm

10
.pre-commit-config.yaml
View file

@ -1,13 +1,13 @@
repos:
# Automatically add trailing commas to calls and literals.
- repo: https://github.com/asottile/add-trailing-comma
rev: v4.0.0
rev: v3.1.0
hooks:
- id: add-trailing-comma
# Some out-of-the-box hooks for pre-commit.
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v6.0.0
rev: v5.0.0
hooks:
- id: check-added-large-files
- id: check-ast
@ -31,14 +31,14 @@ repos:
# Run Pyupgrade on all Python files. This will upgrade the code to Python 3.12.
- repo: https://github.com/asottile/pyupgrade
rev: v3.21.2
rev: v3.19.1
hooks:
- id: pyupgrade
args: ["--py312-plus"]
# An extremely fast Python linter and formatter.
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.15.5
rev: v0.9.5
hooks:
- id: ruff-format
- id: ruff
@ -46,6 +46,6 @@ repos:
# Static checker for GitHub Actions workflow files.
- repo: https://github.com/rhysd/actionlint
rev: v1.7.11
rev: v1.7.7
hooks:
- id: actionlint

6
.vscode/launch.json vendored
View file

@ -8,11 +8,7 @@
"module": "uvicorn",
"args": [
"discord_rss_bot.main:app",
"--reload",
"--host",
"0.0.0.0",
"--port",
"3000",
"--reload"
],
"jinja": true,
"justMyCode": true

8
.vscode/settings.json vendored
View file

@ -1,19 +1,13 @@
{
"cSpell.words": [
"autoexport",
"botuser",
"Genshins",
"healthcheck",
"Hoyolab",
"levelname",
"Lovinator",
"markdownified",
"markdownify",
"pipx",
"pyproject",
"thead",
"thelovinator",
"uvicorn"
"thead"
],
"python.analysis.typeCheckingMode": "basic"
}

5
Dockerfile
View file

@ -1,15 +1,14 @@
FROM python:3.14-slim
FROM python:3.13-slim
COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
RUN useradd --create-home botuser && \
mkdir -p /home/botuser/discord-rss-bot/ /home/botuser/.local/share/discord_rss_bot/ && \
chown -R botuser:botuser /home/botuser/
USER botuser
WORKDIR /home/botuser/discord-rss-bot
COPY --chown=botuser:botuser requirements.txt /home/botuser/discord-rss-bot/
RUN --mount=type=cache,target=/root/.cache/uv \
--mount=type=bind,source=pyproject.toml,target=pyproject.toml \
uv sync --no-install-project
COPY --chown=botuser:botuser discord_rss_bot/ /home/botuser/discord-rss-bot/discord_rss_bot/
EXPOSE 5000
VOLUME ["/home/botuser/.local/share/discord_rss_bot/"]
HEALTHCHECK --interval=10m --timeout=5s CMD ["uv", "run", "./discord_rss_bot/healthcheck.py"]
CMD ["uv", "run", "uvicorn", "discord_rss_bot.main:app", "--host=0.0.0.0", "--port=5000", "--proxy-headers", "--forwarded-allow-ips='*'", "--log-level", "debug"]

103
README.md
View file

@ -2,25 +2,8 @@
Subscribe to RSS feeds and get updates to a Discord webhook.
Email: [tlovinator@gmail.com](mailto:tlovinator@gmail.com)
Discord: TheLovinator#9276
## Features
- Subscribe to RSS feeds and get updates to a Discord webhook.
- Web interface to manage subscriptions.
- Customizable message format for each feed.
- Choose between Discord embed or plain text.
- Regex filters for RSS feeds.
- Blacklist/whitelist words in the title/description/author/etc.
- Set different update frequencies for each feed or use a global default.
- Gets extra information from APIs if available, currently for:
- [https://feeds.c3kay.de/](https://feeds.c3kay.de/)
- Genshin Impact News
- Honkai Impact 3rd News
- Honkai Starrail News
- Zenless Zone Zero News
> [!NOTE]
> You should look at [MonitoRSS](https://github.com/synzen/monitorss) for a more feature-rich project.
## Installation
@ -30,7 +13,9 @@ or [install directly on your computer](#install-directly-on-your-computer).
### Docker
- Open a terminal in the repository folder.
- <kbd>Shift</kbd> + <kbd>right-click</kbd> in the folder and `Open PowerShell window here`
- Windows 10: <kbd>Shift</kbd> + <kbd>right-click</kbd> in the folder and select `Open PowerShell window here`
- Windows 11: <kbd>Shift</kbd> + <kbd>right-click</kbd> in the folder and Show more options
and `Open PowerShell window here`
- Run the Docker Compose file:
- `docker-compose up`
- You can stop the bot with <kbd>Ctrl</kbd> + <kbd>c</kbd>.
@ -44,68 +29,34 @@ or [install directly on your computer](#install-directly-on-your-computer).
### Install directly on your computer
- Install the latest of [uv](https://docs.astral.sh/uv/#installation):
- `powershell -ExecutionPolicy ByPass -c "irm <https://astral.sh/uv/install.ps1> | iex"`
This is not recommended if you don't have an init system (e.g., systemd)
- Install the latest version of needed software:
- [Python](https://www.python.org/)
- You should use the latest version.
- You want to add Python to your PATH.
- Windows: Find `App execution aliases` and disable python.exe and python3.exe
- [Poetry](https://python-poetry.org/docs/master/#installation)
- Windows: You have to add `%appdata%\Python\Scripts` to your PATH for Poetry to work.
- Download the project from GitHub with Git or download
the [ZIP](https://github.com/TheLovinator1/discord-rss-bot/archive/refs/heads/master.zip).
- If you want to update the bot, you can run `git pull` in the project folder or download the ZIP again.
- Open a terminal in the repository folder.
- <kbd>Shift</kbd> + <kbd>right-click</kbd> in the folder and `Open PowerShell window here`
- Windows 10: <kbd>Shift</kbd> + <kbd>right-click</kbd> in the folder and select `Open PowerShell window here`
- Windows 11: <kbd>Shift</kbd> + <kbd>right-click</kbd> in the folder and Show more options
and `Open PowerShell window here`
- Install requirements:
- Type `poetry install` into the PowerShell window. Make sure you are
in the repository folder where the [pyproject.toml](pyproject.toml) file is located.
- (You may have to restart your terminal if it can't find the `poetry` command. Also double check it is in
your PATH.)
- Start the bot:
- Type `uv run discord_rss_bot/main.py` into the PowerShell window.
- Type `poetry run python discord_rss_bot/main.py` into the PowerShell window.
- You can stop the bot with <kbd>Ctrl</kbd> + <kbd>c</kbd>.
- Bot is now running on port 3000.
- You should run this bot behind a reverse proxy like [Caddy](https://caddyserver.com/)
or [Nginx](https://www.nginx.com/) if you want to access it from the internet. Remember to add authentication.
- You can access the web interface at `http://localhost:3000/`.
- To run automatically on boot:
- Use [Windows Task Scheduler](https://en.wikipedia.org/wiki/Windows_Task_Scheduler).
- Or add a shortcut to `%userprofile%\AppData\Roaming\Microsoft\Windows\Start Menu\Programs\Startup`.
Note: You will need to run `poetry install` again if [poetry.lock](poetry.lock) has been modified.
## Git Backup (State Version Control)
## Contact
The bot can commit every configuration change (adding/removing feeds, webhook
changes, blacklist/whitelist updates) to a separate private Git repository so
you get a full, auditable history of state changes — similar to `etckeeper`.
### Configuration
Set the following environment variables (e.g. in `docker-compose.yml` or a
`.env` file):
| Variable | Required | Description |
| ------------------- | -------- | ----------------------------------------------------------------------------------------------------------------------------------- |
| `GIT_BACKUP_PATH` | Yes | Local path where the backup git repository is stored. The bot will initialise it automatically if it does not yet exist. |
| `GIT_BACKUP_REMOTE` | No | Remote URL to push to after each commit (e.g. `git@github.com:you/private-config.git`). Leave unset to keep the history local only. |
### What is backed up
After every relevant change a `state.json` file is written and committed.
The file contains:
- All feed URLs together with their webhook URL, custom message, embed
settings, and any blacklist/whitelist filters.
- The global list of Discord webhooks.
### Docker example
```yaml
services:
discord-rss-bot:
image: ghcr.io/thelovinator1/discord-rss-bot:latest
volumes:
- ./data:/data
environment:
- GIT_BACKUP_PATH=/data/backup
- GIT_BACKUP_REMOTE=git@github.com:you/private-config.git
```
For SSH-based remotes mount your SSH key into the container and make sure the
host key is trusted, e.g.:
```yaml
volumes:
- ./data:/data
- ~/.ssh:/root/.ssh:ro
```
Email: [mailto:tlovinator@gmail.com](tlovinator@gmail.com)
Discord: TheLovinator#9276

19
discord_rss_bot/custom_filters.py
View file

@ -4,14 +4,15 @@ import urllib.parse
from functools import lru_cache
from typing import TYPE_CHECKING
from discord_rss_bot.filter.blacklist import entry_should_be_skipped
from discord_rss_bot.filter.blacklist import feed_has_blacklist_tags
from discord_rss_bot.filter.whitelist import has_white_tags
from discord_rss_bot.filter.whitelist import should_be_sent
from discord_rss_bot.filter.blacklist import entry_should_be_skipped, feed_has_blacklist_tags
from discord_rss_bot.filter.whitelist import has_white_tags, should_be_sent
from discord_rss_bot.settings import get_reader
if TYPE_CHECKING:
from reader import Entry
from reader import Reader
from reader import Entry, Reader
# Our reader
reader: Reader = get_reader()
@lru_cache
@ -30,12 +31,11 @@ def encode_url(url_to_quote: str) -> str:
return urllib.parse.quote(string=url_to_quote) if url_to_quote else ""
def entry_is_whitelisted(entry_to_check: Entry, reader: Reader) -> bool:
def entry_is_whitelisted(entry_to_check: Entry) -> bool:
"""Check if the entry is whitelisted.
Args:
entry_to_check: The feed to check.
reader: Custom Reader instance.
Returns:
bool: True if the feed is whitelisted, False otherwise.
@ -44,12 +44,11 @@ def entry_is_whitelisted(entry_to_check: Entry, reader: Reader) -> bool:
return bool(has_white_tags(reader, entry_to_check.feed) and should_be_sent(reader, entry_to_check))
def entry_is_blacklisted(entry_to_check: Entry, reader: Reader) -> bool:
def entry_is_blacklisted(entry_to_check: Entry) -> bool:
"""Check if the entry is blacklisted.
Args:
entry_to_check: The feed to check.
reader: Custom Reader instance.
Returns:
bool: True if the feed is blacklisted, False otherwise.

142
discord_rss_bot/custom_message.py
View file

@ -1,27 +1,18 @@
from __future__ import annotations
import html
import json
import logging
import re
from dataclasses import dataclass
from typing import TYPE_CHECKING
from bs4 import BeautifulSoup
from bs4 import Tag
from bs4 import BeautifulSoup, Tag
from markdownify import markdownify
from reader import Entry, Feed, Reader, TagNotFoundError
from discord_rss_bot.is_url_valid import is_url_valid
if TYPE_CHECKING:
from reader import Entry
from reader import Feed
from reader import Reader
from discord_rss_bot.settings import get_reader
logger: logging.Logger = logging.getLogger(__name__)
DISCORD_TIMESTAMP_TAG_RE: re.Pattern[str] = re.compile(r"<t:\d+(?::[tTdDfFrRsS])?>")
@dataclass(slots=True)
class CustomEmbed:
@ -55,80 +46,18 @@ def try_to_replace(custom_message: str, template: str, replace_with: str) -> str
return custom_message
def _preserve_discord_timestamp_tags(text: str) -> tuple[str, dict[str, str]]:
"""Replace Discord timestamp tags with placeholders before markdown conversion.
Args:
text: The text to replace tags in.
Returns:
The text with Discord timestamp tags replaced by placeholders and a mapping of placeholders to original tags.
"""
replacements: dict[str, str] = {}
def replace_match(match: re.Match[str]) -> str:
placeholder: str = f"DISCORDTIMESTAMPPLACEHOLDER{len(replacements)}"
replacements[placeholder] = match.group(0)
return placeholder
return DISCORD_TIMESTAMP_TAG_RE.sub(replace_match, text), replacements
def _restore_discord_timestamp_tags(text: str, replacements: dict[str, str]) -> str:
"""Restore preserved Discord timestamp tags after markdown conversion.
Args:
text: The text to restore tags in.
replacements: A mapping of placeholders to original Discord timestamp tags.
Returns:
The text with placeholders replaced by the original Discord timestamp tags.
"""
for placeholder, original_value in replacements.items():
text = text.replace(placeholder, original_value)
return text
def format_entry_html_for_discord(text: str) -> str:
"""Convert entry HTML to Discord-friendly markdown while preserving Discord timestamp tags.
Args:
text: The HTML text to format.
Returns:
The formatted text with Discord timestamp tags preserved.
"""
if not text:
return ""
unescaped_text: str = html.unescape(text)
protected_text, replacements = _preserve_discord_timestamp_tags(unescaped_text)
formatted_text: str = markdownify(
html=protected_text,
strip=["img", "table", "td", "tr", "tbody", "thead"],
escape_misc=False,
heading_style="ATX",
)
if "[https://" in formatted_text or "[https://www." in formatted_text:
formatted_text = formatted_text.replace("[https://", "[")
formatted_text = formatted_text.replace("[https://www.", "[")
return _restore_discord_timestamp_tags(formatted_text, replacements)
def replace_tags_in_text_message(entry: Entry, reader: Reader) -> str:
def replace_tags_in_text_message(entry: Entry) -> str:
"""Replace tags in custom_message.
Args:
entry: The entry to get the tags from.
reader: Custom Reader instance.
Returns:
Returns the custom_message with the tags replaced.
"""
feed: Feed = entry.feed
custom_message: str = get_custom_message(feed=feed, reader=reader)
custom_reader: Reader = get_reader()
custom_message: str = get_custom_message(feed=feed, custom_reader=custom_reader)
content = ""
if entry.content:
@ -139,8 +68,16 @@ def replace_tags_in_text_message(entry: Entry, reader: Reader) -> str:
first_image: str = get_first_image(summary, content)
summary = format_entry_html_for_discord(summary)
content = format_entry_html_for_discord(content)
summary = markdownify(html=summary, strip=["img", "table", "td", "tr", "tbody", "thead"], escape_misc=False)
content = markdownify(html=content, strip=["img", "table", "td", "tr", "tbody", "thead"], escape_misc=False)
if "[https://" in content or "[https://www." in content:
content = content.replace("[https://", "[")
content = content.replace("[https://www.", "[")
if "[https://" in summary or "[https://www." in summary:
summary = summary.replace("[https://", "[")
summary = summary.replace("[https://www.", "[")
feed_added: str = feed.added.strftime("%Y-%m-%d %H:%M:%S") if feed.added else "Never"
feed_last_exception: str = feed.last_exception.value_str if feed.last_exception else ""
@ -215,6 +152,13 @@ def get_first_image(summary: str | None, content: str | None) -> str:
logger.warning("Invalid URL: %s", src)
continue
# Genshins first image is a divider, so we ignore it.
# https://hyl-static-res-prod.hoyolab.com/divider_config/PC/line_3.png
skip_images: list[str] = [
"https://img-os-static.hoyolab.com/divider_config/",
"https://hyl-static-res-prod.hoyolab.com/divider_config/",
]
if not str(image.attrs["src"]).startswith(tuple(skip_images)):
return str(image.attrs["src"])
if summary and (images := BeautifulSoup(summary, features="lxml").find_all("img")):
for image in images:
@ -226,22 +170,24 @@ def get_first_image(summary: str | None, content: str | None) -> str:
logger.warning("Invalid URL: %s", image.attrs["src"])
continue
# Genshins first image is a divider, so we ignore it.
if not str(image.attrs["src"]).startswith("https://img-os-static.hoyolab.com/divider_config"):
return str(image.attrs["src"])
return ""
def replace_tags_in_embed(feed: Feed, entry: Entry, reader: Reader) -> CustomEmbed:
def replace_tags_in_embed(feed: Feed, entry: Entry) -> CustomEmbed:
"""Replace tags in embed.
Args:
feed: The feed to get the tags from.
entry: The entry to get the tags from.
reader: Custom Reader instance.
Returns:
Returns the embed with the tags replaced.
"""
embed: CustomEmbed = get_embed(feed=feed, reader=reader)
custom_reader: Reader = get_reader()
embed: CustomEmbed = get_embed(feed=feed, custom_reader=custom_reader)
content = ""
if entry.content:
@ -252,8 +198,16 @@ def replace_tags_in_embed(feed: Feed, entry: Entry, reader: Reader) -> CustomEmb
first_image: str = get_first_image(summary, content)
summary = format_entry_html_for_discord(summary)
content = format_entry_html_for_discord(content)
summary = markdownify(html=summary, strip=["img", "table", "td", "tr", "tbody", "thead"], escape_misc=False)
content = markdownify(html=content, strip=["img", "table", "td", "tr", "tbody", "thead"], escape_misc=False)
if "[https://" in content or "[https://www." in content:
content = content.replace("[https://", "[")
content = content.replace("[https://www.", "[")
if "[https://" in summary or "[https://www." in summary:
summary = summary.replace("[https://", "[")
summary = summary.replace("[https://www.", "[")
feed_added: str = feed.added.strftime("%Y-%m-%d %H:%M:%S") if feed.added else "Never"
feed_last_updated: str = feed.last_updated.strftime("%Y-%m-%d %H:%M:%S") if feed.last_updated else "Never"
@ -332,29 +286,31 @@ def _replace_embed_tags(embed: CustomEmbed, template: str, replace_with: str) ->
embed.footer_icon_url = try_to_replace(embed.footer_icon_url, template, replace_with)
def get_custom_message(reader: Reader, feed: Feed) -> str:
def get_custom_message(custom_reader: Reader, feed: Feed) -> str:
"""Get custom_message tag from feed.
Args:
reader: What Reader to use.
custom_reader: What Reader to use.
feed: The feed to get the tag from.
Returns:
Returns the contents from the custom_message tag.
"""
try:
custom_message: str = str(reader.get_tag(feed, "custom_message", ""))
custom_message: str = str(custom_reader.get_tag(feed, "custom_message"))
except TagNotFoundError:
custom_message = ""
except ValueError:
custom_message = ""
return custom_message
def save_embed(reader: Reader, feed: Feed, embed: CustomEmbed) -> None:
def save_embed(custom_reader: Reader, feed: Feed, embed: CustomEmbed) -> None:
"""Set embed tag in feed.
Args:
reader: What Reader to use.
custom_reader: What Reader to use.
feed: The feed to set the tag in.
embed: The embed to set.
"""
@ -370,20 +326,20 @@ def save_embed(reader: Reader, feed: Feed, embed: CustomEmbed) -> None:
"footer_text": embed.footer_text,
"footer_icon_url": embed.footer_icon_url,
}
reader.set_tag(feed, "embed", json.dumps(embed_dict)) # pyright: ignore[reportArgumentType]
custom_reader.set_tag(feed, "embed", json.dumps(embed_dict)) # pyright: ignore[reportArgumentType]
def get_embed(reader: Reader, feed: Feed) -> CustomEmbed:
def get_embed(custom_reader: Reader, feed: Feed) -> CustomEmbed:
"""Get embed tag from feed.
Args:
reader: What Reader to use.
custom_reader: What Reader to use.
feed: The feed to get the tag from.
Returns:
Returns the contents from the embed tag.
"""
embed = reader.get_tag(feed, "embed", "")
embed = custom_reader.get_tag(feed, "embed", "")
if embed:
if not isinstance(embed, str):

296
discord_rss_bot/feeds.py
View file

@ -1,45 +1,25 @@
from __future__ import annotations
import datetime
import json
import logging
import os
import pprint
import re
from typing import TYPE_CHECKING
from typing import Any
from urllib.parse import ParseResult
from urllib.parse import urlparse
import tldextract
from discord_webhook import DiscordEmbed
from discord_webhook import DiscordWebhook
from discord_webhook import DiscordEmbed, DiscordWebhook
from fastapi import HTTPException
from markdownify import markdownify
from reader import Entry
from reader import EntryNotFoundError
from reader import Feed
from reader import FeedExistsError
from reader import FeedNotFoundError
from reader import Reader
from reader import ReaderError
from reader import StorageError
from reader import Entry, EntryNotFoundError, Feed, FeedExistsError, Reader, ReaderError, StorageError, TagNotFoundError
from discord_rss_bot.custom_message import CustomEmbed
from discord_rss_bot.custom_message import get_custom_message
from discord_rss_bot.custom_message import replace_tags_in_embed
from discord_rss_bot.custom_message import replace_tags_in_text_message
from discord_rss_bot.custom_message import (
CustomEmbed,
get_custom_message,
replace_tags_in_embed,
replace_tags_in_text_message,
)
from discord_rss_bot.filter.blacklist import entry_should_be_skipped
from discord_rss_bot.filter.whitelist import has_white_tags
from discord_rss_bot.filter.whitelist import should_be_sent
from discord_rss_bot.hoyolab_api import create_hoyolab_webhook
from discord_rss_bot.hoyolab_api import extract_post_id_from_hoyolab_url
from discord_rss_bot.hoyolab_api import fetch_hoyolab_post
from discord_rss_bot.hoyolab_api import is_c3kay_feed
from discord_rss_bot.filter.whitelist import has_white_tags, should_be_sent
from discord_rss_bot.is_url_valid import is_url_valid
from discord_rss_bot.settings import default_custom_embed
from discord_rss_bot.settings import default_custom_message
from discord_rss_bot.settings import get_reader
from discord_rss_bot.missing_tags import add_missing_tags
from discord_rss_bot.settings import default_custom_message, get_reader
if TYPE_CHECKING:
from collections.abc import Iterable
@ -49,159 +29,53 @@ if TYPE_CHECKING:
logger: logging.Logger = logging.getLogger(__name__)
def extract_domain(url: str) -> str: # noqa: PLR0911
"""Extract the domain name from a URL.
Args:
url: The URL to extract the domain from.
Returns:
str: The domain name, formatted for display.
"""
# Check for empty URL first
if not url:
return "Other"
try:
# Special handling for YouTube feeds
if "youtube.com/feeds/videos.xml" in url:
return "YouTube"
# Special handling for Reddit feeds
if "reddit.com" in url and ".rss" in url:
return "Reddit"
# Parse the URL and extract the domain
parsed_url: ParseResult = urlparse(url)
domain: str = parsed_url.netloc
# If we couldn't extract a domain, return "Other"
if not domain:
return "Other"
# Remove www. prefix if present
domain = re.sub(r"^www\.", "", domain)
# Special handling for common domains
domain_mapping: dict[str, str] = {"github.com": "GitHub"}
if domain in domain_mapping:
return domain_mapping[domain]
# Use tldextract to get the domain (SLD)
ext = tldextract.extract(url)
if ext.domain:
return ext.domain.capitalize()
return domain.capitalize()
except (ValueError, AttributeError, TypeError) as e:
logger.warning("Error extracting domain from %s: %s", url, e)
return "Other"
def send_entry_to_discord(entry: Entry, reader: Reader) -> str | None: # noqa: C901
def send_entry_to_discord(entry: Entry, custom_reader: Reader | None = None) -> str | None:
"""Send a single entry to Discord.
Args:
entry: The entry to send to Discord.
reader: The reader to use.
custom_reader: The reader to use. If None, the default reader will be used.
Returns:
str | None: The error message if there was an error, otherwise None.
"""
# Get the default reader if we didn't get a custom one.
reader: Reader = get_reader() if custom_reader is None else custom_reader
# Get the webhook URL for the entry.
webhook_url: str = str(reader.get_tag(entry.feed_url, "webhook", ""))
if not webhook_url:
return "No webhook URL found."
# If https://discord.com/quests/<quest_id> is in the URL, send a separate message with the URL.
send_discord_quest_notification(entry, webhook_url, reader=reader)
# Check if this is a c3kay feed
if is_c3kay_feed(entry.feed.url):
entry_link: str | None = entry.link
if entry_link:
post_id: str | None = extract_post_id_from_hoyolab_url(entry_link)
if post_id:
post_data: dict[str, Any] | None = fetch_hoyolab_post(post_id)
if post_data:
webhook = create_hoyolab_webhook(webhook_url, entry, post_data)
execute_webhook(webhook, entry, reader=reader)
return None
logger.warning(
"Failed to create Hoyolab webhook for feed %s, falling back to regular processing",
entry.feed.url,
)
else:
logger.warning("No entry link found for feed %s, falling back to regular processing", entry.feed.url)
webhook_message: str = ""
# Try to get the custom message for the feed. If the user has none, we will use the default message.
# This has to be a string for some reason so don't change it to "not custom_message.get_custom_message()"
if get_custom_message(reader, entry.feed) != "": # noqa: PLC1901
webhook_message: str = replace_tags_in_text_message(entry=entry, reader=reader)
webhook_message: str = replace_tags_in_text_message(entry=entry)
if not webhook_message:
webhook_message = "No message found."
# Create the webhook.
try:
should_send_embed = bool(reader.get_tag(entry.feed, "should_send_embed", True))
should_send_embed = bool(reader.get_tag(entry.feed, "should_send_embed"))
except TagNotFoundError:
logger.exception("No should_send_embed tag found for feed: %s", entry.feed.url)
should_send_embed = True
except StorageError:
logger.exception("Error getting should_send_embed tag for feed: %s", entry.feed.url)
should_send_embed = True
# YouTube feeds should never use embeds
if is_youtube_feed(entry.feed.url):
should_send_embed = False
if should_send_embed:
webhook = create_embed_webhook(webhook_url, entry, reader=reader)
webhook = create_embed_webhook(webhook_url, entry)
else:
webhook: DiscordWebhook = DiscordWebhook(url=webhook_url, content=webhook_message, rate_limit_retry=True)
execute_webhook(webhook, entry, reader=reader)
execute_webhook(webhook, entry)
return None
def send_discord_quest_notification(entry: Entry, webhook_url: str, reader: Reader) -> None:
"""Send a separate message to Discord if the entry is a quest notification."""
quest_regex: re.Pattern[str] = re.compile(r"https://discord\.com/quests/\d+")
def send_notification(quest_url: str) -> None:
"""Helper function to send quest notification to Discord."""
logger.info("Sending quest notification to Discord: %s", quest_url)
webhook = DiscordWebhook(
url=webhook_url,
content=quest_url,
rate_limit_retry=True,
)
execute_webhook(webhook, entry, reader=reader)
# Iterate through the content of the entry
for content in entry.content:
if content.type == "text" and content.value:
match = quest_regex.search(content.value)
if match:
send_notification(match.group(0))
return
elif content.type == "text/html" and content.value:
# Convert HTML to text and check for quest links
text_value = markdownify(
html=content.value,
strip=["img", "table", "td", "tr", "tbody", "thead"],
escape_misc=False,
heading_style="ATX",
)
match: re.Match[str] | None = quest_regex.search(text_value)
if match:
send_notification(match.group(0))
return
logger.info("No quest notification found in entry: %s", entry.id)
def set_description(custom_embed: CustomEmbed, discord_embed: DiscordEmbed) -> None:
"""Set the description of the embed.
@ -234,17 +108,12 @@ def set_title(custom_embed: CustomEmbed, discord_embed: DiscordEmbed) -> None:
discord_embed.set_title(embed_title) if embed_title else None
def create_embed_webhook( # noqa: C901
webhook_url: str,
entry: Entry,
reader: Reader,
) -> DiscordWebhook:
def create_embed_webhook(webhook_url: str, entry: Entry) -> DiscordWebhook:
"""Create a webhook with an embed.
Args:
webhook_url (str): The webhook URL.
entry (Entry): The entry to send to Discord.
reader (Reader): The Reader instance to use for getting embed data.
Returns:
DiscordWebhook: The webhook with the embed.
@ -253,7 +122,7 @@ def create_embed_webhook( # noqa: C901
feed: Feed = entry.feed
# Get the embed data from the database.
custom_embed: CustomEmbed = replace_tags_in_embed(feed=feed, entry=entry, reader=reader)
custom_embed: CustomEmbed = replace_tags_in_embed(feed=feed, entry=entry)
discord_embed: DiscordEmbed = DiscordEmbed()
@ -315,14 +184,13 @@ def get_webhook_url(reader: Reader, entry: Entry) -> str:
str: The webhook URL.
"""
try:
webhook_url: str = str(reader.get_tag(entry.feed_url, "webhook", ""))
webhook_url: str = str(reader.get_tag(entry.feed_url, "webhook"))
except TagNotFoundError:
logger.exception("No webhook URL found for feed: %s", entry.feed.url)
return ""
except StorageError:
logger.exception("Storage error getting webhook URL for feed: %s", entry.feed.url)
return ""
if not webhook_url:
logger.error("No webhook URL found for feed: %s", entry.feed.url)
return ""
return webhook_url
@ -341,53 +209,44 @@ def set_entry_as_read(reader: Reader, entry: Entry) -> None:
logger.exception("Error setting entry to read: %s", entry.id)
def send_to_discord(reader: Reader | None = None, feed: Feed | None = None, *, do_once: bool = False) -> None: # noqa: C901, PLR0912
def send_to_discord(custom_reader: Reader | None = None, feed: Feed | None = None, *, do_once: bool = False) -> None:
"""Send entries to Discord.
If response was not ok, we will log the error and mark the entry as unread, so it will be sent again next time.
Args:
reader: If we should use a custom reader instead of the default one.
custom_reader: If we should use a custom reader instead of the default one.
feed: The feed to send to Discord.
do_once: If we should only send one entry. This is used in the test.
"""
logger.info("Starting to send entries to Discord.")
# Get the default reader if we didn't get a custom one.
effective_reader: Reader = get_reader() if reader is None else reader
reader: Reader = get_reader() if custom_reader is None else custom_reader
# Check for new entries for every feed.
effective_reader.update_feeds(
scheduled=True,
workers=os.cpu_count() or 1,
)
reader.update_feeds()
# Loop through the unread entries.
entries: Iterable[Entry] = effective_reader.get_entries(feed=feed, read=False)
entries: Iterable[Entry] = reader.get_entries(feed=feed, read=False)
for entry in entries:
set_entry_as_read(effective_reader, entry)
set_entry_as_read(reader, entry)
if entry.added < datetime.datetime.now(tz=entry.added.tzinfo) - datetime.timedelta(days=1):
logger.info("Entry is older than 24 hours: %s from %s", entry.id, entry.feed.url)
continue
webhook_url: str = get_webhook_url(effective_reader, entry)
webhook_url: str = get_webhook_url(reader, entry)
if not webhook_url:
logger.info("No webhook URL found for feed: %s", entry.feed.url)
continue
should_send_embed: bool = should_send_embed_check(effective_reader, entry)
# Youtube feeds only need to send the link
if is_youtube_feed(entry.feed.url):
should_send_embed = False
should_send_embed: bool = should_send_embed_check(reader, entry)
if should_send_embed:
webhook = create_embed_webhook(webhook_url, entry, reader=effective_reader)
webhook = create_embed_webhook(webhook_url, entry)
else:
# If the user has set the custom message to an empty string, we will use the default message, otherwise we
# will use the custom message.
if get_custom_message(effective_reader, entry.feed) != "": # noqa: PLC1901
webhook_message = replace_tags_in_text_message(entry, reader=effective_reader)
if get_custom_message(reader, entry.feed) != "": # noqa: PLC1901
webhook_message = replace_tags_in_text_message(entry)
else:
webhook_message: str = str(default_custom_message)
@ -397,35 +256,19 @@ def send_to_discord(reader: Reader | None = None, feed: Feed | None = None, *, d
webhook: DiscordWebhook = DiscordWebhook(url=webhook_url, content=webhook_message, rate_limit_retry=True)
# Check if the entry is blacklisted, and if it is, we will skip it.
if entry_should_be_skipped(effective_reader, entry):
if entry_should_be_skipped(reader, entry):
logger.info("Entry was blacklisted: %s", entry.id)
continue
# Check if the feed has a whitelist, and if it does, check if the entry is whitelisted.
if has_white_tags(effective_reader, entry.feed) and not should_be_sent(effective_reader, entry):
logger.info("Entry was not whitelisted: %s", entry.id)
if has_white_tags(reader, entry.feed):
if should_be_sent(reader, entry):
execute_webhook(webhook, entry)
return
continue
# Use a custom webhook for Hoyolab feeds.
if is_c3kay_feed(entry.feed.url):
entry_link: str | None = entry.link
if entry_link:
post_id: str | None = extract_post_id_from_hoyolab_url(entry_link)
if post_id:
post_data: dict[str, Any] | None = fetch_hoyolab_post(post_id)
if post_data:
webhook = create_hoyolab_webhook(webhook_url, entry, post_data)
execute_webhook(webhook, entry, reader=effective_reader)
return
logger.warning(
"Failed to create Hoyolab webhook for feed %s, falling back to regular processing",
entry.feed.url,
)
else:
logger.warning("No entry link found for feed %s, falling back to regular processing", entry.feed.url)
# Send the entry to Discord as it is not blacklisted or feed has a whitelist.
execute_webhook(webhook, entry, reader=effective_reader)
execute_webhook(webhook, entry)
# If we only want to send one entry, we will break the loop. This is used when testing this function.
if do_once:
@ -433,27 +276,14 @@ def send_to_discord(reader: Reader | None = None, feed: Feed | None = None, *, d
break
def execute_webhook(webhook: DiscordWebhook, entry: Entry, reader: Reader) -> None:
def execute_webhook(webhook: DiscordWebhook, entry: Entry) -> None:
"""Execute the webhook.
Args:
webhook (DiscordWebhook): The webhook to execute.
entry (Entry): The entry to send to Discord.
reader (Reader): The Reader instance to use for checking feed status.
"""
# If the feed has been paused or deleted, we will not send the entry to Discord.
entry_feed: Feed = entry.feed
if entry_feed.updates_enabled is False:
logger.warning("Feed is paused, not sending entry to Discord: %s", entry_feed.url)
return
try:
reader.get_feed(entry_feed.url)
except FeedNotFoundError:
logger.warning("Feed not found in reader, not sending entry to Discord: %s", entry_feed.url)
return
response: Response = webhook.execute()
if response.status_code not in {200, 204}:
msg: str = f"Error sending entry to Discord: {response.text}\n{pprint.pformat(webhook.json)}"
@ -465,18 +295,6 @@ def execute_webhook(webhook: DiscordWebhook, entry: Entry, reader: Reader) -> No
logger.info("Sent entry to Discord: %s", entry.id)
def is_youtube_feed(feed_url: str) -> bool:
"""Check if the feed is a YouTube feed.
Args:
feed_url: The feed URL to check.
Returns:
bool: True if the feed is a YouTube feed, False otherwise.
"""
return "youtube.com/feeds/videos.xml" in feed_url
def should_send_embed_check(reader: Reader, entry: Entry) -> bool:
"""Check if we should send an embed to Discord.
@ -487,12 +305,11 @@ def should_send_embed_check(reader: Reader, entry: Entry) -> bool:
Returns:
bool: True if we should send an embed, False otherwise.
"""
# YouTube feeds should never use embeds - only links
if is_youtube_feed(entry.feed.url):
return False
try:
should_send_embed = bool(reader.get_tag(entry.feed, "should_send_embed", True))
should_send_embed = bool(reader.get_tag(entry.feed, "should_send_embed"))
except TagNotFoundError:
logger.exception("No should_send_embed tag found for feed: %s", entry.feed.url)
should_send_embed = True
except ReaderError:
logger.exception("Error getting should_send_embed tag for feed: %s", entry.feed.url)
should_send_embed = True
@ -516,7 +333,7 @@ def truncate_webhook_message(webhook_message: str) -> str:
return webhook_message
def create_feed(reader: Reader, feed_url: str, webhook_dropdown: str) -> None: # noqa: C901
def create_feed(reader: Reader, feed_url: str, webhook_dropdown: str) -> None:
"""Add a new feed, update it and mark every entry as read.
Args:
@ -547,7 +364,9 @@ def create_feed(reader: Reader, feed_url: str, webhook_dropdown: str) -> None:
reader.add_feed(clean_feed_url)
except FeedExistsError:
# Add the webhook to an already added feed if it doesn't have a webhook instead of trying to create a new.
if not reader.get_tag(clean_feed_url, "webhook", ""):
try:
reader.get_tag(clean_feed_url, "webhook")
except TagNotFoundError:
reader.set_tag(clean_feed_url, "webhook", webhook_url) # pyright: ignore[reportArgumentType]
except ReaderError as e:
raise HTTPException(status_code=404, detail=f"Error adding feed: {e}") from e
@ -572,8 +391,7 @@ def create_feed(reader: Reader, feed_url: str, webhook_dropdown: str) -> None:
# This is the default message that will be sent to Discord.
reader.set_tag(clean_feed_url, "custom_message", default_custom_message) # pyright: ignore[reportArgumentType]
# Set the default embed tag when creating the feed
reader.set_tag(clean_feed_url, "embed", json.dumps(default_custom_embed))
# Update the full-text search index so our new feed is searchable.
reader.update_search()
add_missing_tags(reader)

94
discord_rss_bot/filter/blacklist.py
View file

@ -2,119 +2,59 @@ from __future__ import annotations
from typing import TYPE_CHECKING
from discord_rss_bot.filter.utils import is_regex_match
from discord_rss_bot.filter.utils import is_word_in_text
if TYPE_CHECKING:
from reader import Entry
from reader import Feed
from reader import Reader
from reader import Entry, Feed, Reader
def feed_has_blacklist_tags(reader: Reader, feed: Feed) -> bool:
def feed_has_blacklist_tags(custom_reader: Reader, feed: Feed) -> bool:
"""Return True if the feed has blacklist tags.
The following tags are checked:
- blacklist_author
- blacklist_content
- blacklist_summary
- blacklist_title
- regex_blacklist_author
- regex_blacklist_content
- regex_blacklist_summary
- regex_blacklist_title
- blacklist_summary
- blacklist_content.
Args:
reader: The reader.
custom_reader: The reader.
feed: The feed to check.
Returns:
bool: If the feed has any of the tags.
"""
blacklist_author: str = str(reader.get_tag(feed, "blacklist_author", "")).strip()
blacklist_content: str = str(reader.get_tag(feed, "blacklist_content", "")).strip()
blacklist_summary: str = str(reader.get_tag(feed, "blacklist_summary", "")).strip()
blacklist_title: str = str(reader.get_tag(feed, "blacklist_title", "")).strip()
blacklist_title: str = str(custom_reader.get_tag(feed, "blacklist_title", ""))
blacklist_summary: str = str(custom_reader.get_tag(feed, "blacklist_summary", ""))
blacklist_content: str = str(custom_reader.get_tag(feed, "blacklist_content", ""))
regex_blacklist_author: str = str(reader.get_tag(feed, "regex_blacklist_author", "")).strip()
regex_blacklist_content: str = str(reader.get_tag(feed, "regex_blacklist_content", "")).strip()
regex_blacklist_summary: str = str(reader.get_tag(feed, "regex_blacklist_summary", "")).strip()
regex_blacklist_title: str = str(reader.get_tag(feed, "regex_blacklist_title", "")).strip()
return bool(
blacklist_title
or blacklist_author
or blacklist_content
or blacklist_summary
or regex_blacklist_author
or regex_blacklist_content
or regex_blacklist_summary
or regex_blacklist_title,
)
return bool(blacklist_title or blacklist_summary or blacklist_content)
def entry_should_be_skipped(reader: Reader, entry: Entry) -> bool: # noqa: PLR0911
def entry_should_be_skipped(custom_reader: Reader, entry: Entry) -> bool:
"""Return True if the entry is in the blacklist.
Args:
reader: The reader.
custom_reader: The reader.
entry: The entry to check.
Returns:
bool: If the entry is in the blacklist.
"""
feed = entry.feed
blacklist_title: str = str(reader.get_tag(feed, "blacklist_title", "")).strip()
blacklist_summary: str = str(reader.get_tag(feed, "blacklist_summary", "")).strip()
blacklist_content: str = str(reader.get_tag(feed, "blacklist_content", "")).strip()
blacklist_author: str = str(reader.get_tag(feed, "blacklist_author", "")).strip()
regex_blacklist_title: str = str(reader.get_tag(feed, "regex_blacklist_title", "")).strip()
regex_blacklist_summary: str = str(reader.get_tag(feed, "regex_blacklist_summary", "")).strip()
regex_blacklist_content: str = str(reader.get_tag(feed, "regex_blacklist_content", "")).strip()
regex_blacklist_author: str = str(reader.get_tag(feed, "regex_blacklist_author", "")).strip()
blacklist_title: str = str(custom_reader.get_tag(entry.feed, "blacklist_title", ""))
blacklist_summary: str = str(custom_reader.get_tag(entry.feed, "blacklist_summary", ""))
blacklist_content: str = str(custom_reader.get_tag(entry.feed, "blacklist_content", ""))
blacklist_author: str = str(custom_reader.get_tag(entry.feed, "blacklist_author", ""))
# TODO(TheLovinator): Also add support for entry_text and more.
# Check regular blacklist
if entry.title and blacklist_title and is_word_in_text(blacklist_title, entry.title):
return True
if entry.summary and blacklist_summary and is_word_in_text(blacklist_summary, entry.summary):
return True
if (
entry.content
and entry.content[0].value
and blacklist_content
and is_word_in_text(blacklist_content, entry.content[0].value)
):
return True
if entry.author and blacklist_author and is_word_in_text(blacklist_author, entry.author):
return True
if (
entry.content
and entry.content[0].value
and blacklist_content
and is_word_in_text(blacklist_content, entry.content[0].value)
):
return True
# Check regex blacklist
if entry.title and regex_blacklist_title and is_regex_match(regex_blacklist_title, entry.title):
return True
if entry.summary and regex_blacklist_summary and is_regex_match(regex_blacklist_summary, entry.summary):
return True
if (
entry.content
and entry.content[0].value
and regex_blacklist_content
and is_regex_match(regex_blacklist_content, entry.content[0].value)
):
return True
if entry.author and regex_blacklist_author and is_regex_match(regex_blacklist_author, entry.author):
return True
return bool(
entry.content
and entry.content[0].value
and regex_blacklist_content
and is_regex_match(regex_blacklist_content, entry.content[0].value),
and blacklist_content
and is_word_in_text(blacklist_content, entry.content[0].value),
)

50
discord_rss_bot/filter/utils.py
View file

@ -1,10 +1,7 @@
from __future__ import annotations
import logging
import re
logger: logging.Logger = logging.getLogger(__name__)
def is_word_in_text(word_string: str, text: str) -> bool:
"""Check if any of the words are in the text.
@ -23,50 +20,3 @@ def is_word_in_text(word_string: str, text: str) -> bool:
# Check if any pattern matches the text.
return any(pattern.search(text) for pattern in patterns)
def is_regex_match(regex_string: str, text: str) -> bool:
"""Check if any of the regex patterns match the text.
Args:
regex_string: A string containing regex patterns, separated by newlines or commas.
text: The text to search in.
Returns:
bool: True if any regex pattern matches the text, otherwise False.
"""
if not regex_string or not text:
return False
# Split by newlines first, then by commas (for backward compatibility)
regex_list: list[str] = []
# First split by newlines
lines: list[str] = regex_string.split("\n")
for line in lines:
stripped_line: str = line.strip()
if stripped_line:
# For backward compatibility, also split by commas if there are any
if "," in stripped_line:
regex_list.extend([part.strip() for part in stripped_line.split(",") if part.strip()])
else:
regex_list.append(stripped_line)
# Attempt to compile and apply each regex pattern
for pattern_str in regex_list:
if not pattern_str:
logger.warning("Empty regex pattern found in the list.")
continue
try:
pattern: re.Pattern[str] = re.compile(pattern_str, re.IGNORECASE)
if pattern.search(text):
logger.info("Regex pattern matched: %s", pattern_str)
return True
except re.error:
logger.warning("Invalid regex pattern: %s", pattern_str)
continue
logger.info("No regex patterns matched.")
return False

80
discord_rss_bot/filter/whitelist.py
View file

@ -2,105 +2,59 @@ from __future__ import annotations
from typing import TYPE_CHECKING
from discord_rss_bot.filter.utils import is_regex_match
from discord_rss_bot.filter.utils import is_word_in_text
if TYPE_CHECKING:
from reader import Entry
from reader import Feed
from reader import Reader
from reader import Entry, Feed, Reader
def has_white_tags(reader: Reader, feed: Feed) -> bool:
def has_white_tags(custom_reader: Reader, feed: Feed) -> bool:
"""Return True if the feed has whitelist tags.
The following tags are checked:
- regex_whitelist_author
- regex_whitelist_content
- regex_whitelist_summary
- regex_whitelist_title
- whitelist_author
- whitelist_content
- whitelist_summary
- whitelist_title
- whitelist_summary
- whitelist_content.
Args:
reader: The reader.
custom_reader: The reader.
feed: The feed to check.
Returns:
bool: If the feed has any of the tags.
"""
whitelist_title: str = str(reader.get_tag(feed, "whitelist_title", "")).strip()
whitelist_summary: str = str(reader.get_tag(feed, "whitelist_summary", "")).strip()
whitelist_content: str = str(reader.get_tag(feed, "whitelist_content", "")).strip()
whitelist_author: str = str(reader.get_tag(feed, "whitelist_author", "")).strip()
whitelist_title: str = str(custom_reader.get_tag(feed, "whitelist_title", ""))
whitelist_summary: str = str(custom_reader.get_tag(feed, "whitelist_summary", ""))
whitelist_content: str = str(custom_reader.get_tag(feed, "whitelist_content", ""))
regex_whitelist_title: str = str(reader.get_tag(feed, "regex_whitelist_title", "")).strip()
regex_whitelist_summary: str = str(reader.get_tag(feed, "regex_whitelist_summary", "")).strip()
regex_whitelist_content: str = str(reader.get_tag(feed, "regex_whitelist_content", "")).strip()
regex_whitelist_author: str = str(reader.get_tag(feed, "regex_whitelist_author", "")).strip()
return bool(
whitelist_title
or whitelist_author
or whitelist_content
or whitelist_summary
or regex_whitelist_author
or regex_whitelist_content
or regex_whitelist_summary
or regex_whitelist_title,
)
return bool(whitelist_title or whitelist_summary or whitelist_content)
def should_be_sent(reader: Reader, entry: Entry) -> bool: # noqa: PLR0911
def should_be_sent(custom_reader: Reader, entry: Entry) -> bool:
"""Return True if the entry is in the whitelist.
Args:
reader: The reader.
custom_reader: The reader.
entry: The entry to check.
Returns:
bool: If the entry is in the whitelist.
"""
feed: Feed = entry.feed
# Regular whitelist tags
whitelist_title: str = str(reader.get_tag(feed, "whitelist_title", "")).strip()
whitelist_summary: str = str(reader.get_tag(feed, "whitelist_summary", "")).strip()
whitelist_content: str = str(reader.get_tag(feed, "whitelist_content", "")).strip()
whitelist_author: str = str(reader.get_tag(feed, "whitelist_author", "")).strip()
whitelist_title: str = str(custom_reader.get_tag(feed, "whitelist_title", ""))
whitelist_summary: str = str(custom_reader.get_tag(feed, "whitelist_summary", ""))
whitelist_content: str = str(custom_reader.get_tag(feed, "whitelist_content", ""))
whitelist_author: str = str(custom_reader.get_tag(feed, "whitelist_author", ""))
# Regex whitelist tags
regex_whitelist_title: str = str(reader.get_tag(feed, "regex_whitelist_title", "")).strip()
regex_whitelist_summary: str = str(reader.get_tag(feed, "regex_whitelist_summary", "")).strip()
regex_whitelist_content: str = str(reader.get_tag(feed, "regex_whitelist_content", "")).strip()
regex_whitelist_author: str = str(reader.get_tag(feed, "regex_whitelist_author", "")).strip()
# Check regular whitelist
if entry.title and whitelist_title and is_word_in_text(whitelist_title, entry.title):
return True
if entry.summary and whitelist_summary and is_word_in_text(whitelist_summary, entry.summary):
return True
if entry.author and whitelist_author and is_word_in_text(whitelist_author, entry.author):
return True
if (
entry.content
and entry.content[0].value
and whitelist_content
and is_word_in_text(whitelist_content, entry.content[0].value)
):
return True
# Check regex whitelist
if entry.title and regex_whitelist_title and is_regex_match(regex_whitelist_title, entry.title):
return True
if entry.summary and regex_whitelist_summary and is_regex_match(regex_whitelist_summary, entry.summary):
return True
if entry.author and regex_whitelist_author and is_regex_match(regex_whitelist_author, entry.author):
return True
return bool(
entry.content
and entry.content[0].value
and regex_whitelist_content
and is_regex_match(regex_whitelist_content, entry.content[0].value),
and whitelist_content
and is_word_in_text(whitelist_content, entry.content[0].value),
)

243
discord_rss_bot/git_backup.py
View file

@ -1,243 +0,0 @@
"""Git backup module for committing bot state changes to a private repository.
Configure the backup by setting these environment variables:
- ``GIT_BACKUP_PATH``: Local filesystem path for the backup git repository.
When set, the bot will initialise a git repo there (if one doesn't exist)
and commit an export of its state after every relevant change.
- ``GIT_BACKUP_REMOTE``: Optional remote URL (e.g. ``git@github.com:you/private-repo.git``).
When set, every commit is followed by a ``git push`` to this remote.
The exported state is written as ``state.json`` inside the backup repo. It
contains the list of feeds together with their webhook URL, filter settings
(blacklist / whitelist, regex variants), custom messages and embed settings.
Global webhooks are also included.
Example docker-compose snippet::
environment:
- GIT_BACKUP_PATH=/data/backup
- GIT_BACKUP_REMOTE=git@github.com:you/private-config.git
"""
from __future__ import annotations
import json
import logging
import os
import shutil
import subprocess # noqa: S404
from pathlib import Path
from typing import TYPE_CHECKING
from typing import Any
if TYPE_CHECKING:
from reader import Reader
logger: logging.Logger = logging.getLogger(__name__)
GIT_EXECUTABLE: str = shutil.which("git") or "git"
type TAG_VALUE = (
dict[str, str | int | float | bool | dict[str, Any] | list[Any] | None]
| list[str | int | float | bool | dict[str, Any] | list[Any] | None]
| None
)
# Tags that are exported per-feed (empty values are omitted).
_FEED_TAGS: tuple[str, ...] = (
"webhook",
"custom_message",
"should_send_embed",
"embed",
"blacklist_title",
"blacklist_summary",
"blacklist_content",
"blacklist_author",
"regex_blacklist_title",
"regex_blacklist_summary",
"regex_blacklist_content",
"regex_blacklist_author",
"whitelist_title",
"whitelist_summary",
"whitelist_content",
"whitelist_author",
"regex_whitelist_title",
"regex_whitelist_summary",
"regex_whitelist_content",
"regex_whitelist_author",
".reader.update",
)
def get_backup_path() -> Path | None:
"""Return the configured backup path, or *None* if not configured.
Returns:
Path to the backup repository, or None if ``GIT_BACKUP_PATH`` is unset.
"""
raw: str = os.environ.get("GIT_BACKUP_PATH", "").strip()
return Path(raw) if raw else None
def get_backup_remote() -> str:
"""Return the configured remote URL, or an empty string if not set.
Returns:
The remote URL string from ``GIT_BACKUP_REMOTE``, or ``""`` if unset.
"""
return os.environ.get("GIT_BACKUP_REMOTE", "").strip()
def setup_backup_repo(backup_path: Path) -> bool:
"""Ensure the backup directory exists and contains a git repository.
If the directory does not yet contain a ``.git`` folder a new repository is
initialised. A basic git identity is configured locally so that commits
succeed even in environments where a global ``~/.gitconfig`` is absent.
Args:
backup_path: Local path for the backup repository.
Returns:
``True`` if the repository is ready, ``False`` on any error.
"""
try:
backup_path.mkdir(parents=True, exist_ok=True)
git_dir: Path = backup_path / ".git"
if not git_dir.exists():
subprocess.run([GIT_EXECUTABLE, "init", str(backup_path)], check=True, capture_output=True) # noqa: S603
logger.info("Initialised git backup repository at %s", backup_path)
# Ensure a local identity exists so that `git commit` always works.
for key, value in (("user.email", "discord-rss-bot@localhost"), ("user.name", "discord-rss-bot")):
result: subprocess.CompletedProcess[bytes] = subprocess.run( # noqa: S603
[GIT_EXECUTABLE, "-C", str(backup_path), "config", "--local", key],
check=False,
capture_output=True,
)
if result.returncode != 0:
subprocess.run( # noqa: S603
[GIT_EXECUTABLE, "-C", str(backup_path), "config", "--local", key, value],
check=True,
capture_output=True,
)
# Configure the remote if GIT_BACKUP_REMOTE is set.
remote_url: str = get_backup_remote()
if remote_url:
# Check if remote "origin" already exists.
check_remote: subprocess.CompletedProcess[bytes] = subprocess.run( # noqa: S603
[GIT_EXECUTABLE, "-C", str(backup_path), "remote", "get-url", "origin"],
check=False,
capture_output=True,
)
if check_remote.returncode != 0:
# Remote doesn't exist, add it.
subprocess.run( # noqa: S603
[GIT_EXECUTABLE, "-C", str(backup_path), "remote", "add", "origin", remote_url],
check=True,
capture_output=True,
)
logger.info("Added remote 'origin' with URL: %s", remote_url)
else:
# Remote exists, update it if the URL has changed.
current_url: str = check_remote.stdout.decode().strip()
if current_url != remote_url:
subprocess.run( # noqa: S603
[GIT_EXECUTABLE, "-C", str(backup_path), "remote", "set-url", "origin", remote_url],
check=True,
capture_output=True,
)
logger.info("Updated remote 'origin' URL from %s to %s", current_url, remote_url)
except Exception:
logger.exception("Failed to set up git backup repository at %s", backup_path)
return False
return True
def export_state(reader: Reader, backup_path: Path) -> None:
"""Serialise the current bot state to ``state.json`` inside *backup_path*.
Args:
reader: The :class:`reader.Reader` instance to read state from.
backup_path: Destination directory for the exported ``state.json``.
"""
feeds_state: list[dict] = []
for feed in reader.get_feeds():
feed_data: dict = {"url": feed.url}
for tag in _FEED_TAGS:
try:
value: TAG_VALUE = reader.get_tag(feed, tag, None)
if value is not None and value != "": # noqa: PLC1901
feed_data[tag] = value
except Exception:
logger.exception("Failed to read tag '%s' for feed '%s' during state export", tag, feed.url)
feeds_state.append(feed_data)
webhooks: list[str | int | float | bool | dict[str, Any] | list[Any] | None] = list(
reader.get_tag((), "webhooks", []),
)
# Export global update interval if set
global_update_interval: dict[str, Any] | None = None
global_update_config = reader.get_tag((), ".reader.update", None)
if isinstance(global_update_config, dict):
global_update_interval = global_update_config
state: dict = {"feeds": feeds_state, "webhooks": webhooks}
if global_update_interval is not None:
state["global_update_interval"] = global_update_interval
state_file: Path = backup_path / "state.json"
state_file.write_text(json.dumps(state, indent=2, default=str), encoding="utf-8")
def commit_state_change(reader: Reader, message: str) -> None:
"""Export current state and commit it to the backup repository.
This is a no-op when ``GIT_BACKUP_PATH`` is not configured. Errors are
logged but never raised so that a backup failure never interrupts normal
bot operation.
Args:
reader: The :class:`reader.Reader` instance to read state from.
message: Commit message describing the change (e.g. ``"Add feed example.com/rss.xml"``).
"""
backup_path: Path | None = get_backup_path()
if backup_path is None:
return
if not setup_backup_repo(backup_path):
return
try:
export_state(reader, backup_path)
subprocess.run([GIT_EXECUTABLE, "-C", str(backup_path), "add", "-A"], check=True, capture_output=True) # noqa: S603
# Only create a commit if there are staged changes.
diff_result: subprocess.CompletedProcess[bytes] = subprocess.run( # noqa: S603
[GIT_EXECUTABLE, "-C", str(backup_path), "diff", "--cached", "--exit-code"],
check=False,
capture_output=True,
)
if diff_result.returncode == 0:
logger.debug("No state changes to commit for: %s", message)
return
subprocess.run( # noqa: S603
[GIT_EXECUTABLE, "-C", str(backup_path), "commit", "-m", message],
check=True,
capture_output=True,
)
logger.info("Committed state change to backup repo: %s", message)
# Push to remote if configured.
if get_backup_remote():
subprocess.run( # noqa: S603
[GIT_EXECUTABLE, "-C", str(backup_path), "push", "origin", "HEAD"],
check=True,
capture_output=True,
)
logger.info("Pushed state change to remote 'origin': %s", message)
except Exception:
logger.exception("Failed to commit state change '%s' to backup repo", message)

195
discord_rss_bot/hoyolab_api.py
View file

@ -1,195 +0,0 @@
from __future__ import annotations
import contextlib
import json
import logging
import re
from typing import TYPE_CHECKING
from typing import Any
import requests
from discord_webhook import DiscordEmbed
from discord_webhook import DiscordWebhook
if TYPE_CHECKING:
from reader import Entry
logger: logging.Logger = logging.getLogger(__name__)
def is_c3kay_feed(feed_url: str) -> bool:
"""Check if the feed is from c3kay.de.
Args:
feed_url: The feed URL to check.
Returns:
bool: True if the feed is from c3kay.de, False otherwise.
"""
return "feeds.c3kay.de" in feed_url
def extract_post_id_from_hoyolab_url(url: str) -> str | None:
"""Extract the post ID from a Hoyolab URL.
Args:
url: The Hoyolab URL to extract the post ID from.
For example: https://www.hoyolab.com/article/38588239
Returns:
str | None: The post ID if found, None otherwise.
"""
try:
match: re.Match[str] | None = re.search(r"/article/(\d+)", url)
if match:
return match.group(1)
except (ValueError, AttributeError, TypeError) as e:
logger.warning("Error extracting post ID from Hoyolab URL %s: %s", url, e)
return None
def fetch_hoyolab_post(post_id: str) -> dict[str, Any] | None:
"""Fetch post data from the Hoyolab API.
Args:
post_id: The post ID to fetch.
Returns:
dict[str, Any] | None: The post data if successful, None otherwise.
"""
if not post_id:
return None
http_ok = 200
try:
url: str = f"https://bbs-api-os.hoyolab.com/community/post/wapi/getPostFull?post_id={post_id}"
response: requests.Response = requests.get(url, timeout=10)
if response.status_code == http_ok:
data: dict[str, Any] = response.json()
if data.get("retcode") == 0 and "data" in data and "post" in data["data"]:
return data["data"]["post"]
logger.warning("Failed to fetch Hoyolab post %s: %s", post_id, response.text)
except (requests.RequestException, ValueError):
logger.exception("Error fetching Hoyolab post %s", post_id)
return None
def create_hoyolab_webhook(webhook_url: str, entry: Entry, post_data: dict[str, Any]) -> DiscordWebhook: # noqa: C901, PLR0912, PLR0914, PLR0915
"""Create a webhook with data from the Hoyolab API.
Args:
webhook_url: The webhook URL.
entry: The entry to send to Discord.
post_data: The post data from the Hoyolab API.
Returns:
DiscordWebhook: The webhook with the embed.
"""
entry_link: str = entry.link or entry.feed.url
webhook = DiscordWebhook(url=webhook_url, rate_limit_retry=True)
# Extract relevant data from the post
post: dict[str, Any] = post_data.get("post", {})
subject: str = post.get("subject", "")
content: str = post.get("content", "{}")
logger.debug("Post subject: %s", subject)
logger.debug("Post content: %s", content)
content_data: dict[str, str] = {}
with contextlib.suppress(json.JSONDecodeError, ValueError):
content_data = json.loads(content)
logger.debug("Content data: %s", content_data)
description: str = content_data.get("describe", "")
if not description:
description = post.get("desc", "")
# Create the embed
discord_embed = DiscordEmbed()
# Set title and description
discord_embed.set_title(subject)
discord_embed.set_url(entry_link)
# Get post.image_list
image_list: list[dict[str, Any]] = post_data.get("image_list", [])
if image_list:
image_url: str = str(image_list[0].get("url", ""))
image_height: int = int(image_list[0].get("height", 1080))
image_width: int = int(image_list[0].get("width", 1920))
logger.debug("Image URL: %s, Height: %s, Width: %s", image_url, image_height, image_width)
discord_embed.set_image(url=image_url, height=image_height, width=image_width)
video: dict[str, str | int | bool] = post_data.get("video", {})
if video and video.get("url"):
video_url: str = str(video.get("url", ""))
logger.debug("Video URL: %s", video_url)
with contextlib.suppress(requests.RequestException):
video_response: requests.Response = requests.get(video_url, stream=True, timeout=10)
if video_response.ok:
webhook.add_file(
file=video_response.content,
filename=f"{entry.id}.mp4",
)
game = post_data.get("game", {})
if game and game.get("color"):
game_color = str(game.get("color", ""))
discord_embed.set_color(game_color.removeprefix("#"))
user: dict[str, str | int | bool] = post_data.get("user", {})
author_name: str = str(user.get("nickname", ""))
avatar_url: str = str(user.get("avatar_url", ""))
if author_name:
webhook.avatar_url = avatar_url
webhook.username = author_name
classification = post_data.get("classification", {})
if classification and classification.get("name"):
footer = str(classification.get("name", ""))
discord_embed.set_footer(text=footer)
webhook.add_embed(discord_embed)
# Only show Youtube URL if available
structured_content: str = post.get("structured_content", "")
if structured_content: # noqa: PLR1702
try:
structured_content_data: list[dict[str, Any]] = json.loads(structured_content)
for item in structured_content_data:
if item.get("insert") and isinstance(item["insert"], dict):
video_url: str = str(item["insert"].get("video", ""))
if video_url:
video_id_match: re.Match[str] | None = re.search(r"embed/([a-zA-Z0-9_-]+)", video_url)
if video_id_match:
video_id: str = video_id_match.group(1)
logger.debug("Video ID: %s", video_id)
webhook.content = f"https://www.youtube.com/watch?v={video_id}"
webhook.remove_embeds()
except (json.JSONDecodeError, ValueError) as e:
logger.warning("Error parsing structured content: %s", e)
event_start_date: str = post.get("event_start_date", "")
if event_start_date and event_start_date != "0":
discord_embed.add_embed_field(name="Start", value=f"<t:{event_start_date}:R>")
event_end_date: str = post.get("event_end_date", "")
if event_end_date and event_end_date != "0":
discord_embed.add_embed_field(name="End", value=f"<t:{event_end_date}:R>")
created_at: str = post.get("created_at", "")
if created_at and created_at != "0":
discord_embed.set_timestamp(timestamp=created_at)
return webhook

3
discord_rss_bot/is_url_valid.py
View file

@ -1,7 +1,6 @@
from __future__ import annotations
from urllib.parse import ParseResult
from urllib.parse import urlparse
from urllib.parse import ParseResult, urlparse
def is_url_valid(url: str) -> bool:

1328
discord_rss_bot/main.py
View file

@ -7,65 +7,48 @@ import typing
import urllib.parse
from contextlib import asynccontextmanager
from dataclasses import dataclass
from datetime import UTC
from datetime import datetime
from datetime import UTC, datetime
from functools import lru_cache
from typing import TYPE_CHECKING
from typing import Annotated
from typing import Any
from typing import cast
from typing import TYPE_CHECKING, Annotated, cast
import httpx
import sentry_sdk
import uvicorn
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from fastapi import Depends
from fastapi import FastAPI
from fastapi import Form
from fastapi import HTTPException
from fastapi import Request
from fastapi import FastAPI, Form, HTTPException, Request
from fastapi.responses import HTMLResponse
from fastapi.staticfiles import StaticFiles
from fastapi.templating import Jinja2Templates
from httpx import Response
from markdownify import markdownify
from reader import Entry
from reader import EntryNotFoundError
from reader import Feed
from reader import FeedExistsError
from reader import FeedNotFoundError
from reader import Reader
from reader import ReaderError
from reader import TagNotFoundError
from reader import Entry, EntryNotFoundError, Feed, FeedNotFoundError, Reader, TagNotFoundError
from starlette.responses import RedirectResponse
from discord_rss_bot import settings
from discord_rss_bot.custom_filters import entry_is_blacklisted
from discord_rss_bot.custom_filters import entry_is_whitelisted
from discord_rss_bot.custom_message import CustomEmbed
from discord_rss_bot.custom_message import get_custom_message
from discord_rss_bot.custom_message import get_embed
from discord_rss_bot.custom_message import get_first_image
from discord_rss_bot.custom_message import replace_tags_in_text_message
from discord_rss_bot.custom_message import save_embed
from discord_rss_bot.feeds import create_feed
from discord_rss_bot.feeds import extract_domain
from discord_rss_bot.feeds import send_entry_to_discord
from discord_rss_bot.feeds import send_to_discord
from discord_rss_bot.git_backup import commit_state_change
from discord_rss_bot.git_backup import get_backup_path
from discord_rss_bot.is_url_valid import is_url_valid
from discord_rss_bot.search import create_search_context
from discord_rss_bot.custom_filters import (
entry_is_blacklisted,
entry_is_whitelisted,
)
from discord_rss_bot.custom_message import (
CustomEmbed,
get_custom_message,
get_embed,
get_first_image,
replace_tags_in_text_message,
save_embed,
)
from discord_rss_bot.feeds import create_feed, send_entry_to_discord, send_to_discord
from discord_rss_bot.missing_tags import add_missing_tags
from discord_rss_bot.search import create_html_for_search_results
from discord_rss_bot.settings import get_reader
if TYPE_CHECKING:
from collections.abc import AsyncGenerator
from collections.abc import Iterable
from reader.types import JSONType
LOGGING_CONFIG: dict[str, Any] = {
LOGGING_CONFIG = {
"version": 1,
"disable_existing_loggers": True,
"formatters": {
@ -101,71 +84,18 @@ LOGGING_CONFIG: dict[str, Any] = {
logging.config.dictConfig(LOGGING_CONFIG)
logger: logging.Logger = logging.getLogger(__name__)
def get_reader_dependency() -> Reader:
"""Provide the app Reader instance as a FastAPI dependency.
Returns:
Reader: The shared Reader instance.
"""
return get_reader()
# Time constants for relative time formatting
SECONDS_PER_MINUTE = 60
SECONDS_PER_HOUR = 3600
SECONDS_PER_DAY = 86400
def relative_time(dt: datetime | None) -> str:
"""Convert a datetime to a relative time string (e.g., '2 hours ago', 'in 5 minutes').
Args:
dt: The datetime to convert (should be timezone-aware).
Returns:
A human-readable relative time string.
"""
if dt is None:
return "Never"
now = datetime.now(tz=UTC)
diff = dt - now
seconds = int(abs(diff.total_seconds()))
is_future = diff.total_seconds() > 0
# Determine the appropriate unit and value
if seconds < SECONDS_PER_MINUTE:
value = seconds
unit = "s"
elif seconds < SECONDS_PER_HOUR:
value = seconds // SECONDS_PER_MINUTE
unit = "m"
elif seconds < SECONDS_PER_DAY:
value = seconds // SECONDS_PER_HOUR
unit = "h"
else:
value = seconds // SECONDS_PER_DAY
unit = "d"
# Format based on future or past
return f"in {value}{unit}" if is_future else f"{value}{unit} ago"
reader: Reader = get_reader()
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncGenerator[None]:
"""Lifespan function for the FastAPI app."""
reader: Reader = get_reader()
scheduler: AsyncIOScheduler = AsyncIOScheduler(timezone=UTC)
scheduler.add_job(
func=send_to_discord,
trigger="interval",
minutes=1,
id="send_to_discord",
max_instances=1,
next_run_time=datetime.now(tz=UTC),
)
async def lifespan(app: FastAPI) -> typing.AsyncGenerator[None]:
"""This is needed for the ASGI server to run."""
add_missing_tags(reader)
scheduler: AsyncIOScheduler = AsyncIOScheduler()
# Update all feeds every 15 minutes.
# TODO(TheLovinator): Make this configurable.
scheduler.add_job(send_to_discord, "interval", minutes=15, next_run_time=datetime.now(tz=UTC))
scheduler.start()
logger.info("Scheduler started.")
yield
@ -180,29 +110,27 @@ templates: Jinja2Templates = Jinja2Templates(directory="discord_rss_bot/template
# Add the filters to the Jinja2 environment so they can be used in html templates.
templates.env.filters["encode_url"] = lambda url: urllib.parse.quote(url) if url else ""
templates.env.filters["entry_is_whitelisted"] = entry_is_whitelisted
templates.env.filters["entry_is_blacklisted"] = entry_is_blacklisted
templates.env.filters["discord_markdown"] = markdownify
templates.env.filters["relative_time"] = relative_time
templates.env.globals["get_backup_path"] = get_backup_path
@app.post("/add_webhook")
async def post_add_webhook(
webhook_name: Annotated[str, Form()],
webhook_url: Annotated[str, Form()],
reader: Annotated[Reader, Depends(get_reader_dependency)],
) -> RedirectResponse:
"""Add a feed to the database.
Args:
webhook_name: The name of the webhook.
webhook_url: The url of the webhook.
reader: The Reader instance.
Returns:
RedirectResponse: Redirect to the index page.
Raises:
HTTPException: If the webhook already exists.
Returns:
RedirectResponse: Redirect to the index page.
"""
# Get current webhooks from the database if they exist otherwise use an empty list.
webhooks = list(reader.get_tag((), "webhooks", []))
@ -219,8 +147,6 @@ async def post_add_webhook(
reader.set_tag((), "webhooks", webhooks) # pyright: ignore[reportArgumentType]
commit_state_change(reader, f"Add webhook {webhook_name.strip()}")
return RedirectResponse(url="/", status_code=303)
# TODO(TheLovinator): Show this error on the page.
@ -229,22 +155,17 @@ async def post_add_webhook(
@app.post("/delete_webhook")
async def post_delete_webhook(
webhook_url: Annotated[str, Form()],
reader: Annotated[Reader, Depends(get_reader_dependency)],
) -> RedirectResponse:
async def post_delete_webhook(webhook_url: Annotated[str, Form()]) -> RedirectResponse:
"""Delete a webhook from the database.
Args:
webhook_url: The url of the webhook.
reader: The Reader instance.
Returns:
RedirectResponse: Redirect to the index page.
Raises:
HTTPException: If the webhook could not be deleted
Returns:
RedirectResponse: Redirect to the index page.
"""
# TODO(TheLovinator): Check if the webhook is in use by any feeds before deleting it.
# TODO(TheLovinator): Replace HTTPException with a custom exception for both of these.
@ -271,8 +192,6 @@ async def post_delete_webhook(
# Add our new list of webhooks to the database.
reader.set_tag((), "webhooks", webhooks) # pyright: ignore[reportArgumentType]
commit_state_change(reader, f"Delete webhook {webhook_url.strip()}")
return RedirectResponse(url="/", status_code=303)
@ -280,34 +199,27 @@ async def post_delete_webhook(
async def post_create_feed(
feed_url: Annotated[str, Form()],
webhook_dropdown: Annotated[str, Form()],
reader: Annotated[Reader, Depends(get_reader_dependency)],
) -> RedirectResponse:
"""Add a feed to the database.
Args:
feed_url: The feed to add.
webhook_dropdown: The webhook to use.
reader: The Reader instance.
Returns:
RedirectResponse: Redirect to the feed page.
"""
clean_feed_url: str = feed_url.strip()
create_feed(reader, feed_url, webhook_dropdown)
commit_state_change(reader, f"Add feed {clean_feed_url}")
return RedirectResponse(url=f"/feed?feed_url={urllib.parse.quote(clean_feed_url)}", status_code=303)
@app.post("/pause")
async def post_pause_feed(
feed_url: Annotated[str, Form()],
reader: Annotated[Reader, Depends(get_reader_dependency)],
) -> RedirectResponse:
async def post_pause_feed(feed_url: Annotated[str, Form()]) -> RedirectResponse:
"""Pause a feed.
Args:
feed_url: The feed to pause.
reader: The Reader instance.
Returns:
RedirectResponse: Redirect to the feed page.
@ -318,15 +230,11 @@ async def post_pause_feed(
@app.post("/unpause")
async def post_unpause_feed(
feed_url: Annotated[str, Form()],
reader: Annotated[Reader, Depends(get_reader_dependency)],
) -> RedirectResponse:
async def post_unpause_feed(feed_url: Annotated[str, Form()]) -> RedirectResponse:
"""Unpause a feed.
Args:
feed_url: The Feed to unpause.
reader: The Reader instance.
Returns:
RedirectResponse: Redirect to the feed page.
@ -338,15 +246,10 @@ async def post_unpause_feed(
@app.post("/whitelist")
async def post_set_whitelist(
reader: Annotated[Reader, Depends(get_reader_dependency)],
whitelist_title: Annotated[str, Form()] = "",
whitelist_summary: Annotated[str, Form()] = "",
whitelist_content: Annotated[str, Form()] = "",
whitelist_author: Annotated[str, Form()] = "",
regex_whitelist_title: Annotated[str, Form()] = "",
regex_whitelist_summary: Annotated[str, Form()] = "",
regex_whitelist_content: Annotated[str, Form()] = "",
regex_whitelist_author: Annotated[str, Form()] = "",
feed_url: Annotated[str, Form()] = "",
) -> RedirectResponse:
"""Set what the whitelist should be sent, if you have this set only words in the whitelist will be sent.
@ -356,12 +259,7 @@ async def post_set_whitelist(
whitelist_summary: Whitelisted words for when checking the summary.
whitelist_content: Whitelisted words for when checking the content.
whitelist_author: Whitelisted words for when checking the author.
regex_whitelist_title: Whitelisted regex for when checking the title.
regex_whitelist_summary: Whitelisted regex for when checking the summary.
regex_whitelist_content: Whitelisted regex for when checking the content.
regex_whitelist_author: Whitelisted regex for when checking the author.
feed_url: The feed we should set the whitelist for.
reader: The Reader instance.
Returns:
RedirectResponse: Redirect to the feed page.
@ -371,28 +269,17 @@ async def post_set_whitelist(
reader.set_tag(clean_feed_url, "whitelist_summary", whitelist_summary) # pyright: ignore[reportArgumentType][call-overload]
reader.set_tag(clean_feed_url, "whitelist_content", whitelist_content) # pyright: ignore[reportArgumentType][call-overload]
reader.set_tag(clean_feed_url, "whitelist_author", whitelist_author) # pyright: ignore[reportArgumentType][call-overload]
reader.set_tag(clean_feed_url, "regex_whitelist_title", regex_whitelist_title) # pyright: ignore[reportArgumentType][call-overload]
reader.set_tag(clean_feed_url, "regex_whitelist_summary", regex_whitelist_summary) # pyright: ignore[reportArgumentType][call-overload]
reader.set_tag(clean_feed_url, "regex_whitelist_content", regex_whitelist_content) # pyright: ignore[reportArgumentType][call-overload]
reader.set_tag(clean_feed_url, "regex_whitelist_author", regex_whitelist_author) # pyright: ignore[reportArgumentType][call-overload]
commit_state_change(reader, f"Update whitelist for {clean_feed_url}")
return RedirectResponse(url=f"/feed?feed_url={urllib.parse.quote(clean_feed_url)}", status_code=303)
@app.get("/whitelist", response_class=HTMLResponse)
async def get_whitelist(
feed_url: str,
request: Request,
reader: Annotated[Reader, Depends(get_reader_dependency)],
):
async def get_whitelist(feed_url: str, request: Request):
"""Get the whitelist.
Args:
feed_url: What feed we should get the whitelist for.
request: The request object.
reader: The Reader instance.
Returns:
HTMLResponse: The whitelist page.
@ -400,14 +287,11 @@ async def get_whitelist(
clean_feed_url: str = feed_url.strip()
feed: Feed = reader.get_feed(urllib.parse.unquote(clean_feed_url))
# Get previous data, this is used when creating the form.
whitelist_title: str = str(reader.get_tag(feed, "whitelist_title", ""))
whitelist_summary: str = str(reader.get_tag(feed, "whitelist_summary", ""))
whitelist_content: str = str(reader.get_tag(feed, "whitelist_content", ""))
whitelist_author: str = str(reader.get_tag(feed, "whitelist_author", ""))
regex_whitelist_title: str = str(reader.get_tag(feed, "regex_whitelist_title", ""))
regex_whitelist_summary: str = str(reader.get_tag(feed, "regex_whitelist_summary", ""))
regex_whitelist_content: str = str(reader.get_tag(feed, "regex_whitelist_content", ""))
regex_whitelist_author: str = str(reader.get_tag(feed, "regex_whitelist_author", ""))
context = {
"request": request,
@ -416,25 +300,16 @@ async def get_whitelist(
"whitelist_summary": whitelist_summary,
"whitelist_content": whitelist_content,
"whitelist_author": whitelist_author,
"regex_whitelist_title": regex_whitelist_title,
"regex_whitelist_summary": regex_whitelist_summary,
"regex_whitelist_content": regex_whitelist_content,
"regex_whitelist_author": regex_whitelist_author,
}
return templates.TemplateResponse(request=request, name="whitelist.html", context=context)
@app.post("/blacklist")
async def post_set_blacklist(
reader: Annotated[Reader, Depends(get_reader_dependency)],
blacklist_title: Annotated[str, Form()] = "",
blacklist_summary: Annotated[str, Form()] = "",
blacklist_content: Annotated[str, Form()] = "",
blacklist_author: Annotated[str, Form()] = "",
regex_blacklist_title: Annotated[str, Form()] = "",
regex_blacklist_summary: Annotated[str, Form()] = "",
regex_blacklist_content: Annotated[str, Form()] = "",
regex_blacklist_author: Annotated[str, Form()] = "",
feed_url: Annotated[str, Form()] = "",
) -> RedirectResponse:
"""Set the blacklist.
@ -447,12 +322,7 @@ async def post_set_blacklist(
blacklist_summary: Blacklisted words for when checking the summary.
blacklist_content: Blacklisted words for when checking the content.
blacklist_author: Blacklisted words for when checking the author.
regex_blacklist_title: Blacklisted regex for when checking the title.
regex_blacklist_summary: Blacklisted regex for when checking the summary.
regex_blacklist_content: Blacklisted regex for when checking the content.
regex_blacklist_author: Blacklisted regex for when checking the author.
feed_url: What feed we should set the blacklist for.
reader: The Reader instance.
Returns:
RedirectResponse: Redirect to the feed page.
@ -462,40 +332,28 @@ async def post_set_blacklist(
reader.set_tag(clean_feed_url, "blacklist_summary", blacklist_summary) # pyright: ignore[reportArgumentType][call-overload]
reader.set_tag(clean_feed_url, "blacklist_content", blacklist_content) # pyright: ignore[reportArgumentType][call-overload]
reader.set_tag(clean_feed_url, "blacklist_author", blacklist_author) # pyright: ignore[reportArgumentType][call-overload]
reader.set_tag(clean_feed_url, "regex_blacklist_title", regex_blacklist_title) # pyright: ignore[reportArgumentType][call-overload]
reader.set_tag(clean_feed_url, "regex_blacklist_summary", regex_blacklist_summary) # pyright: ignore[reportArgumentType][call-overload]
reader.set_tag(clean_feed_url, "regex_blacklist_content", regex_blacklist_content) # pyright: ignore[reportArgumentType][call-overload]
reader.set_tag(clean_feed_url, "regex_blacklist_author", regex_blacklist_author) # pyright: ignore[reportArgumentType][call-overload]
commit_state_change(reader, f"Update blacklist for {clean_feed_url}")
return RedirectResponse(url=f"/feed?feed_url={urllib.parse.quote(clean_feed_url)}", status_code=303)
@app.get("/blacklist", response_class=HTMLResponse)
async def get_blacklist(
feed_url: str,
request: Request,
reader: Annotated[Reader, Depends(get_reader_dependency)],
):
async def get_blacklist(feed_url: str, request: Request):
"""Get the blacklist.
Args:
feed_url: What feed we should get the blacklist for.
request: The request object.
reader: The Reader instance.
Returns:
HTMLResponse: The blacklist page.
"""
feed: Feed = reader.get_feed(urllib.parse.unquote(feed_url))
# Get previous data, this is used when creating the form.
blacklist_title: str = str(reader.get_tag(feed, "blacklist_title", ""))
blacklist_summary: str = str(reader.get_tag(feed, "blacklist_summary", ""))
blacklist_content: str = str(reader.get_tag(feed, "blacklist_content", ""))
blacklist_author: str = str(reader.get_tag(feed, "blacklist_author", ""))
regex_blacklist_title: str = str(reader.get_tag(feed, "regex_blacklist_title", ""))
regex_blacklist_summary: str = str(reader.get_tag(feed, "regex_blacklist_summary", ""))
regex_blacklist_content: str = str(reader.get_tag(feed, "regex_blacklist_content", ""))
regex_blacklist_author: str = str(reader.get_tag(feed, "regex_blacklist_author", ""))
context = {
"request": request,
@ -504,10 +362,6 @@ async def get_blacklist(
"blacklist_summary": blacklist_summary,
"blacklist_content": blacklist_content,
"blacklist_author": blacklist_author,
"regex_blacklist_title": regex_blacklist_title,
"regex_blacklist_summary": regex_blacklist_summary,
"regex_blacklist_content": regex_blacklist_content,
"regex_blacklist_author": regex_blacklist_author,
}
return templates.TemplateResponse(request=request, name="blacklist.html", context=context)
@ -515,7 +369,6 @@ async def get_blacklist(
@app.post("/custom")
async def post_set_custom(
feed_url: Annotated[str, Form()],
reader: Annotated[Reader, Depends(get_reader_dependency)],
custom_message: Annotated[str, Form()] = "",
) -> RedirectResponse:
"""Set the custom message, this is used when sending the message.
@ -523,7 +376,6 @@ async def post_set_custom(
Args:
custom_message: The custom message.
feed_url: The feed we should set the custom message for.
reader: The Reader instance.
Returns:
RedirectResponse: Redirect to the feed page.
@ -540,22 +392,16 @@ async def post_set_custom(
reader.set_tag(feed_url, "custom_message", default_custom_message)
clean_feed_url: str = feed_url.strip()
commit_state_change(reader, f"Update custom message for {clean_feed_url}")
return RedirectResponse(url=f"/feed?feed_url={urllib.parse.quote(clean_feed_url)}", status_code=303)
@app.get("/custom", response_class=HTMLResponse)
async def get_custom(
feed_url: str,
request: Request,
reader: Annotated[Reader, Depends(get_reader_dependency)],
):
async def get_custom(feed_url: str, request: Request):
"""Get the custom message. This is used when sending the message to Discord.
Args:
feed_url: What feed we should get the custom message for.
request: The request object.
reader: The Reader instance.
Returns:
HTMLResponse: The custom message page.
@ -576,17 +422,12 @@ async def get_custom(
@app.get("/embed", response_class=HTMLResponse)
async def get_embed_page(
feed_url: str,
request: Request,
reader: Annotated[Reader, Depends(get_reader_dependency)],
):
async def get_embed_page(feed_url: str, request: Request):
"""Get the custom message. This is used when sending the message to Discord.
Args:
feed_url: What feed we should get the custom message for.
request: The request object.
reader: The Reader instance.
Returns:
HTMLResponse: The embed page.
@ -620,9 +461,8 @@ async def get_embed_page(
@app.post("/embed", response_class=HTMLResponse)
async def post_embed( # noqa: C901
async def post_embed( # noqa: PLR0913, PLR0917
feed_url: Annotated[str, Form()],
reader: Annotated[Reader, Depends(get_reader_dependency)],
title: Annotated[str, Form()] = "",
description: Annotated[str, Form()] = "",
color: Annotated[str, Form()] = "",
@ -648,7 +488,7 @@ async def post_embed( # noqa: C901
author_icon_url: The author icon url of the embed.
footer_text: The footer text of the embed.
footer_icon_url: The footer icon url of the embed.
reader: The Reader instance.
Returns:
RedirectResponse: Redirect to the embed page.
@ -657,245 +497,59 @@ async def post_embed( # noqa: C901
feed: Feed = reader.get_feed(urllib.parse.unquote(clean_feed_url))
custom_embed: CustomEmbed = get_embed(reader, feed)
# Only overwrite fields that the user provided. This prevents accidental
# clearing of previously saved embed data when the form submits empty
# values for fields the user did not change.
if title:
custom_embed.title = title
if description:
custom_embed.description = description
if color:
custom_embed.color = color
if image_url:
custom_embed.image_url = image_url
if thumbnail_url:
custom_embed.thumbnail_url = thumbnail_url
if author_name:
custom_embed.author_name = author_name
if author_url:
custom_embed.author_url = author_url
if author_icon_url:
custom_embed.author_icon_url = author_icon_url
if footer_text:
custom_embed.footer_text = footer_text
if footer_icon_url:
custom_embed.footer_icon_url = footer_icon_url
# Save the data.
save_embed(reader, feed, custom_embed)
commit_state_change(reader, f"Update embed settings for {clean_feed_url}")
return RedirectResponse(url=f"/feed?feed_url={urllib.parse.quote(clean_feed_url)}", status_code=303)
@app.post("/use_embed")
async def post_use_embed(
feed_url: Annotated[str, Form()],
reader: Annotated[Reader, Depends(get_reader_dependency)],
) -> RedirectResponse:
async def post_use_embed(feed_url: Annotated[str, Form()]) -> RedirectResponse:
"""Use embed instead of text.
Args:
feed_url: The feed to change.
reader: The Reader instance.
Returns:
RedirectResponse: Redirect to the feed page.
"""
clean_feed_url: str = feed_url.strip()
reader.set_tag(clean_feed_url, "should_send_embed", True) # pyright: ignore[reportArgumentType]
commit_state_change(reader, f"Enable embed mode for {clean_feed_url}")
return RedirectResponse(url=f"/feed?feed_url={urllib.parse.quote(clean_feed_url)}", status_code=303)
@app.post("/use_text")
async def post_use_text(
feed_url: Annotated[str, Form()],
reader: Annotated[Reader, Depends(get_reader_dependency)],
) -> RedirectResponse:
async def post_use_text(feed_url: Annotated[str, Form()]) -> RedirectResponse:
"""Use text instead of embed.
Args:
feed_url: The feed to change.
reader: The Reader instance.
Returns:
RedirectResponse: Redirect to the feed page.
"""
clean_feed_url: str = feed_url.strip()
reader.set_tag(clean_feed_url, "should_send_embed", False) # pyright: ignore[reportArgumentType]
commit_state_change(reader, f"Disable embed mode for {clean_feed_url}")
return RedirectResponse(url=f"/feed?feed_url={urllib.parse.quote(clean_feed_url)}", status_code=303)
@app.post("/set_update_interval")
async def post_set_update_interval(
feed_url: Annotated[str, Form()],
reader: Annotated[Reader, Depends(get_reader_dependency)],
interval_minutes: Annotated[int | None, Form()] = None,
redirect_to: Annotated[str, Form()] = "",
) -> RedirectResponse:
"""Set the update interval for a feed.
Args:
feed_url: The feed to change.
interval_minutes: The update interval in minutes (None to reset to global default).
redirect_to: Optional redirect URL (defaults to feed page).
reader: The Reader instance.
Returns:
RedirectResponse: Redirect to the specified page or feed page.
"""
clean_feed_url: str = feed_url.strip()
# If no interval specified, reset to global default
if interval_minutes is None:
try:
reader.delete_tag(clean_feed_url, ".reader.update")
commit_state_change(reader, f"Reset update interval to default for {clean_feed_url}")
except TagNotFoundError:
pass
else:
# Validate interval (minimum 1 minute, no maximum)
interval_minutes = max(interval_minutes, 1)
reader.set_tag(clean_feed_url, ".reader.update", {"interval": interval_minutes}) # pyright: ignore[reportArgumentType]
commit_state_change(reader, f"Set update interval to {interval_minutes} minutes for {clean_feed_url}")
# Update the feed immediately to recalculate update_after with the new interval
try:
reader.update_feed(clean_feed_url)
logger.info("Updated feed after interval change: %s", clean_feed_url)
except Exception:
logger.exception("Failed to update feed after interval change: %s", clean_feed_url)
if redirect_to:
return RedirectResponse(url=redirect_to, status_code=303)
return RedirectResponse(url=f"/feed?feed_url={urllib.parse.quote(clean_feed_url)}", status_code=303)
@app.post("/change_feed_url")
async def post_change_feed_url(
old_feed_url: Annotated[str, Form()],
new_feed_url: Annotated[str, Form()],
reader: Annotated[Reader, Depends(get_reader_dependency)],
) -> RedirectResponse:
"""Change the URL for an existing feed.
Args:
old_feed_url: Current feed URL.
new_feed_url: New feed URL to change to.
reader: The Reader instance.
Returns:
RedirectResponse: Redirect to the feed page for the resulting URL.
Raises:
HTTPException: If the old feed is not found, the new URL already exists, or change fails.
"""
clean_old_feed_url: str = old_feed_url.strip()
clean_new_feed_url: str = new_feed_url.strip()
if not clean_old_feed_url or not clean_new_feed_url:
raise HTTPException(status_code=400, detail="Feed URLs cannot be empty")
if clean_old_feed_url == clean_new_feed_url:
return RedirectResponse(url=f"/feed?feed_url={urllib.parse.quote(clean_old_feed_url)}", status_code=303)
try:
reader.change_feed_url(clean_old_feed_url, clean_new_feed_url)
except FeedNotFoundError as e:
raise HTTPException(status_code=404, detail=f"Feed not found: {clean_old_feed_url}") from e
except FeedExistsError as e:
raise HTTPException(status_code=409, detail=f"Feed already exists: {clean_new_feed_url}") from e
except ReaderError as e:
raise HTTPException(status_code=400, detail=f"Failed to change feed URL: {e}") from e
# Update the feed with the new URL so we can discover what entries it returns.
# Then mark all unread entries as read so the scheduler doesn't resend them.
try:
reader.update_feed(clean_new_feed_url)
except Exception:
logger.exception("Failed to update feed after URL change: %s", clean_new_feed_url)
for entry in reader.get_entries(feed=clean_new_feed_url, read=False):
try:
reader.set_entry_read(entry, True)
except Exception:
logger.exception("Failed to mark entry as read after URL change: %s", entry.id)
commit_state_change(reader, f"Change feed URL from {clean_old_feed_url} to {clean_new_feed_url}")
return RedirectResponse(url=f"/feed?feed_url={urllib.parse.quote(clean_new_feed_url)}", status_code=303)
@app.post("/reset_update_interval")
async def post_reset_update_interval(
feed_url: Annotated[str, Form()],
reader: Annotated[Reader, Depends(get_reader_dependency)],
redirect_to: Annotated[str, Form()] = "",
) -> RedirectResponse:
"""Reset the update interval for a feed to use the global default.
Args:
feed_url: The feed to change.
redirect_to: Optional redirect URL (defaults to feed page).
reader: The Reader instance.
Returns:
RedirectResponse: Redirect to the specified page or feed page.
"""
clean_feed_url: str = feed_url.strip()
try:
reader.delete_tag(clean_feed_url, ".reader.update")
commit_state_change(reader, f"Reset update interval to default for {clean_feed_url}")
except TagNotFoundError:
# Tag doesn't exist, which is fine
pass
# Update the feed immediately to recalculate update_after with the new interval
try:
reader.update_feed(clean_feed_url)
logger.info("Updated feed after interval reset: %s", clean_feed_url)
except Exception:
logger.exception("Failed to update feed after interval reset: %s", clean_feed_url)
if redirect_to:
return RedirectResponse(url=redirect_to, status_code=303)
return RedirectResponse(url=f"/feed?feed_url={urllib.parse.quote(clean_feed_url)}", status_code=303)
@app.post("/set_global_update_interval")
async def post_set_global_update_interval(
interval_minutes: Annotated[int, Form()],
reader: Annotated[Reader, Depends(get_reader_dependency)],
) -> RedirectResponse:
"""Set the global default update interval.
Args:
interval_minutes: The update interval in minutes.
reader: The Reader instance.
Returns:
RedirectResponse: Redirect to the settings page.
"""
# Validate interval (minimum 1 minute, no maximum)
interval_minutes = max(interval_minutes, 1)
reader.set_tag((), ".reader.update", {"interval": interval_minutes}) # pyright: ignore[reportArgumentType]
commit_state_change(reader, f"Set global update interval to {interval_minutes} minutes")
return RedirectResponse(url="/settings", status_code=303)
@app.get("/add", response_class=HTMLResponse)
def get_add(
request: Request,
reader: Annotated[Reader, Depends(get_reader_dependency)],
):
def get_add(request: Request):
"""Page for adding a new feed.
Args:
request: The request object.
reader: The Reader instance.
Returns:
HTMLResponse: The add feed page.
@ -908,25 +562,19 @@ def get_add(
@app.get("/feed", response_class=HTMLResponse)
async def get_feed( # noqa: C901, PLR0912, PLR0914, PLR0915
feed_url: str,
request: Request,
reader: Annotated[Reader, Depends(get_reader_dependency)],
starting_after: str = "",
):
async def get_feed(feed_url: str, request: Request, starting_after: str = ""):
"""Get a feed by URL.
Args:
feed_url: The feed to add.
request: The request object.
starting_after: The entry to start after. Used for pagination.
reader: The Reader instance.
Returns:
HTMLResponse: The feed page.
Raises:
HTTPException: If the feed is not found.
Returns:
HTMLResponse: The feed page.
"""
entries_per_page: int = 20
@ -939,7 +587,7 @@ async def get_feed( # noqa: C901, PLR0912, PLR0914, PLR0915
# Only show button if more than 10 entries.
total_entries: int = reader.get_entry_counts(feed=feed).total or 0
is_show_more_entries_button_visible: bool = total_entries > entries_per_page
show_more_entires_button: bool = total_entries > entries_per_page
# Get entries from the feed.
if starting_after:
@ -950,22 +598,7 @@ async def get_feed( # noqa: C901, PLR0912, PLR0914, PLR0915
except EntryNotFoundError as e:
current_entries = list(reader.get_entries(feed=clean_feed_url))
msg: str = f"{e}\n\n{[entry.id for entry in current_entries]}"
html: str = create_html_for_feed(reader=reader, entries=current_entries, current_feed_url=clean_feed_url)
# Get feed and global intervals for error case too
feed_interval: int | None = None
feed_update_config = reader.get_tag(feed, ".reader.update", None)
if isinstance(feed_update_config, dict) and "interval" in feed_update_config:
interval_value = feed_update_config["interval"]
if isinstance(interval_value, int):
feed_interval = interval_value
global_interval: int = 60
global_update_config = reader.get_tag((), ".reader.update", None)
if isinstance(global_update_config, dict) and "interval" in global_update_config:
interval_value = global_update_config["interval"]
if isinstance(interval_value, int):
global_interval = interval_value
html: str = create_html_for_feed(current_entries)
context = {
"request": request,
@ -976,10 +609,8 @@ async def get_feed( # noqa: C901, PLR0912, PLR0914, PLR0915
"should_send_embed": False,
"last_entry": None,
"messages": msg,
"is_show_more_entries_button_visible": is_show_more_entries_button_visible,
"show_more_entires_button": show_more_entires_button,
"total_entries": total_entries,
"feed_interval": feed_interval,
"global_interval": global_interval,
}
return templates.TemplateResponse(request=request, name="feed.html", context=context)
@ -1000,25 +631,13 @@ async def get_feed( # noqa: C901, PLR0912, PLR0914, PLR0915
last_entry = entries[-1]
# Create the html for the entries.
html: str = create_html_for_feed(reader=reader, entries=entries, current_feed_url=clean_feed_url)
html: str = create_html_for_feed(entries)
should_send_embed: bool = bool(reader.get_tag(feed, "should_send_embed", True))
# Get the update interval for this feed
feed_interval: int | None = None
feed_update_config = reader.get_tag(feed, ".reader.update", None)
if isinstance(feed_update_config, dict) and "interval" in feed_update_config:
interval_value = feed_update_config["interval"]
if isinstance(interval_value, int):
feed_interval = interval_value
# Get the global default update interval
global_interval: int = 60 # Default to 60 minutes if not set
global_update_config = reader.get_tag((), ".reader.update", None)
if isinstance(global_update_config, dict) and "interval" in global_update_config:
interval_value = global_update_config["interval"]
if isinstance(interval_value, int):
global_interval = interval_value
try:
should_send_embed: bool = bool(reader.get_tag(feed, "should_send_embed"))
except TagNotFoundError:
add_missing_tags(reader)
should_send_embed: bool = bool(reader.get_tag(feed, "should_send_embed"))
context = {
"request": request,
@ -1028,25 +647,17 @@ async def get_feed( # noqa: C901, PLR0912, PLR0914, PLR0915
"html": html,
"should_send_embed": should_send_embed,
"last_entry": last_entry,
"is_show_more_entries_button_visible": is_show_more_entries_button_visible,
"show_more_entires_button": show_more_entires_button,
"total_entries": total_entries,
"feed_interval": feed_interval,
"global_interval": global_interval,
}
return templates.TemplateResponse(request=request, name="feed.html", context=context)
def create_html_for_feed( # noqa: C901, PLR0914
reader: Reader,
entries: Iterable[Entry],
current_feed_url: str = "",
) -> str:
def create_html_for_feed(entries: Iterable[Entry]) -> str:
"""Create HTML for the search results.
Args:
reader: The Reader instance to use.
entries: The entries to create HTML for.
current_feed_url: The feed URL currently being viewed in /feed.
Returns:
str: The HTML for the search results.
@ -1062,75 +673,31 @@ def create_html_for_feed( # noqa: C901, PLR0914
first_image = get_first_image(summary, content)
text: str = replace_tags_in_text_message(entry, reader=reader) or (
"<div class='text-muted'>No content available.</div>"
)
text: str = replace_tags_in_text_message(entry) or "<div class='text-muted'>No content available.</div>"
published = ""
if entry.published:
published: str = entry.published.strftime("%Y-%m-%d %H:%M:%S")
blacklisted: str = ""
if entry_is_blacklisted(entry, reader=reader):
if entry_is_blacklisted(entry):
blacklisted = "<span class='badge bg-danger'>Blacklisted</span>"
whitelisted: str = ""
if entry_is_whitelisted(entry, reader=reader):
if entry_is_whitelisted(entry):
whitelisted = "<span class='badge bg-success'>Whitelisted</span>"
source_feed_url: str = getattr(entry, "original_feed_url", None) or entry.feed.url
from_another_feed: str = ""
if current_feed_url and source_feed_url != current_feed_url:
from_another_feed = f"<span class='badge bg-warning text-dark'>From another feed: {source_feed_url}</span>"
# Add feed link when viewing from webhook_entries or aggregated views
feed_link: str = ""
if not current_feed_url or source_feed_url != current_feed_url:
encoded_feed_url: str = urllib.parse.quote(source_feed_url)
feed_title: str = entry.feed.title if hasattr(entry.feed, "title") and entry.feed.title else source_feed_url
feed_link = (
f"<a class='text-muted' style='font-size: 0.85em;' "
f"href='/feed?feed_url={encoded_feed_url}'>{feed_title}</a><br>"
)
entry_id: str = urllib.parse.quote(entry.id)
encoded_source_feed_url: str = urllib.parse.quote(source_feed_url)
to_discord_html: str = (
f"<a class='text-muted' href='/post_entry?entry_id={entry_id}&feed_url={encoded_source_feed_url}'>"
"Send to Discord</a>"
)
# Check if this is a YouTube feed entry and the entry has a link
is_youtube_feed = "youtube.com/feeds/videos.xml" in entry.feed.url
video_embed_html = ""
if is_youtube_feed and entry.link:
# Extract the video ID and create an embed if possible
video_id: str | None = extract_youtube_video_id(entry.link)
if video_id:
video_embed_html: str = f"""
<div class="ratio ratio-16x9 mt-3 mb-3">
<iframe src="https://www.youtube.com/embed/{video_id}"
title="{entry.title}"
allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture"
allowfullscreen>
</iframe>
</div>
"""
# Don't use the first image if we have a video embed
first_image = ""
to_discord_html: str = f"<a class='text-muted' href='/post_entry?entry_id={entry_id}'>Send to Discord</a>"
image_html: str = f"<img src='{first_image}' class='img-fluid'>" if first_image else ""
html += f"""<div class="p-2 mb-2 border border-dark">
{blacklisted}{whitelisted}{from_another_feed}<a class="text-muted text-decoration-none" href="{entry.link}"><h2>{entry.title}</h2></a>
{feed_link}{f"By {entry.author} @" if entry.author else ""}{published} - {to_discord_html}
{blacklisted}{whitelisted}<a class="text-muted text-decoration-none" href="{entry.link}"><h2>{entry.title}</h2></a>
{f"By {entry.author} @" if entry.author else ""}{published} - {to_discord_html}
{text}
{video_embed_html}
{image_html}
</div>
""" # noqa: E501
"""
return html.strip()
@ -1169,7 +736,6 @@ def get_data_from_hook_url(hook_name: str, hook_url: str) -> WebhookInfo:
hook_name (str): The webhook name.
hook_url (str): The webhook URL.
Returns:
WebhookInfo: The webhook username, avatar, guild id, etc.
"""
@ -1190,64 +756,12 @@ def get_data_from_hook_url(hook_name: str, hook_url: str) -> WebhookInfo:
return our_hook
@app.get("/settings", response_class=HTMLResponse)
async def get_settings(
request: Request,
reader: Annotated[Reader, Depends(get_reader_dependency)],
):
"""Settings page.
Args:
request: The request object.
reader: The Reader instance.
Returns:
HTMLResponse: The settings page.
"""
# Get the global default update interval
global_interval: int = 60 # Default to 60 minutes if not set
global_update_config = reader.get_tag((), ".reader.update", None)
if isinstance(global_update_config, dict) and "interval" in global_update_config:
interval_value = global_update_config["interval"]
if isinstance(interval_value, int):
global_interval = interval_value
# Get all feeds with their intervals
feeds: Iterable[Feed] = reader.get_feeds()
feed_intervals = []
for feed in feeds:
feed_interval: int | None = None
feed_update_config = reader.get_tag(feed, ".reader.update", None)
if isinstance(feed_update_config, dict) and "interval" in feed_update_config:
interval_value = feed_update_config["interval"]
if isinstance(interval_value, int):
feed_interval = interval_value
feed_intervals.append({
"feed": feed,
"interval": feed_interval,
"effective_interval": feed_interval or global_interval,
"domain": extract_domain(feed.url),
})
context = {
"request": request,
"global_interval": global_interval,
"feed_intervals": feed_intervals,
}
return templates.TemplateResponse(request=request, name="settings.html", context=context)
@app.get("/webhooks", response_class=HTMLResponse)
async def get_webhooks(
request: Request,
reader: Annotated[Reader, Depends(get_reader_dependency)],
):
async def get_webhooks(request: Request):
"""Page for adding a new webhook.
Args:
request: The request object.
reader: The Reader instance.
Returns:
HTMLResponse: The add webhook page.
@ -1268,241 +782,136 @@ async def get_webhooks(
@app.get("/", response_class=HTMLResponse)
def get_index(
request: Request,
reader: Annotated[Reader, Depends(get_reader_dependency)],
message: str = "",
):
def get_index(request: Request):
"""This is the root of the website.
Args:
request: The request object.
message: Optional message to display to the user.
reader: The Reader instance.
Returns:
HTMLResponse: The index page.
"""
return templates.TemplateResponse(
request=request,
name="index.html",
context=make_context_index(request, message, reader),
)
return templates.TemplateResponse(request=request, name="index.html", context=make_context_index(request))
def make_context_index(request: Request, message: str = "", reader: Reader | None = None):
def make_context_index(request: Request):
"""Create the needed context for the index page.
Args:
request: The request object.
message: Optional message to display to the user.
reader: The Reader instance.
Returns:
dict: The context for the index page.
"""
effective_reader: Reader = reader or get_reader_dependency()
hooks: list[dict[str, str]] = cast("list[dict[str, str]]", list(effective_reader.get_tag((), "webhooks", [])))
hooks: list[dict[str, str]] = cast("list[dict[str, str]]", list(reader.get_tag((), "webhooks", [])))
feed_list: list[dict[str, JSONType | Feed | str]] = []
broken_feeds: list[Feed] = []
feeds_without_attached_webhook: list[Feed] = []
feed_list = []
broken_feeds = []
feeds_without_attached_webhook = []
# Get all feeds and organize them
feeds: Iterable[Feed] = effective_reader.get_feeds()
feeds: Iterable[Feed] = reader.get_feeds()
for feed in feeds:
webhook: str = str(effective_reader.get_tag(feed.url, "webhook", ""))
if not webhook:
try:
webhook = reader.get_tag(feed.url, "webhook")
feed_list.append({"feed": feed, "webhook": webhook})
except TagNotFoundError:
broken_feeds.append(feed)
continue
feed_list.append({"feed": feed, "webhook": webhook, "domain": extract_domain(feed.url)})
webhook_list: list[str] = [hook["url"] for hook in hooks]
webhook_list = [hook["url"] for hook in hooks]
if webhook not in webhook_list:
feeds_without_attached_webhook.append(feed)
return {
"request": request,
"feeds": feed_list,
"feed_count": effective_reader.get_feed_counts(),
"entry_count": effective_reader.get_entry_counts(),
"feed_count": reader.get_feed_counts(),
"entry_count": reader.get_entry_counts(),
"webhooks": hooks,
"broken_feeds": broken_feeds,
"feeds_without_attached_webhook": feeds_without_attached_webhook,
"messages": message or None,
}
@app.post("/remove", response_class=HTMLResponse)
async def remove_feed(
feed_url: Annotated[str, Form()],
reader: Annotated[Reader, Depends(get_reader_dependency)],
):
async def remove_feed(feed_url: Annotated[str, Form()]):
"""Get a feed by URL.
Args:
feed_url: The feed to add.
reader: The Reader instance.
Returns:
RedirectResponse: Redirect to the index page.
Raises:
HTTPException: Feed not found
Returns:
RedirectResponse: Redirect to the index page.
"""
try:
reader.delete_feed(urllib.parse.unquote(feed_url))
except FeedNotFoundError as e:
raise HTTPException(status_code=404, detail="Feed not found") from e
commit_state_change(reader, f"Remove feed {urllib.parse.unquote(feed_url)}")
return RedirectResponse(url="/", status_code=303)
@app.get("/update", response_class=HTMLResponse)
async def update_feed(
request: Request,
feed_url: str,
reader: Annotated[Reader, Depends(get_reader_dependency)],
):
"""Update a feed.
Args:
request: The request object.
feed_url: The feed URL to update.
reader: The Reader instance.
Returns:
RedirectResponse: Redirect to the feed page.
Raises:
HTTPException: If the feed is not found.
"""
try:
reader.update_feed(urllib.parse.unquote(feed_url))
except FeedNotFoundError as e:
raise HTTPException(status_code=404, detail="Feed not found") from e
logger.info("Manually updated feed: %s", feed_url)
return RedirectResponse(url="/feed?feed_url=" + urllib.parse.quote(feed_url), status_code=303)
@app.post("/backup")
async def manual_backup(
request: Request,
reader: Annotated[Reader, Depends(get_reader_dependency)],
) -> RedirectResponse:
"""Manually trigger a git backup of the current state.
Args:
request: The request object.
reader: The Reader instance.
Returns:
RedirectResponse: Redirect to the index page with a success or error message.
"""
backup_path = get_backup_path()
if backup_path is None:
message = "Git backup is not configured. Set GIT_BACKUP_PATH environment variable to enable backups."
logger.warning("Manual git backup attempted but GIT_BACKUP_PATH is not configured")
return RedirectResponse(url=f"/?message={urllib.parse.quote(message)}", status_code=303)
try:
commit_state_change(reader, "Manual backup triggered from web UI")
message = "Successfully created git backup!"
logger.info("Manual git backup completed successfully")
except Exception as e:
message = f"Failed to create git backup: {e}"
logger.exception("Manual git backup failed")
return RedirectResponse(url=f"/?message={urllib.parse.quote(message)}", status_code=303)
@app.get("/search", response_class=HTMLResponse)
async def search(
request: Request,
query: str,
reader: Annotated[Reader, Depends(get_reader_dependency)],
):
async def search(request: Request, query: str):
"""Get entries matching a full-text search query.
Args:
query: The query to search for.
request: The request object.
reader: The Reader instance.
Returns:
HTMLResponse: The search page.
"""
reader.update_search()
context = create_search_context(query, reader=reader)
return templates.TemplateResponse(request=request, name="search.html", context={"request": request, **context})
context = {
"request": request,
"search_html": create_html_for_search_results(query),
"query": query,
"search_amount": reader.search_entry_counts(query),
}
return templates.TemplateResponse(request=request, name="search.html", context=context)
@app.get("/post_entry", response_class=HTMLResponse)
async def post_entry(
entry_id: str,
reader: Annotated[Reader, Depends(get_reader_dependency)],
feed_url: str = "",
):
async def post_entry(entry_id: str):
"""Send single entry to Discord.
Args:
entry_id: The entry to send.
feed_url: Optional feed URL used to disambiguate entries with identical IDs.
reader: The Reader instance.
Returns:
RedirectResponse: Redirect to the feed page.
"""
unquoted_entry_id: str = urllib.parse.unquote(entry_id)
clean_feed_url: str = urllib.parse.unquote(feed_url.strip()) if feed_url else ""
# Prefer feed-scoped lookup when feed_url is provided. This avoids ambiguity when
# multiple feeds contain entries with the same ID.
entry: Entry | None = None
if clean_feed_url:
entry = next(
(entry for entry in reader.get_entries(feed=clean_feed_url) if entry.id == unquoted_entry_id),
None,
)
else:
entry = next((entry for entry in reader.get_entries() if entry.id == unquoted_entry_id), None)
entry: Entry | None = next((entry for entry in reader.get_entries() if entry.id == unquoted_entry_id), None)
if entry is None:
return HTMLResponse(status_code=404, content=f"Entry '{entry_id}' not found.")
if result := send_entry_to_discord(entry=entry, reader=reader):
if result := send_entry_to_discord(entry=entry):
return result
# Redirect to the feed page.
redirect_feed_url: str = entry.feed.url.strip()
return RedirectResponse(url=f"/feed?feed_url={urllib.parse.quote(redirect_feed_url)}", status_code=303)
clean_feed_url: str = entry.feed.url.strip()
return RedirectResponse(url=f"/feed?feed_url={urllib.parse.quote(clean_feed_url)}", status_code=303)
@app.post("/modify_webhook", response_class=HTMLResponse)
def modify_webhook(
old_hook: Annotated[str, Form()],
new_hook: Annotated[str, Form()],
reader: Annotated[Reader, Depends(get_reader_dependency)],
redirect_to: Annotated[str, Form()] = "",
):
def modify_webhook(old_hook: Annotated[str, Form()], new_hook: Annotated[str, Form()]):
"""Modify a webhook.
Args:
old_hook: The webhook to modify.
new_hook: The new webhook.
redirect_to: Optional redirect URL after the update.
reader: The Reader instance.
Returns:
RedirectResponse: Redirect to the webhook page.
Raises:
HTTPException: Webhook could not be modified.
Returns:
RedirectResponse: Redirect to the webhook page.
"""
# Get current webhooks from the database if they exist otherwise use an empty list.
webhooks = list(reader.get_tag((), "webhooks", []))
@ -1510,20 +919,15 @@ def modify_webhook(
# Webhooks are stored as a list of dictionaries.
# Example: [{"name": "webhook_name", "url": "webhook_url"}]
webhooks = cast("list[dict[str, str]]", webhooks)
old_hook_clean: str = old_hook.strip()
new_hook_clean: str = new_hook.strip()
webhook_modified: bool = False
for hook in webhooks:
if hook["url"] in old_hook_clean:
hook["url"] = new_hook_clean
if hook["url"] in old_hook.strip():
hook["url"] = new_hook.strip()
# Check if it has been modified.
if hook["url"] != new_hook_clean:
if hook["url"] != new_hook.strip():
raise HTTPException(status_code=500, detail="Webhook could not be modified")
webhook_modified = True
# Add our new list of webhooks to the database.
reader.set_tag((), "webhooks", webhooks) # pyright: ignore[reportArgumentType]
@ -1531,506 +935,16 @@ def modify_webhook(
# matches the old one.
feeds: Iterable[Feed] = reader.get_feeds()
for feed in feeds:
webhook: str = str(reader.get_tag(feed, "webhook", ""))
if webhook == old_hook_clean:
reader.set_tag(feed.url, "webhook", new_hook_clean) # pyright: ignore[reportArgumentType]
if webhook_modified and old_hook_clean != new_hook_clean:
commit_state_change(reader, f"Modify webhook URL from {old_hook_clean} to {new_hook_clean}")
redirect_url: str = redirect_to.strip() or "/webhooks"
if redirect_to:
redirect_url = redirect_url.replace(urllib.parse.quote(old_hook_clean), urllib.parse.quote(new_hook_clean))
redirect_url = redirect_url.replace(old_hook_clean, new_hook_clean)
# Redirect to the requested page.
return RedirectResponse(url=redirect_url, status_code=303)
def extract_youtube_video_id(url: str) -> str | None:
"""Extract YouTube video ID from a YouTube video URL.
Args:
url: The YouTube video URL.
Returns:
The video ID if found, None otherwise.
"""
if not url:
return None
# Handle standard YouTube URLs (youtube.com/watch?v=VIDEO_ID)
if "youtube.com/watch" in url and "v=" in url:
return url.split("v=")[1].split("&", maxsplit=1)[0]
# Handle shortened YouTube URLs (youtu.be/VIDEO_ID)
if "youtu.be/" in url:
return url.split("youtu.be/")[1].split("?", maxsplit=1)[0]
return None
def resolve_final_feed_url(url: str) -> tuple[str, str | None]:
"""Resolve a feed URL by following redirects.
Args:
url: The feed URL to resolve.
Returns:
tuple[str, str | None]: A tuple with (resolved_url, error_message).
error_message is None when resolution succeeded.
"""
clean_url: str = url.strip()
if not clean_url:
return "", "URL is empty"
if not is_url_valid(clean_url):
return clean_url, "URL is invalid"
try:
response: Response = httpx.get(clean_url, follow_redirects=True, timeout=10.0)
except httpx.HTTPError as e:
return clean_url, str(e)
if not response.is_success:
return clean_url, f"HTTP {response.status_code}"
return str(response.url), None
def create_webhook_feed_url_preview(
webhook_feeds: list[Feed],
replace_from: str,
replace_to: str,
resolve_urls: bool, # noqa: FBT001
force_update: bool = False, # noqa: FBT001, FBT002
existing_feed_urls: set[str] | None = None,
) -> list[dict[str, str | bool | None]]:
"""Create preview rows for bulk feed URL replacement.
Args:
webhook_feeds: Feeds attached to a webhook.
replace_from: Text to replace in each URL.
replace_to: Replacement text.
resolve_urls: Whether to resolve resulting URLs via HTTP redirects.
force_update: Whether conflicts should be marked as force-overwritable.
existing_feed_urls: Optional set of all tracked feed URLs used for conflict detection.
Returns:
list[dict[str, str | bool | None]]: Rows used in the preview table.
"""
known_feed_urls: set[str] = existing_feed_urls or {feed.url for feed in webhook_feeds}
preview_rows: list[dict[str, str | bool | None]] = []
for feed in webhook_feeds:
old_url: str = feed.url
has_match: bool = bool(replace_from and replace_from in old_url)
candidate_url: str = old_url
if has_match:
candidate_url = old_url.replace(replace_from, replace_to)
resolved_url: str = candidate_url
resolution_error: str | None = None
if has_match and candidate_url != old_url and resolve_urls:
resolved_url, resolution_error = resolve_final_feed_url(candidate_url)
will_force_ignore_errors: bool = bool(
force_update and bool(resolution_error) and has_match and old_url != candidate_url,
)
target_exists: bool = bool(
has_match and not resolution_error and resolved_url != old_url and resolved_url in known_feed_urls,
)
will_force_overwrite: bool = bool(target_exists and force_update)
will_change: bool = bool(
has_match
and old_url != (candidate_url if will_force_ignore_errors else resolved_url)
and (not target_exists or will_force_overwrite)
and (not resolution_error or will_force_ignore_errors),
)
preview_rows.append({
"old_url": old_url,
"candidate_url": candidate_url,
"resolved_url": resolved_url,
"has_match": has_match,
"will_change": will_change,
"target_exists": target_exists,
"will_force_overwrite": will_force_overwrite,
"will_force_ignore_errors": will_force_ignore_errors,
"resolution_error": resolution_error,
})
return preview_rows
def build_webhook_mass_update_context(
webhook_feeds: list[Feed],
all_feeds: list[Feed],
replace_from: str,
replace_to: str,
resolve_urls: bool, # noqa: FBT001
force_update: bool = False, # noqa: FBT001, FBT002
) -> dict[str, str | bool | int | list[dict[str, str | bool | None]] | dict[str, int]]:
"""Build context data used by the webhook mass URL update preview UI.
Args:
webhook_feeds: Feeds attached to the selected webhook.
all_feeds: All tracked feeds.
replace_from: Text to replace in URLs.
replace_to: Replacement text.
resolve_urls: Whether to resolve resulting URLs.
force_update: Whether to allow overwriting existing target URLs.
Returns:
dict[str, ...]: Context values for rendering preview controls and table.
"""
clean_replace_from: str = replace_from.strip()
clean_replace_to: str = replace_to.strip()
preview_rows: list[dict[str, str | bool | None]] = []
if clean_replace_from:
preview_rows = create_webhook_feed_url_preview(
webhook_feeds=webhook_feeds,
replace_from=clean_replace_from,
replace_to=clean_replace_to,
resolve_urls=resolve_urls,
force_update=force_update,
existing_feed_urls={feed.url for feed in all_feeds},
)
preview_summary: dict[str, int] = {
"total": len(preview_rows),
"matched": sum(1 for row in preview_rows if row["has_match"]),
"will_update": sum(1 for row in preview_rows if row["will_change"]),
"conflicts": sum(1 for row in preview_rows if row["target_exists"] and not row["will_force_overwrite"]),
"force_overwrite": sum(1 for row in preview_rows if row["will_force_overwrite"]),
"force_ignore_errors": sum(1 for row in preview_rows if row["will_force_ignore_errors"]),
"resolve_errors": sum(1 for row in preview_rows if row["resolution_error"]),
}
preview_summary["no_match"] = preview_summary["total"] - preview_summary["matched"]
preview_summary["no_change"] = sum(
1 for row in preview_rows if row["has_match"] and not row["resolution_error"] and not row["will_change"]
)
return {
"replace_from": clean_replace_from,
"replace_to": clean_replace_to,
"resolve_urls": resolve_urls,
"force_update": force_update,
"preview_rows": preview_rows,
"preview_summary": preview_summary,
"preview_change_count": preview_summary["will_update"],
}
@app.get("/webhook_entries_mass_update_preview", response_class=HTMLResponse)
async def get_webhook_entries_mass_update_preview(
webhook_url: str,
request: Request,
reader: Annotated[Reader, Depends(get_reader_dependency)],
replace_from: str = "",
replace_to: str = "",
resolve_urls: bool = True, # noqa: FBT001, FBT002
force_update: bool = False, # noqa: FBT001, FBT002
) -> HTMLResponse:
"""Render the mass-update preview fragment for a webhook using HTMX.
Args:
webhook_url: Webhook URL whose feeds are being updated.
request: The request object.
reader: The Reader instance.
replace_from: Text to find in URLs.
replace_to: Replacement text.
resolve_urls: Whether to resolve resulting URLs.
force_update: Whether to allow overwriting existing target URLs.
Returns:
HTMLResponse: Rendered partial template containing summary + preview table.
"""
clean_webhook_url: str = urllib.parse.unquote(webhook_url.strip())
all_feeds: list[Feed] = list(reader.get_feeds())
webhook_feeds: list[Feed] = [
feed for feed in all_feeds if str(reader.get_tag(feed.url, "webhook", "")) == clean_webhook_url
]
context = {
"request": request,
"webhook_url": clean_webhook_url,
**build_webhook_mass_update_context(
webhook_feeds=webhook_feeds,
all_feeds=all_feeds,
replace_from=replace_from,
replace_to=replace_to,
resolve_urls=resolve_urls,
force_update=force_update,
),
}
return templates.TemplateResponse(request=request, name="_webhook_mass_update_preview.html", context=context)
@app.get("/webhook_entries", response_class=HTMLResponse)
async def get_webhook_entries( # noqa: C901, PLR0914
webhook_url: str,
request: Request,
reader: Annotated[Reader, Depends(get_reader_dependency)],
starting_after: str = "",
replace_from: str = "",
replace_to: str = "",
resolve_urls: bool = True, # noqa: FBT001, FBT002
force_update: bool = False, # noqa: FBT001, FBT002
message: str = "",
) -> HTMLResponse:
"""Get all latest entries from all feeds for a specific webhook.
Args:
webhook_url: The webhook URL to get entries for.
request: The request object.
starting_after: The entry to start after. Used for pagination.
replace_from: Optional URL substring to find for bulk URL replacement preview.
replace_to: Optional replacement substring used in bulk URL replacement preview.
resolve_urls: Whether to resolve replaced URLs by following redirects.
force_update: Whether to allow overwriting existing target URLs during apply.
message: Optional status message shown in the UI.
reader: The Reader instance.
Returns:
HTMLResponse: The webhook entries page.
Raises:
HTTPException: If no feeds are found for this webhook or webhook doesn't exist.
"""
entries_per_page: int = 20
clean_webhook_url: str = urllib.parse.unquote(webhook_url.strip())
# Get the webhook name from the webhooks list
webhooks: list[dict[str, str]] = cast("list[dict[str, str]]", list(reader.get_tag((), "webhooks", [])))
webhook_name: str = ""
for hook in webhooks:
if hook["url"] == clean_webhook_url:
webhook_name = hook["name"]
break
if not webhook_name:
raise HTTPException(status_code=404, detail=f"Webhook not found: {clean_webhook_url}")
hook_info: WebhookInfo = get_data_from_hook_url(hook_name=webhook_name, hook_url=clean_webhook_url)
# Get all feeds associated with this webhook
all_feeds: list[Feed] = list(reader.get_feeds())
webhook_feeds: list[Feed] = []
for feed in all_feeds:
feed_webhook: str = str(reader.get_tag(feed.url, "webhook", ""))
if feed_webhook == clean_webhook_url:
webhook_feeds.append(feed)
# Get all entries from all feeds for this webhook, sorted by published date
all_entries: list[Entry] = [entry for feed in webhook_feeds for entry in reader.get_entries(feed=feed)]
# Sort entries by published date (newest first), with undated entries last.
all_entries.sort(
key=lambda e: (
e.published is not None,
e.published or datetime.min.replace(tzinfo=UTC),
),
reverse=True,
)
# Handle pagination
if starting_after:
try:
start_after_entry: Entry | None = reader.get_entry((
starting_after.split("|", maxsplit=1)[0],
starting_after.split("|")[1],
))
except (FeedNotFoundError, EntryNotFoundError):
start_after_entry = None
else:
start_after_entry = None
# Find the index of the starting entry
start_index: int = 0
if start_after_entry:
for idx, entry in enumerate(all_entries):
if entry.id == start_after_entry.id and entry.feed.url == start_after_entry.feed.url:
start_index = idx + 1
break
# Get the page of entries
paginated_entries: list[Entry] = all_entries[start_index : start_index + entries_per_page]
# Get the last entry for pagination
last_entry: Entry | None = None
if paginated_entries:
last_entry = paginated_entries[-1]
# Create the html for the entries
html: str = create_html_for_feed(reader=reader, entries=paginated_entries)
mass_update_context = build_webhook_mass_update_context(
webhook_feeds=webhook_feeds,
all_feeds=all_feeds,
replace_from=replace_from,
replace_to=replace_to,
resolve_urls=resolve_urls,
force_update=force_update,
)
# Check if there are more entries available
total_entries: int = len(all_entries)
is_show_more_entries_button_visible: bool = (start_index + entries_per_page) < total_entries
context = {
"request": request,
"hook_info": hook_info,
"webhook_name": webhook_name,
"webhook_url": clean_webhook_url,
"webhook_feeds": webhook_feeds,
"entries": paginated_entries,
"html": html,
"last_entry": last_entry,
"is_show_more_entries_button_visible": is_show_more_entries_button_visible,
"total_entries": total_entries,
"feeds_count": len(webhook_feeds),
"message": urllib.parse.unquote(message) if message else "",
**mass_update_context,
}
return templates.TemplateResponse(request=request, name="webhook_entries.html", context=context)
@app.post("/bulk_change_feed_urls", response_class=HTMLResponse)
async def post_bulk_change_feed_urls( # noqa: C901, PLR0914, PLR0912, PLR0915
webhook_url: Annotated[str, Form()],
replace_from: Annotated[str, Form()],
reader: Annotated[Reader, Depends(get_reader_dependency)],
replace_to: Annotated[str, Form()] = "",
resolve_urls: Annotated[bool, Form()] = True, # noqa: FBT002
force_update: Annotated[bool, Form()] = False, # noqa: FBT002
) -> RedirectResponse:
"""Bulk-change feed URLs attached to a webhook.
Args:
webhook_url: The webhook URL whose feeds should be updated.
replace_from: Text to find in each URL.
replace_to: Text to replace with.
resolve_urls: Whether to resolve resulting URLs via redirects.
force_update: Whether existing target feed URLs should be overwritten.
reader: The Reader instance.
Returns:
RedirectResponse: Redirect to webhook detail with status message.
Raises:
HTTPException: If webhook is missing or replace_from is empty.
"""
clean_webhook_url: str = urllib.parse.unquote(webhook_url.strip())
clean_replace_from: str = replace_from.strip()
clean_replace_to: str = replace_to.strip()
if not clean_replace_from:
raise HTTPException(status_code=400, detail="replace_from cannot be empty")
webhooks: list[dict[str, str]] = cast("list[dict[str, str]]", list(reader.get_tag((), "webhooks", [])))
if not any(hook["url"] == clean_webhook_url for hook in webhooks):
raise HTTPException(status_code=404, detail=f"Webhook not found: {clean_webhook_url}")
all_feeds: list[Feed] = list(reader.get_feeds())
webhook_feeds: list[Feed] = []
for feed in all_feeds:
feed_webhook: str = str(reader.get_tag(feed.url, "webhook", ""))
if feed_webhook == clean_webhook_url:
webhook_feeds.append(feed)
preview_rows: list[dict[str, str | bool | None]] = create_webhook_feed_url_preview(
webhook_feeds=webhook_feeds,
replace_from=clean_replace_from,
replace_to=clean_replace_to,
resolve_urls=resolve_urls,
force_update=force_update,
existing_feed_urls={feed.url for feed in all_feeds},
)
changed_count: int = 0
skipped_count: int = 0
failed_count: int = 0
conflict_count: int = 0
force_overwrite_count: int = 0
for row in preview_rows:
if not row["has_match"]:
webhook = reader.get_tag(feed, "webhook")
except TagNotFoundError:
continue
if row["resolution_error"] and not force_update:
skipped_count += 1
continue
if webhook == old_hook.strip():
reader.set_tag(feed.url, "webhook", new_hook.strip()) # pyright: ignore[reportArgumentType]
if row["target_exists"] and not force_update:
conflict_count += 1
skipped_count += 1
continue
old_url: str = str(row["old_url"])
new_url: str = str(row["candidate_url"] if row["will_force_ignore_errors"] else row["resolved_url"])
if old_url == new_url:
skipped_count += 1
continue
if row["target_exists"] and force_update:
try:
reader.delete_feed(new_url)
force_overwrite_count += 1
except FeedNotFoundError:
pass
except ReaderError:
failed_count += 1
continue
try:
reader.change_feed_url(old_url, new_url)
except FeedExistsError:
skipped_count += 1
continue
except FeedNotFoundError:
skipped_count += 1
continue
except ReaderError:
failed_count += 1
continue
try:
reader.update_feed(new_url)
except Exception:
logger.exception("Failed to update feed after URL change: %s", new_url)
for entry in reader.get_entries(feed=new_url, read=False):
try:
reader.set_entry_read(entry, True)
except Exception:
logger.exception("Failed to mark entry as read after URL change: %s", entry.id)
changed_count += 1
if changed_count > 0:
commit_state_change(
reader,
f"Bulk change {changed_count} feed URL(s) for webhook {clean_webhook_url}",
)
status_message: str = (
f"Updated {changed_count} feed URL(s). "
f"Force overwrote {force_overwrite_count}. "
f"Conflicts {conflict_count}. "
f"Skipped {skipped_count}. "
f"Failed {failed_count}."
)
redirect_url: str = (
f"/webhook_entries?webhook_url={urllib.parse.quote(clean_webhook_url)}"
f"&message={urllib.parse.quote(status_message)}"
)
return RedirectResponse(url=redirect_url, status_code=303)
# Redirect to the webhook page.
return RedirectResponse(url="/webhooks", status_code=303)
if __name__ == "__main__":
@ -2043,9 +957,9 @@ if __name__ == "__main__":
uvicorn.run(
"main:app",
log_level="debug",
log_level="info",
host="0.0.0.0", # noqa: S104
port=3000,
port=5000,
proxy_headers=True,
forwarded_allow_ips="*",
)

106
discord_rss_bot/missing_tags.py Normal file
View file

@ -0,0 +1,106 @@
from __future__ import annotations
from reader import Feed, Reader, TagNotFoundError
from discord_rss_bot.settings import default_custom_embed, default_custom_message
def add_custom_message(reader: Reader, feed: Feed) -> None:
"""Add the custom message tag to the feed if it doesn't exist.
Args:
reader: What Reader to use.
feed: The feed to add the tag to.
"""
try:
reader.get_tag(feed, "custom_message")
except TagNotFoundError:
reader.set_tag(feed.url, "custom_message", default_custom_message) # pyright: ignore[reportArgumentType]
reader.set_tag(feed.url, "has_custom_message", True) # pyright: ignore[reportArgumentType]
def add_has_custom_message(reader: Reader, feed: Feed) -> None:
"""Add the has_custom_message tag to the feed if it doesn't exist.
Args:
reader: What Reader to use.
feed: The feed to add the tag to.
"""
try:
reader.get_tag(feed, "has_custom_message")
except TagNotFoundError:
if reader.get_tag(feed, "custom_message") == default_custom_message:
reader.set_tag(feed.url, "has_custom_message", False) # pyright: ignore[reportArgumentType]
else:
reader.set_tag(feed.url, "has_custom_message", True) # pyright: ignore[reportArgumentType]
def add_if_embed(reader: Reader, feed: Feed) -> None:
"""Add the if_embed tag to the feed if it doesn't exist.
Args:
reader: What Reader to use.
feed: The feed to add the tag to.
"""
try:
reader.get_tag(feed, "if_embed")
except TagNotFoundError:
reader.set_tag(feed.url, "if_embed", True) # pyright: ignore[reportArgumentType]
def add_custom_embed(reader: Reader, feed: Feed) -> None:
"""Add the custom embed tag to the feed if it doesn't exist.
Args:
reader: What Reader to use.
feed: The feed to add the tag to.
"""
try:
reader.get_tag(feed, "embed")
except TagNotFoundError:
reader.set_tag(feed.url, "embed", default_custom_embed) # pyright: ignore[reportArgumentType]
reader.set_tag(feed.url, "has_custom_embed", True) # pyright: ignore[reportArgumentType]
def add_has_custom_embed(reader: Reader, feed: Feed) -> None:
"""Add the has_custom_embed tag to the feed if it doesn't exist.
Args:
reader: What Reader to use.
feed: The feed to add the tag to.
"""
try:
reader.get_tag(feed, "has_custom_embed")
except TagNotFoundError:
if reader.get_tag(feed, "embed") == default_custom_embed:
reader.set_tag(feed.url, "has_custom_embed", False) # pyright: ignore[reportArgumentType]
else:
reader.set_tag(feed.url, "has_custom_embed", True) # pyright: ignore[reportArgumentType]
def add_should_send_embed(reader: Reader, feed: Feed) -> None:
"""Add the should_send_embed tag to the feed if it doesn't exist.
Args:
reader: What Reader to use.
feed: The feed to add the tag to.
"""
try:
reader.get_tag(feed, "should_send_embed")
except TagNotFoundError:
reader.set_tag(feed.url, "should_send_embed", True) # pyright: ignore[reportArgumentType]
def add_missing_tags(reader: Reader) -> None:
"""Add missing tags to feeds.
Args:
reader: What Reader to use.
"""
for feed in reader.get_feeds():
add_custom_message(reader, feed)
add_has_custom_message(reader, feed)
add_if_embed(reader, feed)
add_custom_embed(reader, feed)
add_has_custom_embed(reader, feed)
add_should_send_embed(reader, feed)

84
discord_rss_bot/search.py
View file

@ -3,78 +3,66 @@ from __future__ import annotations
import urllib.parse
from typing import TYPE_CHECKING
from discord_rss_bot.settings import get_reader
if TYPE_CHECKING:
from collections.abc import Iterable
from reader import EntrySearchResult
from reader import Feed
from reader import HighlightedString
from reader import Reader
from reader import EntrySearchResult, Feed, HighlightedString, Reader
def create_search_context(query: str, reader: Reader) -> dict:
"""Build context for search.html template.
def create_html_for_search_results(query: str, custom_reader: Reader | None = None) -> str:
"""Create HTML for the search results.
Args:
query (str): The search query.
reader (Reader): Custom Reader instance.
query: Our search query
custom_reader: The reader. If None, we will get the reader from the settings.
Returns:
dict: Context dictionary for rendering the search results.
str: The HTML.
"""
# TODO(TheLovinator): There is a .content that also contains text, we should use that if .summary is not available.
# TODO(TheLovinator): We should also add <span> tags to the title.
# Get the default reader if we didn't get a custom one.
reader: Reader = get_reader() if custom_reader is None else custom_reader
search_results: Iterable[EntrySearchResult] = reader.search_entries(query)
results: list[dict] = []
html: str = ""
for result in search_results:
if ".summary" in result.content:
result_summary: str = add_span_with_slice(result.content[".summary"])
feed: Feed = reader.get_feed(result.feed_url)
feed_url: str = urllib.parse.quote(feed.url)
# Prefer summary, fall back to content
if ".summary" in result.content:
highlighted = result.content[".summary"]
else:
content_keys = [k for k in result.content if k.startswith(".content")]
highlighted = result.content[content_keys[0]] if content_keys else None
html += f"""
<div class="p-2 mb-2 border border-dark">
<a class="text-muted text-decoration-none" href="/feed?feed_url={feed_url}">
<h2>{result.metadata[".title"]}</h2>
</a>
{result_summary}
</div>
"""
summary: str = add_spans(highlighted) if highlighted else "(no preview available)"
results.append({
"title": add_spans(result.metadata.get(".title")),
"summary": summary,
"feed_url": feed_url,
})
return {
"query": query,
"search_amount": {"total": len(results)},
"results": results,
}
return html
def add_spans(highlighted_string: HighlightedString | None) -> str:
"""Wrap all highlighted parts with <span> tags.
def add_span_with_slice(highlighted_string: HighlightedString) -> str:
"""Add span tags to the string to highlight the search results.
Args:
highlighted_string (HighlightedString | None): The highlighted string to process.
highlighted_string: The highlighted string.
Returns:
str: The processed string with <span> tags around highlighted parts.
str: The string with added <span> tags.
"""
if highlighted_string is None:
return ""
value: str = highlighted_string.value
parts: list[str] = []
last_index = 0
# TODO(TheLovinator): We are looping through the highlights and only using the last one. We should use all of them.
before_span, span_part, after_span = "", "", ""
for txt_slice in highlighted_string.highlights:
parts.extend((
value[last_index : txt_slice.start],
f"<span class='bg-warning'>{value[txt_slice.start : txt_slice.stop]}</span>",
))
last_index = txt_slice.stop
before_span: str = f"{highlighted_string.value[: txt_slice.start]}"
span_part: str = f"<span class='bg-warning'>{highlighted_string.value[txt_slice.start : txt_slice.stop]}</span>"
after_span: str = f"{highlighted_string.value[txt_slice.stop :]}"
# add any trailing text
parts.append(value[last_index:])
return "".join(parts)
return f"{before_span}{span_part}{after_span}"

23
discord_rss_bot/settings.py
View file

@ -1,23 +1,16 @@
from __future__ import annotations
import os
import typing
from functools import lru_cache
from pathlib import Path
from platformdirs import user_data_dir
from reader import Reader
from reader import make_reader
from reader import Reader, make_reader
if typing.TYPE_CHECKING:
from reader.types import JSONType
data_dir: str = os.getenv("DISCORD_RSS_BOT_DATA_DIR", "").strip() or user_data_dir(
appname="discord_rss_bot",
appauthor="TheLovinator",
roaming=True,
ensure_exists=True,
)
data_dir: str = user_data_dir(appname="discord_rss_bot", appauthor="TheLovinator", roaming=True, ensure_exists=True)
# TODO(TheLovinator): Add default things to the database and make the edible.
@ -31,7 +24,7 @@ default_custom_embed: dict[str, str] = {
}
@lru_cache(maxsize=1)
@lru_cache
def get_reader(custom_location: Path | None = None) -> Reader:
"""Get the reader.
@ -42,13 +35,5 @@ def get_reader(custom_location: Path | None = None) -> Reader:
The reader.
"""
db_location: Path = custom_location or Path(data_dir) / "db.sqlite"
reader: Reader = make_reader(url=str(db_location))
# https://reader.readthedocs.io/en/latest/api.html#reader.types.UpdateConfig
# Set the default update interval to 15 minutes if not already configured
# Users can change this via the Settings page or per-feed in the feed page
if reader.get_tag((), ".reader.update", None) is None:
# Set default
reader.set_tag((), ".reader.update", {"interval": 15})
return reader
return make_reader(url=str(db_location))

4
discord_rss_bot/static/styles.css
View file

@ -13,7 +13,3 @@ body {
.form-text {
color: #acabab;
}
.interval-input {
max-width: 120px;
}

73
discord_rss_bot/templates/_webhook_mass_update_preview.html
View file

@ -1,73 +0,0 @@
{% if preview_rows %}
<p class="small text-muted mb-1">
{{ preview_change_count }} feed URL{{ 's' if preview_change_count != 1 else '' }} ready to update.
</p>
<div class="small text-muted mb-2 d-flex flex-wrap gap-2">
<span class="badge bg-secondary">Total: {{ preview_summary.total }}</span>
<span class="badge bg-info text-dark">Matched: {{ preview_summary.matched }}</span>
<span class="badge bg-success">Will update: {{ preview_summary.will_update }}</span>
<span class="badge bg-warning text-dark">Conflicts: {{ preview_summary.conflicts }}</span>
<span class="badge bg-warning">Force overwrite: {{ preview_summary.force_overwrite }}</span>
<span class="badge bg-warning text-dark">Force ignore errors: {{ preview_summary.force_ignore_errors }}</span>
<span class="badge bg-danger">Resolve errors: {{ preview_summary.resolve_errors }}</span>
<span class="badge bg-secondary">No change: {{ preview_summary.no_change }}</span>
<span class="badge bg-secondary">No match: {{ preview_summary.no_match }}</span>
</div>
<form action="/bulk_change_feed_urls" method="post" class="mb-2">
<input type="hidden" name="webhook_url" value="{{ webhook_url }}" />
<input type="hidden" name="replace_from" value="{{ replace_from }}" />
<input type="hidden" name="replace_to" value="{{ replace_to }}" />
<input type="hidden"
name="resolve_urls"
value="{{ 'true' if resolve_urls else 'false' }}" />
<input type="hidden"
name="force_update"
value="{{ 'true' if force_update else 'false' }}" />
<button type="submit"
class="btn btn-warning w-100"
{% if preview_change_count == 0 %}disabled{% endif %}
onclick="return confirm('Apply these feed URL updates?');">Apply mass update</button>
</form>
<div class="table-responsive mt-2">
<table class="table table-sm table-dark table-striped align-middle mb-0">
<thead>
<tr>
<th scope="col">Old URL</th>
<th scope="col">New URL</th>
<th scope="col">Status</th>
</tr>
</thead>
<tbody>
{% for row in preview_rows %}
<tr>
<td>
<code>{{ row.old_url }}</code>
</td>
<td>
<code>{{ row.resolved_url if resolve_urls else row.candidate_url }}</code>
</td>
<td>
{% if not row.has_match %}
<span class="badge bg-secondary">No match</span>
{% elif row.will_force_ignore_errors %}
<span class="badge bg-warning text-dark">Will force update (ignore resolve error)</span>
{% elif row.resolution_error %}
<span class="badge bg-danger">{{ row.resolution_error }}</span>
{% elif row.will_force_overwrite %}
<span class="badge bg-warning">Will force overwrite</span>
{% elif row.target_exists %}
<span class="badge bg-warning text-dark">Conflict: target URL exists</span>
{% elif row.will_change %}
<span class="badge bg-success">Will update</span>
{% else %}
<span class="badge bg-secondary">No change</span>
{% endif %}
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
{% elif replace_from %}
<p class="small text-muted mb-0">No preview rows found for that replacement pattern.</p>
{% endif %}

12
discord_rss_bot/templates/base.html
View file

@ -1,5 +1,6 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
@ -17,6 +18,7 @@
{% block head %}
{% endblock head %}
</head>
<body class="text-white-50">
{% include "nav.html" %}
<div class="p-2 mb-2">
@ -25,12 +27,10 @@
{% if messages %}
<div class="alert alert-warning alert-dismissible fade show" role="alert">
<pre>{{ messages }}</pre>
<button type="button"
class="btn-close"
data-bs-dismiss="alert"
aria-label="Close"></button>
<button type="button" class="btn-close" data-bs-dismiss="alert" aria-label="Close"></button>
</div>
{% endif %}
{% block content %}
{% endblock content %}
<footer class="d-flex flex-wrap justify-content-between align-items-center py-3 my-4 border-top">
@ -52,9 +52,7 @@
</div>
</div>
</div>
<script src="https://cdn.jsdelivr.net/npm/htmx.org@2.0.8/dist/htmx.min.js"
integrity="sha384-/TgkGk7p307TH7EXJDuUlgG3Ce1UVolAOFopFekQkkXihi5u/6OCvVKyz1W+idaz"
crossorigin="anonymous"></script>
<script src="/static/bootstrap.min.js" defer></script>
</body>
</html>

43
discord_rss_bot/templates/blacklist.html
View file

@ -42,49 +42,6 @@
<label for="blacklist_author" class="col-sm-6 col-form-label">Blacklist - Author</label>
<input name="blacklist_author" type="text" class="form-control bg-dark border-dark text-muted"
id="blacklist_author" value="{%- if blacklist_author -%}{{ blacklist_author }}{%- endif -%}" />
<div class="mt-4">
<div class="form-text">
<ul class="list-inline">
<li>
Regular expression patterns for advanced filtering. Each pattern should be on a new
line.
</li>
<li>Patterns are case-insensitive.</li>
<li>
Examples:
<code>
<pre>
^New Release:.*
\b(update|version|patch)\s+\d+\.\d+
.*\[(important|notice)\].*
</pre>
</code>
</li>
</ul>
</div>
<label for="regex_blacklist_title" class="col-sm-6 col-form-label">Regex Blacklist - Title</label>
<textarea name="regex_blacklist_title" class="form-control bg-dark border-dark text-muted"
id="regex_blacklist_title"
rows="3">{%- if regex_blacklist_title -%}{{ regex_blacklist_title }}{%- endif -%}</textarea>
<label for="regex_blacklist_summary" class="col-sm-6 col-form-label">Regex Blacklist -
Summary</label>
<textarea name="regex_blacklist_summary" class="form-control bg-dark border-dark text-muted"
id="regex_blacklist_summary"
rows="3">{%- if regex_blacklist_summary -%}{{ regex_blacklist_summary }}{%- endif -%}</textarea>
<label for="regex_blacklist_content" class="col-sm-6 col-form-label">Regex Blacklist -
Content</label>
<textarea name="regex_blacklist_content" class="form-control bg-dark border-dark text-muted"
id="regex_blacklist_content"
rows="3">{%- if regex_blacklist_content -%}{{ regex_blacklist_content }}{%- endif -%}</textarea>
<label for="regex_blacklist_author" class="col-sm-6 col-form-label">Regex Blacklist - Author</label>
<textarea name="regex_blacklist_author" class="form-control bg-dark border-dark text-muted"
id="regex_blacklist_author"
rows="3">{%- if regex_blacklist_author -%}{{ regex_blacklist_author }}{%- endif -%}</textarea>
</div>
</div>
</div>
<!-- Add a hidden feed_url field to the form -->

60
discord_rss_bot/templates/custom.html
View file

@ -14,90 +14,90 @@
<li>You can use \n to create a new line.</li>
<li>
You can remove the embed from links by adding < and> around the link. (For example <
{% raw %} {{entry_link}} {% endraw %}>)
{% raw %} {{ entry_link }} {% endraw %}>)
</li>
<br />
<li>
<code>
{% raw %}
{{feed_author}}
{{ feed_author }}
{% endraw %}
</code>{{ feed.author }}
</li>
<li>
<code>
{% raw %}
{{feed_added}}
{{ feed_added }}
{% endraw %}
</code>{{ feed.added }}
</li>
<li>
<code>
{% raw %}
{{feed_last_exception}}
{{ feed_last_exception }}
{% endraw %}
</code>{{ feed.last_exception }}
</li>
<li>
<code>
{% raw %}
{{feed_last_updated}}
{{ feed_last_updated }}
{% endraw %}
</code>{{ feed.last_updated }}
</li>
<li>
<code>
{% raw %}
{{feed_link}}
{{ feed_link }}
{% endraw %}
</code>{{ feed.link }}
</li>
<li>
<code>
{% raw %}
{{feed_subtitle}}
{{ feed_subtitle }}
{% endraw %}
</code>{{ feed.subtitle }}
</li>
<li>
<code>
{% raw %}
{{feed_title}}
{{ feed_title }}
{% endraw %}
</code>{{ feed.title }}
</li>
<li>
<code>
{% raw %}
{{feed_updated}}
{{ feed_updated }}
{% endraw %}
</code>{{ feed.updated }}
</li>
<li>
<code>
{% raw %}
{{feed_updates_enabled}}
{{ feed_updates_enabled }}
{% endraw %}
</code>{{ feed.updates_enabled }}
</li>
<li>
<code>
{% raw %}
{{feed_url}}
{{ feed_url }}
{% endraw %}
</code>{{ feed.url }}
</li>
<li>
<code>
{% raw %}
{{feed_user_title}}
{{ feed_user_title }}
{% endraw %}
</code>{{ feed.user_title }}
</li>
<li>
<code>
{% raw %}
{{feed_version}}
{{ feed_version }}
{% endraw %}
</code>{{ feed.version }}
</li>
@ -106,14 +106,14 @@
<li>
<code>
{% raw %}
{{entry_added}}
{{ entry_added }}
{% endraw %}
</code>{{ entry.added }}
</li>
<li>
<code>
{% raw %}
{{entry_author}}
{{ entry_author }}
{% endraw %}
</code>{{ entry.author }}
</li>
@ -121,14 +121,14 @@
<li>
<code>
{% raw %}
{{entry_content}}
{{ entry_content }}
{% endraw %}
</code>{{ entry.content[0].value|discord_markdown }}
</li>
<li>
<code>
{% raw %}
{{entry_content_raw}}
{{ entry_content_raw }}
{% endraw %}
</code>{{ entry.content[0].value }}
</li>
@ -136,42 +136,42 @@
<li>
<code>
{% raw %}
{{entry_id}}
{{ entry_id }}
{% endraw %}
</code>{{ entry.id }}
</li>
<li>
<code>
{% raw %}
{{entry_important}}
{{ entry_important }}
{% endraw %}
</code>{{ entry.important }}
</li>
<li>
<code>
{% raw %}
{{entry_link}}
{{ entry_link }}
{% endraw %}
</code>{{ entry.link }}
</li>
<li>
<code>
{% raw %}
{{entry_published}}
{{ entry_published }}
{% endraw %}
</code>{{ entry.published }}
</li>
<li>
<code>
{% raw %}
{{entry_read}}
{{ entry_read }}
{% endraw %}
</code>{{ entry.read }}
</li>
<li>
<code>
{% raw %}
{{entry_read_modified}}
{{ entry_read_modified }}
{% endraw %}
</code>{{ entry.read_modified }}
</li>
@ -179,14 +179,14 @@
<li>
<code>
{% raw %}
{{entry_summary}}
{{ entry_summary }}
{% endraw %}
</code>{{ entry.summary|discord_markdown }}
</li>
<li>
<code>
{% raw %}
{{entry_summary_raw}}
{{ entry_summary_raw }}
{% endraw %}
</code>{{ entry.summary }}
</li>
@ -194,21 +194,21 @@
<li>
<code>
{% raw %}
{{entry_title}}
{{ entry_title }}
{% endraw %}
</code>{{ entry.title }}
</li>
<li>
<code>
{% raw %}
{{entry_text}}
{{ entry_text }}
{% endraw %}
</code> Same as entry_content if it exists, otherwise entry_summary
</li>
<li>
<code>
{% raw %}
{{entry_updated}}
{{ entry_updated }}
{% endraw %}
</code>{{ entry.updated }}
</li>
@ -216,7 +216,7 @@
<li>
<code>
{% raw %}
{{image_1}}
{{ image_1 }}
{% endraw %}
</code>First image in the entry if it exists
</li>
@ -226,7 +226,7 @@
<li>
<code>
{% raw %}
{{feed_title}}\n{{entry_content}}
{{ feed_title }}\n{{ entry_content }}
{% endraw %}
</code>
</li>

156
discord_rss_bot/templates/feed.html
View file

@ -1,172 +1,84 @@
{% extends "base.html" %}
{% block title %}
| {{ feed.title }}
| {{ feed.title }}
{% endblock title %}
{% block content %}
<div class="card mb-3 border border-dark p-3 text-light">
<div class="card mb-3 border border-dark p-3 text-light">
<!-- Feed Title -->
<h2>
<a class="text-muted" href="{{ feed.url }}">{{ feed.title }}</a> ({{ total_entries }} entries)
</h2>
{% if not feed.updates_enabled %}<span class="badge bg-danger">Disabled</span>{% endif %}
{% if not feed.updates_enabled %}
<span class="badge bg-danger">Disabled</span>
{% endif %}
{% if feed.last_exception %}
<div class="mt-3">
<h5 class="text-danger">{{ feed.last_exception.type_name }}:</h5>
<code class="d-block">{{ feed.last_exception.value_str }}</code>
<button class="btn btn-secondary btn-sm mt-2"
type="button"
data-bs-toggle="collapse"
data-bs-target="#exceptionDetails"
aria-expanded="false"
aria-controls="exceptionDetails">Show Traceback</button>
<button class="btn btn-secondary btn-sm mt-2" type="button" data-bs-toggle="collapse"
data-bs-target="#exceptionDetails" aria-expanded="false" aria-controls="exceptionDetails">
Show Traceback
</button>
<div class="collapse" id="exceptionDetails">
<pre><code>{{ feed.last_exception.traceback_str }}</code></pre>
</div>
</div>
{% endif %}
<!-- Feed Actions -->
<div class="mt-3 d-flex flex-wrap gap-2">
<a href="/update?feed_url={{ feed.url|encode_url }}"
class="btn btn-primary btn-sm">Update</a>
<form action="/remove" method="post" class="d-inline">
<button class="btn btn-danger btn-sm"
name="feed_url"
value="{{ feed.url }}"
<button class="btn btn-danger btn-sm" name="feed_url" value="{{ feed.url }}"
onclick="return confirm('Are you sure you want to delete this feed?')">Remove</button>
</form>
{% if not feed.updates_enabled %}
<form action="/unpause" method="post" class="d-inline">
<button class="btn btn-secondary btn-sm"
name="feed_url"
value="{{ feed.url }}">Unpause</button>
<button class="btn btn-secondary btn-sm" name="feed_url" value="{{ feed.url }}">Unpause</button>
</form>
{% else %}
<form action="/pause" method="post" class="d-inline">
<button class="btn btn-danger btn-sm" name="feed_url" value="{{ feed.url }}">Pause</button>
</form>
{% endif %}
{% if not "youtube.com/feeds/videos.xml" in feed.url %}
{% if should_send_embed %}
<form action="/use_text" method="post" class="d-inline">
<button class="btn btn-dark btn-sm" name="feed_url" value="{{ feed.url }}">Send text message instead of embed</button>
<button class="btn btn-dark btn-sm" name="feed_url" value="{{ feed.url }}">
Send text message instead of embed
</button>
</form>
{% else %}
<form action="/use_embed" method="post" class="d-inline">
<button class="btn btn-dark btn-sm" name="feed_url" value="{{ feed.url }}">Send embed instead of text message</button>
<button class="btn btn-dark btn-sm" name="feed_url" value="{{ feed.url }}">
Send embed instead of text message
</button>
</form>
{% endif %}
{% endif %}
</div>
<!-- Additional Links -->
<div class="mt-3">
<a class="text-muted d-block"
href="/whitelist?feed_url={{ feed.url|encode_url }}">Whitelist</a>
<a class="text-muted d-block"
href="/blacklist?feed_url={{ feed.url|encode_url }}">Blacklist</a>
<a class="text-muted d-block"
href="/custom?feed_url={{ feed.url|encode_url }}">
Customize message
{% if not should_send_embed %}(Currently active){% endif %}
<a class="text-muted d-block" href="/whitelist?feed_url={{ feed.url|encode_url }}">Whitelist</a>
<a class="text-muted d-block" href="/blacklist?feed_url={{ feed.url|encode_url }}">Blacklist</a>
<a class="text-muted d-block" href="/custom?feed_url={{ feed.url|encode_url }}">
Customize message {% if not should_send_embed %}(Currently active){% endif %}
</a>
{% if not "youtube.com/feeds/videos.xml" in feed.url %}
<a class="text-muted d-block"
href="/embed?feed_url={{ feed.url|encode_url }}">
Customize embed
{% if should_send_embed %}(Currently active){% endif %}
<a class="text-muted d-block" href="/embed?feed_url={{ feed.url|encode_url }}">
Customize embed {% if should_send_embed %}(Currently active){% endif %}
</a>
{% endif %}
</div>
<!-- Feed URL Configuration -->
<div class="mt-4 border-top border-secondary pt-3">
<h5 class="mb-3">Feed URL</h5>
<form action="/change_feed_url" method="post" class="mb-2">
<input type="hidden" name="old_feed_url" value="{{ feed.url }}" />
<div class="input-group input-group-sm mb-2">
<input type="url"
class="form-control form-control-sm"
name="new_feed_url"
value="{{ feed.url }}"
required />
<button class="btn btn-warning" type="submit">Update URL</button>
</div>
</form>
</div>
<!-- Feed Metadata -->
<div class="mt-4 border-top border-secondary pt-3">
<h5 class="mb-3">Feed Information</h5>
<div class="row text-muted">
<div class="col-md-6 mb-2">
<small><strong>Added:</strong> {{ feed.added | relative_time }}</small>
</div>
<div class="col-md-6 mb-2">
<small><strong>Last Updated:</strong> {{ feed.last_updated | relative_time }}</small>
</div>
<div class="col-md-6 mb-2">
<small><strong>Last Retrieved:</strong> {{ feed.last_retrieved | relative_time }}</small>
</div>
<div class="col-md-6 mb-2">
<small><strong>Next Update:</strong> {{ feed.update_after | relative_time }}</small>
</div>
<div class="col-md-6 mb-2">
<small><strong>Updates:</strong> <span class="badge {{ 'bg-success' if feed.updates_enabled else 'bg-danger' }}">{{ 'Enabled' if feed.updates_enabled else 'Disabled' }}</span></small>
</div>
</div>
</div>
<!-- Update Interval Configuration -->
<div class="mt-4 border-top border-secondary pt-3">
<h5 class="mb-3">
Update Interval <span class="badge
{% if feed_interval %}
bg-info
{% else %}
bg-secondary
{% endif %}">
{% if feed_interval %}
Custom
{% else %}
Using global default
{% endif %}
</span>
</h5>
<div class="d-flex align-items-center gap-2 flex-wrap">
<span class="text-muted">Current: <strong>
{% if feed_interval %}
{{ feed_interval }}
{% if feed_interval >= 60 %}({{ (feed_interval / 60) | round(1) }} hours){% endif %}
{% else %}
{{ global_interval }}
{% if global_interval >= 60 %}({{ (global_interval / 60) | round(1) }} hours){% endif %}
{% endif %}
minutes</strong></span>
<form action="/set_update_interval"
method="post"
class="d-inline-flex gap-2 align-items-center">
<input type="hidden" name="feed_url" value="{{ feed.url }}" />
<input type="number"
class="form-control form-control-sm interval-input"
style="width: 100px"
name="interval_minutes"
placeholder="Minutes"
min="1"
value="{{ feed_interval if feed_interval else global_interval }}"
required />
<button class="btn btn-primary btn-sm" type="submit">Set Interval</button>
</form>
{% if feed_interval %}
<form action="/reset_update_interval" method="post" class="d-inline">
<input type="hidden" name="feed_url" value="{{ feed.url }}" />
<button class="btn btn-secondary btn-sm" type="submit">Reset to Global Default</button>
</form>
{% endif %}
</div>
</div>
</div>
{# Rendered HTML content #}
<pre>{{ html|safe }}</pre>
{% if is_show_more_entries_button_visible %}
<a class="btn btn-dark mt-3"
{% if show_more_entires_button %}
<a class="btn btn-dark mt-3"
href="/feed?feed_url={{ feed.url|encode_url }}&starting_after={{ last_entry.id|encode_url }}">
Show more entries
</a>
</a>
{% endif %}
{% endblock content %}

117
discord_rss_bot/templates/index.html
View file

@ -1,7 +1,7 @@
{% extends "base.html" %}
{% block content %}
<!-- List all feeds -->
<ul>
<!-- List all feeds -->
<ul>
<!-- Check if any feeds -->
{% if feeds %}
<p>
@ -28,77 +28,41 @@
{{ entry_count.averages[2]|round(1) }})
</abbr>
</p>
<!-- Loop through the webhooks and add the feeds grouped by domain -->
<!-- Loop through the webhooks and add the feeds connected to them. -->
{% for hook_from_context in webhooks %}
<div class="p-2 mb-3 border border-dark">
<div class="d-flex justify-content-between align-items-center mb-3">
<h2 class="h5 mb-0">{{ hook_from_context.name }}</h2>
<a class="text-muted fs-6 btn btn-outline-light btn-sm ms-auto me-2"
href="/webhook_entries?webhook_url={{ hook_from_context.url|encode_url }}">Settings</a>
<a class="text-muted fs-6 btn btn-outline-light btn-sm"
href="/webhook_entries?webhook_url={{ hook_from_context.url|encode_url }}">View Latest Entries</a>
</div>
<!-- Group feeds by domain within each webhook -->
{% set feeds_for_hook = [] %}
<div class="p-2 mb-2 border border-dark">
<h2 class="h5">
<a class="text-muted" href="/webhooks">{{ hook_from_context.name }}</a>
</h2>
<ul class="list-group">
{% for feed_webhook in feeds %}
{% if hook_from_context.url == feed_webhook.webhook %}
{% set _ = feeds_for_hook.append(feed_webhook) %}
{% endif %}
{% endfor %}
{% if feeds_for_hook %}
<!-- Create a dictionary to hold feeds grouped by domain -->
{% set domains = {} %}
{% for feed_item in feeds_for_hook %}
{% set feed = feed_item.feed %}
{% set domain = feed_item.domain %}
{% if domain not in domains %}
{% set _ = domains.update({domain: []}) %}
{% endif %}
{% set _ = domains[domain].append(feed) %}
{% endfor %}
<!-- Display domains and their feeds -->
{% for domain, domain_feeds in domains.items() %}
<div class="card bg-dark border border-dark mb-2">
<div class="card-header">
<h3 class="h6 mb-0 text-white-50">{{ domain }} ({{ domain_feeds|length }})</h3>
</div>
<div class="card-body p-2">
<ul class="list-group list-unstyled mb-0">
{% for feed in domain_feeds %}
<li>
<a class="text-muted" href="/feed?feed_url={{ feed.url|encode_url }}">
{% if feed.title %}
{{ feed.title }}
{% else %}
{{ feed.url }}
{% endif %}
</a>
{% set feed = feed_webhook["feed"] %}
{% set hook_from_feed = feed_webhook["webhook"] %}
{% if hook_from_context.url == hook_from_feed %}
<div>
<a class="text-muted" href="/feed?feed_url={{ feed.url|encode_url }}">{{ feed.url }}</a>
{% if not feed.updates_enabled %}<span class="text-warning">Disabled</span>{% endif %}
{% if feed.last_exception %}<span class="text-danger">({{ feed.last_exception.value_str }})</span>{% endif %}
</li>
{% if feed.last_exception %}<span
class="text-danger">({{ feed.last_exception.value_str }})</span>{% endif %}
</div>
{% endif %}
{% endfor %}
</ul>
</div>
</div>
{% endfor %}
{% else %}
<p class="text-muted">No feeds associated with this webhook.</p>
{% endif %}
</div>
{% endfor %}
{% else %}
<p>
Hello there!
<br />
<br />
<br>
You need to add a webhook <a class="text-muted" href="/add_webhook">here</a> to get started. After that, you can
add feeds <a class="text-muted" href="/add">here</a>. You can find both of these links in the navigation bar
above.
<br />
<br />
If you have any questions or suggestions, feel free to contact me on <a class="text-muted" href="mailto:tlovinator@gmail.com">tlovinator@gmail.com</a> or TheLovinator#9276 on Discord.
<br />
<br />
<br>
<br>
If you have any questions or suggestions, feel free to contact me on <a class="text-muted"
href="mailto:tlovinator@gmail.com">tlovinator@gmail.com</a> or TheLovinator#9276 on Discord.
<br>
<br>
Thanks!
</p>
{% endif %}
@ -108,21 +72,7 @@
<ul class="list-group text-danger">
Feeds without webhook:
{% for broken_feed in broken_feeds %}
<a class="text-muted"
href="/feed?feed_url={{ broken_feed.url|encode_url }}">
{# Display username@youtube for YouTube feeds #}
{% if "youtube.com/feeds/videos.xml" in broken_feed.url %}
{% if "user=" in broken_feed.url %}
{{ broken_feed.url.split("user=")[1] }}@youtube
{% elif "channel_id=" in broken_feed.url %}
{{ broken_feed.title if broken_feed.title else broken_feed.url.split("channel_id=")[1] }}@youtube
{% else %}
{{ broken_feed.url }}
{% endif %}
{% else %}
{{ broken_feed.url }}
{% endif %}
</a>
<a class="text-muted" href="/feed?feed_url={{ broken_feed.url|encode_url }}">{{ broken_feed.url }}</a>
{% endfor %}
</ul>
</div>
@ -133,23 +83,10 @@
<ul class="list-group text-danger">
Feeds without attached webhook:
{% for feed in feeds_without_attached_webhook %}
<a class="text-muted" href="/feed?feed_url={{ feed.url|encode_url }}">
{# Display username@youtube for YouTube feeds #}
{% if "youtube.com/feeds/videos.xml" in feed.url %}
{% if "user=" in feed.url %}
{{ feed.url.split("user=")[1] }}@youtube
{% elif "channel_id=" in feed.url %}
{{ feed.title if feed.title else feed.url.split("channel_id=")[1] }}@youtube
{% else %}
{{ feed.url }}
{% endif %}
{% else %}
{{ feed.url }}
{% endif %}
</a>
<a class="text-muted" href="/feed?feed_url={{ feed.url|encode_url }}">{{ feed.url }}</a>
{% endfor %}
</ul>
</div>
{% endif %}
</ul>
</ul>
{% endblock content %}

25
discord_rss_bot/templates/nav.html
View file

@ -1,9 +1,6 @@
<nav class="navbar navbar-expand-md navbar-dark p-2 mb-3 border-bottom border-warning">
<div class="container-fluid">
<button class="navbar-toggler ms-auto"
type="button"
data-bs-toggle="collapse"
data-bs-target="#collapseNavbar">
<button class="navbar-toggler ms-auto" type="button" data-bs-toggle="collapse" data-bs-target="#collapseNavbar">
<span class="navbar-toggler-icon"></span>
</button>
<div class="navbar-collapse collapse" id="collapseNavbar">
@ -19,28 +16,10 @@
<li class="nav-item">
<a class="nav-link" href="/webhooks">Webhooks</a>
</li>
<li class="nav-item nav-link d-none d-md-block">|</li>
<li class="nav-item">
<a class="nav-link" href="/settings">Settings</a>
</li>
{% if get_backup_path() %}
<li class="nav-item nav-link d-none d-md-block">|</li>
<li class="nav-item">
<form action="/backup" method="post" class="d-inline">
<button type="submit"
class="nav-link btn btn-link text-decoration-none"
onclick="return confirm('Create a manual git backup of the current state?');">
Backup
</button>
</form>
</li>
{% endif %}
</ul>
{# Search #}
<form action="/search" method="get" class="ms-auto w-50 input-group">
<input name="query"
class="form-control bg-dark border-dark text-muted"
type="search"
<input name="query" class="form-control bg-dark border-dark text-muted" type="search"
placeholder="Search" />
</form>
{# Donate button #}

18
discord_rss_bot/templates/search.html
View file

@ -1,18 +1,10 @@
{% extends "base.html" %}
{% block title %}
| Search
| Search
{% endblock title %}
{% block content %}
<div class="p-2 border border-dark text-muted">
Your search for "{{ query }}" returned {{ search_amount.total }} results.
</div>
{% for result in results %}
<div class="p-2 mb-2 border border-dark">
<a class="text-muted text-decoration-none"
href="/feed?feed_url={{ result.feed_url }}">
<h2>{{ result.title|safe }}</h2>
</a>
<div class="text-muted">{{ result.summary|safe }}</div>
</div>
{% endfor %}
<div class="p-2 border border-dark text-muted">
Your search for "{{- query -}}" returned {{- search_amount.total -}} results.
</div>
{{- search_html | safe -}}
{% endblock content %}

122
discord_rss_bot/templates/settings.html
View file

@ -1,122 +0,0 @@
{% extends "base.html" %}
{% block title %}
| Settings
{% endblock title %}
{% block content %}
<section>
<div class="text-light">
<div class="d-flex flex-wrap justify-content-between align-items-center gap-2">
<h2 class="mb-0">Global Settings</h2>
</div>
<p class="text-muted mt-2 mb-4">
Set a default interval for all feeds. Individual feeds can still override this value.
</p>
<div class="mb-4">
<div>
Current default is {{ global_interval }} min.
Even though we check ETags and Last-Modified headers, choosing a very low interval may cause issues with some feeds or cause excessive load on the server hosting the feed. Remember to be kind.
</div>
</div>
</div>
<form action="/set_global_update_interval" method="post" class="mb-2">
<div class="settings-form-row mb-2">
<label for="interval_minutes" class="form-label mb-1">Default interval (minutes)</label>
<div class="input-group input-group-lg">
<input id="interval_minutes"
type="number"
class="form-control settings-input"
name="interval_minutes"
placeholder="Minutes"
min="1"
value="{{ global_interval }}"
required />
<button class="btn btn-primary px-4" type="submit">Save</button>
</div>
</div>
</form>
</section>
<section class="mt-5">
<div class="text-light">
<div class="d-flex flex-wrap justify-content-between align-items-center gap-2">
<h2 class="mb-0">Feed Update Intervals</h2>
</div>
<p class="text-muted mt-2 mb-4">
Customize the update interval for individual feeds. Leave empty or reset to use the global default.
</p>
</div>
{% if feed_intervals %}
<div class="table-responsive">
<table class="table table-dark table-hover">
<thead>
<tr>
<th>Feed</th>
<th>Domain</th>
<th>Status</th>
<th>Interval</th>
<th>Last Updated</th>
<th>Next Update</th>
<th>Set Interval (min)</th>
<th>Actions</th>
</tr>
</thead>
<tbody>
{% for item in feed_intervals %}
<tr>
<td>
<a href="/feed?feed_url={{ item.feed.url|encode_url }}"
class="text-light text-decoration-none">{{ item.feed.title }}</a>
</td>
<td>
<span class="text-muted small">{{ item.domain }}</span>
</td>
<td>
<span class="badge {{ 'bg-success' if item.feed.updates_enabled else 'bg-danger' }}">
{{ 'Enabled' if item.feed.updates_enabled else 'Disabled' }}
</span>
</td>
<td>
<span>{{ item.effective_interval }} min</span>
{% if item.interval %}
<span class="badge bg-info ms-1">Custom</span>
{% else %}
<span class="badge bg-secondary ms-1">Global</span>
{% endif %}
</td>
<td>
<small class="text-muted">{{ item.feed.last_updated | relative_time }}</small>
</td>
<td>
<small class="text-muted">{{ item.feed.update_after | relative_time }}</small>
</td>
<td>
<form action="/set_update_interval" method="post" class="d-flex gap-2">
<input type="hidden" name="feed_url" value="{{ item.feed.url }}" />
<input type="hidden" name="redirect_to" value="/settings" />
<input type="number"
class="form-control form-control-sm interval-input"
name="interval_minutes"
placeholder="Minutes"
min="1"
value="{{ item.interval if item.interval else global_interval }}" />
<button class="btn btn-primary btn-sm" type="submit">Set</button>
</form>
</td>
<td>
{% if item.interval %}
<form action="/reset_update_interval" method="post" class="d-inline">
<input type="hidden" name="feed_url" value="{{ item.feed.url }}" />
<input type="hidden" name="redirect_to" value="/settings" />
<button class="btn btn-outline-secondary btn-sm" type="submit">Reset</button>
</form>
{% endif %}
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
{% else %}
<p class="text-muted">No feeds added yet.</p>
{% endif %}
</section>
{% endblock content %}

167
discord_rss_bot/templates/webhook_entries.html
View file

@ -1,167 +0,0 @@
{% extends "base.html" %}
{% block title %}
| {{ webhook_name }}
{% endblock title %}
{% block content %}
{% if message %}<div class="alert alert-info" role="alert">{{ message }}</div>{% endif %}
<div class="card mb-3 border border-dark p-3 text-light">
<div class="d-flex flex-column flex-md-row justify-content-between gap-3">
<div>
<h2 class="mb-2">{{ webhook_name }}</h2>
<p class="text-muted mb-1">
{{ total_entries }} total from {{ feeds_count }} feed{{ 's' if feeds_count != 1 else '' }}
</p>
<p class="text-muted mb-0">
<code>{{ webhook_url }}</code>
</p>
</div>
<div class="d-flex gap-2 align-items-start">
<a class="btn btn-outline-light btn-sm" href="/">Back to dashboard</a>
<a class="btn btn-outline-info btn-sm" href="/webhooks">All webhooks</a>
</div>
</div>
</div>
<div class="row g-3 mb-3">
<div class="col-lg-5">
<div class="card border border-dark p-3 text-light h-100">
<h3 class="h5">Settings</h3>
<ul class="list-unstyled text-muted mb-3">
<li>
<strong>Custom name:</strong> {{ hook_info.custom_name }}
</li>
<li>
<strong>Discord name:</strong> {{ hook_info.name or 'Unavailable' }}
</li>
<li>
<strong>Webhook:</strong>
<a class="text-muted" href="{{ hook_info.url }}">{{ hook_info.url | replace('https://discord.com/api/webhooks', '') }}</a>
</li>
</ul>
<form action="/modify_webhook" method="post" class="row g-3 mb-3">
<input type="hidden" name="old_hook" value="{{ webhook_url }}" />
<input type="hidden"
name="redirect_to"
value="/webhook_entries?webhook_url={{ webhook_url|encode_url }}" />
<div class="col-12">
<label for="new_hook" class="form-label">Modify Webhook</label>
<input type="text"
name="new_hook"
id="new_hook"
class="form-control border text-muted bg-dark"
placeholder="Enter new webhook URL" />
</div>
<div class="col-12">
<button type="submit" class="btn btn-primary w-100">Save Webhook URL</button>
</div>
</form>
<form action="/delete_webhook" method="post">
<input type="hidden" name="webhook_url" value="{{ webhook_url }}" />
<button type="submit"
class="btn btn-danger w-100"
onclick="return confirm('Are you sure you want to delete this webhook?');">
Delete Webhook
</button>
</form>
<hr class="border-secondary my-3" />
<h3 class="h6">Mass update feed URLs</h3>
<p class="text-muted small mb-2">Replace part of feed URLs for all feeds attached to this webhook.</p>
<form action="/webhook_entries"
method="get"
class="row g-2 mb-2"
hx-get="/webhook_entries_mass_update_preview"
hx-target="#mass-update-preview"
hx-swap="innerHTML">
<input type="hidden" name="webhook_url" value="{{ webhook_url|encode_url }}" />
<div class="col-12">
<label for="replace_from" class="form-label small">Replace this</label>
<input type="text"
name="replace_from"
id="replace_from"
class="form-control border text-muted bg-dark"
value="{{ replace_from }}"
placeholder="https://old-domain.example" />
</div>
<div class="col-12">
<label for="replace_to" class="form-label small">With this</label>
<input type="text"
name="replace_to"
id="replace_to"
class="form-control border text-muted bg-dark"
value="{{ replace_to }}"
placeholder="https://new-domain.example" />
</div>
<div class="col-12 form-check ms-1">
<input class="form-check-input"
type="checkbox"
value="true"
id="resolve_urls"
name="resolve_urls"
{% if resolve_urls %}checked{% endif %} />
<label class="form-check-label small" for="resolve_urls">Resolve final URL with redirects</label>
</div>
<div class="col-12 form-check ms-1">
<input class="form-check-input"
type="checkbox"
value="true"
id="force_update"
name="force_update"
{% if force_update %}checked{% endif %} />
<label class="form-check-label small" for="force_update">Force update (overwrite conflicting target feed URLs)</label>
</div>
<div class="col-12">
<button type="submit" class="btn btn-outline-warning w-100">Preview changes</button>
</div>
</form>
<div id="mass-update-preview">{% include "_webhook_mass_update_preview.html" %}</div>
</div>
</div>
<div class="col-lg-7">
<div class="card border border-dark p-3 text-light h-100">
<h3 class="h5">Attached feeds</h3>
{% if webhook_feeds %}
<ul class="list-group list-unstyled mb-0">
{% for feed in webhook_feeds %}
<li class="mb-2">
<a class="text-muted" href="/feed?feed_url={{ feed.url|encode_url }}">
{% if feed.title %}
{{ feed.title }}
{% else %}
{{ feed.url }}
{% endif %}
</a>
{% if feed.title %}<span class="text-muted">- {{ feed.url }}</span>{% endif %}
{% if not feed.updates_enabled %}<span class="text-warning">Disabled</span>{% endif %}
{% if feed.last_exception %}<span class="text-danger">({{ feed.last_exception.value_str }})</span>{% endif %}
</li>
{% endfor %}
</ul>
{% else %}
<p class="text-muted mb-0">No feeds are attached to this webhook yet.</p>
{% endif %}
</div>
</div>
</div>
{# Rendered HTML content #}
{% if entries %}
<h3 class="h5 text-light">Latest entries</h3>
<pre>{{ html|safe }}</pre>
{% if is_show_more_entries_button_visible and last_entry %}
<a class="btn btn-dark mt-3"
href="/webhook_entries?webhook_url={{ webhook_url|encode_url }}&starting_after={{ last_entry.feed.url|encode_url }}|{{ last_entry.id|encode_url }}">
Show more entries
</a>
{% endif %}
{% elif feeds_count == 0 %}
<div>
<p>
No feeds found for {{ webhook_name }}. <a href="/add" class="alert-link">Add feeds</a> to this webhook to see entries here.
</p>
</div>
{% else %}
<div>
<p>
No entries found for {{ webhook_name }}. <a href="/settings" class="alert-link">Update feeds</a> to fetch new entries.
</p>
</div>
{% endif %}
{% endblock content %}

34
discord_rss_bot/templates/webhooks.html
View file

@ -1,9 +1,9 @@
{% extends "base.html" %}
{% block title %}
| Webhooks
| Webhooks
{% endblock title %}
{% block content %}
<div class="container my-4 text-light">
<div class="container my-4 text-light">
{% for hook in hooks_with_data %}
<div class="border border-dark mb-4 shadow-sm p-3">
<div class="text-muted">
@ -17,37 +17,29 @@
</li>
<li>
<strong>Webhook:</strong>
<a class="text-muted" href="{{ hook.url }}">{{ hook.url | replace('https://discord.com/api/webhooks', '') }}</a>
<a class="text-muted"
href="{{ hook.url }}">{{ hook.url | replace("https://discord.com/api/webhooks", "") }}</a>
</li>
</ul>
<hr />
<hr>
<form action="/modify_webhook" method="post" class="row g-3">
<input type="hidden" name="old_hook" value="{{ hook.url }}" />
<div class="col-md-8">
<label for="new_hook" class="form-label">Modify Webhook</label>
<input type="text"
name="new_hook"
id="new_hook"
class="form-control border text-muted bg-dark"
<input type="text" name="new_hook" id="new_hook" class="form-control border text-muted bg-dark"
placeholder="Enter new webhook URL" />
</div>
<div class="col-md-4 d-flex align-items-end">
<button type="submit" class="btn btn-primary w-100">Modify</button>
</div>
</form>
</div>
<div class="d-flex justify-content-between mt-2 gap-2">
<div>
<a href="/webhook_entries?webhook_url={{ hook.url|encode_url }}"
class="btn btn-info btn-sm">View Latest Entries</a>
</div>
<div class="d-flex justify-content-between mt-2">
<form action="/delete_webhook" method="post">
<input type="hidden" name="webhook_url" value="{{ hook.url }}" />
<button type="submit"
class="btn btn-danger"
onclick="return confirm('Are you sure you want to delete this webhook?');">
Delete
</button>
<button type="submit" class="btn btn-danger"
onclick="return confirm('Are you sure you want to delete this webhook?');">Delete</button>
</form>
</div>
</div>
@ -55,9 +47,9 @@
<div class="border border-dark p-3">
You can append <code>?thread_id=THREAD_ID</code> to the URL to send messages to a thread.
</div>
<br />
<br>
<div class="text-end">
<a class="btn btn-primary mb-3" href="/add_webhook">Add New Webhook</a>
</div>
</div>
{% endblock content %}
</div>
{% endblock content %}

45
discord_rss_bot/templates/whitelist.html
View file

@ -1,6 +1,6 @@
{% extends "base.html" %}
{% block title %}
| Whitelist
| Blacklist
{% endblock title %}
{% block content %}
<div class="p-2 border border-dark">
@ -42,49 +42,6 @@
<label for="whitelist_author" class="col-sm-6 col-form-label">Whitelist - Author</label>
<input name="whitelist_author" type="text" class="form-control bg-dark border-dark text-muted"
id="whitelist_author" value="{%- if whitelist_author -%} {{ whitelist_author }} {%- endif -%}" />
<div class="mt-4">
<div class="form-text">
<ul class="list-inline">
<li>
Regular expression patterns for advanced filtering. Each pattern should be on a new
line.
</li>
<li>Patterns are case-insensitive.</li>
<li>
Examples:
<code>
<pre>
^New Release:.*
\b(update|version|patch)\s+\d+\.\d+
.*\[(important|notice)\].*
</pre>
</code>
</li>
</ul>
</div>
<label for="regex_whitelist_title" class="col-sm-6 col-form-label">Regex Whitelist - Title</label>
<textarea name="regex_whitelist_title" class="form-control bg-dark border-dark text-muted"
id="regex_whitelist_title"
rows="3">{%- if regex_whitelist_title -%}{{ regex_whitelist_title }}{%- endif -%}</textarea>
<label for="regex_whitelist_summary" class="col-sm-6 col-form-label">Regex Whitelist -
Summary</label>
<textarea name="regex_whitelist_summary" class="form-control bg-dark border-dark text-muted"
id="regex_whitelist_summary"
rows="3">{%- if regex_whitelist_summary -%}{{ regex_whitelist_summary }}{%- endif -%}</textarea>
<label for="regex_whitelist_content" class="col-sm-6 col-form-label">Regex Whitelist -
Content</label>
<textarea name="regex_whitelist_content" class="form-control bg-dark border-dark text-muted"
id="regex_whitelist_content"
rows="3">{%- if regex_whitelist_content -%}{{ regex_whitelist_content }}{%- endif -%}</textarea>
<label for="regex_whitelist_author" class="col-sm-6 col-form-label">Regex Whitelist - Author</label>
<textarea name="regex_whitelist_author" class="form-control bg-dark border-dark text-muted"
id="regex_whitelist_author"
rows="3">{%- if regex_whitelist_author -%}{{ regex_whitelist_author }}{%- endif -%}</textarea>
</div>
</div>
</div>
<!-- Add a hidden feed_url field to the form -->

2
docker-compose.yml
View file

@ -10,7 +10,7 @@ services:
# - /Docker/Bots/discord-rss-bot:/home/botuser/.local/share/discord_rss_bot/
- data:/home/botuser/.local/share/discord_rss_bot/
healthcheck:
test: [ "CMD", "uv", "run", "./discord_rss_bot/healthcheck.py" ]
test: ["CMD", "python", "discord_rss_bot/healthcheck.py"]
interval: 1m
timeout: 10s
retries: 3

55
pyproject.toml
View file

@ -5,7 +5,7 @@ description = "RSS bot for Discord"
readme = "README.md"
requires-python = ">=3.12"
dependencies = [
"apscheduler>=3.11.0",
"apscheduler",
"discord-webhook",
"fastapi",
"httpx",
@ -17,30 +17,54 @@ dependencies = [
"python-multipart",
"reader",
"sentry-sdk[fastapi]",
"tldextract",
"uvicorn",
]
[dependency-groups]
dev = ["djlint", "pytest", "pytest-randomly", "pytest-xdist"]
dev = ["pytest"]
[tool.poetry]
name = "discord-rss-bot"
version = "1.0.0"
description = "RSS bot for Discord"
authors = ["Joakim Hellsén <tlovinator@gmail.com>"]
[tool.poetry.dependencies]
python = "^3.12"
apscheduler = "*"
discord-webhook = "*"
fastapi = "*"
httpx = "*"
jinja2 = "*"
lxml = "*"
markdownify = "*"
platformdirs = "*"
python-dotenv = "*"
python-multipart = "*"
reader = "*"
sentry-sdk = {version = "*", extras = ["fastapi"]}
uvicorn = "*"
[tool.poetry.group.dev.dependencies]
pytest = "*"
[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"
[tool.djlint]
ignore = "D004,D018,J018,T001,J004"
profile = "jinja"
max_line_length = 120
format_attribute_template_tags = true
[tool.ruff]
preview = true
unsafe-fixes = true
fix = true
line-length = 120
lint.select = ["ALL"]
lint.unfixable = ["F841"] # Don't automatically remove unused variables
lint.pydocstyle.convention = "google"
lint.isort.required-imports = ["from __future__ import annotations"]
lint.isort.force-single-line = true
lint.pycodestyle.ignore-overlong-task-comments = true
lint.ignore = [
"ANN201", # Checks that public functions and methods have return type annotations.
@ -62,8 +86,6 @@ lint.ignore = [
"PLR6301", # Checks for the presence of unused self parameter in methods definitions.
"RUF029", # Checks for functions declared async that do not await or otherwise use features requiring the function to be declared async.
"TD003", # Checks that a TODO comment is associated with a link to a relevant issue or ticket.
"PLR0913", # Checks for function definitions that include too many arguments.
"PLR0917", # Checks for function definitions that include too many positional arguments.
# Conflicting lint rules when using Ruff's formatter
# https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules
@ -86,8 +108,15 @@ lint.ignore = [
[tool.ruff.lint.per-file-ignores]
"tests/*" = ["S101", "D103", "PLR2004"]
[tool.ruff.lint.mccabe]
max-complexity = 15 # Don't judge lol
[tool.pytest.ini_options]
addopts = "-n 5 --dist loadfile"
python_files = ["test_*.py"]
log_cli = true
log_cli_level = "DEBUG"
log_cli_format = "%(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s)"
log_cli_date_format = "%Y-%m-%d %H:%M:%S"
filterwarnings = [
"ignore::bs4.GuessedAtParserWarning",
"ignore:functools\\.partial will be a method descriptor in future Python versions; wrap it in staticmethod\\(\\) if you want to preserve the old behavior:FutureWarning",

13
requirements.txt Normal file
View file

@ -0,0 +1,13 @@
apscheduler
discord-webhook
fastapi
httpx
jinja2
lxml
markdownify
platformdirs
python-dotenv
python-multipart
reader
sentry-sdk[fastapi]
uvicorn

68
tests/conftest.py
View file

@ -1,68 +0,0 @@
from __future__ import annotations
import os
import shutil
import sys
import tempfile
import warnings
from contextlib import suppress
from pathlib import Path
from typing import TYPE_CHECKING
from typing import Any
from bs4 import MarkupResemblesLocatorWarning
if TYPE_CHECKING:
import pytest
def pytest_addoption(parser: pytest.Parser) -> None:
"""Register custom command-line options for optional integration tests."""
parser.addoption(
"--run-real-git-backup-tests",
action="store_true",
default=False,
help="Run tests that push git backup state to a real repository.",
)
def pytest_sessionstart(session: pytest.Session) -> None:
"""Isolate persistent app state per xdist worker to avoid cross-worker test interference."""
worker_id: str = os.environ.get("PYTEST_XDIST_WORKER", "gw0")
worker_data_dir: Path = Path(tempfile.gettempdir()) / "discord-rss-bot-tests" / worker_id
# Start each worker from a clean state.
shutil.rmtree(worker_data_dir, ignore_errors=True)
worker_data_dir.mkdir(parents=True, exist_ok=True)
os.environ["DISCORD_RSS_BOT_DATA_DIR"] = str(worker_data_dir)
# Tests call markdownify which may invoke BeautifulSoup on strings that look
# like URLs; that triggers MarkupResemblesLocatorWarning from bs4. Silence
# that warning during tests to avoid noisy output.
warnings.filterwarnings("ignore", category=MarkupResemblesLocatorWarning)
# If modules were imported before this hook (unlikely), force them to use
# the worker-specific location.
settings_module: Any = sys.modules.get("discord_rss_bot.settings")
if settings_module is not None:
settings_module.data_dir = str(worker_data_dir)
get_reader: Any = getattr(settings_module, "get_reader", None)
if get_reader is not None and hasattr(get_reader, "cache_clear"):
get_reader.cache_clear()
main_module: Any = sys.modules.get("discord_rss_bot.main")
if main_module is not None and settings_module is not None:
with suppress(Exception):
current_reader = getattr(main_module, "reader", None)
if current_reader is not None:
current_reader.close()
get_reader: Any = getattr(settings_module, "get_reader", None)
if callable(get_reader):
get_reader()
def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
"""Skip real git-repo push tests unless explicitly requested."""
if config.getoption("--run-real-git-backup-tests"):
return

100
tests/test_blacklist.py
View file

@ -4,13 +4,9 @@ import tempfile
from pathlib import Path
from typing import TYPE_CHECKING
from reader import Entry
from reader import Feed
from reader import Reader
from reader import make_reader
from reader import Entry, Feed, Reader, make_reader
from discord_rss_bot.filter.blacklist import entry_should_be_skipped
from discord_rss_bot.filter.blacklist import feed_has_blacklist_tags
from discord_rss_bot.filter.blacklist import entry_should_be_skipped, feed_has_blacklist_tags
if TYPE_CHECKING:
from collections.abc import Iterable
@ -38,18 +34,11 @@ def test_has_black_tags() -> None:
# Test feed without any blacklist tags
assert_msg: str = "Feed should not have any blacklist tags"
assert feed_has_blacklist_tags(reader=get_reader(), feed=feed) is False, assert_msg
assert feed_has_blacklist_tags(custom_reader=get_reader(), feed=feed) is False, assert_msg
check_if_has_tag(reader, feed, "blacklist_title")
check_if_has_tag(reader, feed, "blacklist_summary")
check_if_has_tag(reader, feed, "blacklist_content")
check_if_has_tag(reader, feed, "blacklist_author")
# Test regex blacklist tags
check_if_has_tag(reader, feed, "regex_blacklist_title")
check_if_has_tag(reader, feed, "regex_blacklist_summary")
check_if_has_tag(reader, feed, "regex_blacklist_content")
check_if_has_tag(reader, feed, "regex_blacklist_author")
# Clean up
reader.delete_feed(feed_url)
@ -58,11 +47,11 @@ def test_has_black_tags() -> None:
def check_if_has_tag(reader: Reader, feed: Feed, blacklist_name: str) -> None:
reader.set_tag(feed, blacklist_name, "a") # pyright: ignore[reportArgumentType]
assert_msg: str = f"Feed should have blacklist tags: {blacklist_name}"
assert feed_has_blacklist_tags(reader=reader, feed=feed) is True, assert_msg
assert feed_has_blacklist_tags(custom_reader=reader, feed=feed) is True, assert_msg
asset_msg: str = f"Feed should not have any blacklist tags: {blacklist_name}"
reader.delete_tag(feed, blacklist_name)
assert feed_has_blacklist_tags(reader=reader, feed=feed) is False, asset_msg
assert feed_has_blacklist_tags(custom_reader=reader, feed=feed) is False, asset_msg
def test_should_be_skipped() -> None:
@ -85,7 +74,6 @@ def test_should_be_skipped() -> None:
# Test entry without any blacklists
assert entry_should_be_skipped(reader, first_entry[0]) is False, f"Entry should not be skipped: {first_entry[0]}"
# Test standard blacklist functionality
reader.set_tag(feed, "blacklist_title", "fvnnnfnfdnfdnfd") # pyright: ignore[reportArgumentType]
assert entry_should_be_skipped(reader, first_entry[0]) is True, f"Entry should be skipped: {first_entry[0]}"
reader.delete_tag(feed, "blacklist_title")
@ -125,81 +113,3 @@ def test_should_be_skipped() -> None:
assert entry_should_be_skipped(reader, first_entry[0]) is False, f"Entry should not be skipped: {first_entry[0]}"
reader.delete_tag(feed, "blacklist_author")
assert entry_should_be_skipped(reader, first_entry[0]) is False, f"Entry should not be skipped: {first_entry[0]}"
def test_regex_should_be_skipped() -> None:
"""Test the regex filtering functionality for blacklist."""
reader: Reader = get_reader()
# Add feed and update entries
reader.add_feed(feed_url)
feed: Feed = reader.get_feed(feed_url)
reader.update_feeds()
# Get first entry
first_entry: list[Entry] = []
entries: Iterable[Entry] = reader.get_entries(feed=feed)
assert entries is not None, f"Entries should not be None: {entries}"
for entry in entries:
first_entry.append(entry)
break
assert len(first_entry) == 1, f"First entry should be added: {first_entry}"
# Test entry without any regex blacklists
assert entry_should_be_skipped(reader, first_entry[0]) is False, f"Entry should not be skipped: {first_entry[0]}"
# Test regex blacklist for title
reader.set_tag(feed, "regex_blacklist_title", r"fvnnn\w+") # pyright: ignore[reportArgumentType]
assert entry_should_be_skipped(reader, first_entry[0]) is True, (
f"Entry should be skipped with regex title match: {first_entry[0]}"
)
reader.delete_tag(feed, "regex_blacklist_title")
assert entry_should_be_skipped(reader, first_entry[0]) is False, f"Entry should not be skipped: {first_entry[0]}"
# Test regex blacklist for summary
reader.set_tag(feed, "regex_blacklist_summary", r"ffdnfdn\w+") # pyright: ignore[reportArgumentType]
assert entry_should_be_skipped(reader, first_entry[0]) is True, (
f"Entry should be skipped with regex summary match: {first_entry[0]}"
)
reader.delete_tag(feed, "regex_blacklist_summary")
assert entry_should_be_skipped(reader, first_entry[0]) is False, f"Entry should not be skipped: {first_entry[0]}"
# Test regex blacklist for content
reader.set_tag(feed, "regex_blacklist_content", r"ffdnfdnfdn\w+") # pyright: ignore[reportArgumentType]
assert entry_should_be_skipped(reader, first_entry[0]) is True, (
f"Entry should be skipped with regex content match: {first_entry[0]}"
)
reader.delete_tag(feed, "regex_blacklist_content")
assert entry_should_be_skipped(reader, first_entry[0]) is False, f"Entry should not be skipped: {first_entry[0]}"
# Test regex blacklist for author
reader.set_tag(feed, "regex_blacklist_author", r"TheLovinator\d*") # pyright: ignore[reportArgumentType]
assert entry_should_be_skipped(reader, first_entry[0]) is True, (
f"Entry should be skipped with regex author match: {first_entry[0]}"
)
reader.delete_tag(feed, "regex_blacklist_author")
assert entry_should_be_skipped(reader, first_entry[0]) is False, f"Entry should not be skipped: {first_entry[0]}"
# Test invalid regex pattern (should not raise an exception)
reader.set_tag(feed, "regex_blacklist_title", r"[incomplete") # pyright: ignore[reportArgumentType]
assert entry_should_be_skipped(reader, first_entry[0]) is False, (
f"Entry should not be skipped with invalid regex: {first_entry[0]}"
)
reader.delete_tag(feed, "regex_blacklist_title")
# Test multiple regex patterns separated by commas
reader.set_tag(feed, "regex_blacklist_author", r"pattern1,TheLovinator\d*,pattern3") # pyright: ignore[reportArgumentType]
assert entry_should_be_skipped(reader, first_entry[0]) is True, (
f"Entry should be skipped with one matching pattern in list: {first_entry[0]}"
)
reader.delete_tag(feed, "regex_blacklist_author")
assert entry_should_be_skipped(reader, first_entry[0]) is False, f"Entry should not be skipped: {first_entry[0]}"
# Test newline-separated regex patterns
newline_patterns = "pattern1\nTheLovinator\\d*\npattern3"
reader.set_tag(feed, "regex_blacklist_author", newline_patterns) # pyright: ignore[reportArgumentType]
assert entry_should_be_skipped(reader, first_entry[0]) is True, (
f"Entry should be skipped with newline-separated patterns: {first_entry[0]}"
)
reader.delete_tag(feed, "regex_blacklist_author")
assert entry_should_be_skipped(reader, first_entry[0]) is False, f"Entry should not be skipped: {first_entry[0]}"

68
tests/test_custom_filter.py
View file

@ -5,9 +5,7 @@ import tempfile
from pathlib import Path
from typing import TYPE_CHECKING
from discord_rss_bot.custom_filters import encode_url
from discord_rss_bot.custom_filters import entry_is_blacklisted
from discord_rss_bot.custom_filters import entry_is_whitelisted
from discord_rss_bot.custom_filters import encode_url, entry_is_blacklisted, entry_is_whitelisted
from discord_rss_bot.settings import get_reader
if TYPE_CHECKING:
@ -45,39 +43,39 @@ def test_entry_is_whitelisted() -> None:
Path.mkdir(Path(temp_dir), exist_ok=True)
custom_loc: pathlib.Path = pathlib.Path(temp_dir, "custom_loc_db.sqlite")
reader: Reader = get_reader(custom_location=str(custom_loc))
custom_reader: Reader = get_reader(custom_location=str(custom_loc))
# Add a feed to the database.
reader.add_feed("https://lovinator.space/rss_test.xml")
reader.update_feed("https://lovinator.space/rss_test.xml")
custom_reader.add_feed("https://lovinator.space/rss_test.xml")
custom_reader.update_feed("https://lovinator.space/rss_test.xml")
# whitelist_title
reader.set_tag("https://lovinator.space/rss_test.xml", "whitelist_title", "fvnnnfnfdnfdnfd") # pyright: ignore[reportArgumentType]
for entry in reader.get_entries():
if entry_is_whitelisted(entry, reader=reader) is True:
custom_reader.set_tag("https://lovinator.space/rss_test.xml", "whitelist_title", "fvnnnfnfdnfdnfd") # pyright: ignore[reportArgumentType]
for entry in custom_reader.get_entries():
if entry_is_whitelisted(entry) is True:
assert entry.title == "fvnnnfnfdnfdnfd", f"Expected: fvnnnfnfdnfdnfd, Got: {entry.title}"
break
reader.delete_tag("https://lovinator.space/rss_test.xml", "whitelist_title")
custom_reader.delete_tag("https://lovinator.space/rss_test.xml", "whitelist_title")
# whitelist_summary
reader.set_tag("https://lovinator.space/rss_test.xml", "whitelist_summary", "fvnnnfnfdnfdnfd") # pyright: ignore[reportArgumentType]
for entry in reader.get_entries():
if entry_is_whitelisted(entry, reader=reader) is True:
custom_reader.set_tag("https://lovinator.space/rss_test.xml", "whitelist_summary", "fvnnnfnfdnfdnfd") # pyright: ignore[reportArgumentType]
for entry in custom_reader.get_entries():
if entry_is_whitelisted(entry) is True:
assert entry.summary == "fvnnnfnfdnfdnfd", f"Expected: fvnnnfnfdnfdnfd, Got: {entry.summary}"
break
reader.delete_tag("https://lovinator.space/rss_test.xml", "whitelist_summary")
custom_reader.delete_tag("https://lovinator.space/rss_test.xml", "whitelist_summary")
# whitelist_content
reader.set_tag("https://lovinator.space/rss_test.xml", "whitelist_content", "fvnnnfnfdnfdnfd") # pyright: ignore[reportArgumentType]
for entry in reader.get_entries():
if entry_is_whitelisted(entry, reader=reader) is True:
custom_reader.set_tag("https://lovinator.space/rss_test.xml", "whitelist_content", "fvnnnfnfdnfdnfd") # pyright: ignore[reportArgumentType]
for entry in custom_reader.get_entries():
if entry_is_whitelisted(entry) is True:
assert_msg = f"Expected: <p>ffdnfdnfdnfdnfdndfn</p>, Got: {entry.content[0].value}"
assert entry.content[0].value == "<p>ffdnfdnfdnfdnfdndfn</p>", assert_msg
break
reader.delete_tag("https://lovinator.space/rss_test.xml", "whitelist_content")
custom_reader.delete_tag("https://lovinator.space/rss_test.xml", "whitelist_content")
# Close the reader, so we can delete the directory.
reader.close()
custom_reader.close()
def test_entry_is_blacklisted() -> None:
@ -87,36 +85,36 @@ def test_entry_is_blacklisted() -> None:
Path.mkdir(Path(temp_dir), exist_ok=True)
custom_loc: pathlib.Path = pathlib.Path(temp_dir, "custom_loc_db.sqlite")
reader: Reader = get_reader(custom_location=str(custom_loc))
custom_reader: Reader = get_reader(custom_location=str(custom_loc))
# Add a feed to the database.
reader.add_feed("https://lovinator.space/rss_test.xml")
reader.update_feed("https://lovinator.space/rss_test.xml")
custom_reader.add_feed("https://lovinator.space/rss_test.xml")
custom_reader.update_feed("https://lovinator.space/rss_test.xml")
# blacklist_title
reader.set_tag("https://lovinator.space/rss_test.xml", "blacklist_title", "fvnnnfnfdnfdnfd") # pyright: ignore[reportArgumentType]
for entry in reader.get_entries():
if entry_is_blacklisted(entry, reader=reader) is True:
custom_reader.set_tag("https://lovinator.space/rss_test.xml", "blacklist_title", "fvnnnfnfdnfdnfd") # pyright: ignore[reportArgumentType]
for entry in custom_reader.get_entries():
if entry_is_blacklisted(entry) is True:
assert entry.title == "fvnnnfnfdnfdnfd", f"Expected: fvnnnfnfdnfdnfd, Got: {entry.title}"
break
reader.delete_tag("https://lovinator.space/rss_test.xml", "blacklist_title")
custom_reader.delete_tag("https://lovinator.space/rss_test.xml", "blacklist_title")
# blacklist_summary
reader.set_tag("https://lovinator.space/rss_test.xml", "blacklist_summary", "fvnnnfnfdnfdnfd") # pyright: ignore[reportArgumentType]
for entry in reader.get_entries():
if entry_is_blacklisted(entry, reader=reader) is True:
custom_reader.set_tag("https://lovinator.space/rss_test.xml", "blacklist_summary", "fvnnnfnfdnfdnfd") # pyright: ignore[reportArgumentType]
for entry in custom_reader.get_entries():
if entry_is_blacklisted(entry) is True:
assert entry.summary == "fvnnnfnfdnfdnfd", f"Expected: fvnnnfnfdnfdnfd, Got: {entry.summary}"
break
reader.delete_tag("https://lovinator.space/rss_test.xml", "blacklist_summary")
custom_reader.delete_tag("https://lovinator.space/rss_test.xml", "blacklist_summary")
# blacklist_content
reader.set_tag("https://lovinator.space/rss_test.xml", "blacklist_content", "fvnnnfnfdnfdnfd") # pyright: ignore[reportArgumentType]
for entry in reader.get_entries():
if entry_is_blacklisted(entry, reader=reader) is True:
custom_reader.set_tag("https://lovinator.space/rss_test.xml", "blacklist_content", "fvnnnfnfdnfdnfd") # pyright: ignore[reportArgumentType]
for entry in custom_reader.get_entries():
if entry_is_blacklisted(entry) is True:
assert_msg = f"Expected: <p>ffdnfdnfdnfdnfdndfn</p>, Got: {entry.content[0].value}"
assert entry.content[0].value == "<p>ffdnfdnfdnfdnfdndfn</p>", assert_msg
break
reader.delete_tag("https://lovinator.space/rss_test.xml", "blacklist_content")
custom_reader.delete_tag("https://lovinator.space/rss_test.xml", "blacklist_content")
# Close the reader, so we can delete the directory.
reader.close()
custom_reader.close()

140
tests/test_custom_message.py
View file

@ -1,140 +0,0 @@
from __future__ import annotations
import typing
from types import SimpleNamespace
from unittest.mock import MagicMock
from unittest.mock import patch
import pytest
from discord_rss_bot.custom_message import CustomEmbed
from discord_rss_bot.custom_message import format_entry_html_for_discord
from discord_rss_bot.custom_message import replace_tags_in_embed
from discord_rss_bot.custom_message import replace_tags_in_text_message
if typing.TYPE_CHECKING:
from reader import Entry
# https://docs.discord.com/developers/reference#message-formatting
TIMESTAMP_FORMATS: tuple[str, ...] = (
"<t:1773461490>",
"<t:1773461490:F>",
"<t:1773461490:f>",
"<t:1773461490:D>",
"<t:1773461490:d>",
"<t:1773461490:t>",
"<t:1773461490:T>",
"<t:1773461490:R>",
"<t:1773461490:s>",
"<t:1773461490:S>",
)
def make_feed() -> SimpleNamespace:
return SimpleNamespace(
added=None,
author="Feed Author",
last_exception=None,
last_updated=None,
link="https://example.com/feed",
subtitle="",
title="Example Feed",
updated=None,
updates_enabled=True,
url="https://example.com/feed.xml",
user_title="",
version="atom10",
)
def make_entry(summary: str) -> SimpleNamespace:
feed: SimpleNamespace = make_feed()
return SimpleNamespace(
added=None,
author="Entry Author",
content=[],
feed=feed,
feed_url=feed.url,
id="entry-1",
important=False,
link="https://example.com/entry-1",
published=None,
read=False,
read_modified=None,
summary=summary,
title="Entry Title",
updated=None,
)
@pytest.mark.parametrize("timestamp_tag", TIMESTAMP_FORMATS)
def test_format_entry_html_for_discord_preserves_timestamp_tags(timestamp_tag: str) -> None:
escaped_timestamp_tag: str = timestamp_tag.replace("<", "&lt;").replace(">", "&gt;")
html_summary: str = f"<p>Starts: 2026-03-13 23:30 UTC ({escaped_timestamp_tag})</p>"
rendered: str = format_entry_html_for_discord(html_summary)
assert timestamp_tag in rendered
assert "DISCORDTIMESTAMPPLACEHOLDER" not in rendered
def test_format_entry_html_for_discord_empty_text_returns_empty_string() -> None:
rendered: str = format_entry_html_for_discord("")
assert not rendered
def test_format_entry_html_for_discord_cleans_markdownified_https_link_text() -> None:
html_summary: str = "[https://example.com](https://example.com)"
rendered: str = format_entry_html_for_discord(html_summary)
assert "[example.com](https://example.com)" in rendered
assert "[https://example.com]" not in rendered
def test_format_entry_html_for_discord_does_not_preserve_invalid_timestamp_style() -> None:
invalid_timestamp: str = "<t:1773461490:Z>"
html_summary: str = f"<p>Invalid style ({invalid_timestamp.replace('<', '&lt;').replace('>', '&gt;')})</p>"
rendered: str = format_entry_html_for_discord(html_summary)
assert invalid_timestamp not in rendered
@patch("discord_rss_bot.custom_message.get_custom_message")
def test_replace_tags_in_text_message_preserves_timestamp_tags(
mock_get_custom_message: MagicMock,
) -> None:
mock_reader = MagicMock()
mock_get_custom_message.return_value = "{{entry_summary}}"
summary_parts: list[str] = [
f"<p>Format {index}: ({timestamp_tag.replace('<', '&lt;').replace('>', '&gt;')})</p>"
for index, timestamp_tag in enumerate(TIMESTAMP_FORMATS, start=1)
]
entry_ns: SimpleNamespace = make_entry("".join(summary_parts))
entry: Entry = typing.cast("Entry", entry_ns)
rendered: str = replace_tags_in_text_message(entry, reader=mock_reader)
for timestamp_tag in TIMESTAMP_FORMATS:
assert timestamp_tag in rendered
@patch("discord_rss_bot.custom_message.get_embed")
def test_replace_tags_in_embed_preserves_timestamp_tags(
mock_get_embed: MagicMock,
) -> None:
mock_reader = MagicMock()
mock_get_embed.return_value = CustomEmbed(description="{{entry_summary}}")
summary_parts: list[str] = [
f"<p>Format {index}: ({timestamp_tag.replace('<', '&lt;').replace('>', '&gt;')})</p>"
for index, timestamp_tag in enumerate(TIMESTAMP_FORMATS, start=1)
]
entry_ns: SimpleNamespace = make_entry("".join(summary_parts))
entry: Entry = typing.cast("Entry", entry_ns)
embed: CustomEmbed = replace_tags_in_embed(entry_ns.feed, entry, reader=mock_reader)
for timestamp_tag in TIMESTAMP_FORMATS:
assert timestamp_tag in embed.description

201
tests/test_feeds.py
View file

@ -4,20 +4,12 @@ import os
import tempfile
from pathlib import Path
from typing import LiteralString
from unittest.mock import MagicMock
from unittest.mock import patch
import pytest
from reader import Feed
from reader import Reader
from reader import make_reader
from reader import Feed, Reader, make_reader
from discord_rss_bot.feeds import extract_domain
from discord_rss_bot.feeds import is_youtube_feed
from discord_rss_bot.feeds import send_entry_to_discord
from discord_rss_bot.feeds import send_to_discord
from discord_rss_bot.feeds import should_send_embed_check
from discord_rss_bot.feeds import truncate_webhook_message
from discord_rss_bot.feeds import send_to_discord, truncate_webhook_message
from discord_rss_bot.missing_tags import add_missing_tags
def test_send_to_discord() -> None:
@ -34,6 +26,8 @@ def test_send_to_discord() -> None:
# Add a feed to the reader.
reader.add_feed("https://www.reddit.com/r/Python/.rss")
add_missing_tags(reader)
# Update the feed to get the entries.
reader.update_feeds()
@ -55,7 +49,7 @@ def test_send_to_discord() -> None:
assert reader.get_tag(feed, "webhook") == webhook_url, f"The webhook URL should be '{webhook_url}'."
# Send the feed to Discord.
send_to_discord(reader=reader, feed=feed, do_once=True)
send_to_discord(custom_reader=reader, feed=feed, do_once=True)
# Close the reader, so we can delete the directory.
reader.close()
@ -91,186 +85,3 @@ def test_truncate_webhook_message_long_message():
# Test the end of the message
assert_msg = "The end of the truncated message should be '...' to indicate truncation."
assert truncated_message[-half_length:] == "A" * half_length, assert_msg
def test_is_youtube_feed():
"""Test the is_youtube_feed function."""
# YouTube feed URLs
assert is_youtube_feed("https://www.youtube.com/feeds/videos.xml?channel_id=123456") is True
assert is_youtube_feed("https://www.youtube.com/feeds/videos.xml?user=username") is True
# Non-YouTube feed URLs
assert is_youtube_feed("https://www.example.com/feed.xml") is False
assert is_youtube_feed("https://www.youtube.com/watch?v=123456") is False
assert is_youtube_feed("https://www.reddit.com/r/Python/.rss") is False
@patch("discord_rss_bot.feeds.logger")
def test_should_send_embed_check_youtube_feeds(mock_logger: MagicMock) -> None:
"""Test should_send_embed_check returns False for YouTube feeds regardless of settings."""
# Create mocks
mock_reader = MagicMock()
mock_entry = MagicMock()
# Configure a YouTube feed
mock_entry.feed.url = "https://www.youtube.com/feeds/videos.xml?channel_id=123456"
# Set reader to return True for should_send_embed (would normally create an embed)
mock_reader.get_tag.return_value = True
# Result should be False, overriding the feed settings
result = should_send_embed_check(mock_reader, mock_entry)
assert result is False, "YouTube feeds should never use embeds"
# Function should not even call get_tag for YouTube feeds
mock_reader.get_tag.assert_not_called()
@patch("discord_rss_bot.feeds.logger")
def test_should_send_embed_check_normal_feeds(mock_logger: MagicMock) -> None:
"""Test should_send_embed_check returns feed settings for non-YouTube feeds."""
# Create mocks
mock_reader = MagicMock()
mock_entry = MagicMock()
# Configure a normal feed
mock_entry.feed.url = "https://www.example.com/feed.xml"
# Test with should_send_embed set to True
mock_reader.get_tag.return_value = True
result = should_send_embed_check(mock_reader, mock_entry)
assert result is True, "Normal feeds should use embeds when enabled"
# Test with should_send_embed set to False
mock_reader.get_tag.return_value = False
result = should_send_embed_check(mock_reader, mock_entry)
assert result is False, "Normal feeds should not use embeds when disabled"
@patch("discord_rss_bot.feeds.get_reader")
@patch("discord_rss_bot.feeds.get_custom_message")
@patch("discord_rss_bot.feeds.replace_tags_in_text_message")
@patch("discord_rss_bot.feeds.create_embed_webhook")
@patch("discord_rss_bot.feeds.DiscordWebhook")
@patch("discord_rss_bot.feeds.execute_webhook")
def test_send_entry_to_discord_youtube_feed(
mock_execute_webhook: MagicMock,
mock_discord_webhook: MagicMock,
mock_create_embed: MagicMock,
mock_replace_tags: MagicMock,
mock_get_custom_message: MagicMock,
mock_get_reader: MagicMock,
):
"""Test send_entry_to_discord function with YouTube feeds."""
# Set up mocks
mock_reader = MagicMock()
mock_get_reader.return_value = mock_reader
mock_entry = MagicMock()
mock_feed = MagicMock()
# Configure a YouTube feed
mock_entry.feed = mock_feed
mock_entry.feed.url = "https://www.youtube.com/feeds/videos.xml?channel_id=123456"
mock_entry.feed_url = "https://www.youtube.com/feeds/videos.xml?channel_id=123456"
# Mock the tags
mock_reader.get_tag.side_effect = lambda feed, tag, default=None: { # noqa: ARG005
"webhook": "https://discord.com/api/webhooks/123/abc",
"should_send_embed": True, # This should be ignored for YouTube feeds
}.get(tag, default)
# Mock custom message
mock_get_custom_message.return_value = "Custom message"
mock_replace_tags.return_value = "Formatted message with {{entry_link}}"
# Mock webhook
mock_webhook = MagicMock()
mock_discord_webhook.return_value = mock_webhook
# Call the function
send_entry_to_discord(mock_entry, mock_reader)
# Assertions
mock_create_embed.assert_not_called()
mock_discord_webhook.assert_called_once()
# Check webhook was created with the right message
webhook_call_kwargs = mock_discord_webhook.call_args[1]
assert "content" in webhook_call_kwargs, "Webhook should have content"
assert webhook_call_kwargs["url"] == "https://discord.com/api/webhooks/123/abc"
# Verify execute_webhook was called
mock_execute_webhook.assert_called_once_with(mock_webhook, mock_entry, reader=mock_reader)
def test_extract_domain_youtube_feed() -> None:
"""Test extract_domain for YouTube feeds."""
url: str = "https://www.youtube.com/feeds/videos.xml?channel_id=123456"
assert extract_domain(url) == "YouTube", "YouTube feeds should return 'YouTube' as the domain."
def test_extract_domain_reddit_feed() -> None:
"""Test extract_domain for Reddit feeds."""
url: str = "https://www.reddit.com/r/Python/.rss"
assert extract_domain(url) == "Reddit", "Reddit feeds should return 'Reddit' as the domain."
def test_extract_domain_github_feed() -> None:
"""Test extract_domain for GitHub feeds."""
url: str = "https://www.github.com/user/repo"
assert extract_domain(url) == "GitHub", "GitHub feeds should return 'GitHub' as the domain."
def test_extract_domain_custom_domain() -> None:
"""Test extract_domain for custom domains."""
url: str = "https://www.example.com/feed"
assert extract_domain(url) == "Example", "Custom domains should return the capitalized first part of the domain."
def test_extract_domain_no_www_prefix() -> None:
"""Test extract_domain removes 'www.' prefix."""
url: str = "https://www.example.com/feed"
assert extract_domain(url) == "Example", "The 'www.' prefix should be removed from the domain."
def test_extract_domain_no_tld() -> None:
"""Test extract_domain for domains without a TLD."""
url: str = "https://localhost/feed"
assert extract_domain(url) == "Localhost", "Domains without a TLD should return the capitalized domain."
def test_extract_domain_invalid_url() -> None:
"""Test extract_domain for invalid URLs."""
url: str = "not-a-valid-url"
assert extract_domain(url) == "Other", "Invalid URLs should return 'Other' as the domain."
def test_extract_domain_empty_url() -> None:
"""Test extract_domain for empty URLs."""
url: str = ""
assert extract_domain(url) == "Other", "Empty URLs should return 'Other' as the domain."
def test_extract_domain_special_characters() -> None:
"""Test extract_domain for URLs with special characters."""
url: str = "https://www.ex-ample.com/feed"
assert extract_domain(url) == "Ex-ample", "Domains with special characters should return the capitalized domain."
@pytest.mark.parametrize(
argnames=("url", "expected"),
argvalues=[
("https://blog.something.com", "Something"),
("https://www.something.com", "Something"),
("https://subdomain.example.co.uk", "Example"),
("https://github.com/user/repo", "GitHub"),
("https://youtube.com/feeds/videos.xml?channel_id=abc", "YouTube"),
("https://reddit.com/r/python/.rss", "Reddit"),
("", "Other"),
("not a url", "Other"),
("https://www.example.com", "Example"),
("https://foo.bar.baz.com", "Baz"),
],
)
def test_extract_domain(url: str, expected: str) -> None:
assert extract_domain(url) == expected

475
tests/test_git_backup.py
View file

@ -1,475 +0,0 @@
from __future__ import annotations
import contextlib
import json
import shutil
import subprocess # noqa: S404
from pathlib import Path
from typing import TYPE_CHECKING
from typing import Any
from unittest.mock import MagicMock
from unittest.mock import patch
import pytest
from fastapi.testclient import TestClient
from discord_rss_bot.git_backup import commit_state_change
from discord_rss_bot.git_backup import export_state
from discord_rss_bot.git_backup import get_backup_path
from discord_rss_bot.git_backup import get_backup_remote
from discord_rss_bot.git_backup import setup_backup_repo
from discord_rss_bot.main import app
if TYPE_CHECKING:
from pathlib import Path
SKIP_IF_NO_GIT: pytest.MarkDecorator = pytest.mark.skipif(
shutil.which("git") is None,
reason="git executable not found",
)
def test_get_backup_path_unset(monkeypatch: pytest.MonkeyPatch) -> None:
"""get_backup_path returns None when GIT_BACKUP_PATH is not set."""
monkeypatch.delenv("GIT_BACKUP_PATH", raising=False)
assert get_backup_path() is None
def test_get_backup_path_set(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
"""get_backup_path returns a Path when GIT_BACKUP_PATH is set."""
monkeypatch.setenv("GIT_BACKUP_PATH", str(tmp_path))
result: Path | None = get_backup_path()
assert result == tmp_path
def test_get_backup_path_strips_whitespace(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
"""get_backup_path strips surrounding whitespace from the env var value."""
monkeypatch.setenv("GIT_BACKUP_PATH", f" {tmp_path} ")
result: Path | None = get_backup_path()
assert result == tmp_path
def test_get_backup_remote_unset(monkeypatch: pytest.MonkeyPatch) -> None:
"""get_backup_remote returns empty string when GIT_BACKUP_REMOTE is not set."""
monkeypatch.delenv("GIT_BACKUP_REMOTE", raising=False)
assert not get_backup_remote()
def test_get_backup_remote_set(monkeypatch: pytest.MonkeyPatch) -> None:
"""get_backup_remote returns the configured remote URL."""
monkeypatch.setenv("GIT_BACKUP_REMOTE", "git@github.com:user/repo.git")
assert get_backup_remote() == "git@github.com:user/repo.git"
@SKIP_IF_NO_GIT
def test_setup_backup_repo_creates_git_repo(tmp_path: Path) -> None:
"""setup_backup_repo initialises a git repo in a fresh directory."""
backup_path: Path = tmp_path / "backup"
result: bool = setup_backup_repo(backup_path)
assert result is True
assert (backup_path / ".git").exists()
@SKIP_IF_NO_GIT
def test_setup_backup_repo_idempotent(tmp_path: Path) -> None:
"""setup_backup_repo does not fail when called on an existing repo."""
backup_path: Path = tmp_path / "backup"
assert setup_backup_repo(backup_path) is True
assert setup_backup_repo(backup_path) is True
def test_setup_backup_repo_adds_origin_remote(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
"""setup_backup_repo adds remote 'origin' when GIT_BACKUP_REMOTE is set."""
backup_path: Path = tmp_path / "backup"
monkeypatch.setenv("GIT_BACKUP_REMOTE", "git@github.com:user/private.git")
with patch("discord_rss_bot.git_backup.subprocess.run") as mock_run:
# git config --local queries fail initially so setup writes defaults.
mock_run.side_effect = [
MagicMock(returncode=0), # git init
MagicMock(returncode=1), # config user.email read
MagicMock(returncode=0), # config user.email write
MagicMock(returncode=1), # config user.name read
MagicMock(returncode=0), # config user.name write
MagicMock(returncode=1), # remote get-url origin (missing)
MagicMock(returncode=0), # remote add origin <url>
]
assert setup_backup_repo(backup_path) is True
called_commands: list[list[str]] = [call.args[0] for call in mock_run.call_args_list]
assert ["remote", "add", "origin", "git@github.com:user/private.git"] in [
cmd[-4:] for cmd in called_commands if len(cmd) >= 4
]
def test_setup_backup_repo_updates_origin_remote(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
"""setup_backup_repo updates existing origin when URL differs."""
backup_path: Path = tmp_path / "backup"
monkeypatch.setenv("GIT_BACKUP_REMOTE", "git@github.com:user/new-private.git")
with patch("discord_rss_bot.git_backup.subprocess.run") as mock_run:
# Existing repo path: no git init call.
(backup_path / ".git").mkdir(parents=True)
mock_run.side_effect = [
MagicMock(returncode=0), # config user.email read
MagicMock(returncode=0), # config user.name read
MagicMock(returncode=0, stdout=b"git@github.com:user/old-private.git\n"), # remote get-url origin
MagicMock(returncode=0), # remote set-url origin <new>
]
assert setup_backup_repo(backup_path) is True
called_commands: list[list[str]] = [call.args[0] for call in mock_run.call_args_list]
assert ["remote", "set-url", "origin", "git@github.com:user/new-private.git"] in [
cmd[-4:] for cmd in called_commands if len(cmd) >= 4
]
def test_export_state_creates_state_json(tmp_path: Path) -> None:
"""export_state writes a valid state.json to the backup directory."""
mock_reader = MagicMock()
# Feeds
feed1 = MagicMock()
feed1.url = "https://example.com/feed.rss"
mock_reader.get_feeds.return_value = [feed1]
# Tag values: webhook present, everything else absent (returns None)
def get_tag_side_effect(
feed_or_key: tuple | str,
tag: str | None = None,
default: str | None = None,
) -> list[Any] | str | None:
if feed_or_key == () and tag is None:
# Called for global webhooks list
return []
if tag == "webhook":
return "https://discord.com/api/webhooks/123/abc"
return default
mock_reader.get_tag.side_effect = get_tag_side_effect
backup_path: Path = tmp_path / "backup"
backup_path.mkdir()
export_state(mock_reader, backup_path)
state_file: Path = backup_path / "state.json"
assert state_file.exists(), "state.json should be created by export_state"
data: dict[str, Any] = json.loads(state_file.read_text(encoding="utf-8"))
assert "feeds" in data
assert "webhooks" in data
assert data["feeds"][0]["url"] == "https://example.com/feed.rss"
assert data["feeds"][0]["webhook"] == "https://discord.com/api/webhooks/123/abc"
def test_export_state_omits_empty_tags(tmp_path: Path) -> None:
"""export_state does not include tags with empty-string or None values."""
mock_reader = MagicMock()
feed1 = MagicMock()
feed1.url = "https://example.com/feed.rss"
mock_reader.get_feeds.return_value = [feed1]
def get_tag_side_effect(
feed_or_key: tuple | str,
tag: str | None = None,
default: str | None = None,
) -> list[Any] | str | None:
if feed_or_key == ():
return []
# Return empty string for all tags
return default # default is None
mock_reader.get_tag.side_effect = get_tag_side_effect
backup_path: Path = tmp_path / "backup"
backup_path.mkdir()
export_state(mock_reader, backup_path)
data: dict[str, Any] = json.loads((backup_path / "state.json").read_text())
# Only "url" key should be present (no empty-value tags)
assert list(data["feeds"][0].keys()) == ["url"]
def test_commit_state_change_noop_when_not_configured(monkeypatch: pytest.MonkeyPatch) -> None:
"""commit_state_change does nothing when GIT_BACKUP_PATH is not set."""
monkeypatch.delenv("GIT_BACKUP_PATH", raising=False)
mock_reader = MagicMock()
# Should not raise and should not call reader methods for export
commit_state_change(mock_reader, "Add feed example.com/rss")
mock_reader.get_feeds.assert_not_called()
@SKIP_IF_NO_GIT
def test_commit_state_change_commits(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
"""commit_state_change creates a commit in the backup repo."""
backup_path: Path = tmp_path / "backup"
monkeypatch.setenv("GIT_BACKUP_PATH", str(backup_path))
monkeypatch.delenv("GIT_BACKUP_REMOTE", raising=False)
mock_reader = MagicMock()
mock_reader.get_feeds.return_value = []
mock_reader.get_tag.return_value = []
commit_state_change(mock_reader, "Add feed https://example.com/rss")
# Verify a commit was created in the backup repo
git_executable: str | None = shutil.which("git")
assert git_executable is not None, "git executable not found"
result: subprocess.CompletedProcess[str] = subprocess.run( # noqa: S603
[git_executable, "-C", str(backup_path), "log", "--oneline"],
capture_output=True,
text=True,
check=False,
)
assert result.returncode == 0
assert "Add feed https://example.com/rss" in result.stdout
@SKIP_IF_NO_GIT
def test_commit_state_change_no_double_commit(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
"""commit_state_change does not create a commit when state has not changed."""
backup_path: Path = tmp_path / "backup"
monkeypatch.setenv("GIT_BACKUP_PATH", str(backup_path))
monkeypatch.delenv("GIT_BACKUP_REMOTE", raising=False)
mock_reader = MagicMock()
mock_reader.get_feeds.return_value = []
mock_reader.get_tag.return_value = []
commit_state_change(mock_reader, "First commit")
commit_state_change(mock_reader, "Should not appear")
git_executable: str | None = shutil.which("git")
assert git_executable is not None, "git executable not found"
result: subprocess.CompletedProcess[str] = subprocess.run( # noqa: S603
[git_executable, "-C", str(backup_path), "log", "--oneline"],
capture_output=True,
text=True,
check=False,
)
assert result.returncode == 0
assert "First commit" in result.stdout
assert "Should not appear" not in result.stdout
def test_commit_state_change_push_when_remote_set(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
"""commit_state_change calls git push when GIT_BACKUP_REMOTE is configured."""
backup_path: Path = tmp_path / "backup"
monkeypatch.setenv("GIT_BACKUP_PATH", str(backup_path))
monkeypatch.setenv("GIT_BACKUP_REMOTE", "git@github.com:user/private.git")
mock_reader = MagicMock()
mock_reader.get_feeds.return_value = []
mock_reader.get_tag.return_value = []
with patch("discord_rss_bot.git_backup.subprocess.run") as mock_run:
# Make all subprocess calls succeed
mock_run.return_value = MagicMock(returncode=1) # returncode=1 means staged changes exist
commit_state_change(mock_reader, "Add feed https://example.com/rss")
called_commands: list[list[str]] = [call.args[0] for call in mock_run.call_args_list]
push_calls: list[list[str]] = [cmd for cmd in called_commands if "push" in cmd]
assert push_calls, "git push should have been called when GIT_BACKUP_REMOTE is set"
assert any(cmd[-3:] == ["push", "origin", "HEAD"] for cmd in called_commands), (
"git push should target configured remote name 'origin'"
)
def test_commit_state_change_no_push_when_remote_unset(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
"""commit_state_change does not call git push when GIT_BACKUP_REMOTE is not set."""
backup_path: Path = tmp_path / "backup"
monkeypatch.setenv("GIT_BACKUP_PATH", str(backup_path))
monkeypatch.delenv("GIT_BACKUP_REMOTE", raising=False)
mock_reader = MagicMock()
mock_reader.get_feeds.return_value = []
mock_reader.get_tag.return_value = []
with patch("discord_rss_bot.git_backup.subprocess.run") as mock_run:
mock_run.return_value = MagicMock(returncode=1)
commit_state_change(mock_reader, "Add feed https://example.com/rss")
called_commands: list[list[str]] = [call.args[0] for call in mock_run.call_args_list]
push_calls: list[list[str]] = [cmd for cmd in called_commands if "push" in cmd]
assert not push_calls, "git push should NOT be called when GIT_BACKUP_REMOTE is not set"
client: TestClient = TestClient(app)
test_webhook_name: str = "Test Backup Webhook"
test_webhook_url: str = "https://discord.com/api/webhooks/999999999/testbackupwebhook"
test_feed_url: str = "https://lovinator.space/rss_test.xml"
def setup_test_feed() -> None:
"""Set up a test webhook and feed for endpoint tests."""
# Clean up existing test data
with contextlib.suppress(Exception):
client.post(url="/remove", data={"feed_url": test_feed_url})
with contextlib.suppress(Exception):
client.post(url="/delete_webhook", data={"webhook_url": test_webhook_url})
# Create webhook and feed
client.post(
url="/add_webhook",
data={"webhook_name": test_webhook_name, "webhook_url": test_webhook_url},
)
client.post(url="/add", data={"feed_url": test_feed_url, "webhook_dropdown": test_webhook_name})
def test_post_embed_triggers_backup(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
"""Posting to /embed should trigger a git backup with appropriate message."""
# Set up git backup
backup_path: Path = tmp_path / "backup"
monkeypatch.setenv("GIT_BACKUP_PATH", str(backup_path))
monkeypatch.delenv("GIT_BACKUP_REMOTE", raising=False)
setup_test_feed()
with patch("discord_rss_bot.main.commit_state_change") as mock_commit:
response = client.post(
url="/embed",
data={
"feed_url": test_feed_url,
"title": "Custom Title",
"description": "Custom Description",
"color": "#FF5733",
},
)
assert response.status_code == 200, f"Failed to post embed: {response.text}"
mock_commit.assert_called_once()
# Verify the commit message contains the feed URL
call_args = mock_commit.call_args
assert call_args is not None
commit_message: str = call_args[0][1]
assert "Update embed settings" in commit_message
assert test_feed_url in commit_message
def test_post_use_embed_triggers_backup(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
"""Posting to /use_embed should trigger a git backup."""
backup_path: Path = tmp_path / "backup"
monkeypatch.setenv("GIT_BACKUP_PATH", str(backup_path))
monkeypatch.delenv("GIT_BACKUP_REMOTE", raising=False)
setup_test_feed()
with patch("discord_rss_bot.main.commit_state_change") as mock_commit:
response = client.post(url="/use_embed", data={"feed_url": test_feed_url})
assert response.status_code == 200, f"Failed to enable embed: {response.text}"
mock_commit.assert_called_once()
# Verify the commit message
call_args = mock_commit.call_args
assert call_args is not None
commit_message: str = call_args[0][1]
assert "Enable embed mode" in commit_message
assert test_feed_url in commit_message
def test_post_use_text_triggers_backup(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
"""Posting to /use_text should trigger a git backup."""
backup_path: Path = tmp_path / "backup"
monkeypatch.setenv("GIT_BACKUP_PATH", str(backup_path))
monkeypatch.delenv("GIT_BACKUP_REMOTE", raising=False)
setup_test_feed()
with patch("discord_rss_bot.main.commit_state_change") as mock_commit:
response = client.post(url="/use_text", data={"feed_url": test_feed_url})
assert response.status_code == 200, f"Failed to disable embed: {response.text}"
mock_commit.assert_called_once()
# Verify the commit message
call_args = mock_commit.call_args
assert call_args is not None
commit_message: str = call_args[0][1]
assert "Disable embed mode" in commit_message
assert test_feed_url in commit_message
def test_post_custom_message_triggers_backup(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
"""Posting to /custom should trigger a git backup."""
backup_path: Path = tmp_path / "backup"
monkeypatch.setenv("GIT_BACKUP_PATH", str(backup_path))
monkeypatch.delenv("GIT_BACKUP_REMOTE", raising=False)
setup_test_feed()
with patch("discord_rss_bot.main.commit_state_change") as mock_commit:
response = client.post(
url="/custom",
data={
"feed_url": test_feed_url,
"custom_message": "Check out this entry: {entry.title}",
},
)
assert response.status_code == 200, f"Failed to set custom message: {response.text}"
mock_commit.assert_called_once()
# Verify the commit message
call_args = mock_commit.call_args
assert call_args is not None
commit_message: str = call_args[0][1]
assert "Update custom message" in commit_message
assert test_feed_url in commit_message
@SKIP_IF_NO_GIT
def test_embed_backup_end_to_end(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
"""End-to-end test: customizing embed creates a real commit in the backup repo."""
git_executable: str | None = shutil.which("git")
assert git_executable is not None, "git executable not found"
backup_path: Path = tmp_path / "backup"
monkeypatch.setenv("GIT_BACKUP_PATH", str(backup_path))
monkeypatch.delenv("GIT_BACKUP_REMOTE", raising=False)
setup_test_feed()
# Post embed customization
response = client.post(
url="/embed",
data={
"feed_url": test_feed_url,
"title": "{entry.title}",
"description": "{entry.summary}",
"color": "#0099FF",
"image_url": "{entry.image}",
},
)
assert response.status_code == 200, f"Failed to customize embed: {response.text}"
# Verify a commit was created
result: subprocess.CompletedProcess[str] = subprocess.run( # noqa: S603
[git_executable, "-C", str(backup_path), "log", "--oneline"],
capture_output=True,
text=True,
check=False,
)
assert result.returncode == 0, f"Failed to read git log: {result.stderr}"
assert "Update embed settings" in result.stdout, f"Commit not found in log: {result.stdout}"
# Verify state.json contains embed data
state_file: Path = backup_path / "state.json"
assert state_file.exists(), "state.json should exist in backup repo"
state_data: dict[str, Any] = json.loads(state_file.read_text(encoding="utf-8"))
# Find our test feed in the state
test_feed_data = next((feed for feed in state_data["feeds"] if feed["url"] == test_feed_url), None)
assert test_feed_data is not None, f"Test feed not found in state.json: {state_data}"
# The embed settings are stored as a nested dict under custom_embed tag
# This verifies the embed customization was persisted
assert "webhook" in test_feed_data, "Feed should have webhook set"

39
tests/test_hoyolab_api.py
View file

@ -1,39 +0,0 @@
from __future__ import annotations
from discord_rss_bot.hoyolab_api import extract_post_id_from_hoyolab_url
class TestExtractPostIdFromHoyolabUrl:
def test_extract_post_id_from_article_url(self) -> None:
"""Test extracting post ID from a direct article URL."""
test_cases: list[str] = [
"https://www.hoyolab.com/article/38588239",
"http://hoyolab.com/article/12345",
"https://www.hoyolab.com/article/987654321/comments",
]
expected_ids: list[str] = ["38588239", "12345", "987654321"]
for url, expected_id in zip(test_cases, expected_ids, strict=False):
assert extract_post_id_from_hoyolab_url(url) == expected_id
def test_url_without_post_id(self) -> None:
"""Test with a URL that doesn't have a post ID."""
test_cases: list[str] = [
"https://www.hoyolab.com/community",
]
for url in test_cases:
assert extract_post_id_from_hoyolab_url(url) is None
def test_edge_cases(self) -> None:
"""Test edge cases like None, empty string, and malformed URLs."""
test_cases: list[str | None] = [
None,
"",
"not_a_url",
"http:/", # Malformed URL
]
for url in test_cases:
assert extract_post_id_from_hoyolab_url(url) is None # type: ignore

1400
tests/test_main.py
View file

@ -1,29 +1,14 @@
from __future__ import annotations
import re
import urllib.parse
from dataclasses import dataclass
from dataclasses import field
from datetime import UTC
from datetime import datetime
from typing import TYPE_CHECKING
from typing import cast
from unittest.mock import MagicMock
from unittest.mock import patch
from fastapi.testclient import TestClient
import discord_rss_bot.main as main_module
from discord_rss_bot.main import app
from discord_rss_bot.main import create_html_for_feed
from discord_rss_bot.main import get_reader_dependency
if TYPE_CHECKING:
from pathlib import Path
import pytest
from httpx import Response
from reader import Entry
client: TestClient = TestClient(app)
webhook_name: str = "Hello, I am a webhook!"
@ -60,7 +45,7 @@ def test_search() -> None:
# Check that the feed was added.
response = client.get(url="/")
assert response.status_code == 200, f"Failed to get /: {response.text}"
assert encoded_feed_url(feed_url) in response.text, f"Feed not found in /: {response.text}"
assert feed_url in response.text, f"Feed not found in /: {response.text}"
# Search for an entry.
response: Response = client.get(url="/search/?query=a")
@ -87,14 +72,6 @@ def test_add_webhook() -> None:
def test_create_feed() -> None:
"""Test the /create_feed page."""
# Ensure webhook exists for this test regardless of test order.
client.post(url="/delete_webhook", data={"webhook_url": webhook_url})
response: Response = client.post(
url="/add_webhook",
data={"webhook_name": webhook_name, "webhook_url": webhook_url},
)
assert response.status_code == 200, f"Failed to add webhook: {response.text}"
# Remove the feed if it already exists before we run the test.
feeds: Response = client.get(url="/")
if feed_url in feeds.text:
@ -108,19 +85,11 @@ def test_create_feed() -> None:
# Check that the feed was added.
response = client.get(url="/")
assert response.status_code == 200, f"Failed to get /: {response.text}"
assert encoded_feed_url(feed_url) in response.text, f"Feed not found in /: {response.text}"
assert feed_url in response.text, f"Feed not found in /: {response.text}"
def test_get() -> None:
"""Test the /create_feed page."""
# Ensure webhook exists for this test regardless of test order.
client.post(url="/delete_webhook", data={"webhook_url": webhook_url})
response: Response = client.post(
url="/add_webhook",
data={"webhook_name": webhook_name, "webhook_url": webhook_url},
)
assert response.status_code == 200, f"Failed to add webhook: {response.text}"
# Remove the feed if it already exists before we run the test.
feeds: Response = client.get("/")
if feed_url in feeds.text:
@ -134,7 +103,7 @@ def test_get() -> None:
# Check that the feed was added.
response = client.get("/")
assert response.status_code == 200, f"Failed to get /: {response.text}"
assert encoded_feed_url(feed_url) in response.text, f"Feed not found in /: {response.text}"
assert feed_url in response.text, f"Feed not found in /: {response.text}"
response: Response = client.get(url="/add")
assert response.status_code == 200, f"/add failed: {response.text}"
@ -160,23 +129,12 @@ def test_get() -> None:
response: Response = client.get(url="/webhooks")
assert response.status_code == 200, f"/webhooks failed: {response.text}"
response = client.get(url="/webhook_entries", params={"webhook_url": webhook_url})
assert response.status_code == 200, f"/webhook_entries failed: {response.text}"
response: Response = client.get(url="/whitelist", params={"feed_url": encoded_feed_url(feed_url)})
assert response.status_code == 200, f"/whitelist failed: {response.text}"
def test_pause_feed() -> None:
"""Test the /pause_feed page."""
# Ensure webhook exists for this test regardless of test order.
client.post(url="/delete_webhook", data={"webhook_url": webhook_url})
response: Response = client.post(
url="/add_webhook",
data={"webhook_name": webhook_name, "webhook_url": webhook_url},
)
assert response.status_code == 200, f"Failed to add webhook: {response.text}"
# Remove the feed if it already exists before we run the test.
feeds: Response = client.get(url="/")
if feed_url in feeds.text:
@ -185,7 +143,6 @@ def test_pause_feed() -> None:
# Add the feed.
response: Response = client.post(url="/add", data={"feed_url": feed_url, "webhook_dropdown": webhook_name})
assert response.status_code == 200, f"Failed to add feed: {response.text}"
# Unpause the feed if it is paused.
feeds: Response = client.get(url="/")
@ -200,19 +157,11 @@ def test_pause_feed() -> None:
# Check that the feed was paused.
response = client.get(url="/")
assert response.status_code == 200, f"Failed to get /: {response.text}"
assert encoded_feed_url(feed_url) in response.text, f"Feed not found in /: {response.text}"
assert feed_url in response.text, f"Feed not found in /: {response.text}"
def test_unpause_feed() -> None:
"""Test the /unpause_feed page."""
# Ensure webhook exists for this test regardless of test order.
client.post(url="/delete_webhook", data={"webhook_url": webhook_url})
response: Response = client.post(
url="/add_webhook",
data={"webhook_name": webhook_name, "webhook_url": webhook_url},
)
assert response.status_code == 200, f"Failed to add webhook: {response.text}"
# Remove the feed if it already exists before we run the test.
feeds: Response = client.get("/")
if feed_url in feeds.text:
@ -221,7 +170,6 @@ def test_unpause_feed() -> None:
# Add the feed.
response: Response = client.post(url="/add", data={"feed_url": feed_url, "webhook_dropdown": webhook_name})
assert response.status_code == 200, f"Failed to add feed: {response.text}"
# Pause the feed if it is unpaused.
feeds: Response = client.get(url="/")
@ -236,19 +184,11 @@ def test_unpause_feed() -> None:
# Check that the feed was unpaused.
response = client.get(url="/")
assert response.status_code == 200, f"Failed to get /: {response.text}"
assert encoded_feed_url(feed_url) in response.text, f"Feed not found in /: {response.text}"
assert feed_url in response.text, f"Feed not found in /: {response.text}"
def test_remove_feed() -> None:
"""Test the /remove page."""
# Ensure webhook exists for this test regardless of test order.
client.post(url="/delete_webhook", data={"webhook_url": webhook_url})
response: Response = client.post(
url="/add_webhook",
data={"webhook_name": webhook_name, "webhook_url": webhook_url},
)
assert response.status_code == 200, f"Failed to add webhook: {response.text}"
# Remove the feed if it already exists before we run the test.
feeds: Response = client.get(url="/")
if feed_url in feeds.text:
@ -257,7 +197,6 @@ def test_remove_feed() -> None:
# Add the feed.
response: Response = client.post(url="/add", data={"feed_url": feed_url, "webhook_dropdown": webhook_name})
assert response.status_code == 200, f"Failed to add feed: {response.text}"
# Remove the feed.
response: Response = client.post(url="/remove", data={"feed_url": feed_url})
@ -269,186 +208,6 @@ def test_remove_feed() -> None:
assert feed_url not in response.text, f"Feed found in /: {response.text}"
def test_change_feed_url() -> None:
"""Test changing a feed URL from the feed page endpoint."""
new_feed_url = "https://lovinator.space/rss_test_small.xml"
# Ensure test feeds do not already exist.
client.post(url="/remove", data={"feed_url": feed_url})
client.post(url="/remove", data={"feed_url": new_feed_url})
# Ensure webhook exists.
client.post(url="/delete_webhook", data={"webhook_url": webhook_url})
response: Response = client.post(
url="/add_webhook",
data={"webhook_name": webhook_name, "webhook_url": webhook_url},
)
assert response.status_code == 200, f"Failed to add webhook: {response.text}"
# Add the original feed.
response = client.post(url="/add", data={"feed_url": feed_url, "webhook_dropdown": webhook_name})
assert response.status_code == 200, f"Failed to add feed: {response.text}"
# Change feed URL.
response = client.post(
url="/change_feed_url",
data={"old_feed_url": feed_url, "new_feed_url": new_feed_url},
)
assert response.status_code == 200, f"Failed to change feed URL: {response.text}"
# New feed should be accessible.
response = client.get(url="/feed", params={"feed_url": new_feed_url})
assert response.status_code == 200, f"New feed URL is not accessible: {response.text}"
# Old feed should no longer be accessible.
response = client.get(url="/feed", params={"feed_url": feed_url})
assert response.status_code == 404, "Old feed URL should no longer exist"
# Cleanup.
client.post(url="/remove", data={"feed_url": new_feed_url})
def test_change_feed_url_marks_entries_as_read() -> None:
"""After changing a feed URL all entries on the new feed should be marked read to prevent resending."""
new_feed_url = "https://lovinator.space/rss_test_small.xml"
# Ensure feeds do not already exist.
client.post(url="/remove", data={"feed_url": feed_url})
client.post(url="/remove", data={"feed_url": new_feed_url})
# Ensure webhook exists.
client.post(url="/delete_webhook", data={"webhook_url": webhook_url})
client.post(url="/add_webhook", data={"webhook_name": webhook_name, "webhook_url": webhook_url})
# Add the original feed.
response: Response = client.post(url="/add", data={"feed_url": feed_url, "webhook_dropdown": webhook_name})
assert response.status_code == 200, f"Failed to add feed: {response.text}"
# Patch reader on the main module so we can observe calls.
mock_entry_a = MagicMock()
mock_entry_a.id = "entry-a"
mock_entry_b = MagicMock()
mock_entry_b.id = "entry-b"
real_reader = main_module.get_reader_dependency()
# Use a no-redirect client so the POST response is inspected directly; the
# redirect target (/feed?feed_url=…) would 404 because change_feed_url is mocked.
no_redirect_client = TestClient(app, follow_redirects=False)
with (
patch.object(real_reader, "get_entries", return_value=[mock_entry_a, mock_entry_b]) as mock_get_entries,
patch.object(real_reader, "set_entry_read") as mock_set_read,
patch.object(real_reader, "update_feed") as mock_update_feed,
patch.object(real_reader, "change_feed_url"),
):
response = no_redirect_client.post(
url="/change_feed_url",
data={"old_feed_url": feed_url, "new_feed_url": new_feed_url},
)
assert response.status_code == 303, f"Expected 303 redirect, got {response.status_code}: {response.text}"
# update_feed should have been called with the new URL.
mock_update_feed.assert_called_once_with(new_feed_url)
# get_entries should have been called to fetch unread entries on the new URL.
mock_get_entries.assert_called_once_with(feed=new_feed_url, read=False)
# Every returned entry should have been marked as read.
assert mock_set_read.call_count == 2, f"Expected 2 set_entry_read calls, got {mock_set_read.call_count}"
mock_set_read.assert_any_call(mock_entry_a, True)
mock_set_read.assert_any_call(mock_entry_b, True)
# Cleanup.
client.post(url="/remove", data={"feed_url": feed_url})
client.post(url="/remove", data={"feed_url": new_feed_url})
def test_change_feed_url_empty_old_url_returns_400() -> None:
"""Submitting an empty old_feed_url should return HTTP 400."""
response: Response = client.post(
url="/change_feed_url",
data={"old_feed_url": " ", "new_feed_url": "https://example.com/feed.xml"},
)
assert response.status_code == 400, f"Expected 400 for empty old URL, got {response.status_code}"
def test_change_feed_url_empty_new_url_returns_400() -> None:
"""Submitting a blank new_feed_url should return HTTP 400."""
response: Response = client.post(
url="/change_feed_url",
data={"old_feed_url": feed_url, "new_feed_url": " "},
)
assert response.status_code == 400, f"Expected 400 for blank new URL, got {response.status_code}"
def test_change_feed_url_nonexistent_old_url_returns_404() -> None:
"""Trying to rename a feed that does not exist should return HTTP 404."""
non_existent = "https://does-not-exist.example.com/rss.xml"
# Make sure it really is absent.
client.post(url="/remove", data={"feed_url": non_existent})
response: Response = client.post(
url="/change_feed_url",
data={"old_feed_url": non_existent, "new_feed_url": "https://example.com/new.xml"},
)
assert response.status_code == 404, f"Expected 404 for non-existent feed, got {response.status_code}"
def test_change_feed_url_new_url_already_exists_returns_409() -> None:
"""Changing to a URL that is already tracked should return HTTP 409."""
second_feed_url = "https://lovinator.space/rss_test_small.xml"
# Ensure both feeds are absent.
client.post(url="/remove", data={"feed_url": feed_url})
client.post(url="/remove", data={"feed_url": second_feed_url})
# Ensure webhook exists.
client.post(url="/delete_webhook", data={"webhook_url": webhook_url})
client.post(url="/add_webhook", data={"webhook_name": webhook_name, "webhook_url": webhook_url})
# Add both feeds.
client.post(url="/add", data={"feed_url": feed_url, "webhook_dropdown": webhook_name})
client.post(url="/add", data={"feed_url": second_feed_url, "webhook_dropdown": webhook_name})
# Try to rename one to the other.
response: Response = client.post(
url="/change_feed_url",
data={"old_feed_url": feed_url, "new_feed_url": second_feed_url},
)
assert response.status_code == 409, f"Expected 409 when new URL already exists, got {response.status_code}"
# Cleanup.
client.post(url="/remove", data={"feed_url": feed_url})
client.post(url="/remove", data={"feed_url": second_feed_url})
def test_change_feed_url_same_url_redirects_without_error() -> None:
"""Changing a feed's URL to itself should redirect cleanly without any error."""
# Ensure webhook exists.
client.post(url="/delete_webhook", data={"webhook_url": webhook_url})
client.post(url="/add_webhook", data={"webhook_name": webhook_name, "webhook_url": webhook_url})
# Add the feed.
client.post(url="/remove", data={"feed_url": feed_url})
response: Response = client.post(url="/add", data={"feed_url": feed_url, "webhook_dropdown": webhook_name})
assert response.status_code == 200, f"Failed to add feed: {response.text}"
# Submit the same URL as both old and new.
response = client.post(
url="/change_feed_url",
data={"old_feed_url": feed_url, "new_feed_url": feed_url},
)
assert response.status_code == 200, f"Expected 200 redirect for same URL, got {response.status_code}"
# Feed should still be accessible.
response = client.get(url="/feed", params={"feed_url": feed_url})
assert response.status_code == 200, f"Feed should still exist after no-op URL change: {response.text}"
# Cleanup.
client.post(url="/remove", data={"feed_url": feed_url})
def test_delete_webhook() -> None:
"""Test the /delete_webhook page."""
# Remove the feed if it already exists before we run the test.
@ -470,1152 +229,3 @@ def test_delete_webhook() -> None:
response = client.get(url="/webhooks")
assert response.status_code == 200, f"Failed to get /webhooks: {response.text}"
assert webhook_name not in response.text, f"Webhook found in /webhooks: {response.text}"
def test_update_feed_not_found() -> None:
"""Test updating a non-existent feed."""
# Generate a feed URL that does not exist
nonexistent_feed_url = "https://nonexistent-feed.example.com/rss.xml"
# Try to update the non-existent feed
response: Response = client.get(url="/update", params={"feed_url": urllib.parse.quote(nonexistent_feed_url)})
# Check that it returns a 404 status code
assert response.status_code == 404, f"Expected 404 for non-existent feed, got: {response.status_code}"
assert "Feed not found" in response.text
def test_post_entry_send_to_discord() -> None:
"""Test that /post_entry sends an entry to Discord and redirects to the feed page.
Regression test for the bug where the injected reader was not passed to
send_entry_to_discord, meaning the dependency-injected reader was silently ignored.
"""
# Ensure webhook and feed exist.
client.post(url="/delete_webhook", data={"webhook_url": webhook_url})
response: Response = client.post(
url="/add_webhook",
data={"webhook_name": webhook_name, "webhook_url": webhook_url},
)
assert response.status_code == 200, f"Failed to add webhook: {response.text}"
client.post(url="/remove", data={"feed_url": feed_url})
response = client.post(url="/add", data={"feed_url": feed_url, "webhook_dropdown": webhook_name})
assert response.status_code == 200, f"Failed to add feed: {response.text}"
# Retrieve an entry from the feed to get a valid entry ID.
reader: main_module.Reader = main_module.get_reader_dependency()
entries: list[Entry] = list(reader.get_entries(feed=feed_url, limit=1))
assert entries, "Feed should have at least one entry to send"
entry_to_send: main_module.Entry = entries[0]
encoded_id: str = urllib.parse.quote(entry_to_send.id)
no_redirect_client = TestClient(app, follow_redirects=False)
# Patch execute_webhook so no real HTTP requests are made to Discord.
with patch("discord_rss_bot.feeds.execute_webhook") as mock_execute:
response = no_redirect_client.get(
url="/post_entry",
params={"entry_id": encoded_id, "feed_url": urllib.parse.quote(feed_url)},
)
assert response.status_code == 303, f"Expected redirect after sending, got {response.status_code}: {response.text}"
location: str = response.headers.get("location", "")
assert "feed?feed_url=" in location, f"Should redirect to feed page, got: {location}"
assert mock_execute.called, "execute_webhook should have been called to deliver the entry to Discord"
# Cleanup.
client.post(url="/remove", data={"feed_url": feed_url})
def test_post_entry_unknown_id_returns_404() -> None:
"""Test that /post_entry returns 404 when the entry ID does not exist."""
response: Response = client.get(
url="/post_entry",
params={"entry_id": "https://nonexistent.example.com/entry-that-does-not-exist"},
)
assert response.status_code == 404, f"Expected 404 for unknown entry, got {response.status_code}"
def test_post_entry_uses_feed_url_to_disambiguate_duplicate_ids() -> None:
"""When IDs collide across feeds, /post_entry should pick the entry from provided feed_url."""
@dataclass(slots=True)
class DummyFeed:
url: str
@dataclass(slots=True)
class DummyEntry:
id: str
feed: DummyFeed
feed_url: str
feed_a = "https://example.com/feed-a.xml"
feed_b = "https://example.com/feed-b.xml"
shared_id = "https://example.com/shared-entry-id"
entry_a: Entry = cast("Entry", DummyEntry(id=shared_id, feed=DummyFeed(feed_a), feed_url=feed_a))
entry_b: Entry = cast("Entry", DummyEntry(id=shared_id, feed=DummyFeed(feed_b), feed_url=feed_b))
class StubReader:
def get_entries(self, feed: str | None = None) -> list[Entry]:
if feed == feed_a:
return [entry_a]
if feed == feed_b:
return [entry_b]
return [entry_a, entry_b]
selected_feed_urls: list[str] = []
def fake_send_entry_to_discord(entry: Entry, reader: object) -> None:
selected_feed_urls.append(entry.feed.url)
app.dependency_overrides[get_reader_dependency] = StubReader
no_redirect_client = TestClient(app, follow_redirects=False)
try:
with patch("discord_rss_bot.main.send_entry_to_discord", side_effect=fake_send_entry_to_discord):
response: Response = no_redirect_client.get(
url="/post_entry",
params={"entry_id": urllib.parse.quote(shared_id), "feed_url": urllib.parse.quote(feed_b)},
)
assert response.status_code == 303, f"Expected redirect after sending, got {response.status_code}"
assert selected_feed_urls == [feed_b], f"Expected feed-b entry, got: {selected_feed_urls}"
location = response.headers.get("location", "")
assert urllib.parse.quote(feed_b) in location, f"Expected redirect to feed-b page, got: {location}"
finally:
app.dependency_overrides = {}
def test_navbar_backup_link_hidden_when_not_configured(monkeypatch: pytest.MonkeyPatch) -> None:
"""Test that the backup link is not shown in the navbar when GIT_BACKUP_PATH is not set."""
# Ensure GIT_BACKUP_PATH is not set
monkeypatch.delenv("GIT_BACKUP_PATH", raising=False)
# Get the index page
response: Response = client.get(url="/")
assert response.status_code == 200, f"Failed to get /: {response.text}"
# Check that the backup button is not in the response
assert "Backup" not in response.text or 'action="/backup"' not in response.text, (
"Backup button should not be visible when GIT_BACKUP_PATH is not configured"
)
def test_navbar_backup_link_visible_when_configured(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
"""Test that the backup link is shown in the navbar when GIT_BACKUP_PATH is set."""
# Set GIT_BACKUP_PATH
monkeypatch.setenv("GIT_BACKUP_PATH", str(tmp_path))
# Get the index page
response: Response = client.get(url="/")
assert response.status_code == 200, f"Failed to get /: {response.text}"
# Check that the backup button is in the response
assert "Backup" in response.text, "Backup button text should be visible when GIT_BACKUP_PATH is configured"
assert 'action="/backup"' in response.text, "Backup form should be visible when GIT_BACKUP_PATH is configured"
def test_backup_endpoint_returns_error_when_not_configured(monkeypatch: pytest.MonkeyPatch) -> None:
"""Test that the backup endpoint returns an error when GIT_BACKUP_PATH is not set."""
# Ensure GIT_BACKUP_PATH is not set
monkeypatch.delenv("GIT_BACKUP_PATH", raising=False)
# Try to trigger a backup
response: Response = client.post(url="/backup")
# Should redirect to index with error message
assert response.status_code == 200, f"Failed to post /backup: {response.text}"
assert "Git backup is not configured" in response.text or "GIT_BACKUP_PATH" in response.text, (
"Error message about backup not being configured should be shown"
)
def test_show_more_entries_button_visible_when_many_entries() -> None:
"""Test that the 'Show more entries' button is visible when there are more than 20 entries."""
# Add the webhook first
client.post(url="/delete_webhook", data={"webhook_url": webhook_url})
response: Response = client.post(
url="/add_webhook",
data={"webhook_name": webhook_name, "webhook_url": webhook_url},
)
assert response.status_code == 200, f"Failed to add webhook: {response.text}"
# Remove the feed if it already exists
feeds: Response = client.get(url="/")
if feed_url in feeds.text:
client.post(url="/remove", data={"feed_url": feed_url})
# Add the feed
response: Response = client.post(url="/add", data={"feed_url": feed_url, "webhook_dropdown": webhook_name})
assert response.status_code == 200, f"Failed to add feed: {response.text}"
# Get the feed page
response: Response = client.get(url="/feed", params={"feed_url": feed_url})
assert response.status_code == 200, f"Failed to get /feed: {response.text}"
# Check if the feed has more than 20 entries by looking at the response
# The button should be visible if there are more than 20 entries
# We check for both the button text and the link structure
if "Show more entries" in response.text:
# Button is visible - verify it has the correct structure
assert "starting_after=" in response.text, "Show more entries button should contain starting_after parameter"
# The button should be a link to the feed page with pagination
assert (
f'href="/feed?feed_url={urllib.parse.quote(feed_url)}' in response.text
or f'href="/feed?feed_url={encoded_feed_url(feed_url)}' in response.text
), "Show more entries button should link back to the feed page"
def test_show_more_entries_button_not_visible_when_few_entries() -> None:
"""Test that the 'Show more entries' button is not visible when there are 20 or fewer entries."""
# Ensure webhook exists for this test regardless of test order.
client.post(url="/delete_webhook", data={"webhook_url": webhook_url})
response: Response = client.post(
url="/add_webhook",
data={"webhook_name": webhook_name, "webhook_url": webhook_url},
)
assert response.status_code == 200, f"Failed to add webhook: {response.text}"
# Use a feed with very few entries
small_feed_url = "https://lovinator.space/rss_test_small.xml"
# Clean up if exists
client.post(url="/remove", data={"feed_url": small_feed_url})
# Add a small feed (this may not exist, so this test is conditional)
response: Response = client.post(url="/add", data={"feed_url": small_feed_url, "webhook_dropdown": webhook_name})
if response.status_code == 200:
# Get the feed page
response: Response = client.get(url="/feed", params={"feed_url": small_feed_url})
assert response.status_code == 200, f"Failed to get /feed: {response.text}"
# If the feed has 20 or fewer entries, the button should not be visible
# We check the total entry count in the page
if "0 entries" in response.text or " entries)" in response.text:
# Extract entry count and verify button visibility
match: re.Match[str] | None = re.search(r"\((\d+) entries\)", response.text)
if match:
entry_count = int(match.group(1))
if entry_count <= 20:
assert "Show more entries" not in response.text, (
f"Show more entries button should not be visible when there are {entry_count} entries"
)
# Clean up
client.post(url="/remove", data={"feed_url": small_feed_url})
def test_show_more_entries_pagination_works() -> None:
"""Test that pagination with starting_after parameter works correctly."""
# Add the webhook first
client.post(url="/delete_webhook", data={"webhook_url": webhook_url})
response: Response = client.post(
url="/add_webhook",
data={"webhook_name": webhook_name, "webhook_url": webhook_url},
)
assert response.status_code == 200, f"Failed to add webhook: {response.text}"
# Remove the feed if it already exists
feeds: Response = client.get(url="/")
if feed_url in feeds.text:
client.post(url="/remove", data={"feed_url": feed_url})
# Add the feed
response: Response = client.post(url="/add", data={"feed_url": feed_url, "webhook_dropdown": webhook_name})
assert response.status_code == 200, f"Failed to add feed: {response.text}"
# Get the first page
response: Response = client.get(url="/feed", params={"feed_url": feed_url})
assert response.status_code == 200, f"Failed to get /feed: {response.text}"
# Check if pagination is available
if "Show more entries" in response.text and "starting_after=" in response.text:
# Extract the starting_after parameter from the button link
match: re.Match[str] | None = re.search(r'starting_after=([^"&]+)', response.text)
if match:
starting_after_id: str = match.group(1)
# Request the second page
response: Response = client.get(
url="/feed",
params={"feed_url": feed_url, "starting_after": starting_after_id},
)
assert response.status_code == 200, f"Failed to get paginated feed: {response.text}"
# Verify we got a valid response (the page should contain entries)
assert "entries)" in response.text, "Paginated page should show entry count"
def test_show_more_entries_button_context_variable() -> None:
"""Test that the button visibility variable is correctly passed to the template context."""
# Add the webhook first
client.post(url="/delete_webhook", data={"webhook_url": webhook_url})
response: Response = client.post(
url="/add_webhook",
data={"webhook_name": webhook_name, "webhook_url": webhook_url},
)
assert response.status_code == 200, f"Failed to add webhook: {response.text}"
# Remove the feed if it already exists
feeds: Response = client.get(url="/")
if feed_url in feeds.text:
client.post(url="/remove", data={"feed_url": feed_url})
# Add the feed
response: Response = client.post(url="/add", data={"feed_url": feed_url, "webhook_dropdown": webhook_name})
assert response.status_code == 200, f"Failed to add feed: {response.text}"
# Get the feed page
response: Response = client.get(url="/feed", params={"feed_url": feed_url})
assert response.status_code == 200, f"Failed to get /feed: {response.text}"
# Extract the total entries count from the page
match: re.Match[str] | None = re.search(r"\((\d+) entries\)", response.text)
if match:
entry_count = int(match.group(1))
# If more than 20 entries, button should be visible
if entry_count > 20:
assert "Show more entries" in response.text, (
f"Button should be visible when there are {entry_count} entries (more than 20)"
)
# If 20 or fewer entries, button should not be visible
else:
assert "Show more entries" not in response.text, (
f"Button should not be visible when there are {entry_count} entries (20 or fewer)"
)
def test_create_html_marks_entries_from_another_feed(monkeypatch: pytest.MonkeyPatch) -> None:
"""Entries from another feed should be marked in /feed html output."""
@dataclass(slots=True)
class DummyContent:
value: str
@dataclass(slots=True)
class DummyFeed:
url: str
@dataclass(slots=True)
class DummyEntry:
feed: DummyFeed
id: str
original_feed_url: str | None = None
link: str = "https://example.com/post"
title: str = "Example title"
author: str = "Author"
summary: str = "Summary"
content: list[DummyContent] = field(default_factory=lambda: [DummyContent("Content")])
published: None = None
def __post_init__(self) -> None:
if self.original_feed_url is None:
self.original_feed_url = self.feed.url
selected_feed_url = "https://example.com/feed-a.xml"
same_feed_entry = DummyEntry(DummyFeed(selected_feed_url), "same")
# feed.url matches selected feed, but original_feed_url differs; marker should still show.
other_feed_entry = DummyEntry(
DummyFeed(selected_feed_url),
"other",
original_feed_url="https://example.com/feed-b.xml",
)
monkeypatch.setattr(
"discord_rss_bot.main.replace_tags_in_text_message",
lambda _entry, **_kwargs: "Rendered content",
)
monkeypatch.setattr("discord_rss_bot.main.entry_is_blacklisted", lambda _entry, **_kwargs: False)
monkeypatch.setattr("discord_rss_bot.main.entry_is_whitelisted", lambda _entry, **_kwargs: False)
same_feed_entry_typed: Entry = cast("Entry", same_feed_entry)
other_feed_entry_typed: Entry = cast("Entry", other_feed_entry)
html: str = create_html_for_feed(
reader=MagicMock(),
current_feed_url=selected_feed_url,
entries=[
same_feed_entry_typed,
other_feed_entry_typed,
],
)
assert "From another feed: https://example.com/feed-b.xml" in html
assert "From another feed: https://example.com/feed-a.xml" not in html
def test_webhook_entries_webhook_not_found() -> None:
"""Test webhook_entries endpoint returns 404 when webhook doesn't exist."""
nonexistent_webhook_url = "https://discord.com/api/webhooks/999999/nonexistent"
response: Response = client.get(
url="/webhook_entries",
params={"webhook_url": nonexistent_webhook_url},
)
assert response.status_code == 404, f"Expected 404 for non-existent webhook, got: {response.status_code}"
assert "Webhook not found" in response.text
def test_webhook_entries_no_feeds() -> None:
"""Test webhook_entries endpoint displays message when webhook has no feeds."""
# Clean up any existing feeds first
client.post(url="/remove", data={"feed_url": feed_url})
# Clean up and create a webhook
client.post(url="/delete_webhook", data={"webhook_url": webhook_url})
response: Response = client.post(
url="/add_webhook",
data={"webhook_name": webhook_name, "webhook_url": webhook_url},
)
assert response.status_code == 200, f"Failed to add webhook: {response.text}"
# Get webhook_entries without adding any feeds
response = client.get(
url="/webhook_entries",
params={"webhook_url": webhook_url},
)
assert response.status_code == 200, f"Failed to get /webhook_entries: {response.text}"
assert webhook_name in response.text, "Webhook name not found in response"
assert "No feeds found" in response.text or "Add feeds" in response.text, "Expected message about no feeds"
def test_webhook_entries_no_feeds_still_shows_webhook_settings() -> None:
"""The webhook detail view should show settings/actions even with no attached feeds."""
client.post(url="/delete_webhook", data={"webhook_url": webhook_url})
response: Response = client.post(
url="/add_webhook",
data={"webhook_name": webhook_name, "webhook_url": webhook_url},
)
assert response.status_code == 200, f"Failed to add webhook: {response.text}"
response = client.get(
url="/webhook_entries",
params={"webhook_url": webhook_url},
)
assert response.status_code == 200, f"Failed to get /webhook_entries: {response.text}"
assert "Settings" in response.text, "Expected settings card on webhook detail view"
assert "Modify Webhook" in response.text, "Expected modify form on webhook detail view"
assert "Delete Webhook" in response.text, "Expected delete action on webhook detail view"
assert "Back to dashboard" in response.text, "Expected dashboard navigation link"
assert "All webhooks" in response.text, "Expected all webhooks navigation link"
assert f'name="old_hook" value="{webhook_url}"' in response.text, "Expected old_hook hidden input"
assert f'value="/webhook_entries?webhook_url={urllib.parse.quote(webhook_url)}"' in response.text, (
"Expected modify form to redirect back to the current webhook detail view"
)
def test_webhook_entries_with_feeds_no_entries() -> None:
"""Test webhook_entries endpoint when webhook has feeds but no entries yet."""
# Clean up and create fresh webhook
client.post(url="/delete_webhook", data={"webhook_url": webhook_url})
response: Response = client.post(
url="/add_webhook",
data={"webhook_name": webhook_name, "webhook_url": webhook_url},
)
assert response.status_code == 200, f"Failed to add webhook: {response.text}"
# Use a feed URL that exists but has no entries (or clean feed)
empty_feed_url = "https://lovinator.space/empty_feed.xml"
client.post(url="/remove", data={"feed_url": empty_feed_url})
# Add the feed
response = client.post(
url="/add",
data={"feed_url": empty_feed_url, "webhook_dropdown": webhook_name},
)
# Get webhook_entries
response = client.get(
url="/webhook_entries",
params={"webhook_url": webhook_url},
)
assert response.status_code == 200, f"Failed to get /webhook_entries: {response.text}"
assert webhook_name in response.text, "Webhook name not found in response"
# Clean up
client.post(url="/remove", data={"feed_url": empty_feed_url})
def test_webhook_entries_with_entries() -> None:
"""Test webhook_entries endpoint displays entries correctly."""
# Clean up and create webhook
client.post(url="/delete_webhook", data={"webhook_url": webhook_url})
response: Response = client.post(
url="/add_webhook",
data={"webhook_name": webhook_name, "webhook_url": webhook_url},
)
assert response.status_code == 200, f"Failed to add webhook: {response.text}"
# Remove and add the feed
client.post(url="/remove", data={"feed_url": feed_url})
response = client.post(
url="/add",
data={"feed_url": feed_url, "webhook_dropdown": webhook_name},
)
assert response.status_code == 200, f"Failed to add feed: {response.text}"
# Get webhook_entries
response = client.get(
url="/webhook_entries",
params={"webhook_url": webhook_url},
)
assert response.status_code == 200, f"Failed to get /webhook_entries: {response.text}"
assert webhook_name in response.text, "Webhook name not found in response"
# Should show entries (the feed has entries)
assert "total from" in response.text, "Expected to see entry count"
assert "Modify Webhook" in response.text, "Expected webhook settings to be visible"
assert "Attached feeds" in response.text, "Expected attached feeds section to be visible"
def test_webhook_entries_shows_attached_feed_link() -> None:
"""The webhook detail view should list attached feeds linking to their feed pages."""
client.post(url="/delete_webhook", data={"webhook_url": webhook_url})
response: Response = client.post(
url="/add_webhook",
data={"webhook_name": webhook_name, "webhook_url": webhook_url},
)
assert response.status_code == 200, f"Failed to add webhook: {response.text}"
client.post(url="/remove", data={"feed_url": feed_url})
response = client.post(
url="/add",
data={"feed_url": feed_url, "webhook_dropdown": webhook_name},
)
assert response.status_code == 200, f"Failed to add feed: {response.text}"
response = client.get(
url="/webhook_entries",
params={"webhook_url": webhook_url},
)
assert response.status_code == 200, f"Failed to get /webhook_entries: {response.text}"
assert f"/feed?feed_url={urllib.parse.quote(feed_url)}" in response.text, (
"Expected attached feed to link to its feed detail page"
)
assert "Latest entries" in response.text, "Expected latest entries heading on webhook detail view"
client.post(url="/remove", data={"feed_url": feed_url})
def test_webhook_entries_multiple_feeds() -> None:
"""Test webhook_entries endpoint shows feed count correctly."""
# Clean up and create webhook
client.post(url="/delete_webhook", data={"webhook_url": webhook_url})
response: Response = client.post(
url="/add_webhook",
data={"webhook_name": webhook_name, "webhook_url": webhook_url},
)
assert response.status_code == 200, f"Failed to add webhook: {response.text}"
# Remove and add feed
client.post(url="/remove", data={"feed_url": feed_url})
response = client.post(
url="/add",
data={"feed_url": feed_url, "webhook_dropdown": webhook_name},
)
assert response.status_code == 200, f"Failed to add feed: {response.text}"
# Get webhook_entries
response = client.get(
url="/webhook_entries",
params={"webhook_url": webhook_url},
)
assert response.status_code == 200, f"Failed to get /webhook_entries: {response.text}"
assert webhook_name in response.text, "Webhook name not found in response"
# Should show entries and feed count
assert "feed" in response.text.lower(), "Expected to see feed information"
# Clean up
client.post(url="/remove", data={"feed_url": feed_url})
def test_webhook_entries_sort_newest_and_non_null_published_first() -> None:
"""Webhook entries should be sorted newest-first with published=None entries placed last."""
@dataclass(slots=True)
class DummyFeed:
url: str
title: str | None = None
updates_enabled: bool = True
last_exception: None = None
@dataclass(slots=True)
class DummyEntry:
id: str
feed: DummyFeed
published: datetime | None
dummy_feed = DummyFeed(url="https://example.com/feed.xml", title="Example Feed")
# Intentionally unsorted input with two dated entries and two undated entries.
unsorted_entries: list[Entry] = [
cast("Entry", DummyEntry(id="old", feed=dummy_feed, published=datetime(2024, 1, 1, tzinfo=UTC))),
cast("Entry", DummyEntry(id="none-1", feed=dummy_feed, published=None)),
cast("Entry", DummyEntry(id="new", feed=dummy_feed, published=datetime(2024, 2, 1, tzinfo=UTC))),
cast("Entry", DummyEntry(id="none-2", feed=dummy_feed, published=None)),
]
class StubReader:
def get_tag(self, resource: object, key: str, default: object = None) -> object:
if resource == () and key == "webhooks":
return [{"name": webhook_name, "url": webhook_url}]
if key == "webhook" and isinstance(resource, str):
return webhook_url
return default
def get_feeds(self) -> list[DummyFeed]:
return [dummy_feed]
def get_entries(self, **_kwargs: object) -> list[Entry]:
return unsorted_entries
observed_order: list[str] = []
def capture_entries(*, reader: object, entries: list[Entry], current_feed_url: str = "") -> str:
del reader, current_feed_url
observed_order.extend(entry.id for entry in entries)
return ""
app.dependency_overrides[get_reader_dependency] = StubReader
try:
with (
patch(
"discord_rss_bot.main.get_data_from_hook_url",
return_value=main_module.WebhookInfo(custom_name=webhook_name, url=webhook_url),
),
patch("discord_rss_bot.main.create_html_for_feed", side_effect=capture_entries),
):
response: Response = client.get(
url="/webhook_entries",
params={"webhook_url": webhook_url},
)
assert response.status_code == 200, f"Failed to get /webhook_entries: {response.text}"
assert observed_order == ["new", "old", "none-1", "none-2"], (
"Expected newest published entries first and published=None entries last"
)
finally:
app.dependency_overrides = {}
def test_webhook_entries_pagination() -> None:
"""Test webhook_entries endpoint pagination functionality."""
# Clean up and create webhook
client.post(url="/delete_webhook", data={"webhook_url": webhook_url})
response: Response = client.post(
url="/add_webhook",
data={"webhook_name": webhook_name, "webhook_url": webhook_url},
)
assert response.status_code == 200, f"Failed to add webhook: {response.text}"
# Remove and add the feed
client.post(url="/remove", data={"feed_url": feed_url})
response = client.post(
url="/add",
data={"feed_url": feed_url, "webhook_dropdown": webhook_name},
)
assert response.status_code == 200, f"Failed to add feed: {response.text}"
# Get first page of webhook_entries
response = client.get(
url="/webhook_entries",
params={"webhook_url": webhook_url},
)
assert response.status_code == 200, f"Failed to get /webhook_entries: {response.text}"
# Check if pagination button is shown when there are many entries
# The button should be visible if total_entries > 20 (entries_per_page)
if "Load More Entries" in response.text:
# Extract the starting_after parameter from the pagination form
# This is a simple check that pagination elements exist
assert 'name="starting_after"' in response.text, "Expected pagination form with starting_after parameter"
# Clean up
client.post(url="/remove", data={"feed_url": feed_url})
def test_webhook_entries_url_encoding() -> None:
"""Test webhook_entries endpoint handles URL encoding correctly."""
# Clean up and create webhook
client.post(url="/delete_webhook", data={"webhook_url": webhook_url})
response: Response = client.post(
url="/add_webhook",
data={"webhook_name": webhook_name, "webhook_url": webhook_url},
)
assert response.status_code == 200, f"Failed to add webhook: {response.text}"
# Remove and add the feed
client.post(url="/remove", data={"feed_url": feed_url})
response = client.post(
url="/add",
data={"feed_url": feed_url, "webhook_dropdown": webhook_name},
)
assert response.status_code == 200, f"Failed to add feed: {response.text}"
# Get webhook_entries with URL-encoded webhook URL
encoded_webhook_url = urllib.parse.quote(webhook_url)
response = client.get(
url="/webhook_entries",
params={"webhook_url": encoded_webhook_url},
)
assert response.status_code == 200, f"Failed to get /webhook_entries with encoded URL: {response.text}"
assert webhook_name in response.text, "Webhook name not found in response"
# Clean up
client.post(url="/remove", data={"feed_url": feed_url})
def test_dashboard_webhook_name_links_to_webhook_detail() -> None:
"""Webhook names on the dashboard should open the webhook detail view."""
client.post(url="/delete_webhook", data={"webhook_url": webhook_url})
response: Response = client.post(
url="/add_webhook",
data={"webhook_name": webhook_name, "webhook_url": webhook_url},
)
assert response.status_code == 200, f"Failed to add webhook: {response.text}"
client.post(url="/remove", data={"feed_url": feed_url})
response = client.post(url="/add", data={"feed_url": feed_url, "webhook_dropdown": webhook_name})
assert response.status_code == 200, f"Failed to add feed: {response.text}"
response = client.get(url="/")
assert response.status_code == 200, f"Failed to get /: {response.text}"
expected_link = f"/webhook_entries?webhook_url={urllib.parse.quote(webhook_url)}"
assert expected_link in response.text, "Expected dashboard webhook link to point to the webhook detail view"
client.post(url="/remove", data={"feed_url": feed_url})
def test_modify_webhook_redirects_back_to_webhook_detail() -> None:
"""Webhook updates from the detail view should redirect back to that view with the new URL."""
original_webhook_url = "https://discord.com/api/webhooks/1234567890/abcdefghijklmnopqrstuvwxyz"
new_webhook_url = "https://discord.com/api/webhooks/1234567890/updated-token"
client.post(url="/delete_webhook", data={"webhook_url": original_webhook_url})
client.post(url="/delete_webhook", data={"webhook_url": new_webhook_url})
def test_modify_webhook_triggers_git_backup_commit() -> None:
"""Modifying a webhook URL should record a state change for git backup."""
original_webhook_url = "https://discord.com/api/webhooks/1234567890/abcdefghijklmnopqrstuvwxyz"
new_webhook_url = "https://discord.com/api/webhooks/1234567890/updated-token"
client.post(url="/delete_webhook", data={"webhook_url": original_webhook_url})
client.post(url="/delete_webhook", data={"webhook_url": new_webhook_url})
response: Response = client.post(
url="/add_webhook",
data={"webhook_name": webhook_name, "webhook_url": original_webhook_url},
)
assert response.status_code == 200, f"Failed to add webhook: {response.text}"
no_redirect_client = TestClient(app, follow_redirects=False)
with patch("discord_rss_bot.main.commit_state_change") as mock_commit_state_change:
response = no_redirect_client.post(
url="/modify_webhook",
data={
"old_hook": original_webhook_url,
"new_hook": new_webhook_url,
"redirect_to": f"/webhook_entries?webhook_url={urllib.parse.quote(original_webhook_url)}",
},
)
assert response.status_code == 303, f"Expected 303 redirect, got {response.status_code}: {response.text}"
assert mock_commit_state_change.call_count == 1, "Expected webhook modification to trigger git backup commit"
client.post(url="/delete_webhook", data={"webhook_url": new_webhook_url})
response = client.post(
url="/add_webhook",
data={"webhook_name": webhook_name, "webhook_url": original_webhook_url},
)
assert response.status_code == 200, f"Failed to add webhook: {response.text}"
no_redirect_client = TestClient(app, follow_redirects=False)
response = no_redirect_client.post(
url="/modify_webhook",
data={
"old_hook": original_webhook_url,
"new_hook": new_webhook_url,
"redirect_to": f"/webhook_entries?webhook_url={urllib.parse.quote(original_webhook_url)}",
},
)
assert response.status_code == 303, f"Expected 303 redirect, got {response.status_code}: {response.text}"
assert response.headers["location"] == (f"/webhook_entries?webhook_url={urllib.parse.quote(new_webhook_url)}"), (
f"Unexpected redirect location: {response.headers['location']}"
)
client.post(url="/delete_webhook", data={"webhook_url": new_webhook_url})
def test_webhook_entries_mass_update_preview_shows_old_and_new_urls() -> None:
"""Preview should list old->new feed URLs for webhook bulk replacement."""
@dataclass(slots=True)
class DummyFeed:
url: str
title: str | None = None
updates_enabled: bool = True
last_exception: None = None
class StubReader:
def __init__(self) -> None:
self._feeds: list[DummyFeed] = [
DummyFeed(url="https://old.example.com/rss/a.xml", title="A"),
DummyFeed(url="https://old.example.com/rss/b.xml", title="B"),
DummyFeed(url="https://unchanged.example.com/rss/c.xml", title="C"),
]
def get_tag(self, resource: object, key: str, default: object = None) -> object:
if resource == () and key == "webhooks":
return [{"name": webhook_name, "url": webhook_url}]
if key == "webhook" and isinstance(resource, str):
if resource.startswith("https://old.example.com"):
return webhook_url
if resource.startswith("https://unchanged.example.com"):
return webhook_url
return default
def get_feeds(self) -> list[DummyFeed]:
return self._feeds
def get_entries(self, **_kwargs: object) -> list[Entry]:
return []
app.dependency_overrides[get_reader_dependency] = StubReader
try:
with (
patch(
"discord_rss_bot.main.get_data_from_hook_url",
return_value=main_module.WebhookInfo(custom_name=webhook_name, url=webhook_url),
),
patch(
"discord_rss_bot.main.resolve_final_feed_url",
side_effect=lambda url: (url.replace("old.example.com", "new.example.com"), None),
),
):
response: Response = client.get(
url="/webhook_entries",
params={
"webhook_url": webhook_url,
"replace_from": "old.example.com",
"replace_to": "new.example.com",
"resolve_urls": "true",
},
)
assert response.status_code == 200, f"Failed to get preview: {response.text}"
assert "Mass update feed URLs" in response.text
assert "old.example.com/rss/a.xml" in response.text
assert "new.example.com/rss/a.xml" in response.text
assert "Will update" in response.text
assert "Matched: 2" in response.text
assert "Will update: 2" in response.text
finally:
app.dependency_overrides = {}
def test_bulk_change_feed_urls_updates_matching_feeds() -> None:
"""Mass updater should change all matching feed URLs for a webhook."""
@dataclass(slots=True)
class DummyFeed:
url: str
class StubReader:
def __init__(self) -> None:
self._feeds = [
DummyFeed(url="https://old.example.com/rss/a.xml"),
DummyFeed(url="https://old.example.com/rss/b.xml"),
DummyFeed(url="https://unchanged.example.com/rss/c.xml"),
]
self.change_calls: list[tuple[str, str]] = []
self.updated_feeds: list[str] = []
def get_tag(self, resource: object, key: str, default: object = None) -> object:
if resource == () and key == "webhooks":
return [{"name": webhook_name, "url": webhook_url}]
if key == "webhook" and isinstance(resource, str):
return webhook_url
return default
def get_feeds(self) -> list[DummyFeed]:
return self._feeds
def change_feed_url(self, old_url: str, new_url: str) -> None:
self.change_calls.append((old_url, new_url))
def update_feed(self, feed_url: str) -> None:
self.updated_feeds.append(feed_url)
def get_entries(self, **_kwargs: object) -> list[Entry]:
return []
def set_entry_read(self, _entry: Entry, _value: bool) -> None: # noqa: FBT001
return
stub_reader = StubReader()
app.dependency_overrides[get_reader_dependency] = lambda: stub_reader
no_redirect_client = TestClient(app, follow_redirects=False)
try:
with patch(
"discord_rss_bot.main.resolve_final_feed_url",
side_effect=lambda url: (url.replace("old.example.com", "new.example.com"), None),
):
response: Response = no_redirect_client.post(
url="/bulk_change_feed_urls",
data={
"webhook_url": webhook_url,
"replace_from": "old.example.com",
"replace_to": "new.example.com",
"resolve_urls": "true",
},
)
assert response.status_code == 303, f"Expected redirect, got {response.status_code}: {response.text}"
assert "Updated%202%20feed%20URL%28s%29" in response.headers.get("location", "")
assert sorted(stub_reader.change_calls) == sorted([
("https://old.example.com/rss/a.xml", "https://new.example.com/rss/a.xml"),
("https://old.example.com/rss/b.xml", "https://new.example.com/rss/b.xml"),
])
assert sorted(stub_reader.updated_feeds) == sorted([
"https://new.example.com/rss/a.xml",
"https://new.example.com/rss/b.xml",
])
finally:
app.dependency_overrides = {}
def test_webhook_entries_mass_update_preview_fragment_endpoint() -> None:
"""HTMX preview endpoint should render only the mass-update preview fragment."""
@dataclass(slots=True)
class DummyFeed:
url: str
title: str | None = None
updates_enabled: bool = True
last_exception: None = None
class StubReader:
def __init__(self) -> None:
self._feeds: list[DummyFeed] = [
DummyFeed(url="https://old.example.com/rss/a.xml", title="A"),
DummyFeed(url="https://old.example.com/rss/b.xml", title="B"),
]
def get_tag(self, resource: object, key: str, default: object = None) -> object:
if key == "webhook" and isinstance(resource, str):
return webhook_url
return default
def get_feeds(self) -> list[DummyFeed]:
return self._feeds
app.dependency_overrides[get_reader_dependency] = StubReader
try:
with patch(
"discord_rss_bot.main.resolve_final_feed_url",
side_effect=lambda url: (url.replace("old.example.com", "new.example.com"), None),
):
response: Response = client.get(
url="/webhook_entries_mass_update_preview",
params={
"webhook_url": webhook_url,
"replace_from": "old.example.com",
"replace_to": "new.example.com",
"resolve_urls": "true",
},
)
assert response.status_code == 200, f"Failed to get HTMX preview fragment: {response.text}"
assert "Will update: 2" in response.text
assert "<table" in response.text
assert "Mass update feed URLs" not in response.text, "Fragment should not include full page wrapper text"
finally:
app.dependency_overrides = {}
def test_bulk_change_feed_urls_force_update_overwrites_conflict() -> None: # noqa: C901
"""Force update should overwrite conflicting target URLs instead of skipping them."""
@dataclass(slots=True)
class DummyFeed:
url: str
class StubReader:
def __init__(self) -> None:
self._feeds = [
DummyFeed(url="https://old.example.com/rss/a.xml"),
DummyFeed(url="https://new.example.com/rss/a.xml"),
]
self.delete_calls: list[str] = []
self.change_calls: list[tuple[str, str]] = []
def get_tag(self, resource: object, key: str, default: object = None) -> object:
if resource == () and key == "webhooks":
return [{"name": webhook_name, "url": webhook_url}]
if key == "webhook" and isinstance(resource, str):
return webhook_url
return default
def get_feeds(self) -> list[DummyFeed]:
return self._feeds
def delete_feed(self, feed_url: str) -> None:
self.delete_calls.append(feed_url)
def change_feed_url(self, old_url: str, new_url: str) -> None:
self.change_calls.append((old_url, new_url))
def update_feed(self, _feed_url: str) -> None:
return
def get_entries(self, **_kwargs: object) -> list[Entry]:
return []
def set_entry_read(self, _entry: Entry, _value: bool) -> None: # noqa: FBT001
return
stub_reader = StubReader()
app.dependency_overrides[get_reader_dependency] = lambda: stub_reader
no_redirect_client = TestClient(app, follow_redirects=False)
try:
with patch(
"discord_rss_bot.main.resolve_final_feed_url",
side_effect=lambda url: (url.replace("old.example.com", "new.example.com"), None),
):
response: Response = no_redirect_client.post(
url="/bulk_change_feed_urls",
data={
"webhook_url": webhook_url,
"replace_from": "old.example.com",
"replace_to": "new.example.com",
"resolve_urls": "true",
"force_update": "true",
},
)
assert response.status_code == 303, f"Expected redirect, got {response.status_code}: {response.text}"
assert stub_reader.delete_calls == ["https://new.example.com/rss/a.xml"]
assert stub_reader.change_calls == [
(
"https://old.example.com/rss/a.xml",
"https://new.example.com/rss/a.xml",
),
]
assert "Force%20overwrote%201" in response.headers.get("location", "")
finally:
app.dependency_overrides = {}
def test_bulk_change_feed_urls_force_update_ignores_resolution_error() -> None:
"""Force update should proceed even when URL resolution returns an error (e.g. HTTP 404)."""
@dataclass(slots=True)
class DummyFeed:
url: str
class StubReader:
def __init__(self) -> None:
self._feeds = [
DummyFeed(url="https://old.example.com/rss/a.xml"),
]
self.change_calls: list[tuple[str, str]] = []
def get_tag(self, resource: object, key: str, default: object = None) -> object:
if resource == () and key == "webhooks":
return [{"name": webhook_name, "url": webhook_url}]
if key == "webhook" and isinstance(resource, str):
return webhook_url
return default
def get_feeds(self) -> list[DummyFeed]:
return self._feeds
def change_feed_url(self, old_url: str, new_url: str) -> None:
self.change_calls.append((old_url, new_url))
def update_feed(self, _feed_url: str) -> None:
return
def get_entries(self, **_kwargs: object) -> list[Entry]:
return []
def set_entry_read(self, _entry: Entry, _value: bool) -> None: # noqa: FBT001
return
stub_reader = StubReader()
app.dependency_overrides[get_reader_dependency] = lambda: stub_reader
no_redirect_client = TestClient(app, follow_redirects=False)
try:
with patch(
"discord_rss_bot.main.resolve_final_feed_url",
return_value=("https://new.example.com/rss/a.xml", "HTTP 404"),
):
response: Response = no_redirect_client.post(
url="/bulk_change_feed_urls",
data={
"webhook_url": webhook_url,
"replace_from": "old.example.com",
"replace_to": "new.example.com",
"resolve_urls": "true",
"force_update": "true",
},
)
assert response.status_code == 303, f"Expected redirect, got {response.status_code}: {response.text}"
assert stub_reader.change_calls == [
(
"https://old.example.com/rss/a.xml",
"https://new.example.com/rss/a.xml",
),
]
location = response.headers.get("location", "")
assert "Updated%201%20feed%20URL%28s%29" in location
assert "Failed%200" in location
finally:
app.dependency_overrides = {}
def test_reader_dependency_override_is_used() -> None:
"""Reader should be injectable and overridable via FastAPI dependency overrides."""
class StubReader:
def get_tag(self, _resource: str, _key: str, default: str | None = None) -> str | None:
"""Stub get_tag that always returns the default value.
Args:
_resource: Ignored.
_key: Ignored.
default: The value to return.
Returns:
The default value, simulating a missing tag.
"""
return default
app.dependency_overrides[get_reader_dependency] = StubReader
try:
response: Response = client.get(url="/add")
assert response.status_code == 200, f"Expected /add to render with overridden reader: {response.text}"
finally:
app.dependency_overrides = {}

17
tests/test_search.py
View file

@ -4,18 +4,16 @@ import tempfile
from pathlib import Path
from typing import TYPE_CHECKING
from reader import Feed
from reader import Reader
from reader import make_reader
from reader import Feed, Reader, make_reader
from discord_rss_bot.search import create_search_context
from discord_rss_bot.search import create_html_for_search_results
if TYPE_CHECKING:
from collections.abc import Iterable
def test_create_search_context() -> None:
"""Test create_search_context."""
def test_create_html_for_search_results() -> None:
"""Test create_html_for_search_results."""
# Create a reader.
with tempfile.TemporaryDirectory() as temp_dir:
# Create the temp directory.
@ -45,9 +43,10 @@ def test_create_search_context() -> None:
reader.enable_search()
reader.update_search()
# Create the search context.
context: dict = create_search_context("test", reader=reader)
assert context is not None, f"The context should not be None. Got: {context}"
# Create the HTML and check if it is not empty.
search_html: str = create_html_for_search_results("a", reader)
assert search_html is not None, f"The search HTML should not be None. Got: {search_html}"
assert len(search_html) > 10, f"The search HTML should be longer than 10 characters. Got: {len(search_html)}"
# Close the reader, so we can delete the directory.
reader.close()

24
tests/test_settings.py
View file

@ -6,9 +6,7 @@ from pathlib import Path
from reader import Reader
from discord_rss_bot.settings import data_dir
from discord_rss_bot.settings import default_custom_message
from discord_rss_bot.settings import get_reader
from discord_rss_bot.settings import data_dir, default_custom_message, get_reader
def test_reader() -> None:
@ -22,12 +20,12 @@ def test_reader() -> None:
Path.mkdir(Path(temp_dir), exist_ok=True)
custom_loc: pathlib.Path = pathlib.Path(temp_dir, "custom_loc_db.sqlite")
reader: Reader = get_reader(custom_location=str(custom_loc))
assert_msg = f"The custom reader should be an instance of Reader. But it was '{type(reader)}'."
assert isinstance(reader, Reader), assert_msg
custom_reader: Reader = get_reader(custom_location=str(custom_loc))
assert_msg = f"The custom reader should be an instance of Reader. But it was '{type(custom_reader)}'."
assert isinstance(custom_reader, Reader), assert_msg
# Close the reader, so we can delete the directory.
reader.close()
custom_reader.close()
def test_data_dir() -> None:
@ -49,16 +47,16 @@ def test_get_webhook_for_entry() -> None:
Path.mkdir(Path(temp_dir), exist_ok=True)
custom_loc: pathlib.Path = pathlib.Path(temp_dir, "custom_loc_db.sqlite")
reader: Reader = get_reader(custom_location=str(custom_loc))
custom_reader: Reader = get_reader(custom_location=str(custom_loc))
# Add a feed to the database.
reader.add_feed("https://www.reddit.com/r/movies.rss")
reader.update_feed("https://www.reddit.com/r/movies.rss")
custom_reader.add_feed("https://www.reddit.com/r/movies.rss")
custom_reader.update_feed("https://www.reddit.com/r/movies.rss")
# Add a webhook to the database.
reader.set_tag("https://www.reddit.com/r/movies.rss", "webhook", "https://example.com") # pyright: ignore[reportArgumentType]
our_tag = reader.get_tag("https://www.reddit.com/r/movies.rss", "webhook") # pyright: ignore[reportArgumentType]
custom_reader.set_tag("https://www.reddit.com/r/movies.rss", "webhook", "https://example.com") # pyright: ignore[reportArgumentType]
our_tag = custom_reader.get_tag("https://www.reddit.com/r/movies.rss", "webhook") # pyright: ignore[reportArgumentType]
assert our_tag == "https://example.com", f"The tag should be 'https://example.com'. But it was '{our_tag}'."
# Close the reader, so we can delete the directory.
reader.close()
custom_reader.close()

101
tests/test_update_interval.py
View file

@ -1,101 +0,0 @@
from __future__ import annotations
import urllib.parse
from typing import TYPE_CHECKING
from fastapi.testclient import TestClient
from discord_rss_bot.main import app
if TYPE_CHECKING:
from httpx import Response
client: TestClient = TestClient(app)
webhook_name: str = "Test Webhook for Update Interval"
webhook_url: str = "https://discord.com/api/webhooks/1234567890/test_update_interval"
feed_url: str = "https://lovinator.space/rss_test.xml"
def test_global_update_interval() -> None:
"""Test setting the global update interval."""
# Set global update interval to 30 minutes
response: Response = client.post("/set_global_update_interval", data={"interval_minutes": "30"})
assert response.status_code == 200, f"Failed to set global interval: {response.text}"
# Check that the settings page shows the new interval
response = client.get("/settings")
assert response.status_code == 200, f"Failed to get settings page: {response.text}"
assert "30" in response.text, "Global interval not updated on settings page"
def test_per_feed_update_interval() -> None:
"""Test setting per-feed update interval."""
# Clean up any existing feed/webhook
client.post(url="/delete_webhook", data={"webhook_url": webhook_url})
client.post(url="/remove", data={"feed_url": feed_url})
# Add webhook and feed
response: Response = client.post(
url="/add_webhook",
data={"webhook_name": webhook_name, "webhook_url": webhook_url},
)
assert response.status_code == 200, f"Failed to add webhook: {response.text}"
response = client.post(url="/add", data={"feed_url": feed_url, "webhook_dropdown": webhook_name})
assert response.status_code == 200, f"Failed to add feed: {response.text}"
# Set feed-specific update interval to 15 minutes
response = client.post("/set_update_interval", data={"feed_url": feed_url, "interval_minutes": "15"})
assert response.status_code == 200, f"Failed to set feed interval: {response.text}"
# Check that the feed page shows the custom interval
encoded_url = urllib.parse.quote(feed_url)
response = client.get(f"/feed?feed_url={encoded_url}")
assert response.status_code == 200, f"Failed to get feed page: {response.text}"
assert "15" in response.text, "Feed interval not displayed on feed page"
assert "Custom" in response.text, "Custom badge not shown for feed-specific interval"
def test_reset_feed_update_interval() -> None:
"""Test resetting feed update interval to global default."""
# Ensure feed/webhook setup exists regardless of test order
client.post(url="/delete_webhook", data={"webhook_url": webhook_url})
client.post(url="/remove", data={"feed_url": feed_url})
response: Response = client.post(
url="/add_webhook",
data={"webhook_name": webhook_name, "webhook_url": webhook_url},
)
assert response.status_code == 200, f"Failed to add webhook: {response.text}"
response = client.post(url="/add", data={"feed_url": feed_url, "webhook_dropdown": webhook_name})
assert response.status_code == 200, f"Failed to add feed: {response.text}"
# First set a custom interval
response = client.post("/set_update_interval", data={"feed_url": feed_url, "interval_minutes": "15"})
assert response.status_code == 200, f"Failed to set feed interval: {response.text}"
# Reset to global default
response = client.post("/reset_update_interval", data={"feed_url": feed_url})
assert response.status_code == 200, f"Failed to reset feed interval: {response.text}"
# Check that the feed page shows global default
encoded_url = urllib.parse.quote(feed_url)
response = client.get(f"/feed?feed_url={encoded_url}")
assert response.status_code == 200, f"Failed to get feed page: {response.text}"
assert "Using global default" in response.text, "Global default badge not shown after reset"
def test_update_interval_validation() -> None:
"""Test that update interval validation works."""
# Try to set an interval below minimum (should be clamped to 1)
response: Response = client.post("/set_global_update_interval", data={"interval_minutes": "0"})
assert response.status_code == 200, f"Failed to handle minimum interval: {response.text}"
# Try to set an interval above maximum (should be clamped to 10080)
response = client.post("/set_global_update_interval", data={"interval_minutes": "20000"})
assert response.status_code == 200, f"Failed to handle maximum interval: {response.text}"
# Clean up
client.post(url="/remove", data={"feed_url": feed_url})
client.post(url="/delete_webhook", data={"webhook_url": webhook_url})

49
tests/test_utils.py
View file

@ -1,6 +1,5 @@
from __future__ import annotations
from discord_rss_bot.filter.utils import is_regex_match
from discord_rss_bot.filter.utils import is_word_in_text
@ -15,51 +14,3 @@ def test_is_word_in_text() -> None:
assert is_word_in_text("Alert,Forma", "Outbreak - Mutagen Mass - Rhea (Saturn)") is False, msg_false
assert is_word_in_text("Alert,Forma", "Outbreak - Mutagen Mass - Rhea (Saturn)") is False, msg_false
assert is_word_in_text("word1,word2", "This is a sample text containing none of the words.") is False, msg_false
def test_is_regex_match() -> None:
msg_true = "Should return True"
msg_false = "Should return False"
# Test basic regex patterns
assert is_regex_match(r"word\d+", "This text contains word123") is True, msg_true
assert is_regex_match(r"^Hello", "Hello world") is True, msg_true
assert is_regex_match(r"world$", "Hello world") is True, msg_true
# Test case insensitivity
assert is_regex_match(r"hello", "This text contains HELLO") is True, msg_true
# Test comma-separated patterns
assert is_regex_match(r"pattern1,pattern2", "This contains pattern2") is True, msg_true
assert is_regex_match(r"pattern1, pattern2", "This contains pattern1") is True, msg_true
# Test regex that shouldn't match
assert is_regex_match(r"^start", "This doesn't start with the pattern") is False, msg_false
assert is_regex_match(r"end$", "This doesn't end with the pattern") is False, msg_false
# Test with empty input
assert is_regex_match("", "Some text") is False, msg_false
assert is_regex_match("pattern", "") is False, msg_false
# Test with invalid regex (should not raise an exception and return False)
assert is_regex_match(r"[incomplete", "Some text") is False, msg_false
# Test with multiple patterns where one is invalid
assert is_regex_match(r"valid, [invalid, \w+", "Contains word") is True, msg_true
# Test newline-separated patterns
newline_patterns = "pattern1\n^start\ncontains\\d+"
assert is_regex_match(newline_patterns, "This contains123 text") is True, msg_true
assert is_regex_match(newline_patterns, "start of line") is True, msg_true
assert is_regex_match(newline_patterns, "pattern1 is here") is True, msg_true
assert is_regex_match(newline_patterns, "None of these match") is False, msg_false
# Test mixed newline and comma patterns (for backward compatibility)
mixed_patterns = "pattern1\npattern2,pattern3\npattern4"
assert is_regex_match(mixed_patterns, "Contains pattern3") is True, msg_true
assert is_regex_match(mixed_patterns, "Contains pattern4") is True, msg_true
# Test with empty lines and spaces
whitespace_patterns = "\\s+\n \n\npattern\n\n"
assert is_regex_match(whitespace_patterns, "text with spaces") is True, msg_true
assert is_regex_match(whitespace_patterns, "text with pattern") is True, msg_true

85
tests/test_whitelist.py
View file

@ -4,13 +4,9 @@ import tempfile
from pathlib import Path
from typing import TYPE_CHECKING
from reader import Entry
from reader import Feed
from reader import Reader
from reader import make_reader
from reader import Entry, Feed, Reader, make_reader
from discord_rss_bot.filter.whitelist import has_white_tags
from discord_rss_bot.filter.whitelist import should_be_sent
from discord_rss_bot.filter.whitelist import has_white_tags, should_be_sent
if TYPE_CHECKING:
from collections.abc import Iterable
@ -37,18 +33,11 @@ def test_has_white_tags() -> None:
reader.update_feeds()
# Test feed without any whitelist tags
assert has_white_tags(reader=get_reader(), feed=feed) is False, "Feed should not have any whitelist tags"
assert has_white_tags(custom_reader=get_reader(), feed=feed) is False, "Feed should not have any whitelist tags"
check_if_has_tag(reader, feed, "whitelist_title")
check_if_has_tag(reader, feed, "whitelist_summary")
check_if_has_tag(reader, feed, "whitelist_content")
check_if_has_tag(reader, feed, "whitelist_author")
# Test regex whitelist tags
check_if_has_tag(reader, feed, "regex_whitelist_title")
check_if_has_tag(reader, feed, "regex_whitelist_summary")
check_if_has_tag(reader, feed, "regex_whitelist_content")
check_if_has_tag(reader, feed, "regex_whitelist_author")
# Clean up
reader.delete_feed(feed_url)
@ -56,9 +45,9 @@ def test_has_white_tags() -> None:
def check_if_has_tag(reader: Reader, feed: Feed, whitelist_name: str) -> None:
reader.set_tag(feed, whitelist_name, "a") # pyright: ignore[reportArgumentType]
assert has_white_tags(reader=reader, feed=feed) is True, "Feed should have whitelist tags"
assert has_white_tags(custom_reader=reader, feed=feed) is True, "Feed should have whitelist tags"
reader.delete_tag(feed, whitelist_name)
assert has_white_tags(reader=reader, feed=feed) is False, "Feed should not have any whitelist tags"
assert has_white_tags(custom_reader=reader, feed=feed) is False, "Feed should not have any whitelist tags"
def test_should_be_sent() -> None:
@ -120,67 +109,3 @@ def test_should_be_sent() -> None:
assert should_be_sent(reader, first_entry[0]) is False, "Entry should not be sent"
reader.delete_tag(feed, "whitelist_author")
assert should_be_sent(reader, first_entry[0]) is False, "Entry should not be sent"
def test_regex_should_be_sent() -> None:
"""Test the regex filtering functionality for whitelist."""
reader: Reader = get_reader()
# Add feed and update entries
reader.add_feed(feed_url)
feed: Feed = reader.get_feed(feed_url)
reader.update_feeds()
# Get first entry
first_entry: list[Entry] = []
entries: Iterable[Entry] = reader.get_entries(feed=feed)
assert entries is not None, "Entries should not be None"
for entry in entries:
first_entry.append(entry)
break
assert len(first_entry) == 1, "First entry should be added"
# Test entry without any regex whitelists
assert should_be_sent(reader, first_entry[0]) is False, "Entry should not be sent"
# Test regex whitelist for title
reader.set_tag(feed, "regex_whitelist_title", r"fvnnn\w+") # pyright: ignore[reportArgumentType]
assert should_be_sent(reader, first_entry[0]) is True, "Entry should be sent with regex title match"
reader.delete_tag(feed, "regex_whitelist_title")
assert should_be_sent(reader, first_entry[0]) is False, "Entry should not be sent"
# Test regex whitelist for summary
reader.set_tag(feed, "regex_whitelist_summary", r"ffdnfdn\w+") # pyright: ignore[reportArgumentType]
assert should_be_sent(reader, first_entry[0]) is True, "Entry should be sent with regex summary match"
reader.delete_tag(feed, "regex_whitelist_summary")
assert should_be_sent(reader, first_entry[0]) is False, "Entry should not be sent"
# Test regex whitelist for content
reader.set_tag(feed, "regex_whitelist_content", r"ffdnfdnfdn\w+") # pyright: ignore[reportArgumentType]
assert should_be_sent(reader, first_entry[0]) is True, "Entry should be sent with regex content match"
reader.delete_tag(feed, "regex_whitelist_content")
assert should_be_sent(reader, first_entry[0]) is False, "Entry should not be sent"
# Test regex whitelist for author
reader.set_tag(feed, "regex_whitelist_author", r"TheLovinator\d*") # pyright: ignore[reportArgumentType]
assert should_be_sent(reader, first_entry[0]) is True, "Entry should be sent with regex author match"
reader.delete_tag(feed, "regex_whitelist_author")
assert should_be_sent(reader, first_entry[0]) is False, "Entry should not be sent"
# Test invalid regex pattern (should not raise an exception)
reader.set_tag(feed, "regex_whitelist_title", r"[incomplete") # pyright: ignore[reportArgumentType]
assert should_be_sent(reader, first_entry[0]) is False, "Entry should not be sent with invalid regex"
reader.delete_tag(feed, "regex_whitelist_title")
# Test multiple regex patterns separated by commas
reader.set_tag(feed, "regex_whitelist_author", r"pattern1,TheLovinator\d*,pattern3") # pyright: ignore[reportArgumentType]
assert should_be_sent(reader, first_entry[0]) is True, "Entry should be sent with one matching pattern in list"
reader.delete_tag(feed, "regex_whitelist_author")
assert should_be_sent(reader, first_entry[0]) is False, "Entry should not be sent"
# Test newline-separated regex patterns
newline_patterns = "pattern1\nTheLovinator\\d*\npattern3"
reader.set_tag(feed, "regex_whitelist_author", newline_patterns) # pyright: ignore[reportArgumentType]
assert should_be_sent(reader, first_entry[0]) is True, "Entry should be sent with newline-separated patterns"
reader.delete_tag(feed, "regex_whitelist_author")
assert should_be_sent(reader, first_entry[0]) is False, "Entry should not be sent"