Remove test for updating an existing feed

.gitea/workflows/build.yml

@@ -0,0 +1,98 @@
+---
+name: Test and build Docker image
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+  workflow_dispatch:
+  schedule:
+    - cron: "@daily"
+
+env:
+  TEST_WEBHOOK_URL: ${{ secrets.TEST_WEBHOOK_URL }}
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      # GitHub Container Registry
+      - uses: https://github.com/docker/login-action@v3
+        if: github.event_name != 'pull_request'
+        with:
+          registry: ghcr.io
+          username: thelovinator1
+          password: ${{ secrets.PACKAGES_WRITE_GITHUB_TOKEN }}
+
+      # Gitea Container Registry
+      - uses: https://github.com/docker/login-action@v3
+        if: github.event_name != 'pull_request'
+        with:
+          registry: git.lovinator.space
+          username: thelovinator
+          password: ${{ secrets.PACKAGES_WRITE_GITEA_TOKEN }}
+
+      # Download the latest commit from the master branch
+      - uses: https://github.com/actions/checkout@v4
+
+      # Set up QEMU
+      - id: qemu
+        uses: https://github.com/docker/setup-qemu-action@v3
+        with:
+          image: tonistiigi/binfmt:master
+          platforms: linux/amd64,linux/arm64
+          cache-image: false
+
+      # Set up Buildx so we can build multi-arch images
+      - uses: https://github.com/docker/setup-buildx-action@v3
+
+      # Install the latest version of ruff
+      - uses: https://github.com/astral-sh/ruff-action@v3
+        with:
+          version: "latest"
+
+      # Lint the Python code using ruff
+      - run: ruff check --exit-non-zero-on-fix --verbose
+
+      # Check if the Python code needs formatting
+      - run: ruff format --check --verbose
+
+      # Lint Dockerfile
+      - run: docker build --check .
+
+      # Set up Python 3.13
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.13
+
+      # Install dependencies
+      - uses: astral-sh/setup-uv@v5
+        with:
+          version: "latest"
+      - run: uv sync --all-extras --all-groups
+
+      # Run tests
+      - run: uv run pytest
+
+      # Extract metadata (tags, labels) from Git reference and GitHub events for Docker
+      - id: meta
+        uses: https://github.com/docker/metadata-action@v5
+        env:
+          DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index
+        with:
+          images: |
+            ghcr.io/thelovinator1/discord-rss-bot
+            git.lovinator.space/thelovinator/discord-rss-bot
+          tags: |
+            type=raw,value=latest,enable=${{ gitea.ref == format('refs/heads/{0}', 'master') }}
+            type=raw,value=master,enable=${{ gitea.ref == format('refs/heads/{0}', 'master') }}
+
+     # Build and push the Docker image
+      - uses: https://github.com/docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: ${{ gitea.event_name != 'pull_request' }}
+          labels: ${{ steps.meta.outputs.labels }}
+          tags: ${{ steps.meta.outputs.tags }}
+          annotations: ${{ steps.meta.outputs.annotations }}

.github/workflows/build.yml

@@ -1,64 +0,0 @@
----
-name: Test and build Docker image
-on:
-  push:
-  pull_request:
-  workflow_dispatch:
-  schedule:
-    - cron: "0 6 * * *"
-
-env:
-  TEST_WEBHOOK_URL: ${{ secrets.TEST_WEBHOOK_URL }}
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: 3.12
-      - uses: astral-sh/setup-uv@v5
-        with:
-          version: "latest"
-      - run: uv sync --all-extras --all-groups
-      - run: uv run pytest
-  ruff:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: astral-sh/ruff-action@v3
-        with:
-          version: "latest"
-      - run: ruff check --exit-non-zero-on-fix --verbose
-      - run: ruff format --check --verbose
-
-  build:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-    if: github.event_name != 'pull_request'
-    concurrency:
-      group: ${{ github.workflow }}-${{ github.ref }}
-      cancel-in-progress: true
-    needs: [test, ruff]
-    steps:
-      - uses: actions/checkout@v4
-      - uses: docker/setup-qemu-action@v3
-        with:
-          platforms: all
-      - uses: docker/setup-buildx-action@v3
-      - uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      - uses: docker/build-push-action@v6
-        with:
-          context: .
-          platforms: linux/amd64, linux/arm64
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: |
-            ghcr.io/thelovinator1/discord-rss-bot:latest
-            ghcr.io/thelovinator1/discord-rss-bot:master

.vscode/launch.json

@@ -8,7 +8,11 @@
       "module": "uvicorn",
       "args": [
         "discord_rss_bot.main:app",
-        "--reload"
+        "--reload",
+        "--host",
+        "0.0.0.0",
+        "--port",
+        "5000",
       ],
       "jinja": true,
       "justMyCode": true

Dockerfile

@@ -12,4 +12,5 @@ RUN --mount=type=cache,target=/root/.cache/uv \
 COPY --chown=botuser:botuser discord_rss_bot/ /home/botuser/discord-rss-bot/discord_rss_bot/
 EXPOSE 5000
 VOLUME ["/home/botuser/.local/share/discord_rss_bot/"]
+HEALTHCHECK --interval=10m --timeout=5s CMD ["uv", "run", "./discord_rss_bot/healthcheck.py"]
 CMD ["uv", "run", "uvicorn", "discord_rss_bot.main:app", "--host=0.0.0.0", "--port=5000", "--proxy-headers", "--forwarded-allow-ips='*'", "--log-level", "debug"]

discord_rss_bot/custom_message.py

@@ -68,8 +68,18 @@ def replace_tags_in_text_message(entry: Entry) -> str:
 
     first_image: str = get_first_image(summary, content)
 
-    summary = markdownify(html=summary, strip=["img", "table", "td", "tr", "tbody", "thead"], escape_misc=False)
+    summary = markdownify(
-    content = markdownify(html=content, strip=["img", "table", "td", "tr", "tbody", "thead"], escape_misc=False)
+        html=summary,
+        strip=["img", "table", "td", "tr", "tbody", "thead"],
+        escape_misc=False,
+        heading_style="ATX",
+    )
+    content = markdownify(
+        html=content,
+        strip=["img", "table", "td", "tr", "tbody", "thead"],
+        escape_misc=False,
+        heading_style="ATX",
+    )
 
     if "[https://" in content or "[https://www." in content:
         content = content.replace("[https://", "[")
@@ -189,8 +199,18 @@ def replace_tags_in_embed(feed: Feed, entry: Entry) -> CustomEmbed:
 
     first_image: str = get_first_image(summary, content)
 
-    summary = markdownify(html=summary, strip=["img", "table", "td", "tr", "tbody", "thead"], escape_misc=False)
+    summary = markdownify(
-    content = markdownify(html=content, strip=["img", "table", "td", "tr", "tbody", "thead"], escape_misc=False)
+        html=summary,
+        strip=["img", "table", "td", "tr", "tbody", "thead"],
+        escape_misc=False,
+        heading_style="ATX",
+    )
+    content = markdownify(
+        html=content,
+        strip=["img", "table", "td", "tr", "tbody", "thead"],
+        escape_misc=False,
+        heading_style="ATX",
+    )
 
     if "[https://" in content or "[https://www." in content:
         content = content.replace("[https://", "[")

discord_rss_bot/feeds.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import datetime
 import logging
+import os
 import pprint
 import re
 from typing import TYPE_CHECKING, Any
@@ -289,7 +290,7 @@ def set_entry_as_read(reader: Reader, entry: Entry) -> None:
         logger.exception("Error setting entry to read: %s", entry.id)
 
 
-def send_to_discord(custom_reader: Reader | None = None, feed: Feed | None = None, *, do_once: bool = False) -> None:
+def send_to_discord(custom_reader: Reader | None = None, feed: Feed | None = None, *, do_once: bool = False) -> None:  # noqa: PLR0912
     """Send entries to Discord.
 
     If response was not ok, we will log the error and mark the entry as unread, so it will be sent again next time.
@@ -303,7 +304,10 @@ def send_to_discord(custom_reader: Reader | None = None, feed: Feed | None = Non
     reader: Reader = get_reader() if custom_reader is None else custom_reader
 
     # Check for new entries for every feed.
-    reader.update_feeds()
+    reader.update_feeds(
+        scheduled=True,
+        workers=os.cpu_count() or 1,
+    )
 
     # Loop through the unread entries.
     entries: Iterable[Entry] = reader.get_entries(feed=feed, read=False)
@@ -320,6 +324,11 @@ def send_to_discord(custom_reader: Reader | None = None, feed: Feed | None = Non
             continue
 
         should_send_embed: bool = should_send_embed_check(reader, entry)
+
+        # Youtube feeds only need to send the link
+        if is_youtube_feed(entry.feed.url):
+            should_send_embed = False
+
         if should_send_embed:
             webhook = create_embed_webhook(webhook_url, entry)
         else:
@@ -341,11 +350,27 @@ def send_to_discord(custom_reader: Reader | None = None, feed: Feed | None = Non
             continue
 
         # Check if the feed has a whitelist, and if it does, check if the entry is whitelisted.
-        if has_white_tags(reader, entry.feed):
+        if has_white_tags(reader, entry.feed) and not should_be_sent(reader, entry):
-            if should_be_sent(reader, entry):
+            logger.info("Entry was not whitelisted: %s", entry.id)
+            continue
+
+        # Use a custom webhook for Hoyolab feeds.
+        if is_c3kay_feed(entry.feed.url):
+            entry_link: str | None = entry.link
+            if entry_link:
+                post_id: str | None = extract_post_id_from_hoyolab_url(entry_link)
+                if post_id:
+                    post_data: dict[str, Any] | None = fetch_hoyolab_post(post_id)
+                    if post_data:
+                        webhook = create_hoyolab_webhook(webhook_url, entry, post_data)
                         execute_webhook(webhook, entry)
                         return
-            continue
+                    logger.warning(
+                        "Failed to create Hoyolab webhook for feed %s, falling back to regular processing",
+                        entry.feed.url,
+                    )
+            else:
+                logger.warning("No entry link found for feed %s, falling back to regular processing", entry.feed.url)
 
         # Send the entry to Discord as it is not blacklisted or feed has a whitelist.
         execute_webhook(webhook, entry)

discord_rss_bot/main.py

@@ -100,9 +100,9 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None]:
     add_missing_tags(reader)
     scheduler: AsyncIOScheduler = AsyncIOScheduler()
 
-    # Update all feeds every 15 minutes.
+    # Run job every minute to check for new entries. Feeds will be checked every 15 minutes.
     # TODO(TheLovinator): Make this configurable.
-    scheduler.add_job(send_to_discord, "interval", minutes=15, next_run_time=datetime.now(tz=UTC))
+    scheduler.add_job(send_to_discord, "interval", minutes=1, next_run_time=datetime.now(tz=UTC))
     scheduler.start()
     logger.info("Scheduler started.")
     yield
@@ -921,6 +921,29 @@ async def remove_feed(feed_url: Annotated[str, Form()]):
     return RedirectResponse(url="/", status_code=303)
 
 
+@app.get("/update", response_class=HTMLResponse)
+async def update_feed(request: Request, feed_url: str):
+    """Update a feed.
+
+    Args:
+        request: The request object.
+        feed_url: The feed URL to update.
+
+    Raises:
+        HTTPException: If the feed is not found.
+
+    Returns:
+        RedirectResponse: Redirect to the feed page.
+    """
+    try:
+        reader.update_feed(urllib.parse.unquote(feed_url))
+    except FeedNotFoundError as e:
+        raise HTTPException(status_code=404, detail="Feed not found") from e
+
+    logger.info("Manually updated feed: %s", feed_url)
+    return RedirectResponse(url="/feed?feed_url=" + urllib.parse.quote(feed_url), status_code=303)
+
+
 @app.get("/search", response_class=HTMLResponse)
 async def search(request: Request, query: str):
     """Get entries matching a full-text search query.

discord_rss_bot/settings.py

@@ -24,7 +24,7 @@ default_custom_embed: dict[str, str] = {
 }
 
 
-@lru_cache
+@lru_cache(maxsize=1)
 def get_reader(custom_location: Path | None = None) -> Reader:
     """Get the reader.
 
@@ -35,5 +35,10 @@ def get_reader(custom_location: Path | None = None) -> Reader:
         The reader.
     """
     db_location: Path = custom_location or Path(data_dir) / "db.sqlite"
+    reader: Reader = make_reader(url=str(db_location))
 
-    return make_reader(url=str(db_location))
+    # https://reader.readthedocs.io/en/latest/api.html#reader.types.UpdateConfig
+    # Set the update interval to 15 minutes
+    reader.set_tag((), ".reader.update", {"interval": 15})
+
+    return reader

discord_rss_bot/templates/feed.html

@@ -28,6 +28,8 @@
 
     <!-- Feed Actions -->
     <div class="mt-3 d-flex flex-wrap gap-2">
+        <a href="/update?feed_url={{ feed.url|encode_url }}" class="btn btn-primary btn-sm">Update</a>
+
         <form action="/remove" method="post" class="d-inline">
             <button class="btn btn-danger btn-sm" name="feed_url" value="{{ feed.url }}"
                 onclick="return confirm('Are you sure you want to delete this feed?')">Remove</button>

docker-compose.yml

@@ -10,7 +10,7 @@ services:
       # - /Docker/Bots/discord-rss-bot:/home/botuser/.local/share/discord_rss_bot/
       - data:/home/botuser/.local/share/discord_rss_bot/
     healthcheck:
-      test: ["CMD", "python", "discord_rss_bot/healthcheck.py"]
+      test: [ "CMD", "uv", "run", "./discord_rss_bot/healthcheck.py" ]
       interval: 1m
       timeout: 10s
       retries: 3

tests/test_main.py

@@ -229,3 +229,16 @@ def test_delete_webhook() -> None:
     response = client.get(url="/webhooks")
     assert response.status_code == 200, f"Failed to get /webhooks: {response.text}"
     assert webhook_name not in response.text, f"Webhook found in /webhooks: {response.text}"
+
+
+def test_update_feed_not_found() -> None:
+    """Test updating a non-existent feed."""
+    # Generate a feed URL that does not exist
+    nonexistent_feed_url = "https://nonexistent-feed.example.com/rss.xml"
+
+    # Try to update the non-existent feed
+    response: Response = client.get(url="/update", params={"feed_url": urllib.parse.quote(nonexistent_feed_url)})
+
+    # Check that it returns a 404 status code
+    assert response.status_code == 404, f"Expected 404 for non-existent feed, got: {response.status_code}"
+    assert "Feed not found" in response.text