Compare commits
3 commits
a02b5d5f66
...
08dbefa417
| Author | SHA1 | Date | |
|---|---|---|---|
|
08dbefa417 |
|||
|
ff70afa6c3 |
|||
|
0e1624c2c9 |
8 changed files with 224 additions and 4 deletions
|
|
@ -1,34 +1,106 @@
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
from django.core.management.base import BaseCommand
|
from django.core.management.base import BaseCommand
|
||||||
|
|
||||||
|
from feeds.models import Entry
|
||||||
from feeds.models import Feed
|
from feeds.models import Feed
|
||||||
from feeds.services import fetch_and_archive_feed
|
from feeds.services import fetch_and_archive_feed
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from django.core.management.base import CommandParser
|
from django.core.management.base import CommandParser
|
||||||
|
from pytest_django.asserts import QuerySet
|
||||||
|
|
||||||
|
|
||||||
class Command(BaseCommand):
|
class Command(BaseCommand):
|
||||||
"""Django management command to fetch and archive a feed by URL."""
|
"""Django management command to fetch and archive a feed by URL."""
|
||||||
|
|
||||||
help = "Fetch and archive a feed by URL."
|
help = "Fetch and archive a feed by URL."
|
||||||
|
amount_to_show: int = 10
|
||||||
|
|
||||||
def add_arguments(self, parser: CommandParser) -> None:
|
def add_arguments(self, parser: CommandParser) -> None:
|
||||||
"""Add URL argument to the command."""
|
"""Add URL argument and --reset option to the command."""
|
||||||
parser.add_argument("url", type=str, help="Feed URL to fetch and archive.")
|
parser.add_argument(
|
||||||
|
"url",
|
||||||
|
type=str,
|
||||||
|
help="Feed URL to fetch and archive.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--reset",
|
||||||
|
action="store_true",
|
||||||
|
help="Remove all entries for this feed before archiving.",
|
||||||
|
)
|
||||||
|
|
||||||
def handle(self, *args, **options) -> None: # noqa: ARG002
|
def handle(self, *args, **options) -> None: # noqa: ARG002
|
||||||
"""Handle the command execution."""
|
"""Handle the command execution."""
|
||||||
url: str = options["url"]
|
url: str = options["url"]
|
||||||
|
reset: bool = options.get("reset", False)
|
||||||
|
|
||||||
feed, created = Feed.objects.get_or_create(url=url)
|
feed, created = Feed.objects.get_or_create(url=url)
|
||||||
|
|
||||||
if created:
|
if created:
|
||||||
self.stdout.write(self.style.SUCCESS(f"Created new feed for URL: {url}"))
|
msg = f"Created new feed for URL: {url}"
|
||||||
|
self.stdout.write(self.style.SUCCESS(msg))
|
||||||
|
|
||||||
|
if reset:
|
||||||
|
entries_qs: QuerySet[Entry, Entry] = Entry.objects.filter(feed=feed)
|
||||||
|
count: int = entries_qs.count()
|
||||||
|
|
||||||
|
if count == 0:
|
||||||
|
msg = f"No entries found for feed: {url}"
|
||||||
|
self.stdout.write(self.style.WARNING(msg))
|
||||||
|
|
||||||
|
else:
|
||||||
|
msg = f"The following {count} entries will be removed for feed: {url}"
|
||||||
|
self.stdout.write(self.style.WARNING(msg))
|
||||||
|
|
||||||
|
entries = entries_qs.order_by("-published_at")[: self.amount_to_show]
|
||||||
|
for entry in entries:
|
||||||
|
title: str | None = get_entry_title(entry)
|
||||||
|
|
||||||
|
msg = f"- entry_id: {entry.entry_id}, published_at: {entry.published_at}, title: {title}"
|
||||||
|
self.stdout.write(self.style.WARNING(msg))
|
||||||
|
|
||||||
|
if count > self.amount_to_show:
|
||||||
|
self.stdout.write(f"...and {count - self.amount_to_show} more.")
|
||||||
|
|
||||||
|
prompt = "Are you sure you want to delete these entries? Type 'yes' to confirm: "
|
||||||
|
confirm: str = input(prompt)
|
||||||
|
|
||||||
|
if confirm.strip().lower() == "yes":
|
||||||
|
deleted, _ = entries_qs.delete()
|
||||||
|
|
||||||
|
msg = f"Deleted {deleted} entr{'y' if deleted == 1 else 'ies'} for feed: {url}"
|
||||||
|
self.stdout.write(self.style.SUCCESS(msg))
|
||||||
|
|
||||||
|
else:
|
||||||
|
msg = "Aborted reset. No entries were deleted."
|
||||||
|
self.stdout.write(self.style.ERROR(msg))
|
||||||
|
return
|
||||||
|
|
||||||
new_entries: int = fetch_and_archive_feed(feed)
|
new_entries: int = fetch_and_archive_feed(feed)
|
||||||
if new_entries:
|
if new_entries:
|
||||||
msg: str = f"Archived {new_entries} new entr{'y' if new_entries == 1 else 'ies'} for URL: {url}"
|
msg: str = f"Archived {new_entries} new entr{'y' if new_entries == 1 else 'ies'} for URL: {url}"
|
||||||
self.stdout.write(self.style.SUCCESS(msg))
|
self.stdout.write(self.style.SUCCESS(msg))
|
||||||
|
|
||||||
else:
|
else:
|
||||||
msg: str = "\tFeed is up to date, but no new entries were archived."
|
msg: str = "\tFeed is up to date, but no new entries were archived."
|
||||||
self.stdout.write(self.style.WARNING(msg))
|
self.stdout.write(self.style.WARNING(msg))
|
||||||
|
|
||||||
|
|
||||||
|
def get_entry_title(entry: Entry) -> str | None:
|
||||||
|
"""Get the title from an entry's data.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
entry (Entry): The Entry object from which to extract the title.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
str | None: The title of the entry if available, otherwise None.
|
||||||
|
"""
|
||||||
|
# entry_data is a JSONField
|
||||||
|
entry_data: dict[str, Any] | list[Any] | None = entry.data
|
||||||
|
|
||||||
|
if not isinstance(entry_data, dict):
|
||||||
|
return None
|
||||||
|
|
||||||
|
return entry_data.get("title")
|
||||||
|
|
|
||||||
39
feeds/tests/test_archive_feed_reset.py
Normal file
39
feeds/tests/test_archive_feed_reset.py
Normal file
|
|
@ -0,0 +1,39 @@
|
||||||
|
import builtins
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from django.core.management import call_command
|
||||||
|
|
||||||
|
from feeds.models import Entry
|
||||||
|
from feeds.models import Feed
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from collections.abc import Callable
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_reset_option_removes_only_feed_entries(db: None) -> None:
|
||||||
|
"""Test that the --reset option in the archive_feed command only removes entries for the specified feed."""
|
||||||
|
url1 = "http://example.com/feed1.xml"
|
||||||
|
url2 = "http://example.com/feed2.xml"
|
||||||
|
feed1: Feed = Feed.objects.create(url=url1, domain="example.com")
|
||||||
|
feed2: Feed = Feed.objects.create(url=url2, domain="example.com")
|
||||||
|
|
||||||
|
# Create entries for both feeds
|
||||||
|
_e1: Entry = Entry.objects.create(feed=feed1, entry_id="a", content_hash=1)
|
||||||
|
_e2: Entry = Entry.objects.create(feed=feed1, entry_id="b", content_hash=2)
|
||||||
|
_e3: Entry = Entry.objects.create(feed=feed2, entry_id="c", content_hash=3)
|
||||||
|
assert Entry.objects.filter(feed=feed1).count() == 2
|
||||||
|
assert Entry.objects.filter(feed=feed2).count() == 1
|
||||||
|
|
||||||
|
# Simulate user confirmation by patching input
|
||||||
|
orig_input: Callable[[object], str] = builtins.input
|
||||||
|
builtins.input = lambda _: "yes"
|
||||||
|
try:
|
||||||
|
call_command("archive_feed", url1, "--reset")
|
||||||
|
finally:
|
||||||
|
builtins.input = orig_input
|
||||||
|
|
||||||
|
# Only feed1's entries should be deleted
|
||||||
|
assert Entry.objects.filter(feed=feed1).count() == 0
|
||||||
|
assert Entry.objects.filter(feed=feed2).count() == 1
|
||||||
|
|
@ -12,4 +12,9 @@ if TYPE_CHECKING:
|
||||||
urlpatterns: list[URLPattern | URLResolver] = [
|
urlpatterns: list[URLPattern | URLResolver] = [
|
||||||
path("", views.feed_list, name="feed-list"),
|
path("", views.feed_list, name="feed-list"),
|
||||||
path("feeds/<int:feed_id>/", views.feed_detail, name="feed-detail"),
|
path("feeds/<int:feed_id>/", views.feed_detail, name="feed-detail"),
|
||||||
|
path(
|
||||||
|
"feeds/<int:feed_id>/entries/<int:entry_id>/",
|
||||||
|
views.entry_detail,
|
||||||
|
name="entry-detail",
|
||||||
|
),
|
||||||
]
|
]
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,5 @@
|
||||||
|
import html
|
||||||
|
import json
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from django.http import HttpResponse
|
from django.http import HttpResponse
|
||||||
|
|
@ -47,6 +49,7 @@ def feed_detail(request: HttpRequest, feed_id: int) -> HttpResponse:
|
||||||
"-published_at",
|
"-published_at",
|
||||||
"-fetched_at",
|
"-fetched_at",
|
||||||
)[:50]
|
)[:50]
|
||||||
|
|
||||||
html: list[str] = [
|
html: list[str] = [
|
||||||
"<!DOCTYPE html>",
|
"<!DOCTYPE html>",
|
||||||
f"<html><head><title>FeedVault - {feed.url}</title></head><body>",
|
f"<html><head><title>FeedVault - {feed.url}</title></head><body>",
|
||||||
|
|
@ -64,7 +67,51 @@ def feed_detail(request: HttpRequest, feed_id: int) -> HttpResponse:
|
||||||
summary: str | None = entry.data.get("summary") if entry.data else None
|
summary: str | None = entry.data.get("summary") if entry.data else None
|
||||||
snippet: str = title or summary or "[no title]"
|
snippet: str = title or summary or "[no title]"
|
||||||
html.append(
|
html.append(
|
||||||
f"<li><b>{entry.published_at or entry.fetched_at}:</b> {snippet} <small>(id: {entry.entry_id})</small></li>",
|
f"<li><b>{entry.published_at or entry.fetched_at}:</b> "
|
||||||
|
f'<a href="/feeds/{feed.pk}/entries/{entry.pk}/">{snippet}</a> '
|
||||||
|
f"<small>(id: {entry.entry_id})</small></li>",
|
||||||
)
|
)
|
||||||
html.extend(("</ul>", '<p><a href="/">Back to list</a></p>', "</body></html>"))
|
html.extend(("</ul>", '<p><a href="/">Back to list</a></p>', "</body></html>"))
|
||||||
return HttpResponse("\n".join(html))
|
return HttpResponse("\n".join(html))
|
||||||
|
|
||||||
|
|
||||||
|
def entry_detail(request: HttpRequest, feed_id: int, entry_id: int) -> HttpResponse:
|
||||||
|
"""View to display the details of a specific entry.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
request (HttpRequest): The HTTP request object.
|
||||||
|
feed_id (int): The ID of the feed the entry belongs to.
|
||||||
|
entry_id (int): The ID of the entry to display.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
HttpResponse: An HTML response containing the entry details.
|
||||||
|
"""
|
||||||
|
feed: Feed = get_object_or_404(Feed, id=feed_id)
|
||||||
|
entry: Entry = get_object_or_404(Entry, id=entry_id, feed=feed)
|
||||||
|
|
||||||
|
# Render images if present in entry.data
|
||||||
|
entry_data_html: str = ""
|
||||||
|
if entry.data:
|
||||||
|
formatted_json: str = json.dumps(entry.data, indent=2, ensure_ascii=False)
|
||||||
|
escaped_json: str = html.escape(formatted_json)
|
||||||
|
note: str = '<div style="font-size:small;color:#666;margin-bottom:4px;">Note: HTML in the JSON is escaped for display and will not be rendered as HTML.</div>'
|
||||||
|
entry_data_html: str = f"{note}<pre>{escaped_json}</pre>"
|
||||||
|
else:
|
||||||
|
entry_data_html: str = "<p>[No data]</p>"
|
||||||
|
|
||||||
|
html_lines: list[str] = [
|
||||||
|
"<!DOCTYPE html>",
|
||||||
|
f"<html><head><title>FeedVault - Entry {entry.entry_id}</title></head><body>",
|
||||||
|
"<h1>Entry Detail</h1>",
|
||||||
|
f"<p><b>Feed:</b> <a href='/feeds/{feed.pk}/'>{feed.url}</a></p>",
|
||||||
|
f"<p><b>Entry ID:</b> {entry.entry_id}</p>",
|
||||||
|
f"<p><b>Published:</b> {entry.published_at}</p>",
|
||||||
|
f"<p><b>Fetched:</b> {entry.fetched_at}</p>",
|
||||||
|
f"<p><b>Content Hash:</b> {entry.content_hash}</p>",
|
||||||
|
f"<p><b>Error Message:</b> {entry.error_message or '[none]'} </p>",
|
||||||
|
"<h2>Entry Data</h2>",
|
||||||
|
entry_data_html,
|
||||||
|
f'<p><a href="/feeds/{feed.pk}/">Back to feed</a></p>',
|
||||||
|
"</body></html>",
|
||||||
|
]
|
||||||
|
return HttpResponse("\n".join(html_lines))
|
||||||
|
|
|
||||||
10
tools/systemd/feedvault-backup.service
Normal file
10
tools/systemd/feedvault-backup.service
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
[Unit]
|
||||||
|
Description=FeedVault database backup
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
User=feedvault
|
||||||
|
Group=feedvault
|
||||||
|
WorkingDirectory=/home/feedvault/feedvault
|
||||||
|
EnvironmentFile=/home/feedvault/feedvault/.env
|
||||||
|
ExecStart=/usr/bin/uv run python manage.py backup_db
|
||||||
9
tools/systemd/feedvault-backup.timer
Normal file
9
tools/systemd/feedvault-backup.timer
Normal file
|
|
@ -0,0 +1,9 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Nightly FeedVault database backup
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnCalendar=*-*-* 03:15:00
|
||||||
|
Persistent=true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
27
tools/systemd/feedvault.service
Normal file
27
tools/systemd/feedvault.service
Normal file
|
|
@ -0,0 +1,27 @@
|
||||||
|
[Unit]
|
||||||
|
Description=FeedVault
|
||||||
|
Requires=feedvault.socket
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
User=feedvault
|
||||||
|
Group=feedvault
|
||||||
|
WorkingDirectory=/home/feedvault/feedvault
|
||||||
|
EnvironmentFile=/home/feedvault/feedvault/.env
|
||||||
|
RuntimeDirectory=feedvault
|
||||||
|
ExecStart=/usr/bin/uv run gunicorn config.wsgi:application --bind unix:/run/feedvault/feedvault.sock --workers 13 --name feedvault --max-requests-jitter 50 --max-requests 1200
|
||||||
|
ReadWritePaths=/home/feedvault/feedvault /run/feedvault
|
||||||
|
|
||||||
|
NoNewPrivileges=yes
|
||||||
|
PrivateTmp=yes
|
||||||
|
ProtectSystem=full
|
||||||
|
ProtectHome=no
|
||||||
|
CapabilityBoundingSet=
|
||||||
|
AmbientCapabilities=
|
||||||
|
RestrictRealtime=yes
|
||||||
|
LockPersonality=yes
|
||||||
|
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=5
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
11
tools/systemd/feedvault.socket
Normal file
11
tools/systemd/feedvault.socket
Normal file
|
|
@ -0,0 +1,11 @@
|
||||||
|
[Unit]
|
||||||
|
Description=FeedVault Socket
|
||||||
|
|
||||||
|
[Socket]
|
||||||
|
ListenStream=/run/feedvault/feedvault.sock
|
||||||
|
SocketUser=feedvault
|
||||||
|
SocketGroup=feedvault
|
||||||
|
SocketMode=0660
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=sockets.target
|
||||||
Loading…
Add table
Add a link
Reference in a new issue