Add command for updating feeds

.gitignore

@@ -158,3 +158,4 @@ cython_debug/
 # FeedVault directories
 data/
 media/
+staticfiles/

.vscode/settings.json

@@ -50,6 +50,7 @@
     "localbattle",
     "localdomain",
     "lscr",
+    "makemigrations",
     "malformedurl",
     "meowning",
     "mmcdole",

README.md

@@ -30,6 +30,25 @@ Please create a new issue before submitting a big pull request. I am probably ok
 
 Try to minimize the number of dependencies you add to the project. If you need to add a new dependency, please create an issue first.
 
+
+## Development
+
+- [Python](https://www.python.org/)
+- [Poetry](https://python-poetry.org/)
+
+```bash
+poetry install
+poetry shell
+python manage.py test
+python manage.py collectstatic
+python manage.py makemigrations
+python manage.py migrate
+python manage.py runserver
+
+# Update feeds
+python manage.py update_feeds
+```
+
 ## Contact
 
 For any inquiries or support, please create an issue on GitHub.

feedvault/feeds.py

@@ -335,25 +335,34 @@ def populate_feed(url: str | None, user: AbstractBaseUser | AnonymousUser) -> Fe
     return feed
 
 
-def grab_entries(feed: Feed) -> None:
+def grab_entries(feed: Feed) -> None | list[Entry]:
     """Grab the entries from a feed.
 
     Args:
         feed: The feed to grab the entries from.
+
+    Returns:
+        The entries that were added. If no entries were added, None is returned.
     """
+    # Set the last checked time to now.
+    feed.last_checked = timezone.now()
+    feed.save()
+
+    entries_added: list[Entry] = []
     # Parse the feed.
     parsed_feed: dict | None = parse_feed(url=feed.feed_url)
     if not parsed_feed:
-        return
+        return None
 
     entries = parsed_feed.get("entries", [])
     for entry in entries:
         added_entry: Entry | None = add_entry(feed=feed, entry=entry)
         if not added_entry:
             continue
+        entries_added.append(added_entry)
 
-    logger.info("Grabbed entries for feed: %s", feed)
+    logger.info("Added entries: %s", entries_added)
-    return
+    return entries_added
 
 
 def add_url(url: str, user: AbstractBaseUser | AnonymousUser) -> FeedAddResult:

feedvault/management/commands/update_feeds.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from datetime import timedelta
+
+from django.core.management.base import BaseCommand, no_translations
+from django.db.models import Q
+from django.utils import timezone
+
+from feedvault.feeds import grab_entries
+from feedvault.models import Entry, Feed
+
+
+class Command(BaseCommand):
+    help = "Check for new entries in feeds"
+    requires_migrations_checks = True
+
+    @no_translations
+    def handle(self, *args, **options) -> None:  # noqa: ANN002, ANN003, ARG002
+        new_entries: int = 0
+
+        # Grab feeds that haven't been checked in 15 minutes OR haven't been checked at all
+        for feed in Feed.objects.filter(
+            Q(last_checked__lte=timezone.now() - timedelta(minutes=15)) | Q(last_checked__isnull=True),
+        ):
+            entries: None | list[Entry] = grab_entries(feed)
+            if not entries:
+                self.stdout.write(f"No new entries for {feed.title}")
+                continue
+
+            self.stdout.write(f"Updated {feed}")
+            self.stdout.write(f"Added {len(entries)} new entries for {feed}")
+            new_entries += len(entries)
+
+        if new_entries:
+            self.stdout.write(self.style.SUCCESS(f"Successfully updated feeds. Added {new_entries} new entries"))
+
+        self.stdout.write("No new entries found")

feedvault/models.py

@@ -293,7 +293,7 @@ class Entry(models.Model):
 
     def __str__(self) -> str:
         """Return string representation of the entry."""
-        return f"{self.feed} - {self.title}"
+        return f"{self.feed.feed_url} - {self.title}"
 
 
 def get_upload_path(instance: UserUploadedFile, filename: str) -> str:

feedvault/settings.py

@@ -9,6 +9,9 @@ from dotenv import find_dotenv, load_dotenv
 
 load_dotenv(dotenv_path=find_dotenv(), verbose=True)
 
+# Is True when running tests, used for not spamming Discord when new users are created
+TESTING: bool = len(sys.argv) > 1 and sys.argv[1] == "test"
+
 DEBUG: bool = os.getenv(key="DEBUG", default="True").lower() == "true"
 BASE_DIR: Path = Path(__file__).resolve().parent.parent
 SECRET_KEY: str = os.getenv("SECRET_KEY", default="")
@@ -42,7 +45,12 @@ ROOT_URLCONF = "feedvault.urls"
 WSGI_APPLICATION = "feedvault.wsgi.application"
 NINJA_PAGINATION_PER_PAGE = 1000
 STATIC_URL = "static/"
-STATIC_ROOT: Path = BASE_DIR / "static"
+STATIC_ROOT: Path = BASE_DIR / "staticfiles"
+STATICFILES_STORAGE = (
+    "django.contrib.staticfiles.storage.StaticFilesStorage"
+    if TESTING
+    else "whitenoise.storage.CompressedManifestStaticFilesStorage"
+)
 STATIC_ROOT.mkdir(parents=True, exist_ok=True)
 MEDIA_URL = "media/"
 MEDIA_ROOT: Path = BASE_DIR / "media"
@@ -50,8 +58,6 @@ MEDIA_ROOT.mkdir(parents=True, exist_ok=True)
 LOGIN_REDIRECT_URL = "/"
 LOGOUT_REDIRECT_URL = "/"
 
-# Is True when running tests, used for not spamming Discord when new users are created
-TESTING: bool = len(sys.argv) > 1 and sys.argv[1] == "test"
 
 INSTALLED_APPS: list[str] = [
     "feedvault.apps.FeedVaultConfig",

feedvault/tests.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from pathlib import Path
 from typing import TYPE_CHECKING
 
 from django.contrib.auth.models import User
@@ -7,7 +8,7 @@ from django.http.response import HttpResponse
 from django.test import Client, TestCase
 from django.urls import reverse
 
-from feedvault.models import Domain, Entry, Feed
+from feedvault.models import Domain, Entry, Feed, UserUploadedFile
 from feedvault.stats import get_db_size
 
 if TYPE_CHECKING:
@@ -63,17 +64,46 @@ class TestFeedsPage(TestCase):
 
 
 class TestAddPage(TestCase):
+    def setUp(self) -> None:
+        """Create a test user."""
+        self.user: User = User.objects.create_user(
+            username="testuser",
+            email="hello@feedvault.se",
+            password="testpassword",  # noqa: S106
+        )
+
+        self.client.force_login(user=self.user)
+
     def test_add_page(self) -> None:
         """Test if the add page is accessible."""
-        response: HttpResponse = self.client.get(reverse("add"))
+        response: HttpResponse = self.client.post(reverse("add"), {"urls": "https://feedvault.se/feed.xml"})
         assert response.status_code == 200, f"Expected 200, got {response.status_code}"
 
 
 class TestUploadPage(TestCase):
+    def setUp(self) -> None:
+        """Create a test user."""
+        self.user: User = User.objects.create_user(
+            username="testuser",
+            email="hello@feedvault.se",
+            password="testpassword",  # noqa: S106
+        )
+
+        self.client.force_login(user=self.user)
+
     def test_upload_page(self) -> None:
         """Test if the upload page is accessible."""
-        response: HttpResponse = self.client.get(reverse("upload"))
+        # Check the amounts of files in the database
-        assert response.status_code == 200, f"Expected 200, got {response.status_code}"
+        assert UserUploadedFile.objects.count() == 0, f"Expected 0, got {UserUploadedFile.objects.count()}"
+
+        # Open this file and upload it
+        current_file = __file__
+        with Path(current_file).open("rb") as file:
+            response: HttpResponse = self.client.post(reverse("upload"), {"file": file})
+            assert response.status_code == 200, f"Expected 200, got {response.status_code}: {response.content}"
+
+        # Check if the file is in the database
+        assert UserUploadedFile.objects.count() == 1, f"Expected 1, got {UserUploadedFile.objects.count()}"
 
 
 class TestRobotsPage(TestCase):

feedvault/views.py

@@ -22,7 +22,7 @@ from django.views import View
 from django.views.generic.edit import CreateView
 from django.views.generic.list import ListView
 
-from feedvault.add_feeds import add_url
+from feedvault.feeds import add_url
 from feedvault.models import Domain, Entry, Feed, FeedAddResult, UserUploadedFile
 
 if TYPE_CHECKING: