From d04fe12f8079475f763daef86337df241b72141a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Hells=C3=A9n?= Date: Sun, 17 Mar 2024 20:22:19 +0100 Subject: [PATCH] Add command for updating feeds --- .gitignore | 1 + .vscode/settings.json | 1 + README.md | 19 ++++++++++ feedvault/{add_feeds.py => feeds.py} | 17 +++++++-- feedvault/management/commands/__init__.py | 0 feedvault/management/commands/update_feeds.py | 37 ++++++++++++++++++ feedvault/models.py | 2 +- feedvault/settings.py | 12 ++++-- feedvault/tests.py | 38 +++++++++++++++++-- feedvault/views.py | 2 +- 10 files changed, 116 insertions(+), 13 deletions(-) rename feedvault/{add_feeds.py => feeds.py} (96%) create mode 100644 feedvault/management/commands/__init__.py create mode 100644 feedvault/management/commands/update_feeds.py diff --git a/.gitignore b/.gitignore index d2848f9..7ec6ce5 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,4 @@ cython_debug/ # FeedVault directories data/ media/ +staticfiles/ diff --git a/.vscode/settings.json b/.vscode/settings.json index 18ecfb6..23cd12f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -50,6 +50,7 @@ "localbattle", "localdomain", "lscr", + "makemigrations", "malformedurl", "meowning", "mmcdole", diff --git a/README.md b/README.md index 5ee8bbf..c439838 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,25 @@ Please create a new issue before submitting a big pull request. I am probably ok Try to minimize the number of dependencies you add to the project. If you need to add a new dependency, please create an issue first. + +## Development + +- [Python](https://www.python.org/) +- [Poetry](https://python-poetry.org/) + +```bash +poetry install +poetry shell +python manage.py test +python manage.py collectstatic +python manage.py makemigrations +python manage.py migrate +python manage.py runserver + +# Update feeds +python manage.py update_feeds +``` + ## Contact For any inquiries or support, please create an issue on GitHub. diff --git a/feedvault/add_feeds.py b/feedvault/feeds.py similarity index 96% rename from feedvault/add_feeds.py rename to feedvault/feeds.py index b697340..dbe3135 100644 --- a/feedvault/add_feeds.py +++ b/feedvault/feeds.py @@ -335,25 +335,34 @@ def populate_feed(url: str | None, user: AbstractBaseUser | AnonymousUser) -> Fe return feed -def grab_entries(feed: Feed) -> None: +def grab_entries(feed: Feed) -> None | list[Entry]: """Grab the entries from a feed. Args: feed: The feed to grab the entries from. + + Returns: + The entries that were added. If no entries were added, None is returned. """ + # Set the last checked time to now. + feed.last_checked = timezone.now() + feed.save() + + entries_added: list[Entry] = [] # Parse the feed. parsed_feed: dict | None = parse_feed(url=feed.feed_url) if not parsed_feed: - return + return None entries = parsed_feed.get("entries", []) for entry in entries: added_entry: Entry | None = add_entry(feed=feed, entry=entry) if not added_entry: continue + entries_added.append(added_entry) - logger.info("Grabbed entries for feed: %s", feed) - return + logger.info("Added entries: %s", entries_added) + return entries_added def add_url(url: str, user: AbstractBaseUser | AnonymousUser) -> FeedAddResult: diff --git a/feedvault/management/commands/__init__.py b/feedvault/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/feedvault/management/commands/update_feeds.py b/feedvault/management/commands/update_feeds.py new file mode 100644 index 0000000..b688aac --- /dev/null +++ b/feedvault/management/commands/update_feeds.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from datetime import timedelta + +from django.core.management.base import BaseCommand, no_translations +from django.db.models import Q +from django.utils import timezone + +from feedvault.feeds import grab_entries +from feedvault.models import Entry, Feed + + +class Command(BaseCommand): + help = "Check for new entries in feeds" + requires_migrations_checks = True + + @no_translations + def handle(self, *args, **options) -> None: # noqa: ANN002, ANN003, ARG002 + new_entries: int = 0 + + # Grab feeds that haven't been checked in 15 minutes OR haven't been checked at all + for feed in Feed.objects.filter( + Q(last_checked__lte=timezone.now() - timedelta(minutes=15)) | Q(last_checked__isnull=True), + ): + entries: None | list[Entry] = grab_entries(feed) + if not entries: + self.stdout.write(f"No new entries for {feed.title}") + continue + + self.stdout.write(f"Updated {feed}") + self.stdout.write(f"Added {len(entries)} new entries for {feed}") + new_entries += len(entries) + + if new_entries: + self.stdout.write(self.style.SUCCESS(f"Successfully updated feeds. Added {new_entries} new entries")) + + self.stdout.write("No new entries found") diff --git a/feedvault/models.py b/feedvault/models.py index c5d7cd3..f8c04ad 100644 --- a/feedvault/models.py +++ b/feedvault/models.py @@ -293,7 +293,7 @@ class Entry(models.Model): def __str__(self) -> str: """Return string representation of the entry.""" - return f"{self.feed} - {self.title}" + return f"{self.feed.feed_url} - {self.title}" def get_upload_path(instance: UserUploadedFile, filename: str) -> str: diff --git a/feedvault/settings.py b/feedvault/settings.py index 4a074cc..bb99103 100644 --- a/feedvault/settings.py +++ b/feedvault/settings.py @@ -9,6 +9,9 @@ from dotenv import find_dotenv, load_dotenv load_dotenv(dotenv_path=find_dotenv(), verbose=True) +# Is True when running tests, used for not spamming Discord when new users are created +TESTING: bool = len(sys.argv) > 1 and sys.argv[1] == "test" + DEBUG: bool = os.getenv(key="DEBUG", default="True").lower() == "true" BASE_DIR: Path = Path(__file__).resolve().parent.parent SECRET_KEY: str = os.getenv("SECRET_KEY", default="") @@ -42,7 +45,12 @@ ROOT_URLCONF = "feedvault.urls" WSGI_APPLICATION = "feedvault.wsgi.application" NINJA_PAGINATION_PER_PAGE = 1000 STATIC_URL = "static/" -STATIC_ROOT: Path = BASE_DIR / "static" +STATIC_ROOT: Path = BASE_DIR / "staticfiles" +STATICFILES_STORAGE = ( + "django.contrib.staticfiles.storage.StaticFilesStorage" + if TESTING + else "whitenoise.storage.CompressedManifestStaticFilesStorage" +) STATIC_ROOT.mkdir(parents=True, exist_ok=True) MEDIA_URL = "media/" MEDIA_ROOT: Path = BASE_DIR / "media" @@ -50,8 +58,6 @@ MEDIA_ROOT.mkdir(parents=True, exist_ok=True) LOGIN_REDIRECT_URL = "/" LOGOUT_REDIRECT_URL = "/" -# Is True when running tests, used for not spamming Discord when new users are created -TESTING: bool = len(sys.argv) > 1 and sys.argv[1] == "test" INSTALLED_APPS: list[str] = [ "feedvault.apps.FeedVaultConfig", diff --git a/feedvault/tests.py b/feedvault/tests.py index 0cf5fc5..445a8c6 100644 --- a/feedvault/tests.py +++ b/feedvault/tests.py @@ -1,5 +1,6 @@ from __future__ import annotations +from pathlib import Path from typing import TYPE_CHECKING from django.contrib.auth.models import User @@ -7,7 +8,7 @@ from django.http.response import HttpResponse from django.test import Client, TestCase from django.urls import reverse -from feedvault.models import Domain, Entry, Feed +from feedvault.models import Domain, Entry, Feed, UserUploadedFile from feedvault.stats import get_db_size if TYPE_CHECKING: @@ -63,17 +64,46 @@ class TestFeedsPage(TestCase): class TestAddPage(TestCase): + def setUp(self) -> None: + """Create a test user.""" + self.user: User = User.objects.create_user( + username="testuser", + email="hello@feedvault.se", + password="testpassword", # noqa: S106 + ) + + self.client.force_login(user=self.user) + def test_add_page(self) -> None: """Test if the add page is accessible.""" - response: HttpResponse = self.client.get(reverse("add")) + response: HttpResponse = self.client.post(reverse("add"), {"urls": "https://feedvault.se/feed.xml"}) assert response.status_code == 200, f"Expected 200, got {response.status_code}" class TestUploadPage(TestCase): + def setUp(self) -> None: + """Create a test user.""" + self.user: User = User.objects.create_user( + username="testuser", + email="hello@feedvault.se", + password="testpassword", # noqa: S106 + ) + + self.client.force_login(user=self.user) + def test_upload_page(self) -> None: """Test if the upload page is accessible.""" - response: HttpResponse = self.client.get(reverse("upload")) - assert response.status_code == 200, f"Expected 200, got {response.status_code}" + # Check the amounts of files in the database + assert UserUploadedFile.objects.count() == 0, f"Expected 0, got {UserUploadedFile.objects.count()}" + + # Open this file and upload it + current_file = __file__ + with Path(current_file).open("rb") as file: + response: HttpResponse = self.client.post(reverse("upload"), {"file": file}) + assert response.status_code == 200, f"Expected 200, got {response.status_code}: {response.content}" + + # Check if the file is in the database + assert UserUploadedFile.objects.count() == 1, f"Expected 1, got {UserUploadedFile.objects.count()}" class TestRobotsPage(TestCase): diff --git a/feedvault/views.py b/feedvault/views.py index 7134402..3d7056f 100644 --- a/feedvault/views.py +++ b/feedvault/views.py @@ -22,7 +22,7 @@ from django.views import View from django.views.generic.edit import CreateView from django.views.generic.list import ListView -from feedvault.add_feeds import add_url +from feedvault.feeds import add_url from feedvault.models import Domain, Entry, Feed, FeedAddResult, UserUploadedFile if TYPE_CHECKING: