diff --git a/README.md b/README.md index 688be33..c74054a 100644 --- a/README.md +++ b/README.md @@ -26,31 +26,6 @@ _Note: Some features are currently in development._ Feel free to contribute to the project. If you have any questions, please open an issue. -## Dependencies - -- [Python](https://www.python.org/) -- [Poetry](https://python-poetry.org/) -- [PostgreSQL 16](https://www.postgresql.org/) - -## tl;dr - -```bash -poetry install -poetry shell -python manage.py test -python manage.py collectstatic -python manage.py makemigrations -python manage.py migrate -python manage.py runserver - -# Optional -pre-commit install -pre-commit run --all-files - -# Update feeds -python manage.py update_feeds -``` - ## Contact If you have any questions, please open an issue. diff --git a/app/cli.py b/app/cli.py index 3102de9..f86cb28 100644 --- a/app/cli.py +++ b/app/cli.py @@ -1,16 +1,18 @@ from __future__ import annotations +import sys +import traceback +from datetime import datetime, timedelta from pathlib import Path from typing import TYPE_CHECKING import click -from reader import Feed, Reader, UpdateError, UpdateResult -from reader.types import UpdatedFeed +from reader import Feed, ParseError, Reader, StorageError, UpdateError, UpdateResult from app.dependencies import get_reader if TYPE_CHECKING: - from reader import UpdatedFeed + from collections.abc import Iterable def add_broken_feed_to_csv(feed: Feed | UpdateResult | None) -> None: @@ -29,28 +31,47 @@ def update_feeds() -> None: reader: Reader = get_reader() click.echo("Updating feeds...") - for feed in reader.update_feeds_iter(updates_enabled=True, workers=100): - url: str = feed.url - value: UpdatedFeed | None | UpdateError = feed.value + all_feeds: Iterable[Feed] = reader.get_feeds(updates_enabled=True) + feeds = [] - if isinstance(value, UpdateError): + # Only get feeds that hasn't been updated in the last 30 minutes. + for feed in all_feeds: + if feed.last_updated: + now: datetime = datetime.now(tz=feed.last_updated.tzinfo) + delta: timedelta = now - feed.last_updated + + thirty_minutes: int = 60 * 30 # 30 minutes + if delta.total_seconds() < thirty_minutes: + feeds.append(feed) + else: + feeds.append(feed) + + click.echo(f"Feeds to update: {len(feeds)}") + + for feed in feeds: + try: + reader.update_feed(feed) + click.echo(f"Updated feed: {feed.url}") + except ParseError: + # An error occurred while retrieving/parsing the feed. + click.echo(f"Error parsing feed: {feed.url}", err=True) + except UpdateError: + # An error occurred while updating the feed. + # Parent of all update-related exceptions. + click.echo(f"Error updating feed: {feed.url}", err=True) + except StorageError as e: + # An exception was raised by the underlying storage. + click.echo(f"Error updating feed: {feed.url}", err=True) + click.echo(f"Storage error: {e}", err=True) + except AssertionError: + # An assertion failed. + click.echo(f"Assertion error: {feed.url}", err=True) + traceback.print_exc(file=sys.stderr) + reader.disable_feed_updates(feed) add_broken_feed_to_csv(feed) - reader.disable_feed_updates(url) - continue - - if value is None: - click.echo(f"Feed not updated: {url}") - continue - - click.echo(f"Updated feed: {url}") click.echo("Feeds updated.") if __name__ == "__main__": - reader: Reader = get_reader() - - for feed in reader.get_feeds(updates_enabled=False): - reader.enable_feed_updates(feed) - update_feeds() diff --git a/poetry.lock b/poetry.lock index d91395d..48152df 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1146,13 +1146,13 @@ files = [ [[package]] name = "requests" -version = "2.32.1" +version = "2.32.2" description = "Python HTTP for Humans." optional = false python-versions = ">=3.8" files = [ - {file = "requests-2.32.1-py3-none-any.whl", hash = "sha256:21ac9465cdf8c1650fe1ecde8a71669a93d4e6f147550483a2967d08396a56a5"}, - {file = "requests-2.32.1.tar.gz", hash = "sha256:eb97e87e64c79e64e5b8ac75cee9dd1f97f49e289b083ee6be96268930725685"}, + {file = "requests-2.32.2-py3-none-any.whl", hash = "sha256:fc06670dd0ed212426dfeb94fc1b983d917c4f9847c863f313c9dfaaffb7c23c"}, + {file = "requests-2.32.2.tar.gz", hash = "sha256:dd951ff5ecf3e3b3aa26b40703ba77495dab41da839ae72ef3c8e5d8e2433289"}, ] [package.dependencies] @@ -1724,4 +1724,4 @@ watchdog = ["watchdog (>=2.3)"] [metadata] lock-version = "2.0" python-versions = "^3.12" -content-hash = "3c725158636587ce40985c025ae09fbdc68dfd5dacd98e6fe9b4736f052fecfc" +content-hash = "069d3a9e95892d8e057e97858f342469cb3543f8bde8d893c3592583b0d36948" diff --git a/pyproject.toml b/pyproject.toml index 36a3401..427a5f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,6 +4,7 @@ version = "0.1.0" description = "RSS Archive" authors = ["Joakim Hellsén "] readme = "README.md" +packages = [{ include = "app" }] [tool.poetry.dependencies] python = "^3.12"