From 400b72dbf4c94de7a61e92eb438a1e94a5121275 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Hells=C3=A9n?= Date: Fri, 2 Dec 2022 16:47:47 +0100 Subject: [PATCH] Add config file, dropdown for webhooks, and sends stuff to Discord --- discord_rss_bot/feeds.py | 34 ++++++ discord_rss_bot/main.py | 148 +++++++++++++++------------ discord_rss_bot/settings.py | 103 ++++++++++++++++--- discord_rss_bot/templates/feed.html | 33 ++++++ discord_rss_bot/templates/index.html | 70 +++++++++++++ discord_rss_bot/webhooks.py | 26 ----- poetry.lock | 14 ++- pyproject.toml | 4 +- templates/feed.html | 33 ------ templates/index.html | 47 --------- tests/test_discord_rss_bot.py | 3 +- 11 files changed, 328 insertions(+), 187 deletions(-) create mode 100644 discord_rss_bot/feeds.py create mode 100644 discord_rss_bot/templates/feed.html create mode 100644 discord_rss_bot/templates/index.html delete mode 100644 discord_rss_bot/webhooks.py delete mode 100644 templates/feed.html delete mode 100644 templates/index.html diff --git a/discord_rss_bot/feeds.py b/discord_rss_bot/feeds.py new file mode 100644 index 0000000..3097d6d --- /dev/null +++ b/discord_rss_bot/feeds.py @@ -0,0 +1,34 @@ +from discord_webhook import DiscordWebhook + +from discord_rss_bot.settings import logger, reader + + +def check_feeds() -> None: + """Check all feeds""" + reader.update_feeds() + entries = reader.get_entries(read=False) + _check_feed(entries) + + +def check_feed(feed_url: str) -> None: + """Check a single feed""" + reader.update_feeds() + entry = reader.get_entries(feed=feed_url, read=False) + _check_feed(entry, feed_url) + + +def _check_feed(entries, feed_url: str) -> None: + for entry in entries: + reader.mark_entry_as_read(entry) + logger.debug(f"New entry: {entry.title}") + + webhook_url = reader.get_tag(feed_url, "webhook") + if webhook_url: + logger.debug(f"Sending to webhook: {webhook_url}") + webhook = DiscordWebhook(url=str(webhook_url), content=f":robot: :mega: New entry: {entry.title}\n" + f"{entry.link}", rate_limit_retry=True) + response = webhook.execute() + if not response.ok: + # TODO: Send error to discord + logger.error(f"Error: {response.status_code} {response.reason}") + reader.mark_entry_as_unread(entry) diff --git a/discord_rss_bot/main.py b/discord_rss_bot/main.py index 837ac46..abcf83e 100644 --- a/discord_rss_bot/main.py +++ b/discord_rss_bot/main.py @@ -1,31 +1,28 @@ -import logging +import enum +import sys +import uvicorn from apscheduler.schedulers.background import BackgroundScheduler -from discord_webhook import DiscordWebhook from fastapi import FastAPI, Form, Request from fastapi.responses import HTMLResponse from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates -from reader import make_reader +from reader import FeedExistsError -logging.basicConfig( - level=logging.DEBUG, - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", -) -logger = logging.getLogger(__name__) +from discord_rss_bot.feeds import _check_feed +from discord_rss_bot.settings import logger, read_settings_file, reader app = FastAPI() app.mount("/static", StaticFiles(directory="static"), name="static") templates = Jinja2Templates(directory="templates") -reader = make_reader("db.sqlite") @app.post("/check", response_class=HTMLResponse) -def read_check_feed(request: Request, feed_url: str = Form()): +def check_feed(request: Request, feed_url: str = Form()): """Check all feeds""" reader.update_feeds() entry = reader.get_entries(feed=feed_url, read=False) - _check_feed(entry) + _check_feed(entry, feed_url) logger.info(f"Get feed: {feed_url}") feed = reader.get_feed(feed_url) @@ -33,46 +30,25 @@ def read_check_feed(request: Request, feed_url: str = Form()): return templates.TemplateResponse("feed.html", {"request": request, "feed": feed}) -def check_feeds() -> None: - """Check all feeds""" - reader.update_feeds() - entries = reader.get_entries(read=False) - _check_feed(entries) - - -def check_feed(feed_url: str) -> None: - """Check a single feed""" - reader.update_feeds() - entry = reader.get_entries(feed=feed_url, read=False) - _check_feed(entry) - - -def _check_feed(entries): - for entry in entries: - reader.mark_entry_as_read(entry) - print(f"New entry: {entry.title}") - - webhook_url = reader.get_tag((), "webhook") - if webhook_url: - print(f"Sending to webhook: {webhook_url}") - webhook = DiscordWebhook(url=str(webhook_url), content=f":robot: :mega: New entry: {entry.title}\n" - f"{entry.link}", rate_limit_retry=True) - response = webhook.execute() - if not response.ok: - # TODO: Send error to discord - print(f"Error: {response.status_code} {response.reason}") - reader.mark_entry_as_unread(entry) - - @app.on_event('startup') -def init_data(): - """Run on startup""" +def startup(): + """This is called when the server starts. + + It reads the settings file and starts the scheduler.""" + settings = read_settings_file() + + if not settings["webhooks"]: + logger.critical("No webhooks found in settings file.") + sys.exit() + for key in settings["webhooks"]: + logger.info(f"Webhook name: {key} with URL: {settings['webhooks'][key]}") + scheduler = BackgroundScheduler() scheduler.start() @app.get("/", response_class=HTMLResponse) -def read_root(request: Request): +def index(request: Request): """ This is the root of the website. @@ -82,14 +58,39 @@ def read_root(request: Request): Returns: HTMLResponse: The HTML response. """ + context = make_context_index(request) + return templates.TemplateResponse("index.html", context) + + +def make_context_index(request) -> dict: + """ + Create the needed context for the index page. + + Used by / and /add. + Args: + request: The request. + + Returns: + dict: The context. + + """ + hooks = create_list_of_webhooks() + for hook in hooks: + logger.info(f"Webhook name: {hook.name}") + + feed_list = list() feeds = reader.get_feeds() + for feed in feeds: + feed_list.append(feed) + feed_count = reader.get_feed_counts() entry_count = reader.get_entry_counts() context = {"request": request, - "feeds": feeds, + "feeds": feed_list, "feed_count": feed_count, - "entry_count": entry_count} - return templates.TemplateResponse("index.html", context) + "entry_count": entry_count, + "webhooks": hooks} + return context @app.post("/remove", response_class=HTMLResponse) @@ -129,36 +130,53 @@ async def get_feed(request: Request, feed_url: str = Form()): return templates.TemplateResponse("feed.html", {"request": request, "feed": feed}) -@app.post("/global_webhook", response_class=HTMLResponse) -async def add_global_webhook(request: Request, webhook_url: str = Form()): - """ - Add a global webhook. +def create_list_of_webhooks(): + """List with webhooks.""" + logger.info("Creating list with webhooks.") + settings = read_settings_file() + list_of_webhooks = dict() + for hook in settings["webhooks"]: + logger.info(f"Webhook name: {hook} with URL: {settings['webhooks'][hook]}") + list_of_webhooks[hook] = settings["webhooks"][hook] - Args: - request: The request. - webhook_url: The webhook URL. + logger.info(f"List of webhooks: {list_of_webhooks}") + return enum.Enum("DiscordWebhooks", list_of_webhooks) - Returns: - HTMLResponse: The HTML response. - """ - logger.info(f"Add global webhook: {webhook_url}") - reader.set_tag("webhook", webhook_url) - return templates.TemplateResponse("index.html", {"request": request}) + +def get_hook_by_name(name): + """Get a webhook by name.""" + settings = read_settings_file() + logger.debug(f"Webhook name: {name} with URL: {settings['webhooks'][name]}") + return settings["webhooks"][name] @app.post("/add") -async def create_feed(feed_url: str = Form()): +async def create_feed(feed_url: str = Form(), webhook_dropdown: str = Form()): """ Add a feed to the database. Args: feed_url: The feed to add. - default_webhook: The default webhook to use. + webhook_dropdown: The webhook to use. Returns: dict: The feed that was added. """ - reader.add_feed(feed_url) + logger.info(f"Add feed: {feed_url}") + logger.info(f"Webhook: {webhook_dropdown}") + try: + reader.add_feed(feed_url) + except FeedExistsError as error: + logger.error(f"Feed already exists: {error}") + return {"error": "Feed already exists."} reader.update_feed(feed_url) + webhook_url = get_hook_by_name(webhook_dropdown) + reader.set_tag(feed_url, "webhook", webhook_url) - return {"feed_url": str(feed_url), "status": "added"} + new_tag = reader.get_tag(feed_url, "webhook") + logger.info(f"New tag: {new_tag}") + return {"feed_url": str(feed_url), "status": "added", "webhook": webhook_url} + + +if __name__ == "__main__": + uvicorn.run("main:app", log_level="debug") diff --git a/discord_rss_bot/settings.py b/discord_rss_bot/settings.py index 336ffcd..f0038ed 100644 --- a/discord_rss_bot/settings.py +++ b/discord_rss_bot/settings.py @@ -1,28 +1,44 @@ +import functools +import logging import os from pathlib import Path from platformdirs import user_data_dir +from reader import make_reader +from tomlkit import TOMLDocument, comment, document, parse, table + +logging.basicConfig( + level=logging.DEBUG, + format="[%(asctime)s] [%(funcName)s:%(lineno)d] %(message)s", +) +logger = logging.getLogger(__name__) + +# For get_data_dir() +data_directory = user_data_dir(appname="discord_rss_bot", appauthor="TheLovinator", roaming=True) -def get_app_dir(app_dir: str = user_data_dir("discord_rss_bot")) -> Path: +def get_data_dir(data_dir: str = data_directory) -> Path: """ - Get the application directory. This is where the database file is stored. + Get the data directory. This is where the database file and config file are stored. Args: - app_dir: The application directory, defaults to user_data_dir(). + data_dir: The application directory, defaults to user_data_dir(). Returns: Path: The application directory. """ - print(f"Data directory: {app_dir}") + if data_dir != user_data_dir("discord_rss_bot"): + logger.info(f"Using custom data directory: {data_dir}") # Use the environment variable if it exists instead of the default app dir. - app_dir = os.getenv("DATABASE_LOCATION") or app_dir + data_dir = os.getenv("DATA_DIR") or data_dir + + logger.debug(f"Data directory: {data_dir}") # Create the data directory if it doesn't exist - os.makedirs(app_dir, exist_ok=True) + os.makedirs(data_dir, exist_ok=True) - return Path(app_dir) + return Path(data_dir) def get_db_file(custom_db_name: str = "db.sqlite") -> Path: @@ -34,13 +50,76 @@ def get_db_file(custom_db_name: str = "db.sqlite") -> Path: Returns: Path: The database file. """ + if custom_db_name != "db.sqlite": + logger.info(f"Using custom database file: {custom_db_name}") + # Store the database file in the data directory - app_dir = get_app_dir() + data_dir = get_data_dir() + db_location: Path = Path(os.path.join(data_dir, custom_db_name)) # Use the environment variable if it exists instead of the default db name. - db_name = os.getenv("DATABASE_NAME") or custom_db_name - - db_file: Path = Path(os.path.join(app_dir, db_name)) - print(f"Database file: {db_file}") + db_file = os.getenv("DATABASE_LOCATION") or db_location + logger.debug(f"Database file: {db_file}") return Path(db_file) + + +def _create_settings_file(settings_file) -> None: + """Create the settings file if it doesn't exist.""" + logger.debug(f"Settings file: {settings_file}") + + # [webhooks] + # Both options are commented out by default. + webhooks = table() + webhooks.add(comment('"First webhook" = "https://discord.com/api/webhooks/1234567890/abcdefghijklmnopqrstuvwxyz"')) + webhooks.add(comment('"Second webhook" = "https://discord.com/api/webhooks/1234567890/abcdefghijklmnopqrstuvwxyz"')) + + # [database] + # Option is commented out by default. + database = table() + database.add(comment('"location" = "/path/to/database/file"')) + + doc = document() + doc.add("webhooks", webhooks) + doc.add("database", database) + + logger.debug(f"Settings file: {doc}") + logger.debug(f"Settings file as TOML: {doc.as_string()}") + + # Write the settings file + with open(settings_file, "w") as f: + f.write(doc.as_string()) + + +def read_settings_file(custom_settings_name: str = "settings.toml") -> TOMLDocument: + """Read the settings file + + Args: + custom_settings_name: The name of the settings file, defaults to settings.toml. + + Returns: + dict: The settings file as a dict. + """ + if custom_settings_name != "settings.toml": + logger.info(f"Using custom name for settings file: {custom_settings_name}") + + # Store the database file in the data directory + data_dir = get_data_dir() + settings_file_location: Path = Path(os.path.join(data_dir, custom_settings_name)) + + # Use the environment variable if it exists instead of the default db name. + settings_file = os.getenv("SETTINGS_FILE_LOCATION") or settings_file_location + logger.debug(f"Settings file: {settings_file}") + + # Create the settings file if it doesn't exist + if not os.path.exists(settings_file): + _create_settings_file(settings_file) + + with open(settings_file, encoding="utf-8") as f: + data = parse(f.read()) + logger.debug(f"Contents of settings file: {data}") + + return data + + +reader = make_reader(str(get_db_file())) diff --git a/discord_rss_bot/templates/feed.html b/discord_rss_bot/templates/feed.html new file mode 100644 index 0000000..fe0fb11 --- /dev/null +++ b/discord_rss_bot/templates/feed.html @@ -0,0 +1,33 @@ + + + + + Feed + + +URL: {{ feed.url }}
+Title: {{ feed.title }}
+Updated: {{ feed.updated }}
+Link: {{ feed.link }}
+Author: {{ feed.author }}
+Subtitle: {{ feed.subtitle }}
+Version: {{ feed.version }}
+User title: {{ feed.user_title }}
+Added on: {{ feed.added }}
+Last update: {{ feed.last_update }}
+Last exception: {{ feed.last_exception }}
+Updates enabled: {{ feed.updates_enabled }}
+ +
+ +
+ +
+ +
+ + \ No newline at end of file diff --git a/discord_rss_bot/templates/index.html b/discord_rss_bot/templates/index.html new file mode 100644 index 0000000..58352ff --- /dev/null +++ b/discord_rss_bot/templates/index.html @@ -0,0 +1,70 @@ + + + + + Index + + + + + +
+ + + + +
+ + +{% for tag in tags %} + + {{ tag }} + +{% endfor %} + + + + +
+ + + + + + \ No newline at end of file diff --git a/discord_rss_bot/webhooks.py b/discord_rss_bot/webhooks.py deleted file mode 100644 index a8f2113..0000000 --- a/discord_rss_bot/webhooks.py +++ /dev/null @@ -1,26 +0,0 @@ -import sys -from contextlib import closing - -from reader import make_reader - -from discord_rss_bot.discord_rss_bot import db_file_str - - -def webhook_get() -> None: - """Get the webhook url""" - # TODO: Add name to output - with closing(make_reader(db_file_str)) as reader: - try: - webhook_url = reader.get_tag((), "webhook") - print(f"Webhook: {webhook_url}") - except Exception as e: - print("No webhook was found. Use `webhook add` to add one.") - print(f"Error: {e}\nPlease report this error to the developer.") - sys.exit() - - -def webhook_add(webhook_url: str) -> None: - """Add a webhook to the database""" - with closing(make_reader(db_file_str)) as reader: - reader.set_tag((), "webhook", webhook_url) - print(f"Webhook set to {webhook_url}") diff --git a/poetry.lock b/poetry.lock index 6c956fd..4a29488 100644 --- a/poetry.lock +++ b/poetry.lock @@ -467,6 +467,14 @@ category = "dev" optional = false python-versions = ">=3.7" +[[package]] +name = "tomlkit" +version = "0.11.6" +description = "Style preserving TOML library" +category = "main" +optional = false +python-versions = ">=3.6" + [[package]] name = "typing-extensions" version = "4.4.0" @@ -569,7 +577,7 @@ python-versions = ">=3.7" [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "20299ab5e43c85e7cc4f2c4e5fcae9040226147c867404f7d125ec76956fc0bc" +content-hash = "2fbde06d11319f076323fea66c8d3e6396b2483106c69bcbbd7dec5a3282d727" [metadata.files] anyio = [ @@ -875,6 +883,10 @@ tomli = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +tomlkit = [ + {file = "tomlkit-0.11.6-py3-none-any.whl", hash = "sha256:07de26b0d8cfc18f871aec595fda24d95b08fef89d147caa861939f37230bf4b"}, + {file = "tomlkit-0.11.6.tar.gz", hash = "sha256:71b952e5721688937fb02cf9d354dbcf0785066149d2855e44531ebdd2b65d73"}, +] typing-extensions = [ {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, diff --git a/pyproject.toml b/pyproject.toml index 49ef904..aa235ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,10 +10,12 @@ reader = "^3.2" discord-webhook = "^1.0.0" platformdirs = "^2.5.4" fastapi = "^0.88.0" -uvicorn = {extras = ["standard"], version = "^0.20.0"} +uvicorn = { extras = ["standard"], version = "^0.20.0" } jinja2 = "^3.1.2" apscheduler = "^3.9.1.post1" python-multipart = "^0.0.5" +python-dotenv = "^0.21.0" +tomlkit = "^0.11.6" [tool.poetry.group.dev.dependencies] pytest = "^7.1.3" diff --git a/templates/feed.html b/templates/feed.html deleted file mode 100644 index 3a5b0f5..0000000 --- a/templates/feed.html +++ /dev/null @@ -1,33 +0,0 @@ - - - - - Feed - - -URL: {{feed.url}}
-Title: {{feed.title}}
-Updated: {{feed.updated}}
-Link: {{feed.link}}
-Author: {{feed.author}}
-Subtitle: {{feed.subtitle}}
-Version: {{feed.version}}
-User title: {{feed.user_title}}
-Added on: {{feed.added}}
-Last update: {{feed.last_update}}
-Last exception: {{feed.last_exception}}
-Updates enabled: {{feed.updates_enabled}}
- -
- -
- -
- -
- - \ No newline at end of file diff --git a/templates/index.html b/templates/index.html deleted file mode 100644 index 16b5fdf..0000000 --- a/templates/index.html +++ /dev/null @@ -1,47 +0,0 @@ - - - - - Index - - - - -
- - -
- - - - - - - \ No newline at end of file diff --git a/tests/test_discord_rss_bot.py b/tests/test_discord_rss_bot.py index 0928cd3..4b71937 100644 --- a/tests/test_discord_rss_bot.py +++ b/tests/test_discord_rss_bot.py @@ -1,8 +1,7 @@ import os -from typer.testing import CliRunner - from discord_rss_bot.discord_rss_bot import app, app_dir +from typer.testing import CliRunner runner = CliRunner()