From bc00be6f71bd2e2db6269ff2c8c08ee0a3be4eb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Hells=C3=A9n?= Date: Sun, 29 Jan 2023 02:16:36 +0100 Subject: [PATCH] Refactor whitelist/blacklist --- discord_rss_bot/filter/blacklist.py | 74 +++-------------------- discord_rss_bot/filter/whitelist.py | 72 ++-------------------- discord_rss_bot/main.py | 14 ++--- tests/test_blacklist.py | 81 ++++++------------------- tests/test_whitelist.py | 94 +++++++++++++++++++++++++++++ 5 files changed, 130 insertions(+), 205 deletions(-) create mode 100644 tests/test_whitelist.py diff --git a/discord_rss_bot/filter/blacklist.py b/discord_rss_bot/filter/blacklist.py index 302d409..bde6de1 100644 --- a/discord_rss_bot/filter/blacklist.py +++ b/discord_rss_bot/filter/blacklist.py @@ -1,4 +1,4 @@ -from reader import Entry, Feed, Reader, TagNotFoundError +from reader import Entry, Feed, Reader from discord_rss_bot.filter.utils import is_word_in_text @@ -17,9 +17,9 @@ def has_black_tags(custom_reader: Reader, feed: Feed) -> bool: Returns: bool: If the feed has any of the tags. """ - blacklist_title: str = get_blacklist_title(custom_reader, feed) - blacklist_summary: str = get_blacklist_summary(custom_reader, feed) - blacklist_content: str = get_blacklist_content(custom_reader, feed) + blacklist_title: str = str(custom_reader.get_tag(feed, "blacklist_title", "")) + blacklist_summary: str = str(custom_reader.get_tag(feed, "blacklist_summary", "")) + blacklist_content: str = str(custom_reader.get_tag(feed, "blacklist_content", "")) return bool(blacklist_title or blacklist_summary or blacklist_content) @@ -36,9 +36,9 @@ def should_be_skipped(custom_reader: Reader, entry: Entry) -> bool: bool: If the entry is in the blacklist. """ feed: Feed = entry.feed - blacklist_title: str = get_blacklist_title(custom_reader, feed) - blacklist_summary: str = get_blacklist_summary(custom_reader, feed) - # blacklist_content: str = get_blacklist_content(custom_reader, feed) + blacklist_title: str = str(custom_reader.get_tag(feed, "blacklist_title", "")) + blacklist_summary: str = str(custom_reader.get_tag(feed, "blacklist_summary", "")) + # blacklist_content: str = str(custom_reader.get_tag(feed, "blacklist_content", "")) # TODO: Fix content # TODO: Check author # TODO: Also add support for entry_text @@ -48,63 +48,3 @@ def should_be_skipped(custom_reader: Reader, entry: Entry) -> bool: elif entry.summary and blacklist_summary and is_word_in_text(blacklist_summary, entry.summary): return True return False - - -def get_blacklist_content(custom_reader: Reader, feed: Feed) -> str: - """ - Get the blacklist_content tag from the feed. - - Args: - custom_reader: The reader. - feed: The feed to get the tag from. - - Returns: - str: The blacklist_content tag. - """ - try: - blacklist_content: str = custom_reader.get_tag(feed, "blacklist_content") # type: ignore - except TagNotFoundError: - blacklist_content: str = "" - except ValueError: - blacklist_content: str = "" - return blacklist_content - - -def get_blacklist_summary(custom_reader: Reader, feed: Feed) -> str: - """ - Get the blacklist_summary tag from the feed. - - Args: - custom_reader: The reader. - feed: The feed to get the tag from. - - Returns: - str: The blacklist_summary tag. - """ - try: - blacklist_summary: str = custom_reader.get_tag(feed, "blacklist_summary") # type: ignore - except TagNotFoundError: - blacklist_summary: str = "" - except ValueError: - blacklist_summary: str = "" - return blacklist_summary - - -def get_blacklist_title(custom_reader: Reader, feed: Feed) -> str: - """ - Get the blacklist_title tag from the feed. - - Args: - custom_reader: The reader. - feed: The feed to get the tag from. - - Returns: - str: The blacklist_title tag. - """ - try: - blacklist_title: str = custom_reader.get_tag(feed, "blacklist_title") # type: ignore - except TagNotFoundError: - blacklist_title: str = "" - except ValueError: - blacklist_title: str = "" - return blacklist_title diff --git a/discord_rss_bot/filter/whitelist.py b/discord_rss_bot/filter/whitelist.py index 0b58669..d183219 100644 --- a/discord_rss_bot/filter/whitelist.py +++ b/discord_rss_bot/filter/whitelist.py @@ -1,4 +1,4 @@ -from reader import Entry, Feed, Reader, TagNotFoundError +from reader import Entry, Feed, Reader from discord_rss_bot.filter.utils import is_word_in_text @@ -17,9 +17,9 @@ def has_white_tags(custom_reader: Reader, feed: Feed) -> bool: Returns: bool: If the feed has any of the tags. """ - whitelist_title: str = get_whitelist_title(custom_reader, feed) - whitelist_summary: str = get_whitelist_summary(custom_reader, feed) - whitelist_content: str = get_whitelist_content(custom_reader, feed) + whitelist_title: str = str(custom_reader.get_tag(feed, "whitelist_title", "")) + whitelist_summary: str = str(custom_reader.get_tag(feed, "whitelist_summary", "")) + whitelist_content: str = str(custom_reader.get_tag(feed, "whitelist_content", "")) return bool(whitelist_title or whitelist_summary or whitelist_content) @@ -36,8 +36,8 @@ def should_be_sent(custom_reader: Reader, entry: Entry) -> bool: bool: If the entry is in the whitelist. """ feed: Feed = entry.feed - whitelist_title: str = get_whitelist_title(custom_reader, feed) - whitelist_summary: str = get_whitelist_summary(custom_reader, feed) + whitelist_title: str = str(custom_reader.get_tag(feed, "whitelist_title", "")) + whitelist_summary: str = str(custom_reader.get_tag(feed, "whitelist_summary", "")) # whitelist_content: str = get_whitelist_content(custom_reader, feed) # TODO: Fix content # TODO: Check author @@ -47,63 +47,3 @@ def should_be_sent(custom_reader: Reader, entry: Entry) -> bool: elif entry.summary and whitelist_summary and is_word_in_text(whitelist_summary, entry.summary): return True return False - - -def get_whitelist_content(custom_reader: Reader, feed: Feed) -> str: - """ - Get the whitelist_content tag from the feed. - - Args: - custom_reader: The reader. - feed: The feed to get the tag from. - - Returns: - str: The whitelist_content tag. - """ - try: - whitelist_content: str = custom_reader.get_tag(feed, "whitelist_content") # type: ignore - except TagNotFoundError: - whitelist_content: str = "" - except ValueError: - whitelist_content: str = "" - return whitelist_content - - -def get_whitelist_summary(custom_reader: Reader, feed: Feed) -> str: - """ - Get the whitelist_summary tag from the feed. - - Args: - custom_reader: The reader. - feed: The feed to get the tag from. - - Returns: - str: The whitelist_summary tag. - """ - try: - whitelist_summary: str = custom_reader.get_tag(feed, "whitelist_summary") # type: ignore - except TagNotFoundError: - whitelist_summary: str = "" - except ValueError: - whitelist_summary: str = "" - return whitelist_summary - - -def get_whitelist_title(custom_reader: Reader, feed: Feed) -> str: - """ - Get the whitelist_title tag from the feed. - - Args: - custom_reader: The reader. - feed: The feed to get the tag from. - - Returns: - str: The whitelist_title tag. - """ - try: - whitelist_title: str = custom_reader.get_tag(feed, "whitelist_title") # type: ignore - except TagNotFoundError: - whitelist_title: str = "" - except ValueError: - whitelist_title: str = "" - return whitelist_title diff --git a/discord_rss_bot/main.py b/discord_rss_bot/main.py index cb239e2..dca35c0 100644 --- a/discord_rss_bot/main.py +++ b/discord_rss_bot/main.py @@ -27,8 +27,6 @@ from discord_rss_bot.custom_message import ( save_embed, ) from discord_rss_bot.feeds import create_feed, send_entry_to_discord, send_to_discord -from discord_rss_bot.filter.blacklist import get_blacklist_content, get_blacklist_summary, get_blacklist_title -from discord_rss_bot.filter.whitelist import get_whitelist_content, get_whitelist_summary, get_whitelist_title from discord_rss_bot.markdown import convert_html_to_md from discord_rss_bot.missing_tags import add_missing_tags from discord_rss_bot.search import create_html_for_search_results @@ -147,9 +145,9 @@ async def get_whitelist(feed_url: str, request: Request): feed: Feed = reader.get_feed(urllib.parse.unquote(clean_feed_url)) # Get previous data, this is used when creating the form. - whitelist_title: str = get_whitelist_title(reader, feed) - whitelist_summary: str = get_whitelist_summary(reader, feed) - whitelist_content: str = get_whitelist_content(reader, feed) + whitelist_title: str = str(reader.get_tag(feed, "whitelist_title", "")) + whitelist_summary: str = str(reader.get_tag(feed, "whitelist_summary", "")) + whitelist_content: str = str(reader.get_tag(feed, "whitelist_content", "")) context = { "request": request, @@ -193,9 +191,9 @@ async def get_blacklist(feed_url: str, request: Request): feed: Feed = reader.get_feed(urllib.parse.unquote(feed_url)) # Get previous data, this is used when creating the form. - blacklist_title: str = get_blacklist_title(reader, feed) - blacklist_summary: str = get_blacklist_summary(reader, feed) - blacklist_content: str = get_blacklist_content(reader, feed) + blacklist_title: str = str(reader.get_tag(feed, "blacklist_title", "")) + blacklist_summary: str = str(reader.get_tag(feed, "blacklist_summary", "")) + blacklist_content: str = str(reader.get_tag(feed, "blacklist_content", "")) context = { "request": request, diff --git a/tests/test_blacklist.py b/tests/test_blacklist.py index eddd9d4..fd9eba7 100644 --- a/tests/test_blacklist.py +++ b/tests/test_blacklist.py @@ -4,15 +4,9 @@ from typing import Iterable from reader import Entry, Feed, Reader, make_reader -from discord_rss_bot.filter.blacklist import ( - get_blacklist_content, - get_blacklist_summary, - get_blacklist_title, - has_black_tags, - should_be_skipped, -) +from discord_rss_bot.filter.blacklist import has_black_tags, should_be_skipped -feed_url = "https://lovinator.space/rss_test.xml" +feed_url: str = "https://lovinator.space/rss_test.xml" # Create the database @@ -25,7 +19,7 @@ def get_reader() -> Reader: return reader -def test_has_black_tags(): +def test_has_black_tags() -> None: reader: Reader = get_reader() # Add feed and update entries @@ -44,14 +38,14 @@ def test_has_black_tags(): reader.delete_feed(feed_url) -def check_if_has_tag(reader, feed, blacklist_name): - reader.set_tag(feed, blacklist_name, "a") +def check_if_has_tag(reader: Reader, feed: Feed, blacklist_name: str) -> None: + reader.set_tag(feed, blacklist_name, "a") # type: ignore assert has_black_tags(custom_reader=reader, feed=feed) is True reader.delete_tag(feed, blacklist_name) assert has_black_tags(custom_reader=reader, feed=feed) is False -def test_should_be_skipped(): +def test_should_be_skipped() -> None: reader: Reader = get_reader() # Add feed and update entries @@ -76,66 +70,25 @@ def test_should_be_skipped(): reader.delete_tag(feed, "blacklist_title") assert should_be_skipped(reader, first_entry[0]) is False + reader.set_tag(feed, "blacklist_title", "åäö") # type: ignore + assert should_be_skipped(reader, first_entry[0]) is False + reader.delete_tag(feed, "blacklist_title") + assert should_be_skipped(reader, first_entry[0]) is False + reader.set_tag(feed, "blacklist_summary", "ffdnfdnfdnfdnfdndfn") # type: ignore assert should_be_skipped(reader, first_entry[0]) is True reader.delete_tag(feed, "blacklist_summary") assert should_be_skipped(reader, first_entry[0]) is False + reader.set_tag(feed, "blacklist_summary", "åäö") # type: ignore + assert should_be_skipped(reader, first_entry[0]) is False + reader.delete_tag(feed, "blacklist_summary") + assert should_be_skipped(reader, first_entry[0]) is False + reader.set_tag(feed, "blacklist_content", "ffdnfdnfdnfdnfdndfn") # type: ignore - # TODO: This is not impelemented yes + # TODO: This is not impelemented yet assert should_be_skipped(reader, first_entry[0]) is False reader.delete_tag(feed, "blacklist_content") assert should_be_skipped(reader, first_entry[0]) is False # TODO: Also add support for entry_text - - -def test_get_blacklist_content(): - reader: Reader = get_reader() - - # Add feed and update entries - reader.add_feed(feed_url) - feed: Feed = reader.get_feed(feed_url) - reader.update_feeds() - - assert get_blacklist_content(reader, feed) == "" # type: ignore - - reader.set_tag(feed, "blacklist_content", "ffdnfdnfdnfdnfdndfn") # type: ignore - assert get_blacklist_content(reader, feed) == "ffdnfdnfdnfdnfdndfn" # type: ignore - - reader.delete_tag(feed, "blacklist_content") - assert get_blacklist_content(reader, feed) == "" # type: ignore - - -def test_get_blacklist_summary(): - reader: Reader = get_reader() - - # Add feed and update entries - reader.add_feed(feed_url) - feed: Feed = reader.get_feed(feed_url) - reader.update_feeds() - - assert get_blacklist_summary(reader, feed) == "" # type: ignore - - reader.set_tag(feed, "blacklist_summary", "ffdnfdnfdnfdnfdndfn") # type: ignore - assert get_blacklist_summary(reader, feed) == "ffdnfdnfdnfdnfdndfn" # type: ignore - - reader.delete_tag(feed, "blacklist_summary") - assert get_blacklist_summary(reader, feed) == "" # type: ignore - - -def test_get_blacklist_title(): - reader: Reader = get_reader() - - # Add feed and update entries - reader.add_feed(feed_url) - feed: Feed = reader.get_feed(feed_url) - reader.update_feeds() - - assert get_blacklist_title(reader, feed) == "" # type: ignore - - reader.set_tag(feed, "blacklist_title", "ffdnfdnfdnfdnfdndfn") # type: ignore - assert get_blacklist_title(reader, feed) == "ffdnfdnfdnfdnfdndfn" # type: ignore - - reader.delete_tag(feed, "blacklist_title") - assert get_blacklist_title(reader, feed) == "" # type: ignore diff --git a/tests/test_whitelist.py b/tests/test_whitelist.py new file mode 100644 index 0000000..4ea9deb --- /dev/null +++ b/tests/test_whitelist.py @@ -0,0 +1,94 @@ +import tempfile +from pathlib import Path +from typing import Iterable + +from reader import Entry, Feed, Reader, make_reader + +from discord_rss_bot.filter.whitelist import has_white_tags, should_be_sent + +feed_url: str = "https://lovinator.space/rss_test.xml" + + +# Create the database +def get_reader() -> Reader: + tempdir: Path = Path(tempfile.mkdtemp()) + + reader_database: Path = tempdir / "test.sqlite" + reader: Reader = make_reader(url=str(reader_database)) + + return reader + + +def test_has_white_tags() -> None: + reader: Reader = get_reader() + + # Add feed and update entries + reader.add_feed(feed_url) + feed: Feed = reader.get_feed(feed_url) + reader.update_feeds() + + # Test feed without any whitelist tags + assert has_white_tags(custom_reader=get_reader(), feed=feed) is False + + check_if_has_tag(reader, feed, "whitelist_title") + check_if_has_tag(reader, feed, "whitelist_summary") + check_if_has_tag(reader, feed, "whitelist_content") + + # Clean up + reader.delete_feed(feed_url) + + +def check_if_has_tag(reader: Reader, feed: Feed, whitelist_name: str) -> None: + reader.set_tag(feed, whitelist_name, "a") # type: ignore + assert has_white_tags(custom_reader=reader, feed=feed) is True + reader.delete_tag(feed, whitelist_name) + assert has_white_tags(custom_reader=reader, feed=feed) is False + + +def test_should_be_sent() -> None: + reader: Reader = get_reader() + + # Add feed and update entries + reader.add_feed(feed_url) + feed: Feed = reader.get_feed(feed_url) + reader.update_feeds() + + # Get first entry + first_entry: list[Entry] = [] + entries: Iterable[Entry] = reader.get_entries(feed=feed) + assert entries is not None + for entry in entries: + first_entry.append(entry) + break + assert len(first_entry) == 1 + + # Test entry without any whitelists + assert should_be_sent(reader, first_entry[0]) is False + + reader.set_tag(feed, "whitelist_title", "fvnnnfnfdnfdnfd") # type: ignore + assert should_be_sent(reader, first_entry[0]) is True + reader.delete_tag(feed, "whitelist_title") + assert should_be_sent(reader, first_entry[0]) is False + + reader.set_tag(feed, "whitelist_title", "åäö") # type: ignore + assert should_be_sent(reader, first_entry[0]) is False + reader.delete_tag(feed, "whitelist_title") + assert should_be_sent(reader, first_entry[0]) is False + + reader.set_tag(feed, "whitelist_summary", "ffdnfdnfdnfdnfdndfn") # type: ignore + assert should_be_sent(reader, first_entry[0]) is True + reader.delete_tag(feed, "whitelist_summary") + assert should_be_sent(reader, first_entry[0]) is False + + reader.set_tag(feed, "whitelist_summary", "åäö") # type: ignore + assert should_be_sent(reader, first_entry[0]) is False + reader.delete_tag(feed, "whitelist_summary") + assert should_be_sent(reader, first_entry[0]) is False + + reader.set_tag(feed, "whitelist_content", "ffdnfdnfdnfdnfdndfn") # type: ignore + # TODO: This is not impelemented yet + assert should_be_sent(reader, first_entry[0]) is False + reader.delete_tag(feed, "whitelist_content") + assert should_be_sent(reader, first_entry[0]) is False + + # TODO: Also add support for entry_text