TheLovinator/discord-rss-bot
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor whitelist/blacklist

Browse Source
This commit is contained in:
Joakim Hellsén
2023-01-29 02:16:36 +01:00
parent d5ac378664
commit bc00be6f71
5 changed files with 130 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
discord_rss_bot/filter/blacklist.py
View File

@ -1,4 +1,4 @@
from reader import Entry, Feed, Reader, TagNotFoundError
from reader import Entry, Feed, Reader
from discord_rss_bot.filter.utils import is_word_in_text
@ -17,9 +17,9 @@ def has_black_tags(custom_reader: Reader, feed: Feed) -> bool:
Returns:
bool: If the feed has any of the tags.
"""
blacklist_title: str = get_blacklist_title(custom_reader, feed)
blacklist_summary: str = get_blacklist_summary(custom_reader, feed)
blacklist_content: str = get_blacklist_content(custom_reader, feed)
blacklist_title: str = str(custom_reader.get_tag(feed, "blacklist_title", ""))
blacklist_summary: str = str(custom_reader.get_tag(feed, "blacklist_summary", ""))
blacklist_content: str = str(custom_reader.get_tag(feed, "blacklist_content", ""))
return bool(blacklist_title or blacklist_summary or blacklist_content)
@ -36,9 +36,9 @@ def should_be_skipped(custom_reader: Reader, entry: Entry) -> bool:
bool: If the entry is in the blacklist.
"""
feed: Feed = entry.feed
blacklist_title: str = get_blacklist_title(custom_reader, feed)
blacklist_summary: str = get_blacklist_summary(custom_reader, feed)
# blacklist_content: str = get_blacklist_content(custom_reader, feed)
blacklist_title: str = str(custom_reader.get_tag(feed, "blacklist_title", ""))
blacklist_summary: str = str(custom_reader.get_tag(feed, "blacklist_summary", ""))
# blacklist_content: str = str(custom_reader.get_tag(feed, "blacklist_content", ""))
# TODO: Fix content
# TODO: Check author
# TODO: Also add support for entry_text
@ -48,63 +48,3 @@ def should_be_skipped(custom_reader: Reader, entry: Entry) -> bool:
elif entry.summary and blacklist_summary and is_word_in_text(blacklist_summary, entry.summary):
return True
return False
def get_blacklist_content(custom_reader: Reader, feed: Feed) -> str:
"""
Get the blacklist_content tag from the feed.
Args:
custom_reader: The reader.
feed: The feed to get the tag from.
Returns:
str: The blacklist_content tag.
"""
try:
blacklist_content: str = custom_reader.get_tag(feed, "blacklist_content") # type: ignore
except TagNotFoundError:
blacklist_content: str = ""
except ValueError:
blacklist_content: str = ""
return blacklist_content
def get_blacklist_summary(custom_reader: Reader, feed: Feed) -> str:
"""
Get the blacklist_summary tag from the feed.
Args:
custom_reader: The reader.
feed: The feed to get the tag from.
Returns:
str: The blacklist_summary tag.
"""
try:
blacklist_summary: str = custom_reader.get_tag(feed, "blacklist_summary") # type: ignore
except TagNotFoundError:
blacklist_summary: str = ""
except ValueError:
blacklist_summary: str = ""
return blacklist_summary
def get_blacklist_title(custom_reader: Reader, feed: Feed) -> str:
"""
Get the blacklist_title tag from the feed.
Args:
custom_reader: The reader.
feed: The feed to get the tag from.
Returns:
str: The blacklist_title tag.
"""
try:
blacklist_title: str = custom_reader.get_tag(feed, "blacklist_title") # type: ignore
except TagNotFoundError:
blacklist_title: str = ""
except ValueError:
blacklist_title: str = ""
return blacklist_title

72
discord_rss_bot/filter/whitelist.py
View File

@ -1,4 +1,4 @@
from reader import Entry, Feed, Reader, TagNotFoundError
from reader import Entry, Feed, Reader
from discord_rss_bot.filter.utils import is_word_in_text
@ -17,9 +17,9 @@ def has_white_tags(custom_reader: Reader, feed: Feed) -> bool:
Returns:
bool: If the feed has any of the tags.
"""
whitelist_title: str = get_whitelist_title(custom_reader, feed)
whitelist_summary: str = get_whitelist_summary(custom_reader, feed)
whitelist_content: str = get_whitelist_content(custom_reader, feed)
whitelist_title: str = str(custom_reader.get_tag(feed, "whitelist_title", ""))
whitelist_summary: str = str(custom_reader.get_tag(feed, "whitelist_summary", ""))
whitelist_content: str = str(custom_reader.get_tag(feed, "whitelist_content", ""))
return bool(whitelist_title or whitelist_summary or whitelist_content)
@ -36,8 +36,8 @@ def should_be_sent(custom_reader: Reader, entry: Entry) -> bool:
bool: If the entry is in the whitelist.
"""
feed: Feed = entry.feed
whitelist_title: str = get_whitelist_title(custom_reader, feed)
whitelist_summary: str = get_whitelist_summary(custom_reader, feed)
whitelist_title: str = str(custom_reader.get_tag(feed, "whitelist_title", ""))
whitelist_summary: str = str(custom_reader.get_tag(feed, "whitelist_summary", ""))
# whitelist_content: str = get_whitelist_content(custom_reader, feed)
# TODO: Fix content
# TODO: Check author
@ -47,63 +47,3 @@ def should_be_sent(custom_reader: Reader, entry: Entry) -> bool:
elif entry.summary and whitelist_summary and is_word_in_text(whitelist_summary, entry.summary):
return True
return False
def get_whitelist_content(custom_reader: Reader, feed: Feed) -> str:
"""
Get the whitelist_content tag from the feed.
Args:
custom_reader: The reader.
feed: The feed to get the tag from.
Returns:
str: The whitelist_content tag.
"""
try:
whitelist_content: str = custom_reader.get_tag(feed, "whitelist_content") # type: ignore
except TagNotFoundError:
whitelist_content: str = ""
except ValueError:
whitelist_content: str = ""
return whitelist_content
def get_whitelist_summary(custom_reader: Reader, feed: Feed) -> str:
"""
Get the whitelist_summary tag from the feed.
Args:
custom_reader: The reader.
feed: The feed to get the tag from.
Returns:
str: The whitelist_summary tag.
"""
try:
whitelist_summary: str = custom_reader.get_tag(feed, "whitelist_summary") # type: ignore
except TagNotFoundError:
whitelist_summary: str = ""
except ValueError:
whitelist_summary: str = ""
return whitelist_summary
def get_whitelist_title(custom_reader: Reader, feed: Feed) -> str:
"""
Get the whitelist_title tag from the feed.
Args:
custom_reader: The reader.
feed: The feed to get the tag from.
Returns:
str: The whitelist_title tag.
"""
try:
whitelist_title: str = custom_reader.get_tag(feed, "whitelist_title") # type: ignore
except TagNotFoundError:
whitelist_title: str = ""
except ValueError:
whitelist_title: str = ""
return whitelist_title

14
discord_rss_bot/main.py
View File

@ -27,8 +27,6 @@ from discord_rss_bot.custom_message import (
save_embed,
)
from discord_rss_bot.feeds import create_feed, send_entry_to_discord, send_to_discord
from discord_rss_bot.filter.blacklist import get_blacklist_content, get_blacklist_summary, get_blacklist_title
from discord_rss_bot.filter.whitelist import get_whitelist_content, get_whitelist_summary, get_whitelist_title
from discord_rss_bot.markdown import convert_html_to_md
from discord_rss_bot.missing_tags import add_missing_tags
from discord_rss_bot.search import create_html_for_search_results
@ -147,9 +145,9 @@ async def get_whitelist(feed_url: str, request: Request):
feed: Feed = reader.get_feed(urllib.parse.unquote(clean_feed_url))
# Get previous data, this is used when creating the form.
whitelist_title: str = get_whitelist_title(reader, feed)
whitelist_summary: str = get_whitelist_summary(reader, feed)
whitelist_content: str = get_whitelist_content(reader, feed)
whitelist_title: str = str(reader.get_tag(feed, "whitelist_title", ""))
whitelist_summary: str = str(reader.get_tag(feed, "whitelist_summary", ""))
whitelist_content: str = str(reader.get_tag(feed, "whitelist_content", ""))
context = {
"request": request,
@ -193,9 +191,9 @@ async def get_blacklist(feed_url: str, request: Request):
feed: Feed = reader.get_feed(urllib.parse.unquote(feed_url))
# Get previous data, this is used when creating the form.
blacklist_title: str = get_blacklist_title(reader, feed)
blacklist_summary: str = get_blacklist_summary(reader, feed)
blacklist_content: str = get_blacklist_content(reader, feed)
blacklist_title: str = str(reader.get_tag(feed, "blacklist_title", ""))
blacklist_summary: str = str(reader.get_tag(feed, "blacklist_summary", ""))
blacklist_content: str = str(reader.get_tag(feed, "blacklist_content", ""))
context = {
"request": request,

81
tests/test_blacklist.py
View File

@ -4,15 +4,9 @@ from typing import Iterable
from reader import Entry, Feed, Reader, make_reader
from discord_rss_bot.filter.blacklist import (
get_blacklist_content,
get_blacklist_summary,
get_blacklist_title,
has_black_tags,
should_be_skipped,
)
from discord_rss_bot.filter.blacklist import has_black_tags, should_be_skipped
feed_url = "https://lovinator.space/rss_test.xml"
feed_url: str = "https://lovinator.space/rss_test.xml"
# Create the database
@ -25,7 +19,7 @@ def get_reader() -> Reader:
return reader
def test_has_black_tags():
def test_has_black_tags() -> None:
reader: Reader = get_reader()
# Add feed and update entries
@ -44,14 +38,14 @@ def test_has_black_tags():
reader.delete_feed(feed_url)
def check_if_has_tag(reader, feed, blacklist_name):
reader.set_tag(feed, blacklist_name, "a")
def check_if_has_tag(reader: Reader, feed: Feed, blacklist_name: str) -> None:
reader.set_tag(feed, blacklist_name, "a") # type: ignore
assert has_black_tags(custom_reader=reader, feed=feed) is True
reader.delete_tag(feed, blacklist_name)
assert has_black_tags(custom_reader=reader, feed=feed) is False
def test_should_be_skipped():
def test_should_be_skipped() -> None:
reader: Reader = get_reader()
# Add feed and update entries
@ -76,66 +70,25 @@ def test_should_be_skipped():
reader.delete_tag(feed, "blacklist_title")
assert should_be_skipped(reader, first_entry[0]) is False
reader.set_tag(feed, "blacklist_title", "åäö") # type: ignore
assert should_be_skipped(reader, first_entry[0]) is False
reader.delete_tag(feed, "blacklist_title")
assert should_be_skipped(reader, first_entry[0]) is False
reader.set_tag(feed, "blacklist_summary", "ffdnfdnfdnfdnfdndfn") # type: ignore
assert should_be_skipped(reader, first_entry[0]) is True
reader.delete_tag(feed, "blacklist_summary")
assert should_be_skipped(reader, first_entry[0]) is False
reader.set_tag(feed, "blacklist_summary", "åäö") # type: ignore
assert should_be_skipped(reader, first_entry[0]) is False
reader.delete_tag(feed, "blacklist_summary")
assert should_be_skipped(reader, first_entry[0]) is False
reader.set_tag(feed, "blacklist_content", "ffdnfdnfdnfdnfdndfn") # type: ignore
# TODO: This is not impelemented yes
# TODO: This is not impelemented yet
assert should_be_skipped(reader, first_entry[0]) is False
reader.delete_tag(feed, "blacklist_content")
assert should_be_skipped(reader, first_entry[0]) is False
# TODO: Also add support for entry_text
def test_get_blacklist_content():
reader: Reader = get_reader()
# Add feed and update entries
reader.add_feed(feed_url)
feed: Feed = reader.get_feed(feed_url)
reader.update_feeds()
assert get_blacklist_content(reader, feed) == "" # type: ignore
reader.set_tag(feed, "blacklist_content", "ffdnfdnfdnfdnfdndfn") # type: ignore
assert get_blacklist_content(reader, feed) == "ffdnfdnfdnfdnfdndfn" # type: ignore
reader.delete_tag(feed, "blacklist_content")
assert get_blacklist_content(reader, feed) == "" # type: ignore
def test_get_blacklist_summary():
reader: Reader = get_reader()
# Add feed and update entries
reader.add_feed(feed_url)
feed: Feed = reader.get_feed(feed_url)
reader.update_feeds()
assert get_blacklist_summary(reader, feed) == "" # type: ignore
reader.set_tag(feed, "blacklist_summary", "ffdnfdnfdnfdnfdndfn") # type: ignore
assert get_blacklist_summary(reader, feed) == "ffdnfdnfdnfdnfdndfn" # type: ignore
reader.delete_tag(feed, "blacklist_summary")
assert get_blacklist_summary(reader, feed) == "" # type: ignore
def test_get_blacklist_title():
reader: Reader = get_reader()
# Add feed and update entries
reader.add_feed(feed_url)
feed: Feed = reader.get_feed(feed_url)
reader.update_feeds()
assert get_blacklist_title(reader, feed) == "" # type: ignore
reader.set_tag(feed, "blacklist_title", "ffdnfdnfdnfdnfdndfn") # type: ignore
assert get_blacklist_title(reader, feed) == "ffdnfdnfdnfdnfdndfn" # type: ignore
reader.delete_tag(feed, "blacklist_title")
assert get_blacklist_title(reader, feed) == "" # type: ignore

94
tests/test_whitelist.py Normal file
View File

@ -0,0 +1,94 @@
import tempfile
from pathlib import Path
from typing import Iterable
from reader import Entry, Feed, Reader, make_reader
from discord_rss_bot.filter.whitelist import has_white_tags, should_be_sent
feed_url: str = "https://lovinator.space/rss_test.xml"
# Create the database
def get_reader() -> Reader:
tempdir: Path = Path(tempfile.mkdtemp())
reader_database: Path = tempdir / "test.sqlite"
reader: Reader = make_reader(url=str(reader_database))
return reader
def test_has_white_tags() -> None:
reader: Reader = get_reader()
# Add feed and update entries
reader.add_feed(feed_url)
feed: Feed = reader.get_feed(feed_url)
reader.update_feeds()
# Test feed without any whitelist tags
assert has_white_tags(custom_reader=get_reader(), feed=feed) is False
check_if_has_tag(reader, feed, "whitelist_title")
check_if_has_tag(reader, feed, "whitelist_summary")
check_if_has_tag(reader, feed, "whitelist_content")
# Clean up
reader.delete_feed(feed_url)
def check_if_has_tag(reader: Reader, feed: Feed, whitelist_name: str) -> None:
reader.set_tag(feed, whitelist_name, "a") # type: ignore
assert has_white_tags(custom_reader=reader, feed=feed) is True
reader.delete_tag(feed, whitelist_name)
assert has_white_tags(custom_reader=reader, feed=feed) is False
def test_should_be_sent() -> None:
reader: Reader = get_reader()
# Add feed and update entries
reader.add_feed(feed_url)
feed: Feed = reader.get_feed(feed_url)
reader.update_feeds()
# Get first entry
first_entry: list[Entry] = []
entries: Iterable[Entry] = reader.get_entries(feed=feed)
assert entries is not None
for entry in entries:
first_entry.append(entry)
break
assert len(first_entry) == 1
# Test entry without any whitelists
assert should_be_sent(reader, first_entry[0]) is False
reader.set_tag(feed, "whitelist_title", "fvnnnfnfdnfdnfd") # type: ignore
assert should_be_sent(reader, first_entry[0]) is True
reader.delete_tag(feed, "whitelist_title")
assert should_be_sent(reader, first_entry[0]) is False
reader.set_tag(feed, "whitelist_title", "åäö") # type: ignore
assert should_be_sent(reader, first_entry[0]) is False
reader.delete_tag(feed, "whitelist_title")
assert should_be_sent(reader, first_entry[0]) is False
reader.set_tag(feed, "whitelist_summary", "ffdnfdnfdnfdnfdndfn") # type: ignore
assert should_be_sent(reader, first_entry[0]) is True
reader.delete_tag(feed, "whitelist_summary")
assert should_be_sent(reader, first_entry[0]) is False
reader.set_tag(feed, "whitelist_summary", "åäö") # type: ignore
assert should_be_sent(reader, first_entry[0]) is False
reader.delete_tag(feed, "whitelist_summary")
assert should_be_sent(reader, first_entry[0]) is False
reader.set_tag(feed, "whitelist_content", "ffdnfdnfdnfdnfdndfn") # type: ignore
# TODO: This is not impelemented yet
assert should_be_sent(reader, first_entry[0]) is False
reader.delete_tag(feed, "whitelist_content")
assert should_be_sent(reader, first_entry[0]) is False
# TODO: Also add support for entry_text