TheLovinator/twitch-drop-notifier
1
Fork 0
Code Issues Actions Packages Releases Activity

Rewrite everything

Browse Source
This commit is contained in:
Joakim Hellsén
2024-09-09 05:38:43 +02:00
parent 5334b6904e
commit ed0a7755bf
18 changed files with 991 additions and 622 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
core/management/commands/scrape_local.py
View File

@ -20,19 +20,17 @@ class Command(BaseCommand):
*args: Variable length argument list.
**kwargs: Arbitrary keyword arguments.
"""
dirs: list[str] = ["drop_campaigns", "reward_campaigns", "drop_campaigns"]
for dir_name in dirs:
logger.info("Scraping %s", dir_name)
for num, file in enumerate(Path(dir_name).rglob("*.json")):
logger.info("Processing %s", file)
dir_name = Path("json")
for num, file in enumerate(Path(dir_name).rglob("*.json")):
logger.info("Processing %s", file)
with file.open(encoding="utf-8") as f:
try:
load_json = json.load(f)
except json.JSONDecodeError:
logger.exception("Failed to load JSON from %s", file)
continue
asyncio.run(main=process_json_data(num=num, campaign=load_json, local=True))
with file.open(encoding="utf-8") as f:
try:
load_json = json.load(f)
except json.JSONDecodeError:
logger.exception("Failed to load JSON from %s", file)
continue
asyncio.run(main=process_json_data(num=num, campaign=load_json, local=True))
if __name__ == "__main__":

158
core/management/commands/scrape_twitch.py
View File

@ -10,7 +10,7 @@ from platformdirs import user_data_dir
from playwright.async_api import Playwright, async_playwright
from playwright.async_api._generated import Response
from core.models import Benefit, DropCampaign, Game, Owner, Reward, RewardCampaign, TimeBasedDrop
from core.models import Benefit, DropCampaign, Game, Owner, RewardCampaign, TimeBasedDrop
if TYPE_CHECKING:
from playwright.async_api._generated import BrowserContext, Page
@ -19,46 +19,36 @@ if TYPE_CHECKING:
logger: logging.Logger = logging.getLogger(__name__)
def get_data_dir() -> Path:
"""Get the data directory.
Returns:
Path: The data directory.
"""
return Path(
user_data_dir(
appname="TTVDrops",
appauthor="TheLovinator",
roaming=True,
ensure_exists=True,
),
)
def get_profile_dir() -> Path:
"""Get the profile directory for the browser.
Returns:
Path: The profile directory.
"""
profile_dir = Path(get_data_dir() / "chrome-profile")
data_dir = Path(
user_data_dir(appname="TTVDrops", appauthor="TheLovinator", roaming=True, ensure_exists=True),
)
profile_dir: Path = data_dir / "chrome-profile"
profile_dir.mkdir(parents=True, exist_ok=True)
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Launching Chrome browser with user data directory: %s", profile_dir)
return profile_dir
def save_json(campaign: dict | None, dir_name: str) -> None:
def save_json(campaign: dict | None, *, local: bool) -> None:
"""Save JSON data to a file.
Args:
campaign (dict): The JSON data to save.
dir_name (Path): The directory to save the JSON data to.
local (bool): Only save JSON data if we are scraping from the web.
"""
if local:
return
if not campaign:
return
save_dir = Path(dir_name)
save_dir = Path("json")
save_dir.mkdir(parents=True, exist_ok=True)
# File name is the hash of the JSON data
@ -68,27 +58,19 @@ def save_json(campaign: dict | None, dir_name: str) -> None:
json.dump(campaign, f, indent=4)
async def add_reward_campaign(campaign: dict | None) -> None:
async def add_reward_campaign(reward_campaign: dict | None) -> None:
"""Add a reward campaign to the database.
Args:
campaign (dict): The reward campaign to add.
reward_campaign (dict): The reward campaign to add.
"""
if not campaign:
if not reward_campaign:
return
if "data" in campaign and "rewardCampaignsAvailableToUser" in campaign["data"]:
for reward_campaign in campaign["data"]["rewardCampaignsAvailableToUser"]:
our_reward_campaign, created = await RewardCampaign.objects.aupdate_or_create(id=reward_campaign["id"])
await our_reward_campaign.import_json(reward_campaign)
if created:
logger.info("Added reward campaign %s", our_reward_campaign)
if "rewards" in reward_campaign:
for reward in reward_campaign["rewards"]:
reward_instance, created = await Reward.objects.aupdate_or_create(id=reward["id"])
await reward_instance.import_json(reward, our_reward_campaign)
if created:
logger.info("Added reward %s", reward_instance)
our_reward_campaign, created = await RewardCampaign.objects.aupdate_or_create(twitch_id=reward_campaign["id"])
await our_reward_campaign.aimport_json(reward_campaign)
if created:
logger.info("Added reward campaign %s", our_reward_campaign)
async def add_drop_campaign(drop_campaign: dict | None) -> None:
@ -100,23 +82,37 @@ async def add_drop_campaign(drop_campaign: dict | None) -> None:
if not drop_campaign:
return
if drop_campaign.get("game"):
owner, created = await Owner.objects.aupdate_or_create(id=drop_campaign["owner"]["id"])
owner.import_json(drop_campaign["owner"])
game, created = await Game.objects.aupdate_or_create(id=drop_campaign["game"]["id"])
await game.import_json(drop_campaign["game"], owner)
if created:
logger.info("Added game %s", game)
our_drop_campaign, created = await DropCampaign.objects.aupdate_or_create(id=drop_campaign["id"])
await our_drop_campaign.import_json(drop_campaign, game)
if not drop_campaign.get("owner", {}):
logger.error("Owner not found in drop campaign %s", drop_campaign)
return
owner, created = await Owner.objects.aupdate_or_create(twitch_id=drop_campaign["owner"]["id"])
await owner.aimport_json(data=drop_campaign["owner"])
if created:
logger.info("Added drop campaign %s", our_drop_campaign.id)
logger.info("Added owner %s", owner.twitch_id)
if not drop_campaign.get("game", {}):
logger.error("Game not found in drop campaign %s", drop_campaign)
return
game, created = await Game.objects.aupdate_or_create(twitch_id=drop_campaign["game"]["id"])
await game.aimport_json(data=drop_campaign["game"], owner=owner)
if created:
logger.info("Added game %s", game)
our_drop_campaign, created = await DropCampaign.objects.aupdate_or_create(twitch_id=drop_campaign["id"])
await our_drop_campaign.aimport_json(drop_campaign, game)
if created:
logger.info("Added drop campaign %s", our_drop_campaign.twitch_id)
await add_time_based_drops(drop_campaign, our_drop_campaign)
# Check if eventBasedDrops exist
if drop_campaign.get("eventBasedDrops"):
# TODO(TheLovinator): Add event-based drops # noqa: TD003
msg = "Not implemented: Add event-based drops"
raise NotImplementedError(msg)
async def add_time_based_drops(drop_campaign: dict, our_drop_campaign: DropCampaign) -> None:
"""Add time-based drops to the database.
@ -126,33 +122,61 @@ async def add_time_based_drops(drop_campaign: dict, our_drop_campaign: DropCampa
our_drop_campaign (DropCampaign): The drop campaign object in the database.
"""
for time_based_drop in drop_campaign.get("timeBasedDrops", []):
time_based_drop: dict[str, typing.Any]
if time_based_drop.get("preconditionDrops"):
# TODO(TheLovinator): Add precondition drops to time-based drop # noqa: TD003
# TODO(TheLovinator): Send JSON to Discord # noqa: TD003
msg = "Not implemented: Add precondition drops to time-based drop"
raise NotImplementedError(msg)
our_time_based_drop, created = await TimeBasedDrop.objects.aupdate_or_create(id=time_based_drop["id"])
await our_time_based_drop.import_json(time_based_drop, our_drop_campaign)
our_time_based_drop, created = await TimeBasedDrop.objects.aupdate_or_create(twitch_id=time_based_drop["id"])
await our_time_based_drop.aimport_json(time_based_drop, our_drop_campaign)
if created:
logger.info("Added time-based drop %s", our_time_based_drop.id)
logger.info("Added time-based drop %s", our_time_based_drop.twitch_id)
if our_time_based_drop and time_based_drop.get("benefitEdges"):
for benefit_edge in time_based_drop["benefitEdges"]:
benefit, created = await Benefit.objects.aupdate_or_create(id=benefit_edge["benefit"])
await benefit.import_json(benefit_edge["benefit"], our_time_based_drop)
benefit, created = await Benefit.objects.aupdate_or_create(twitch_id=benefit_edge["benefit"])
await benefit.aimport_json(benefit_edge["benefit"], our_time_based_drop)
if created:
logger.info("Added benefit %s", benefit.id)
logger.info("Added benefit %s", benefit.twitch_id)
async def process_json_data(num: int, campaign: dict | None) -> None:
async def handle_drop_campaigns(drop_campaign: dict) -> None:
"""Handle drop campaigns.
We need to grab the game image in data.currentUser.dropCampaigns.game.boxArtURL.
Args:
drop_campaign (dict): The drop campaign to handle.
"""
if not drop_campaign:
return
if drop_campaign.get("game", {}).get("boxArtURL"):
owner_id = drop_campaign.get("owner", {}).get("id")
if not owner_id:
logger.error("Owner ID not found in drop campaign %s", drop_campaign)
return
owner, created = await Owner.objects.aupdate_or_create(twitch_id=drop_campaign["owner"]["id"])
await owner.aimport_json(drop_campaign["owner"])
if created:
logger.info("Added owner %s", owner.twitch_id)
game_obj, created = await Game.objects.aupdate_or_create(twitch_id=drop_campaign["game"]["id"])
await game_obj.aimport_json(data=drop_campaign["game"], owner=owner)
if created:
logger.info("Added game %s", game_obj.twitch_id)
async def process_json_data(num: int, campaign: dict | None, *, local: bool) -> None:
"""Process JSON data.
Args:
num (int): The number of the JSON data.
campaign (dict): The JSON data to process.
local (bool): Only save JSON data if we are scraping from the web.
"""
logger.info("Processing JSON %d", num)
if not campaign:
@ -163,20 +187,18 @@ async def process_json_data(num: int, campaign: dict | None) -> None:
logger.warning("Campaign is not a dictionary. %s", campaign)
return
# This is a Reward Campaign
if "rewardCampaignsAvailableToUser" in campaign.get("data", {}):
save_json(campaign=campaign, dir_name="reward_campaigns")
await add_reward_campaign(campaign=campaign)
save_json(campaign=campaign, local=local)
if "dropCampaign" in campaign.get("data", {}).get("user", {}):
save_json(campaign=campaign, dir_name="drop_campaign")
if campaign.get("data", {}).get("user", {}).get("dropCampaign"):
await add_drop_campaign(drop_campaign=campaign["data"]["user"]["dropCampaign"])
if campaign.get("data", {}).get("rewardCampaignsAvailableToUser"):
for reward_campaign in campaign["data"]["rewardCampaignsAvailableToUser"]:
await add_reward_campaign(reward_campaign=reward_campaign)
if "dropCampaigns" in campaign.get("data", {}).get("currentUser", {}):
if campaign.get("data", {}).get("user", {}).get("dropCampaign"):
await add_drop_campaign(drop_campaign=campaign["data"]["user"]["dropCampaign"])
if campaign.get("data", {}).get("currentUser", {}).get("dropCampaigns"):
for drop_campaign in campaign["data"]["currentUser"]["dropCampaigns"]:
save_json(campaign=campaign, dir_name="drop_campaigns")
await add_drop_campaign(drop_campaign=drop_campaign)
await handle_drop_campaigns(drop_campaign=drop_campaign)
class Command(BaseCommand):
@ -232,7 +254,7 @@ class Command(BaseCommand):
await browser.close()
for num, campaign in enumerate(json_data, start=1):
await process_json_data(num=num, campaign=campaign)
await process_json_data(num=num, campaign=campaign, local=True)
return json_data