TheLovinator/twitch-drop-notifier
1
Fork 0
Code Issues Actions Packages Releases Activity

WIP

Browse Source
This commit is contained in:
Joakim Hellsén
2024-08-02 05:10:23 +02:00
parent 9252e41a15
commit 09b21d4f43
13 changed files with 940 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
twitch_app/management/commands/scrape_twitch.py
View File

@ -28,6 +28,7 @@ from twitch_app.models import (
if TYPE_CHECKING:
from playwright.async_api._generated import BrowserContext, Page
import json
# Where to store the Chrome profile
data_dir = Path(
@ -66,6 +67,7 @@ async def add_or_get_game(json_data: dict, name: str) -> tuple[Game | None, bool
"slug": json_data.get("slug"),
"display_name": json_data.get("displayName"),
"typename": json_data.get("__typename"),
"box_art_url": json_data.get("boxArtURL"), # Only for RewardCampaigns
},
)
@ -204,6 +206,9 @@ async def add_or_get_drop_campaign(
logger.warning("No drop campaign data found")
return None, False
if drop_campaign_data.get("__typename") != "Game":
logger.error("__typename is not 'Game' for %s", drop_campaign_data.get("name", "Unknown Drop Campaign"))
drop_campaign, _ = await DropCampaign.objects.aupdate_or_create(
id=drop_campaign_data["id"],
defaults={
@ -546,16 +551,35 @@ class Command(BaseCommand):
continue
if "rewardCampaignsAvailableToUser" in campaign["data"]:
# Save to folder named "reward_campaigns"
dir_name: Path = Path("reward_campaigns")
dir_name.mkdir(parents=True, exist_ok=True)
with open(file=Path(dir_name / f"reward_campaign_{num}.json"), mode="w", encoding="utf-8") as f:
json.dump(campaign, f, indent=4)
await add_reward_campaign(campaign)
if "dropCampaign" in campaign.get("data", {}).get("user", {}):
if not campaign["data"]["user"]["dropCampaign"]:
logger.warning("No drop campaign found")
continue
# Save to folder named "drop_campaign"
dir_name: Path = Path("drop_campaign")
dir_name.mkdir(parents=True, exist_ok=True)
with open(file=Path(dir_name / f"drop_campaign_{num}.json"), mode="w", encoding="utf-8") as f:
json.dump(campaign, f, indent=4)
await add_drop_campaign(campaign)
if "dropCampaigns" in campaign.get("data", {}).get("user", {}):
for drop_campaign in campaign["data"]["user"]["dropCampaigns"]:
# Save to folder named "drop_campaigns"
dir_name: Path = Path("drop_campaigns")
dir_name.mkdir(parents=True, exist_ok=True)
with open(file=Path(dir_name / f"drop_campaign_{num}.json"), mode="w", encoding="utf-8") as f:
json.dump(drop_campaign, f, indent=4)
await add_drop_campaign(drop_campaign)
return json_data

135
twitch_app/migrations/0010_frontendchannel_frontendgame_frontendorg_and_more.py Normal file
View File

@ -0,0 +1,135 @@
# Generated by Django 5.1rc1 on 2024-08-02 01:20
import django.db.models.deletion
import django.db.models.manager
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("twitch_app", "0009_alter_benefit_entitlement_limit_and_more"),
]
operations = [
migrations.CreateModel(
name="FrontEndChannel",
fields=[
("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
("name", models.TextField(blank=True, null=True)),
("twitch_url", models.URLField(blank=True, null=True)),
("live", models.BooleanField(default=False)),
],
options={
"abstract": False,
"base_manager_name": "prefetch_manager",
},
managers=[
("objects", django.db.models.manager.Manager()),
("prefetch_manager", django.db.models.manager.Manager()),
],
),
migrations.CreateModel(
name="FrontEndGame",
fields=[
("twitch_id", models.TextField(primary_key=True, serialize=False)),
("game_url", models.URLField(blank=True, null=True)),
("display_name", models.TextField(blank=True, null=True)),
],
options={
"abstract": False,
"base_manager_name": "prefetch_manager",
},
managers=[
("objects", django.db.models.manager.Manager()),
("prefetch_manager", django.db.models.manager.Manager()),
],
),
migrations.CreateModel(
name="FrontEndOrg",
fields=[
("id", models.TextField(primary_key=True, serialize=False)),
("name", models.TextField(blank=True, null=True)),
("url", models.TextField(blank=True, null=True)),
],
options={
"abstract": False,
"base_manager_name": "prefetch_manager",
},
managers=[
("objects", django.db.models.manager.Manager()),
("prefetch_manager", django.db.models.manager.Manager()),
],
),
migrations.AddField(
model_name="game",
name="box_art_url",
field=models.URLField(blank=True, null=True),
),
migrations.CreateModel(
name="FrontEndDropCampaign",
fields=[
("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
("account_link_url", models.URLField(blank=True, null=True)),
("about_url", models.URLField(blank=True, null=True)),
("ends_at", models.DateTimeField(null=True)),
("starts_at", models.DateTimeField(null=True)),
("channels", models.ManyToManyField(related_name="drop_campaigns", to="twitch_app.frontendchannel")),
(
"game",
models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="drop_campaigns",
to="twitch_app.frontendgame",
),
),
],
options={
"abstract": False,
"base_manager_name": "prefetch_manager",
},
managers=[
("objects", django.db.models.manager.Manager()),
("prefetch_manager", django.db.models.manager.Manager()),
],
),
migrations.CreateModel(
name="FrontEndDrop",
fields=[
("id", models.TextField(primary_key=True, serialize=False)),
("created_at", models.DateTimeField(null=True)),
("name", models.TextField(blank=True, null=True)),
("image_url", models.URLField(blank=True, null=True)),
("limit", models.PositiveBigIntegerField(null=True)),
("is_ios_available", models.BooleanField(null=True)),
("minutes_watched", models.PositiveBigIntegerField(null=True)),
(
"drop_campaign",
models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="drops",
to="twitch_app.frontenddropcampaign",
),
),
],
options={
"abstract": False,
"base_manager_name": "prefetch_manager",
},
managers=[
("objects", django.db.models.manager.Manager()),
("prefetch_manager", django.db.models.manager.Manager()),
],
),
migrations.AddField(
model_name="frontendgame",
name="org",
field=models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="games",
to="twitch_app.frontendorg",
),
),
]

69
twitch_app/models.py
View File

@ -1,3 +1,5 @@
import typing
import auto_prefetch
from django.db import models
@ -25,6 +27,7 @@ class Game(auto_prefetch.Model):
id = models.AutoField(primary_key=True)
slug = models.TextField(null=True, blank=True)
display_name = models.TextField(null=True, blank=True)
box_art_url = models.URLField(null=True, blank=True)
typename = models.TextField(null=True, blank=True)
def __str__(self) -> str:
@ -502,6 +505,11 @@ class DropCampaign(auto_prefetch.Model):
typename (str): The type name of the object, typically "DropCampaign".
"""
STATUS_CHOICES: typing.ClassVar[list[tuple[str, str]]] = [
("ACTIVE", "Active"),
("EXPIRED", "Expired"),
]
id = models.TextField(primary_key=True)
allow = auto_prefetch.ForeignKey(Allow, on_delete=models.CASCADE, related_name="drop_campaigns", null=True)
account_link_url = models.URLField(null=True, blank=True)
@ -514,9 +522,68 @@ class DropCampaign(auto_prefetch.Model):
name = models.TextField(null=True, blank=True)
owner = auto_prefetch.ForeignKey(Owner, on_delete=models.CASCADE, related_name="drop_campaigns", null=True)
starts_at = models.DateTimeField(null=True)
status = models.TextField(null=True, blank=True)
status = models.TextField(choices=STATUS_CHOICES, null=True, blank=True)
time_based_drops = models.ManyToManyField(TimeBasedDrop, related_name="drop_campaigns")
typename = models.TextField(null=True, blank=True)
def __str__(self) -> str:
return self.name or "Unknown"
class FrontEndChannel(auto_prefetch.Model):
"""This is the channel we will see on the front end."""
name = models.TextField(null=True, blank=True)
twitch_url = models.URLField(null=True, blank=True)
live = models.BooleanField(default=False)
class FrontEndOrg(auto_prefetch.Model):
"""Drops are group by organization -> by game -> by drop campaign."""
id = models.TextField(primary_key=True)
name = models.TextField(null=True, blank=True)
url = models.TextField(null=True, blank=True)
class FrontEndGame(auto_prefetch.Model):
"""This is the game we will see on the front end."""
twitch_id = models.TextField(primary_key=True)
game_url = models.URLField(null=True, blank=True)
display_name = models.TextField(null=True, blank=True)
org = models.ForeignKey(FrontEndOrg, on_delete=models.CASCADE, related_name="games", null=True)
def __str__(self) -> str:
return self.display_name or "Unknown"
class FrontEndDropCampaign(auto_prefetch.Model):
"""This is the drop campaign we will see on the front end."""
account_link_url = models.URLField(null=True, blank=True)
about_url = models.URLField(null=True, blank=True)
ends_at = models.DateTimeField(null=True)
starts_at = models.DateTimeField(null=True)
game = models.ForeignKey(FrontEndGame, on_delete=models.CASCADE, related_name="drop_campaigns", null=True)
channels = models.ManyToManyField(FrontEndChannel, related_name="drop_campaigns")
class FrontEndDrop(auto_prefetch.Model):
"""This is the drop we will see on the front end."""
id = models.TextField(primary_key=True)
created_at = models.DateTimeField(null=True)
name = models.TextField(null=True, blank=True)
image_url = models.URLField(null=True, blank=True)
drop_campaign = models.ForeignKey(FrontEndDropCampaign, on_delete=models.CASCADE, related_name="drops", null=True)
limit = models.PositiveBigIntegerField(null=True)
is_ios_available = models.BooleanField(null=True)
minutes_watched = models.PositiveBigIntegerField(null=True)