TheLovinator/twitch-drop-notifier
1
Fork 0
Code Issues Actions Packages Releases Activity

Fix scraper and other things

Browse Source
This commit is contained in:
Joakim Hellsén
2024-08-01 16:52:14 +02:00
parent 7ac017e4ca
commit 9252e41a15
7 changed files with 410 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
core/templates/partials/reward_campaign_card.html
View File

@ -10,12 +10,12 @@
</div>
<div class="col-md-10">
<div class="card-body">
<h2 class="card-title h5">{{ campaign.name }}</h2>
<h2 class="card-title h5" id="#reward-{{ campaign.id }}">
<a href="#campaign-{{ campaign.id }}" class="plain-text-item">{{ campaign.name }}</a>
</h2>
<p class="card-text text-muted">{{ campaign.summary }}</p>
<p>
Starts at: <abbr title="{{ campaign.starts_at|date:'l d F H:i e' }}">{{ campaign.starts_at }}</abbr>
<br>
Ends at: <abbr title="{{ campaign.ends_at|date:'l d F H:i e' }}">{{ campaign.ends_at|timeuntil }}</abbr>
<p class="mb-2 text-muted">
Ends in: <abbr title="{{ campaign.starts_at|date:'l d F H:i' }} - {{ campaign.ends_at|date:'l d F H:i e' }}">{{ campaign.ends_at|timeuntil }}</abbr>
</p>
<a href="{{ campaign.external_url }}"
class="btn btn-primary"
@ -40,10 +40,6 @@
loading="lazy">
<div>
<strong>{{ reward.name }}</strong>
<br>
<a href="{{ reward.redemption_url }}"
class="btn btn-sm btn-link"
target="_blank">Redeem</a>
</div>
</div>
{% endfor %}

12
core/templates/partials/reward_campaigns_toc.html Normal file
View File

@ -0,0 +1,12 @@
<div class="position-sticky d-none d-lg-block toc">
<div class="card">
<div class="card-body">
<div id="toc-list" class="list-group">
{% for campaign in reward_campaigns %}
<a class="list-group-item list-group-item-action plain-text-item"
href="#reward-{{ campaign.id }}">{{ campaign }}</a>
{% endfor %}
</div>
</div>
</div>
</div>

13
core/templates/reward_campaigns.html
View File

@ -3,11 +3,14 @@
{% block content %}
<div class="container mt-4">
<div class="row">
<h2>Reward Campaigns</h2>
<div>
{% for campaign in reward_campaigns %}
{% include "partials/reward_campaign_card.html" %}
{% endfor %}
<div class="col-lg-3">{% include "partials/reward_campaigns_toc.html" %}</div>
<div class="col-lg-9">
<h2>Reward Campaigns</h2>
<div>
{% for campaign in reward_campaigns %}
{% include "partials/reward_campaign_card.html" %}
{% endfor %}
</div>
</div>
</div>
</div>

26
twitch_app/management/commands/scrape_twitch.py
View File

@ -279,10 +279,12 @@ async def add_drop_campaign(json_data: dict) -> None:
# Add channels to Allow
if allow:
channel_data: list[dict] = allow_data.get("channels", [])
for json_channel in channel_data:
channel, _ = await add_or_get_channel(json_channel)
if channel:
await allow.channels.aadd(channel)
if channel_data:
for json_channel in channel_data:
channel, _ = await add_or_get_channel(json_channel)
if channel:
await allow.channels.aadd(channel)
# Add or get TimeBasedDrops
time_based_drops_data = drop_campaign_data.get("timeBasedDrops", [])
@ -446,13 +448,13 @@ async def add_reward_campaign(json_data: dict) -> None:
defaults={
"name": campaign.get("name"),
"brand": campaign.get("brand"),
"starts_at": campaign.get("startAt"),
"ends_at": campaign.get("endAt"),
"starts_at": campaign.get("startsAt"),
"ends_at": campaign.get("endsAt"),
"status": campaign.get("status"),
"summary": campaign.get("summary"),
"instructions": campaign.get("instructions"),
"external_url": campaign.get("externalURL"),
"reward_value_url_params": campaign.get("rewardValueURLParams"),
"reward_value_url_param": campaign.get("rewardValueURLParam"),
"about_url": campaign.get("aboutURL"),
"is_sitewide": campaign.get("isSitewide"),
"game": game,
@ -477,7 +479,7 @@ class Command(BaseCommand):
self,
playwright: Playwright,
) -> list[dict[str, typing.Any]]:
args = []
args: list[str] = []
# disable navigator.webdriver:true flag
args.append("--disable-blink-features=AutomationControlled")
@ -546,13 +548,15 @@ class Command(BaseCommand):
if "rewardCampaignsAvailableToUser" in campaign["data"]:
await add_reward_campaign(campaign)
if "dropCampaign" in campaign.get("data", {}).get("user", {}): # noqa: SIM102
if "dropCampaign" in campaign.get("data", {}).get("user", {}):
if not campaign["data"]["user"]["dropCampaign"]:
logger.warning("No drop campaign found")
continue
await add_drop_campaign(campaign)
if "dropCampaigns" in campaign.get("data", {}).get("user", {}):
msg = "Multiple dropCampaigns not supported"
raise NotImplementedError(msg)
for drop_campaign in campaign["data"]["user"]["dropCampaigns"]:
await add_drop_campaign(drop_campaign)
return json_data

278
twitch_app/migrations/0008_alter_allow_options_alter_benefit_options_and_more.py Normal file
View File

@ -0,0 +1,278 @@
# Generated by Django 5.1rc1 on 2024-08-01 14:27
import auto_prefetch
import django.db.models.deletion
import django.db.models.manager
from django.db import migrations
from django.db.migrations.operations.base import Operation
class Migration(migrations.Migration):
dependencies: list[tuple[str, str]] = [
("twitch_app", "0007_alter_dropcampaign_time_based_drops"),
]
operations: list[Operation] = [
migrations.AlterModelOptions(
name="allow",
options={"base_manager_name": "prefetch_manager"},
),
migrations.AlterModelOptions(
name="benefit",
options={"base_manager_name": "prefetch_manager"},
),
migrations.AlterModelOptions(
name="benefitedge",
options={"base_manager_name": "prefetch_manager"},
),
migrations.AlterModelOptions(
name="channel",
options={"base_manager_name": "prefetch_manager"},
),
migrations.AlterModelOptions(
name="dropcampaign",
options={"base_manager_name": "prefetch_manager"},
),
migrations.AlterModelOptions(
name="game",
options={"base_manager_name": "prefetch_manager"},
),
migrations.AlterModelOptions(
name="image",
options={"base_manager_name": "prefetch_manager"},
),
migrations.AlterModelOptions(
name="owner",
options={"base_manager_name": "prefetch_manager"},
),
migrations.AlterModelOptions(
name="reward",
options={"base_manager_name": "prefetch_manager"},
),
migrations.AlterModelOptions(
name="rewardcampaign",
options={"base_manager_name": "prefetch_manager"},
),
migrations.AlterModelOptions(
name="timebaseddrop",
options={"base_manager_name": "prefetch_manager"},
),
migrations.AlterModelOptions(
name="unlockrequirements",
options={"base_manager_name": "prefetch_manager"},
),
migrations.AlterModelManagers(
name="allow",
managers=[
("objects", django.db.models.manager.Manager()),
("prefetch_manager", django.db.models.manager.Manager()),
],
),
migrations.AlterModelManagers(
name="benefit",
managers=[
("objects", django.db.models.manager.Manager()),
("prefetch_manager", django.db.models.manager.Manager()),
],
),
migrations.AlterModelManagers(
name="benefitedge",
managers=[
("objects", django.db.models.manager.Manager()),
("prefetch_manager", django.db.models.manager.Manager()),
],
),
migrations.AlterModelManagers(
name="channel",
managers=[
("objects", django.db.models.manager.Manager()),
("prefetch_manager", django.db.models.manager.Manager()),
],
),
migrations.AlterModelManagers(
name="dropcampaign",
managers=[
("objects", django.db.models.manager.Manager()),
("prefetch_manager", django.db.models.manager.Manager()),
],
),
migrations.AlterModelManagers(
name="game",
managers=[
("objects", django.db.models.manager.Manager()),
("prefetch_manager", django.db.models.manager.Manager()),
],
),
migrations.AlterModelManagers(
name="image",
managers=[
("objects", django.db.models.manager.Manager()),
("prefetch_manager", django.db.models.manager.Manager()),
],
),
migrations.AlterModelManagers(
name="owner",
managers=[
("objects", django.db.models.manager.Manager()),
("prefetch_manager", django.db.models.manager.Manager()),
],
),
migrations.AlterModelManagers(
name="reward",
managers=[
("objects", django.db.models.manager.Manager()),
("prefetch_manager", django.db.models.manager.Manager()),
],
),
migrations.AlterModelManagers(
name="rewardcampaign",
managers=[
("objects", django.db.models.manager.Manager()),
("prefetch_manager", django.db.models.manager.Manager()),
],
),
migrations.AlterModelManagers(
name="timebaseddrop",
managers=[
("objects", django.db.models.manager.Manager()),
("prefetch_manager", django.db.models.manager.Manager()),
],
),
migrations.AlterModelManagers(
name="unlockrequirements",
managers=[
("objects", django.db.models.manager.Manager()),
("prefetch_manager", django.db.models.manager.Manager()),
],
),
migrations.AlterField(
model_name="benefit",
name="game",
field=auto_prefetch.ForeignKey(
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="benefits",
to="twitch_app.game",
),
),
migrations.AlterField(
model_name="benefit",
name="owner_organization",
field=auto_prefetch.ForeignKey(
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="benefits",
to="twitch_app.owner",
),
),
migrations.AlterField(
model_name="benefitedge",
name="benefit",
field=auto_prefetch.ForeignKey(
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="benefit_edges",
to="twitch_app.benefit",
),
),
migrations.AlterField(
model_name="dropcampaign",
name="allow",
field=auto_prefetch.ForeignKey(
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="drop_campaigns",
to="twitch_app.allow",
),
),
migrations.AlterField(
model_name="dropcampaign",
name="game",
field=auto_prefetch.ForeignKey(
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="drop_campaigns",
to="twitch_app.game",
),
),
migrations.AlterField(
model_name="dropcampaign",
name="owner",
field=auto_prefetch.ForeignKey(
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="drop_campaigns",
to="twitch_app.owner",
),
),
migrations.AlterField(
model_name="reward",
name="banner_image",
field=auto_prefetch.ForeignKey(
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="banner_rewards",
to="twitch_app.image",
),
),
migrations.AlterField(
model_name="reward",
name="thumbnail_image",
field=auto_prefetch.ForeignKey(
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="thumbnail_rewards",
to="twitch_app.image",
),
),
migrations.AlterField(
model_name="rewardcampaign",
name="game",
field=auto_prefetch.ForeignKey(
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="reward_campaigns",
to="twitch_app.game",
),
),
migrations.AlterField(
model_name="rewardcampaign",
name="image",
field=auto_prefetch.ForeignKey(
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="reward_campaigns",
to="twitch_app.image",
),
),
migrations.AlterField(
model_name="rewardcampaign",
name="unlock_requirements",
field=auto_prefetch.ForeignKey(
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="reward_campaigns",
to="twitch_app.unlockrequirements",
),
),
migrations.AlterField(
model_name="timebaseddrop",
name="game",
field=auto_prefetch.ForeignKey(
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="time_based_drops",
to="twitch_app.game",
),
),
migrations.AlterField(
model_name="timebaseddrop",
name="owner_organization",
field=auto_prefetch.ForeignKey(
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="time_based_drops",
to="twitch_app.owner",
),
),
]

48
twitch_app/migrations/0009_alter_benefit_entitlement_limit_and_more.py Normal file
View File

@ -0,0 +1,48 @@
# Generated by Django 5.1rc1 on 2024-08-01 14:48
from django.db import migrations, models
from django.db.migrations.operations.base import Operation
class Migration(migrations.Migration):
dependencies: list[tuple[str, str]] = [
("twitch_app", "0008_alter_allow_options_alter_benefit_options_and_more"),
]
operations: list[Operation] = [
migrations.AlterField(
model_name="benefit",
name="entitlement_limit",
field=models.TextField(null=True),
),
migrations.AlterField(
model_name="benefitedge",
name="entitlement_limit",
field=models.TextField(null=True),
),
migrations.AlterField(
model_name="channel",
name="id",
field=models.TextField(primary_key=True, serialize=False),
),
migrations.AlterField(
model_name="owner",
name="id",
field=models.TextField(primary_key=True, serialize=False),
),
migrations.AlterField(
model_name="timebaseddrop",
name="entitlement_limit",
field=models.TextField(null=True),
),
migrations.AlterField(
model_name="unlockrequirements",
name="minute_watched_goal",
field=models.TextField(null=True),
),
migrations.AlterField(
model_name="unlockrequirements",
name="subs_goal",
field=models.TextField(null=True),
),
]

77
twitch_app/models.py
View File

@ -1,7 +1,8 @@
import auto_prefetch
from django.db import models
class Game(models.Model):
class Game(auto_prefetch.Model):
"""The game that the reward is for.
Used for reward campaigns (buy subs) and drop campaigns (watch games).
@ -33,7 +34,7 @@ class Game(models.Model):
return f"https://www.twitch.tv/directory/game/{self.slug}"
class Image(models.Model):
class Image(auto_prefetch.Model):
"""An image model representing URLs and type.
Attributes:
@ -54,7 +55,7 @@ class Image(models.Model):
return self.image1_x_url or "Unknown"
class Reward(models.Model):
class Reward(auto_prefetch.Model):
"""The actual reward you get when you complete the requirements.
Attributes:
@ -88,8 +89,13 @@ class Reward(models.Model):
id = models.TextField(primary_key=True)
name = models.TextField(null=True, blank=True)
banner_image = models.ForeignKey(Image, related_name="banner_rewards", on_delete=models.CASCADE, null=True)
thumbnail_image = models.ForeignKey(Image, related_name="thumbnail_rewards", on_delete=models.CASCADE, null=True)
banner_image = auto_prefetch.ForeignKey(Image, related_name="banner_rewards", on_delete=models.CASCADE, null=True)
thumbnail_image = auto_prefetch.ForeignKey(
Image,
related_name="thumbnail_rewards",
on_delete=models.CASCADE,
null=True,
)
earnable_until = models.DateTimeField(null=True)
redemption_instructions = models.TextField(null=True, blank=True)
redemption_url = models.URLField(null=True, blank=True)
@ -99,7 +105,7 @@ class Reward(models.Model):
return self.name or "Unknown"
class UnlockRequirements(models.Model):
class UnlockRequirements(auto_prefetch.Model):
"""Requirements to unlock a reward.
Attributes:
@ -115,15 +121,15 @@ class UnlockRequirements(models.Model):
}
"""
subs_goal = models.PositiveBigIntegerField(null=True)
minute_watched_goal = models.PositiveBigIntegerField(null=True)
subs_goal = models.TextField(null=True)
minute_watched_goal = models.TextField(null=True)
typename = models.TextField(null=True, blank=True)
def __str__(self) -> str:
return f"{self.subs_goal} subs and {self.minute_watched_goal} minutes watched"
class RewardCampaign(models.Model):
class RewardCampaign(auto_prefetch.Model):
"""Represents a reward campaign.
Attributes:
@ -207,14 +213,14 @@ class RewardCampaign(models.Model):
reward_value_url_param = models.TextField(null=True, blank=True)
about_url = models.URLField(null=True, blank=True)
is_sitewide = models.BooleanField(null=True)
game = models.ForeignKey(Game, on_delete=models.CASCADE, related_name="reward_campaigns", null=True)
unlock_requirements = models.ForeignKey(
game = auto_prefetch.ForeignKey(Game, on_delete=models.CASCADE, related_name="reward_campaigns", null=True)
unlock_requirements = auto_prefetch.ForeignKey(
UnlockRequirements,
on_delete=models.CASCADE,
related_name="reward_campaigns",
null=True,
)
image = models.ForeignKey(Image, on_delete=models.CASCADE, related_name="reward_campaigns", null=True)
image = auto_prefetch.ForeignKey(Image, on_delete=models.CASCADE, related_name="reward_campaigns", null=True)
rewards = models.ManyToManyField(Reward, related_name="reward_campaigns")
typename = models.TextField(null=True, blank=True)
@ -222,7 +228,7 @@ class RewardCampaign(models.Model):
return self.name or "Unknown"
class Channel(models.Model):
class Channel(auto_prefetch.Model):
"""Represents a Twitch channel.
Attributes:
@ -240,7 +246,7 @@ class Channel(models.Model):
}
"""
id = models.PositiveBigIntegerField(primary_key=True)
id = models.TextField(primary_key=True)
display_name = models.TextField(null=True, blank=True)
name = models.TextField(null=True, blank=True)
typename = models.TextField(null=True, blank=True)
@ -252,7 +258,7 @@ class Channel(models.Model):
return f"https://www.twitch.tv/{self.name}"
class Allow(models.Model):
class Allow(auto_prefetch.Model):
"""List of channels that you can watch to earn rewards.
Attributes:
@ -283,7 +289,7 @@ class Allow(models.Model):
return f"{self.channels.count()} channels"
class Owner(models.Model):
class Owner(auto_prefetch.Model):
"""Represents the owner of the reward campaign.
Attributes:
@ -294,14 +300,14 @@ class Owner(models.Model):
JSON example:
"game": {
"id": "491487",
"id": "491487", # Can also be a string like 'c57a089c-088f-4402-b02d-c13281b3397e'
"slug": "dead-by-daylight",
"displayName": "Dead by Daylight",
"__typename": "Game"
},"
"""
id = models.PositiveBigIntegerField(primary_key=True)
id = models.TextField(primary_key=True)
slug = models.TextField(null=True, blank=True)
display_name = models.TextField(null=True, blank=True)
typename = models.TextField(null=True, blank=True)
@ -313,7 +319,7 @@ class Owner(models.Model):
return f"https://www.twitch.tv/{self.slug}"
class Benefit(models.Model):
class Benefit(auto_prefetch.Model):
"""Represents a benefit that you can earn.
Attributes:
@ -351,19 +357,19 @@ class Benefit(models.Model):
id = models.TextField(primary_key=True)
created_at = models.DateTimeField(null=True)
entitlement_limit = models.PositiveBigIntegerField(null=True)
game = models.ForeignKey(Game, on_delete=models.CASCADE, related_name="benefits", null=True)
entitlement_limit = models.TextField(null=True)
game = auto_prefetch.ForeignKey(Game, on_delete=models.CASCADE, related_name="benefits", null=True)
image_asset_url = models.URLField(null=True, blank=True)
is_ios_available = models.BooleanField(null=True)
name = models.TextField(null=True, blank=True)
owner_organization = models.ForeignKey(Owner, on_delete=models.CASCADE, related_name="benefits", null=True)
owner_organization = auto_prefetch.ForeignKey(Owner, on_delete=models.CASCADE, related_name="benefits", null=True)
typename = models.TextField(null=True, blank=True)
def __str__(self) -> str:
return self.name or "Unknown"
class BenefitEdge(models.Model):
class BenefitEdge(auto_prefetch.Model):
"""Represents a benefit edge.
Attributes:
@ -400,8 +406,8 @@ class BenefitEdge(models.Model):
],
"""
benefit = models.ForeignKey(Benefit, on_delete=models.CASCADE, related_name="benefit_edges", null=True)
entitlement_limit = models.PositiveBigIntegerField(null=True)
benefit = auto_prefetch.ForeignKey(Benefit, on_delete=models.CASCADE, related_name="benefit_edges", null=True)
entitlement_limit = models.TextField(null=True)
typename = models.TextField(null=True, blank=True)
def __str__(self) -> str:
@ -409,7 +415,7 @@ class BenefitEdge(models.Model):
return f"{benefit_name} - {self.entitlement_limit}"
class TimeBasedDrop(models.Model):
class TimeBasedDrop(auto_prefetch.Model):
"""Represents a time-based drop.
Attributes:
@ -459,19 +465,24 @@ class TimeBasedDrop(models.Model):
id = models.TextField(primary_key=True)
created_at = models.DateTimeField(null=True)
entitlement_limit = models.PositiveBigIntegerField(null=True)
game = models.ForeignKey(Game, on_delete=models.CASCADE, related_name="time_based_drops", null=True)
entitlement_limit = models.TextField(null=True)
game = auto_prefetch.ForeignKey(Game, on_delete=models.CASCADE, related_name="time_based_drops", null=True)
image_asset_url = models.URLField(null=True, blank=True)
is_ios_available = models.BooleanField(null=True)
name = models.TextField(null=True, blank=True)
owner_organization = models.ForeignKey(Owner, on_delete=models.CASCADE, related_name="time_based_drops", null=True)
owner_organization = auto_prefetch.ForeignKey(
Owner,
on_delete=models.CASCADE,
related_name="time_based_drops",
null=True,
)
typename = models.TextField(null=True, blank=True)
def __str__(self) -> str:
return self.name or "Unknown"
class DropCampaign(models.Model):
class DropCampaign(auto_prefetch.Model):
"""Represents a drop campaign.
Attributes:
@ -492,16 +503,16 @@ class DropCampaign(models.Model):
"""
id = models.TextField(primary_key=True)
allow = models.ForeignKey(Allow, on_delete=models.CASCADE, related_name="drop_campaigns", null=True)
allow = auto_prefetch.ForeignKey(Allow, on_delete=models.CASCADE, related_name="drop_campaigns", null=True)
account_link_url = models.URLField(null=True, blank=True)
description = models.TextField(null=True, blank=True)
details_url = models.URLField(null=True, blank=True)
ends_at = models.DateTimeField(null=True)
# event_based_drops = ????
game = models.ForeignKey(Game, on_delete=models.CASCADE, related_name="drop_campaigns", null=True)
game = auto_prefetch.ForeignKey(Game, on_delete=models.CASCADE, related_name="drop_campaigns", null=True)
image_url = models.URLField(null=True, blank=True)
name = models.TextField(null=True, blank=True)
owner = models.ForeignKey(Owner, on_delete=models.CASCADE, related_name="drop_campaigns", null=True)
owner = auto_prefetch.ForeignKey(Owner, on_delete=models.CASCADE, related_name="drop_campaigns", null=True)
starts_at = models.DateTimeField(null=True)
status = models.TextField(null=True, blank=True)
time_based_drops = models.ManyToManyField(TimeBasedDrop, related_name="drop_campaigns")