TheLovinator/twitch-drop-notifier
1
Fork 0
Code Issues Actions Packages Releases Activity

Save JSON in DB

Browse Source
This commit is contained in:
Joakim Hellsén
2024-09-22 11:42:20 +02:00
parent 2be1191a03
commit f914ca6597
6 changed files with 149 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
core/management/commands/scrape_local.py
View File

@ -20,7 +20,7 @@ class Command(BaseCommand):
*args: Variable length argument list.
**kwargs: Arbitrary keyword arguments.
"""
dir_name = Path("json")
dir_name = Path("json2")
for num, file in enumerate(Path(dir_name).rglob("*.json")):
logger.info("Processing %s", file)

2
core/management/commands/scrape_twitch.py
View File

@ -255,7 +255,7 @@ class Command(BaseCommand):
await browser.close()
for num, campaign in enumerate(json_data, start=1):
await process_json_data(num=num, campaign=campaign, local=True)
await process_json_data(num=num, campaign=campaign, local=False)
return json_data

28
core/migrations/0008_dropcampaign_scraped_json_and_more.py Normal file
View File

@ -0,0 +1,28 @@
# Generated by Django 5.1.1 on 2024-09-21 16:51
from __future__ import annotations
from typing import TYPE_CHECKING
from django.db import migrations, models
if TYPE_CHECKING:
from django.db.migrations.operations.base import Operation
class Migration(migrations.Migration):
dependencies: list[tuple[str, str]] = [
("core", "0007_alter_game_options_alter_owner_options_and_more"),
]
operations: list[Operation] = [
migrations.AddField(
model_name="dropcampaign",
name="scraped_json",
field=models.JSONField(help_text="The JSON data from the Twitch API.", null=True),
),
migrations.AddField(
model_name="rewardcampaign",
name="scraped_json",
field=models.JSONField(help_text="The JSON data from the Twitch API.", null=True),
),
]

50
core/migrations/0009_scrapedjson_alter_dropcampaign_scraped_json_and_more.py Normal file
View File

@ -0,0 +1,50 @@
# Generated by Django 5.1.1 on 2024-09-21 23:56
from __future__ import annotations
from typing import TYPE_CHECKING
import django.db.models.deletion
from django.db import migrations, models
if TYPE_CHECKING:
from django.db.migrations.operations.base import Operation
class Migration(migrations.Migration):
dependencies: list[tuple[str, str]] = [
("core", "0008_dropcampaign_scraped_json_and_more"),
]
operations: list[Operation] = [
migrations.CreateModel(
name="ScrapedJson",
fields=[
("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
("json_data", models.JSONField(help_text="The JSON data from the Twitch API.", unique=True)),
("created_at", models.DateTimeField(auto_now_add=True)),
],
options={
"ordering": ["-created_at"],
},
),
migrations.AlterField(
model_name="dropcampaign",
name="scraped_json",
field=models.ForeignKey(
help_text="Reference to the JSON data from the Twitch API.",
null=True,
on_delete=django.db.models.deletion.SET_NULL,
to="core.scrapedjson",
),
),
migrations.AlterField(
model_name="rewardcampaign",
name="scraped_json",
field=models.ForeignKey(
help_text="Reference to the JSON data from the Twitch API.",
null=True,
on_delete=django.db.models.deletion.SET_NULL,
to="core.scrapedjson",
),
),
]

75
core/models.py
View File

@ -22,6 +22,26 @@ logger: logging.Logger = logging.getLogger(__name__)
image_file_format: Literal["webp"] = "webp"
async def update_scraped_json(model_instance: models.Model, new_json_data: dict) -> None:
"""Update the JSON data for the drop campaign.
Args:
model_instance (models.Model): The Django model instance which must have a 'scraped_json' attribute.
new_json_data (dict): The new JSON data to be updated.
"""
if await sync_to_async(hasattr)(model_instance, "scraped_json"):
if model_instance.scraped_json: # type: ignore[attr-defined]
model_instance.scraped_json.json_data = new_json_data # type: ignore[attr-defined]
await model_instance.scraped_json.asave() # type: ignore[attr-defined]
else:
scraped_json_instance: ScrapedJson = await ScrapedJson.objects.acreate(json_data=new_json_data)
model_instance.scraped_json = scraped_json_instance # type: ignore[attr-defined]
await model_instance.asave()
else:
logger.error("The model instance does not have a 'scraped_json' attribute.")
model_instance.save()
def wrong_typename(data: dict, expected: str) -> bool:
"""Check if the data is the expected type.
@ -170,6 +190,17 @@ def fetch_image(image_url: str) -> bytes | None:
class User(AbstractUser): ...
class ScrapedJson(models.Model):
json_data = models.JSONField(unique=True, help_text="The JSON data from the Twitch API.")
created_at = models.DateTimeField(auto_now_add=True)
class Meta:
ordering: ClassVar[list[str]] = ["-created_at"]
def __str__(self) -> str:
return f"{self.pk}"
class Owner(models.Model):
"""The company or person that owns the game.
@ -296,7 +327,7 @@ class Game(models.Model):
logger.info("Updated %s fields for %s", updated, self)
# Handle the owner of the game.
if owner:
if owner and await sync_to_async(lambda: self not in owner.games.all())(): # type: ignore[attr-defined]
await owner.games.aadd(self) # type: ignore # noqa: PGH003
await self.asave()
logger.info("Added game %s for %s", self, owner)
@ -363,6 +394,13 @@ class DropCampaign(models.Model):
game = models.ForeignKey(Game, on_delete=models.CASCADE, related_name="drop_campaigns", null=True)
scraped_json = models.ForeignKey(
ScrapedJson,
null=True,
on_delete=models.SET_NULL,
help_text="Reference to the JSON data from the Twitch API.",
)
class Meta:
ordering: ClassVar[list[str]] = ["ends_at"]
@ -405,6 +443,9 @@ class DropCampaign(models.Model):
if wrong_typename(data, "DropCampaign"):
return self
# Save the JSON data for debugging purposes.
await update_scraped_json(model_instance=self, new_json_data=data)
field_mapping: dict[str, str] = {
"name": "name",
"accountLinkURL": "account_link_url", # TODO(TheLovinator): Should archive site. # noqa: TD003
@ -601,7 +642,7 @@ class Benefit(models.Model):
if updated > 0:
logger.info("Updated %s fields for %s", updated, self)
if time_based_drop:
if time_based_drop and await sync_to_async(lambda: self not in time_based_drop.benefits.all())(): # type: ignore[attr-defined]
await time_based_drop.benefits.aadd(self) # type: ignore # noqa: PGH003
await time_based_drop.asave()
logger.info("Added benefit %s for %s", self, time_based_drop)
@ -685,6 +726,13 @@ class RewardCampaign(models.Model):
game = models.ForeignKey(Game, on_delete=models.CASCADE, related_name="reward_campaigns", null=True)
scraped_json = models.ForeignKey(
ScrapedJson,
null=True,
on_delete=models.SET_NULL,
help_text="Reference to the JSON data from the Twitch API.",
)
class Meta:
ordering: ClassVar[list[str]] = ["-starts_at"]
@ -719,6 +767,9 @@ class RewardCampaign(models.Model):
if wrong_typename(data, "RewardCampaign"):
return self
# Save the JSON data for debugging purposes.
await update_scraped_json(model_instance=self, new_json_data=data)
field_mapping: dict[str, str] = {
"name": "name",
"brand": "brand",
@ -910,18 +961,18 @@ class Reward(models.Model):
logger.info("Updated %s fields for %s", updated, self)
banner_image_url = data.get("bannerImage", {}).get("image1xURL")
if banner_image_url:
await sync_to_async(self.download_banner_image)()
if banner_image_url != self.banner_image_url:
self.banner_image_url = banner_image_url
await self.asave()
if banner_image_url and banner_image_url != self.banner_image_url:
# await sync_to_async(self.download_banner_image)()
# TODO(TheLovinator): Download the image. # noqa: TD003
self.banner_image_url = banner_image_url
await self.asave()
thumbnail_image_url = data.get("thumbnailImage", {}).get("image1xURL")
if thumbnail_image_url:
await sync_to_async(self.download_thumbnail_image)()
if thumbnail_image_url != self.thumbnail_image_url:
self.thumbnail_image_url = thumbnail_image_url
await self.asave()
if thumbnail_image_url and thumbnail_image_url != self.thumbnail_image_url:
# await sync_to_async(self.download_thumbnail_image)()
# TODO(TheLovinator): Download the image. # noqa: TD003
self.thumbnail_image_url = thumbnail_image_url
await self.asave()
if reward_campaign and await sync_to_async(lambda: reward_campaign != self.campaign)():
self.campaign = reward_campaign

6
core/templates/game.html
View File

@ -46,6 +46,12 @@
<tr>
<td><img src="{{ drop_campaign.image_url }}" alt="{{ drop_campaign.name }} image"></td>
<td>
<p><strong>Created at:</strong>
{{ drop_campaign.created_at }}
</p>
<p><strong>Modified at:</strong>
{{ drop_campaign.modified_at }}
</p>
<p><strong>Status:</strong>
{{ drop_campaign.status }}
</p>