TheLovinator/feedvault.se
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add empty Go project

Browse source
This commit is contained in:
Joakim Hellsén 2024-02-03 02:21:51 +01:00
commit 3445466197
30 changed files with 19 additions and 4183 deletions
Download patch file Download diff file Expand all files Collapse all files

4
feeds/__init__.py
View file

@ -1,4 +0,0 @@
"""This package contains the main Django app.
https://docs.djangoproject.com/en/5.0/ref/applications/
"""

69
feeds/admin.py
View file

@ -1,69 +0,0 @@
"""Admin interface for feeds app.
https://docs.djangoproject.com/en/5.0/ref/contrib/admin/
"""
from __future__ import annotations
from typing import TYPE_CHECKING, ClassVar
from django.contrib import admin
from feeds.models import (
Author,
Blocklist,
Cloud,
Contributor,
Feed,
Generator,
Image,
Info,
Link,
Publisher,
Rights,
Subtitle,
Tags,
TextInput,
Title,
)
from feeds.validator import update_blocklist
if TYPE_CHECKING:
from django.db.models.query import QuerySet
from django.http import HttpRequest
admin.site.register(Author)
admin.site.register(Cloud)
admin.site.register(Contributor)
admin.site.register(Feed)
admin.site.register(Generator)
admin.site.register(Image)
admin.site.register(Info)
admin.site.register(Link)
admin.site.register(Publisher)
admin.site.register(Rights)
admin.site.register(Subtitle)
admin.site.register(Tags)
admin.site.register(TextInput)
admin.site.register(Title)
# Add button to update blocklist on the admin page
@admin.register(Blocklist)
class BlocklistAdmin(admin.ModelAdmin):
"""Admin interface for blocklist."""
actions: ClassVar[list[str]] = ["_update_blocklist", "delete_all_blocklist"]
list_display: ClassVar[list[str]] = ["url", "active"]
@admin.action(description="Update blocklist")
def _update_blocklist(self: admin.ModelAdmin, request: HttpRequest, queryset: QuerySet) -> None: # noqa: ARG002
"""Update blocklist."""
msg: str = update_blocklist()
self.message_user(request=request, message=msg)
@admin.action(description="Delete all blocklists")
def delete_all_blocklist(self: admin.ModelAdmin, request: HttpRequest, queryset: QuerySet) -> None: # noqa: ARG002
"""Delete all blocklist from database."""
Blocklist.objects.all().delete()
self.message_user(request=request, message="Deleted all blocklists")

16
feeds/apps.py
View file

@ -1,16 +0,0 @@
"""Django application configuration for the main feeds app."""
from typing import Literal
from django.apps import AppConfig
class FeedsConfig(AppConfig):
"""Configuration object for the main feeds app.
Args:
AppConfig: The base configuration object for Django applications.
"""
default_auto_field: Literal["django.db.models.BigAutoField"] = "django.db.models.BigAutoField"
name: Literal["feeds"] = "feeds"

19
feeds/forms.py
View file

@ -1,19 +0,0 @@
"""https://docs.djangoproject.com/en/5.0/topics/forms/."""
from __future__ import annotations
from django import forms
class UploadOPMLForm(forms.Form):
"""Upload OPML.
Args:
forms: A collection of Fields, plus their associated data.
"""
file = forms.FileField(
label="Select an OPML file",
help_text="max. 100 megabytes",
widget=forms.FileInput(attrs={"accept": ".opml"}),
)

279
feeds/migrations/0001_initial.py
View file

@ -1,279 +0,0 @@
# Generated by Django 5.0.1 on 2024-01-30 01:03
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.CreateModel(
name='Author',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('name', models.TextField(blank=True, help_text='The name of the feed author.')),
('email', models.EmailField(blank=True, help_text='The email address of the feed author.', max_length=254)),
('href', models.URLField(blank=True, help_text="The URL of the feed author. This can be the author's home page, or a contact page with a webmail form.")),
],
options={
'verbose_name': 'Feed author',
'verbose_name_plural': 'Feed authors',
'db_table_comment': 'Details about the author of a feed.',
},
),
migrations.CreateModel(
name='Cloud',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('domain', models.CharField(blank=True, help_text='The domain of the cloud. Should be just the domain name, not including the http:// protocol. All clouds are presumed to operate over HTTP. The cloud specification does not support secure clouds over HTTPS, nor can clouds operate over other protocols.', max_length=255)),
('port', models.CharField(blank=True, help_text='The port of the cloud. Should be an integer, but Universal Feed Parser currently returns it as a string.', max_length=255)),
('path', models.CharField(blank=True, help_text='The URL path of the cloud.', max_length=255)),
('register_procedure', models.CharField(blank=True, help_text='The name of the procedure to call on the cloud.', max_length=255)),
('protocol', models.CharField(blank=True, help_text='The protocol of the cloud. Documentation differs on what the acceptable values are. Acceptable values definitely include xml-rpc and soap, although only in lowercase, despite both being acronyms. There is no way for a publisher to specify the version number of the protocol to use. soap refers to SOAP 1.1; the cloud interface does not support SOAP 1.0 or 1.2. post or http-post might also be acceptable values; nobody really knows for sure.', max_length=255)),
],
options={
'verbose_name': 'Feed cloud',
'verbose_name_plural': 'Feed clouds',
'db_table_comment': 'No one really knows what a cloud is.',
},
),
migrations.CreateModel(
name='Contributor',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('name', models.TextField(blank=True, help_text='The name of this contributor.')),
('email', models.EmailField(blank=True, help_text='The email address of this contributor.', max_length=254)),
('href', models.URLField(blank=True, help_text="The URL of this contributor. This can be the contributor's home page, or a contact page with a webmail form.")),
],
options={
'verbose_name': 'Feed contributor',
'verbose_name_plural': 'Feed contributors',
'db_table_comment': 'Details about the contributor to a feed.',
},
),
migrations.CreateModel(
name='Generator',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('name', models.TextField(blank=True, help_text='Human-readable name of the application used to generate the feed.')),
('href', models.URLField(blank=True, help_text='The URL of the application used to generate the feed.')),
('version', models.CharField(blank=True, help_text='The version number of the application used to generate the feed. There is no required format for this, but most applications use a MAJOR.MINOR version number.', max_length=255)),
],
options={
'verbose_name': 'Feed generator',
'verbose_name_plural': 'Feed generators',
'db_table_comment': 'Details about the software used to generate the feed.',
},
),
migrations.CreateModel(
name='Image',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('title', models.TextField(blank=True, help_text='The alternate text of the feed image, which would go in the alt attribute if you rendered the feed image as an HTML img element.')),
('href', models.URLField(blank=True, help_text='The URL of the feed image itself, which would go in the src attribute if you rendered the feed image as an HTML img element.')),
('link', models.URLField(blank=True, help_text='The URL which the feed image would point to. If you rendered the feed image as an HTML img element, you would wrap it in an a element and put this in the href attribute.')),
('width', models.IntegerField(default=0, help_text='The width of the feed image, which would go in the width attribute if you rendered the feed image as an HTML img element.')),
('height', models.IntegerField(default=0, help_text='The height of the feed image, which would go in the height attribute if you rendered the feed image as an HTML img element.')),
('description', models.TextField(blank=True, help_text='A short description of the feed image, which would go in the title attribute if you rendered the feed image as an HTML img element. This element is rare; it was available in Netscape RSS 0.91 but was dropped from Userland RSS 0.91.')),
],
options={
'verbose_name': 'Feed image',
'verbose_name_plural': 'Feed images',
'db_table_comment': 'A feed image can be a logo, banner, or a picture of the author.',
},
),
migrations.CreateModel(
name='Info',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('value', models.TextField(blank=True, help_text='Free-form human-readable description of the feed format itself. Intended for people who view the feed in a browser, to explain what they just clicked on.')),
('info_type', models.CharField(blank=True, help_text='The content type of the feed info. Most likely text/plain, text/html, or application/xhtml+xml.', max_length=255)),
('language', models.CharField(blank=True, help_text='The language of the feed info.', max_length=255)),
('base', models.URLField(blank=True, help_text='The original base URI for links within the feed copyright.')),
],
options={
'verbose_name': 'Feed information',
'verbose_name_plural': 'Feed information',
'db_table_comment': 'Details about the feed.',
},
),
migrations.CreateModel(
name='Link',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('rel', models.CharField(blank=True, help_text='The relationship of this feed link.', max_length=255)),
('link_type', models.CharField(blank=True, help_text='The content type of the page that this feed link points to.', max_length=255)),
('href', models.URLField(blank=True, help_text='The URL of the page that this feed link points to.')),
('title', models.TextField(blank=True, help_text='The title of this feed link.')),
],
options={
'verbose_name': 'Feed link',
'verbose_name_plural': 'Feed links',
'db_table_comment': 'A list of dictionaries with details on the links associated with the feed.',
},
),
migrations.CreateModel(
name='Publisher',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('name', models.TextField(blank=True, help_text="The name of this feed's publisher.")),
('email', models.EmailField(blank=True, help_text="The email address of this feed's publisher.", max_length=254)),
('href', models.URLField(blank=True, help_text="The URL of this feed's publisher. This can be the publisher's home page, or a contact page with a webmail form.")),
],
options={
'verbose_name': 'Feed publisher',
'verbose_name_plural': 'Feed publishers',
'db_table_comment': 'Details about the publisher of a feed.',
},
),
migrations.CreateModel(
name='Rights',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('value', models.TextField(blank=True, help_text='A human-readable copyright statement for the feed.')),
('rights_type', models.CharField(blank=True, help_text='The content type of the feed copyright. Most likely text/plain, text/html, or application/xhtml+xml.', max_length=255)),
('language', models.CharField(blank=True, help_text='The language of the feed rights.', max_length=255)),
('base', models.URLField(blank=True, help_text='The original base URI for links within the feed copyright.')),
],
options={
'verbose_name': 'Feed rights',
'verbose_name_plural': 'Feed rights',
'db_table_comment': 'Details about the rights of a feed.',
},
),
migrations.CreateModel(
name='Subtitle',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('value', models.TextField(blank=True, help_text='A subtitle, tagline, slogan, or other short description of the feed.')),
('subtitle_type', models.CharField(blank=True, help_text='The content type of the feed subtitle. Most likely text/plain, text/html, or application/xhtml+xml.', max_length=255)),
('language', models.CharField(blank=True, help_text='The language of the feed subtitle.', max_length=255)),
('base', models.URLField(blank=True, help_text='The original base URI for links within the feed subtitle.')),
],
options={
'verbose_name': 'Feed subtitle',
'verbose_name_plural': 'Feed subtitles',
'db_table_comment': 'A subtitle, tagline, slogan, or other short description of the feed.',
},
),
migrations.CreateModel(
name='Tags',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('term', models.TextField(blank=True, help_text='The category term (keyword).')),
('scheme', models.CharField(blank=True, help_text='The category scheme (domain).', max_length=255)),
('label', models.TextField(blank=True, help_text='A human-readable label for the category.')),
],
options={
'verbose_name': 'Feed tag',
'verbose_name_plural': 'Feed tags',
'db_table_comment': 'A list of tags associated with the feed.',
},
),
migrations.CreateModel(
name='TextInput',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('title', models.TextField(blank=True, help_text="The title of the text input form, which would go in the value attribute of the form's submit button.")),
('link', models.URLField(blank=True, help_text='The link of the script which processes the text input form, which would go in the action attribute of the form.')),
('name', models.TextField(blank=True, help_text="The name of the text input box in the form, which would go in the name attribute of the form's input box.")),
('description', models.TextField(blank=True, help_text='A short description of the text input form, which would go in the label element of the form.')),
],
options={
'verbose_name': 'Feed text input',
'verbose_name_plural': 'Feed text inputs',
'db_table_comment': 'A text input form. No one actually uses this. Why are you?',
},
),
migrations.CreateModel(
name='Title',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('value', models.TextField(blank=True, help_text='The title of the feed.')),
('title_type', models.CharField(blank=True, help_text='The content type of the feed title. Most likely text/plain, text/html, or application/xhtml+xml.', max_length=255)),
('language', models.CharField(blank=True, help_text='The language of the feed title.', max_length=255)),
('base', models.URLField(blank=True, help_text='The original base URI for links within the feed title.')),
],
options={
'verbose_name': 'Feed title',
'verbose_name_plural': 'Feed titles',
'db_table_comment': 'Details about the title of a feed.',
},
),
migrations.CreateModel(
name='Feed',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('url', models.URLField(help_text='The feed URL.', unique=True)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('error', models.BooleanField(default=False)),
('error_message', models.TextField(default='')),
('error_at', models.DateTimeField(null=True)),
('bozo', models.BooleanField(default=False, help_text='Is the feed well-formed XML?')),
('bozo_exception', models.TextField(default='', help_text='The exception raised when attempting to parse a non-well-formed feed.')),
('encoding', models.CharField(default='', help_text='The character encoding that was used to parse the feed.', max_length=255)),
('etag', models.CharField(default='', help_text='The ETag of the feed, as specified in the HTTP headers.', max_length=255)),
('headers', models.TextField(default='', help_text='HTTP headers received from the web server when retrieving the feed.')),
('href', models.URLField(default='', help_text='The final URL of the feed that was parsed. If the feed was redirected from the original requested address, href will contain the final (redirected) address.')),
('last_modified', models.DateTimeField(help_text='The last-modified date of the feed, as specified in the HTTP headers.', null=True)),
('namespaces', models.TextField(default='', help_text='A dictionary of all XML namespaces defined in the feed, as {prefix: namespaceURI}.')),
('http_status_code', models.IntegerField(default=0, help_text='The HTTP status code that was returned by the web server when the feed was fetched.')),
('version', models.CharField(choices=[('atom', 'Atom (unknown or unrecognized version)'), ('atom01', 'Atom 0.1'), ('atom02', 'Atom 0.2'), ('atom03', 'Atom 0.3'), ('atom10', 'Atom 1.0'), ('cdf', 'CDF'), ('rss', 'RSS (unknown or unrecognized version)'), ('rss090', 'RSS 0.90'), ('rss091n', 'Netscape RSS 0.91'), ('rss091u', 'Userland RSS 0.91'), ('rss092', 'RSS 0.92'), ('rss093', 'RSS 0.93'), ('rss094', 'RSS 0.94 (no accurate specification is known to exist)'), ('rss10', 'RSS 1.0'), ('rss20', 'RSS 2.0')], default='', help_text='The version of the feed, as determined by Universal Feed Parser.', max_length=255)),
('feed_data', models.TextField(default='', help_text='A dictionary of data about the feed.')),
('docs', models.URLField(blank=True, help_text='A URL pointing to the specification which this feed conforms to.')),
('error_reports_to', models.EmailField(blank=True, help_text='An email address for reporting errors in the feed itself.', max_length=254)),
('feed_id', models.CharField(blank=True, help_text='A globally unique identifier for this feed.', max_length=255)),
('language', models.CharField(blank=True, help_text='The primary language of the feed.', max_length=255)),
('license', models.URLField(blank=True, help_text='A URL pointing to the license of the feed.')),
('logo', models.URLField(blank=True, help_text='A URL to a graphic representing a logo for the feed.')),
('published', models.CharField(blank=True, help_text='The date the feed was published, as a string in the same format as it was published in the original feed.', max_length=255)),
('published_parsed', models.DateTimeField(help_text='The date the feed was published.', null=True)),
('ttl', models.CharField(blank=True, help_text='According to the RSS specification, “None"', max_length=255)),
('updated', models.CharField(blank=True, help_text='The date the feed was last updated, as a string in the same format as it was published in the original feed.', max_length=255)),
('updated_parsed', models.DateTimeField(help_text='The date the feed was last updated.', null=True)),
('author', models.ForeignKey(help_text='The author of the feed.', null=True, on_delete=django.db.models.deletion.CASCADE, to='feeds.author')),
('cloud', models.ForeignKey(help_text='Cloud enables realtime push notifications or distributed publish/subscribe communication for feeds.', null=True, on_delete=django.db.models.deletion.CASCADE, to='feeds.cloud')),
('contributors', models.ManyToManyField(help_text='A list of contributors (secondary authors) to this feed.', to='feeds.contributor')),
('created_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)),
('generator', models.ForeignKey(help_text='Details about the software used to generate the feed.', null=True, on_delete=django.db.models.deletion.CASCADE, to='feeds.generator')),
('image', models.ForeignKey(help_text='A feed image can be a logo, banner, or a picture of the author.', null=True, on_delete=django.db.models.deletion.CASCADE, to='feeds.image')),
('info', models.ForeignKey(help_text='Details about the feed.', null=True, on_delete=django.db.models.deletion.CASCADE, to='feeds.info')),
('link', models.ForeignKey(help_text='A list of dictionaries with details on the links associated with the feed.', null=True, on_delete=django.db.models.deletion.CASCADE, to='feeds.link')),
('publisher', models.ForeignKey(help_text='The publisher of the feed.', null=True, on_delete=django.db.models.deletion.CASCADE, to='feeds.publisher')),
('rights', models.ForeignKey(help_text='Details about the rights of a feed.', null=True, on_delete=django.db.models.deletion.CASCADE, to='feeds.rights')),
('subtitle', models.ForeignKey(help_text='A subtitle, tagline, slogan, or other short description of the feed.', null=True, on_delete=django.db.models.deletion.CASCADE, to='feeds.subtitle')),
('tags', models.ManyToManyField(help_text='A list of tags associated with the feed.', to='feeds.tags')),
('text_input', models.ForeignKey(help_text='A text input form. No one actually uses this. Why are you?', null=True, on_delete=django.db.models.deletion.CASCADE, to='feeds.textinput')),
('title', models.ForeignKey(help_text='Details about the title of a feed.', null=True, on_delete=django.db.models.deletion.CASCADE, to='feeds.title')),
],
options={
'verbose_name': 'Feed',
'verbose_name_plural': 'Feeds',
'db_table_comment': 'A feed. This is the main model of the app.',
},
),
]

25
feeds/migrations/0002_blocklist.py
View file

@ -1,25 +0,0 @@
# Generated by Django 5.0.1 on 2024-01-30 16:41
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('feeds', '0001_initial'),
]
operations = [
migrations.CreateModel(
name='Blocklist',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('url', models.URLField(help_text='The URL to block.', unique=True)),
],
options={
'verbose_name': 'Blocklist',
'verbose_name_plural': 'Blocklists',
'db_table_comment': 'A list of URLs to block.',
},
),
]

18
feeds/migrations/0003_blocklist_active.py
View file

@ -1,18 +0,0 @@
# Generated by Django 5.0.1 on 2024-01-30 16:45
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('feeds', '0002_blocklist'),
]
operations = [
migrations.AddField(
model_name='blocklist',
name='active',
field=models.BooleanField(default=True, help_text='Is this URL still blocked?'),
),
]

18
feeds/migrations/0004_alter_blocklist_url.py
View file

@ -1,18 +0,0 @@
# Generated by Django 5.0.1 on 2024-01-30 18:22
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('feeds', '0003_blocklist_active'),
]
operations = [
migrations.AlterField(
model_name='blocklist',
name='url',
field=models.CharField(help_text='The URL to block.', max_length=2000, unique=True),
),
]

0
feeds/migrations/__init__.py
View file

983
feeds/models.py
View file

@ -1,983 +0,0 @@
"""Django models for the feeds app."""
from __future__ import annotations
import typing
from django.contrib.auth.models import User
from django.db import models
FEED_VERSION_CHOICES: tuple = (
("atom", "Atom (unknown or unrecognized version)"),
("atom01", "Atom 0.1"),
("atom02", "Atom 0.2"),
("atom03", "Atom 0.3"),
("atom10", "Atom 1.0"),
("cdf", "CDF"),
("rss", "RSS (unknown or unrecognized version)"),
("rss090", "RSS 0.90"),
("rss091n", "Netscape RSS 0.91"),
("rss091u", "Userland RSS 0.91"),
("rss092", "RSS 0.92"),
("rss093", "RSS 0.93"),
("rss094", "RSS 0.94 (no accurate specification is known to exist)"),
("rss10", "RSS 1.0"),
("rss20", "RSS 2.0"),
)
class Author(models.Model):
"""Details about the author of a feed.
Comes from the following elements:
- /atom03:feed/atom03:author
- /atom10:feed/atom10:author
- /rdf:RDF/rdf:channel/dc:author
- /rdf:RDF/rdf:channel/dc:creator
- /rss/channel/dc:author
- /rss/channel/dc:creator
- /rss/channel/itunes:author
- /rss/channel/managingEditor
"""
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
name = models.TextField(blank=True, help_text="The name of the feed author.")
email = models.EmailField(blank=True, help_text="The email address of the feed author.")
# If this is a relative URI, it is resolved according to a set of rules:
# https://feedparser.readthedocs.io/en/latest/resolving-relative-links.html#advanced-base
href = models.URLField(
blank=True,
help_text="The URL of the feed author. This can be the author's home page, or a contact page with a webmail form.", # noqa: E501
)
class Meta:
"""Author meta."""
verbose_name: typing.ClassVar[str] = "Feed author"
verbose_name_plural: typing.ClassVar[str] = "Feed authors"
db_table_comment: typing.ClassVar[str] = "Details about the author of a feed."
def __str__(self: Author) -> str:
"""Author."""
return f"{self.name} {self.email} {self.href}"
class Contributor(models.Model):
"""Details about the contributor to a feed.
Comes from the following elements:
- /atom03:feed/atom03:contributor
- /atom10:feed/atom10:contributor
- /rss/channel/dc:contributor
"""
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
name = models.TextField(blank=True, help_text="The name of this contributor.")
email = models.EmailField(blank=True, help_text="The email address of this contributor.")
# If this is a relative URI, it is resolved according to a set of rules:
# https://feedparser.readthedocs.io/en/latest/resolving-relative-links.html#advanced-base
href = models.URLField(
blank=True,
help_text="The URL of this contributor. This can be the contributor's home page, or a contact page with a webmail form.", # noqa: E501
)
class Meta:
"""Contributor meta."""
verbose_name: typing.ClassVar[str] = "Feed contributor"
verbose_name_plural: typing.ClassVar[str] = "Feed contributors"
db_table_comment: typing.ClassVar[str] = "Details about the contributor to a feed."
def __str__(self: Contributor) -> str:
"""Contributor."""
return f"{self.name} {self.email} {self.href}"
class Cloud(models.Model):
"""No one really knows what a cloud is.
Comes from the following elements:
- /rss/channel/cloud
"""
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
domain = models.CharField(
max_length=255,
blank=True,
help_text="The domain of the cloud. Should be just the domain name, not including the http:// protocol. All clouds are presumed to operate over HTTP. The cloud specification does not support secure clouds over HTTPS, nor can clouds operate over other protocols.", # noqa: E501
)
port = models.CharField(
max_length=255,
blank=True,
help_text="The port of the cloud. Should be an integer, but Universal Feed Parser currently returns it as a string.", # noqa: E501
)
path = models.CharField(
max_length=255,
blank=True,
help_text="The URL path of the cloud.",
)
register_procedure = models.CharField(
max_length=255,
blank=True,
help_text="The name of the procedure to call on the cloud.",
)
protocol = models.CharField(
max_length=255,
blank=True,
help_text="The protocol of the cloud. Documentation differs on what the acceptable values are. Acceptable values definitely include xml-rpc and soap, although only in lowercase, despite both being acronyms. There is no way for a publisher to specify the version number of the protocol to use. soap refers to SOAP 1.1; the cloud interface does not support SOAP 1.0 or 1.2. post or http-post might also be acceptable values; nobody really knows for sure.", # noqa: E501
)
class Meta:
"""Cloud meta."""
verbose_name: typing.ClassVar[str] = "Feed cloud"
verbose_name_plural: typing.ClassVar[str] = "Feed clouds"
db_table_comment: typing.ClassVar[str] = "No one really knows what a cloud is."
def __str__(self: Cloud) -> str:
"""Cloud domain."""
return f"{self.register_procedure} {self.protocol}://{self.domain}{self.path}:{self.port}"
class Generator(models.Model):
"""Details about the software used to generate the feed.
Comes from the following elements:
- /atom03:feed/atom03:generator
- /atom10:feed/atom10:generator
- /rdf:RDF/rdf:channel/admin:generatorAgent/@rdf:resource
- /rss/channel/generator
"""
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
name = models.TextField(blank=True, help_text="Human-readable name of the application used to generate the feed.")
# If this is a relative URI, it is resolved according to a set of rules:
# https://feedparser.readthedocs.io/en/latest/resolving-relative-links.html#advanced-base
href = models.URLField(
blank=True,
help_text="The URL of the application used to generate the feed.",
)
version = models.CharField(
max_length=255,
blank=True,
help_text="The version number of the application used to generate the feed. There is no required format for this, but most applications use a MAJOR.MINOR version number.", # noqa: E501
)
class Meta:
"""Generator meta."""
verbose_name: typing.ClassVar[str] = "Feed generator"
verbose_name_plural: typing.ClassVar[str] = "Feed generators"
db_table_comment: typing.ClassVar[str] = "Details about the software used to generate the feed."
def __str__(self: Generator) -> str:
"""Generator name."""
return f"{self.name} {self.version} {self.href}"
class Image(models.Model):
"""A feed image can be a logo, banner, or a picture of the author.
Comes from the following elements:
- /rdf:RDF/rdf:image
- /rss/channel/image
Example:
```xml
<image>
<title>Feed logo</title>
<url>http://example.org/logo.png</url>
<link>http://example.org/</link>
<width>80</width>
<height>15</height>
<description>Visit my home page</description>
</image>
```
This feed image could be rendered in HTML as this:
```html
<a href="http://example.org/">
<img src="http://example.org/logo.png"
width="80"
height="15"
alt="Feed logo"
title="Visit my home page">
</a>
```
"""
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
title = models.TextField(
blank=True,
help_text="The alternate text of the feed image, which would go in the alt attribute if you rendered the feed image as an HTML img element.", # noqa: E501
)
# If this is a relative URI, it is resolved according to a set of rules:
# https://feedparser.readthedocs.io/en/latest/resolving-relative-links.html#advanced-base
href = models.URLField(
blank=True,
help_text="The URL of the feed image itself, which would go in the src attribute if you rendered the feed image as an HTML img element.", # noqa: E501
)
# If this is a relative URI, it is resolved according to a set of rules:
# https://feedparser.readthedocs.io/en/latest/resolving-relative-links.html#advanced-base
link = models.URLField(
blank=True,
help_text="The URL which the feed image would point to. If you rendered the feed image as an HTML img element, you would wrap it in an a element and put this in the href attribute.", # noqa: E501
)
width = models.IntegerField(
default=0,
help_text="The width of the feed image, which would go in the width attribute if you rendered the feed image as an HTML img element.", # noqa: E501
)
height = models.IntegerField(
default=0,
help_text="The height of the feed image, which would go in the height attribute if you rendered the feed image as an HTML img element.", # noqa: E501
)
description = models.TextField(
blank=True,
help_text="A short description of the feed image, which would go in the title attribute if you rendered the feed image as an HTML img element. This element is rare; it was available in Netscape RSS 0.91 but was dropped from Userland RSS 0.91.", # noqa: E501
)
class Meta:
"""Image meta."""
verbose_name: typing.ClassVar[str] = "Feed image"
verbose_name_plural: typing.ClassVar[str] = "Feed images"
db_table_comment: typing.ClassVar[str] = "A feed image can be a logo, banner, or a picture of the author."
def __str__(self: Image) -> str:
"""Image title."""
return f"{self.title} {self.href} {self.link} {self.width}x{self.height} {self.description}"
class Info(models.Model):
"""Details about the feed.
Comes from the following elements:
- /atom03:feed/atom03:info
- /rss/channel/feedburner:browserFriendly
"""
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
# This element is generally ignored by feed readers.
# If this contains HTML or XHTML, it is sanitized by default.
# If this contains HTML or XHTML, certain (X)HTML elements within this value may
# contain relative URI (Uniform Resource Identifier)'s. These relative URI's are
# resolved according to a set of rules: https://feedparser.readthedocs.io/en/latest/resolving-relative-links.html#advanced-base
value = models.TextField(
blank=True,
help_text="Free-form human-readable description of the feed format itself. Intended for people who view the feed in a browser, to explain what they just clicked on.", # noqa: E501
)
# For Atom feeds, the content type is taken from the type attribute, which defaults to text/plain if not specified.
# For RSS feeds, the content type is auto-determined by inspecting the content, and defaults to text/html.
# Note that this may cause silent data loss if the value contains plain text with angle brackets.
# Future enhancement: some versions of RSS clearly specify that certain values default to text/plain,
# and Universal Feed Parser should respect this, but it doesn't yet.
info_type = models.CharField(
max_length=255,
blank=True,
help_text="The content type of the feed info. Most likely text/plain, text/html, or application/xhtml+xml.",
)
# Supposed to be a language code, but publishers have been known to publish random values like “English” or “German”. # noqa: E501
# May come from the element's xml:lang attribute, or it may inherit from a parent element's xml:lang, or the Content-Language HTTP header # noqa: E501
language = models.CharField(
max_length=255,
blank=True,
help_text="The language of the feed info.",
)
# is only useful in rare situations and can usually be ignored. It is the original base URI for this value, as
# specified by the element's xml:base attribute, or a parent element's xml:base, or the appropriate HTTP header,
# or the URI of the feed. By the time you see it, Universal Feed Parser has already resolved relative links in all
# values where it makes sense to do so. Clients should never need to manually resolve relative links.
base = models.URLField(
blank=True,
help_text="The original base URI for links within the feed copyright.",
)
class Meta:
"""Info meta."""
verbose_name: typing.ClassVar[str] = "Feed information"
verbose_name_plural: typing.ClassVar[str] = "Feed information"
db_table_comment: typing.ClassVar[str] = "Details about the feed."
def __str__(self: Info) -> str:
"""Info value."""
return f"{self.value} {self.info_type} {self.language} {self.base}"
class Link(models.Model):
"""A list of dictionaries with details on the links associated with the feed.
Each link has a rel (relationship), type (content type), and href (the URL that the link points to).
Some links may also have a title.
Comes from
/atom03:feed/atom03:link
/atom10:feed/atom10:link
/rdf:RDF/rdf:channel/rdf:link
/rss/channel/link
"""
# Atom 1.0 defines five standard link relationships and describes the process for registering others.
# Here are the five standard rel values:
# alternate
# enclosure
# related
# self
# via
rel = models.CharField(
max_length=255,
blank=True,
help_text="The relationship of this feed link.",
)
link_type = models.CharField(
max_length=255,
blank=True,
help_text="The content type of the page that this feed link points to.",
)
href = models.URLField(
blank=True,
help_text="The URL of the page that this feed link points to.",
)
title = models.TextField(
blank=True,
help_text="The title of this feed link.",
)
class Meta:
"""Link meta."""
verbose_name: typing.ClassVar[str] = "Feed link"
verbose_name_plural: typing.ClassVar[str] = "Feed links"
db_table_comment: typing.ClassVar[str] = (
"A list of dictionaries with details on the links associated with the feed."
)
def __str__(self: Link) -> str:
"""Link href."""
return f"{self.href}"
class Publisher(models.Model):
"""The publisher of the feed.
Comes from the following elements:
/rdf:RDF/rdf:channel/dc:publisher
/rss/channel/dc:publisher
/rss/channel/itunes:owner
/rss/channel/webMaster
"""
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
name = models.TextField(blank=True, help_text="The name of this feed's publisher.")
email = models.EmailField(blank=True, help_text="The email address of this feed's publisher.")
# If this is a relative URI, it is resolved according to a set of rules:
# https://feedparser.readthedocs.io/en/latest/resolving-relative-links.html#advanced-base
href = models.URLField(
blank=True,
help_text="The URL of this feed's publisher. This can be the publisher's home page, or a contact page with a webmail form.", # noqa: E501
)
class Meta:
"""Publisher meta."""
verbose_name: typing.ClassVar[str] = "Feed publisher"
verbose_name_plural: typing.ClassVar[str] = "Feed publishers"
db_table_comment: typing.ClassVar[str] = "Details about the publisher of a feed."
def __str__(self: Publisher) -> str:
"""Publisher."""
return f"{self.name} {self.email} {self.href}"
class Rights(models.Model):
"""Details about the rights of a feed.
For machine-readable copyright information, see feed.license.
Comes from the following elements:
/atom03:feed/atom03:copyright
/atom10:feed/atom10:rights
/rdf:RDF/rdf:channel/dc:rights
/rss/channel/copyright
/rss/channel/dc:rights
"""
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
# If this contains HTML or XHTML, it is sanitized by default.
# If this contains HTML or XHTML, certain (X)HTML elements within this value may
# contain relative URI (Uniform Resource Identifier)'s. These relative URI's are
# resolved according to a set of rules: https://feedparser.readthedocs.io/en/latest/resolving-relative-links.html#advanced-base
value = models.TextField(
blank=True,
help_text="A human-readable copyright statement for the feed.",
)
# For Atom feeds, the content type is taken from the type attribute, which defaults to text/plain if not specified.
# For RSS feeds, the content type is auto-determined by inspecting the content, and defaults to text/html.
# Note that this may cause silent data loss if the value contains plain text with angle brackets.
# Future enhancement: some versions of RSS clearly specify that certain values default to text/plain,
# and Universal Feed Parser should respect this, but it doesn't yet.
rights_type = models.CharField(
max_length=255,
blank=True,
help_text="The content type of the feed copyright. Most likely text/plain, text/html, or application/xhtml+xml.", # noqa: E501
)
# Supposed to be a language code, but publishers have been known to publish random values like “English” or “German”. # noqa: E501
# May come from the element's xml:lang attribute, or it may inherit from a parent element's xml:lang, or the Content-Language HTTP header # noqa: E501
language = models.CharField(
max_length=255,
blank=True,
help_text="The language of the feed rights.",
)
base = models.URLField(
blank=True,
help_text="The original base URI for links within the feed copyright.",
)
class Meta:
"""Rights meta."""
verbose_name: typing.ClassVar[str] = "Feed rights"
verbose_name_plural: typing.ClassVar[str] = "Feed rights"
db_table_comment: typing.ClassVar[str] = "Details about the rights of a feed."
def __str__(self: Rights) -> str:
"""Rights value."""
return f"{self.value} {self.rights_type} {self.language} {self.base}"
class Subtitle(models.Model):
"""A subtitle, tagline, slogan, or other short description of the feed.
Comes from
/atom03:feed/atom03:tagline
/atom10:feed/atom10:subtitle
/rdf:RDF/rdf:channel/dc:description
/rdf:RDF/rdf:channel/rdf:description
/rss/channel/dc:description
/rss/channel/description
/rss/channel/itunes:subtitle
"""
value = models.TextField(
blank=True,
help_text="A subtitle, tagline, slogan, or other short description of the feed.",
)
subtitle_type = models.CharField(
max_length=255,
blank=True,
help_text="The content type of the feed subtitle. Most likely text/plain, text/html, or application/xhtml+xml.",
)
language = models.CharField(
max_length=255,
blank=True,
help_text="The language of the feed subtitle.",
)
base = models.URLField(
blank=True,
help_text="The original base URI for links within the feed subtitle.",
)
class Meta:
"""Subtitle meta."""
verbose_name: typing.ClassVar[str] = "Feed subtitle"
verbose_name_plural: typing.ClassVar[str] = "Feed subtitles"
db_table_comment: typing.ClassVar[str] = "A subtitle, tagline, slogan, or other short description of the feed."
def __str__(self: Subtitle) -> str:
"""Subtitle value."""
return f"{self.value} {self.subtitle_type} {self.language} {self.base}"
class Tags(models.Model):
"""A list of tags associated with the feed.
Comes from
/atom03:feed/dc:subject
/atom10:feed/category
/rdf:RDF/rdf:channel/dc:subject
/rss/channel/category
/rss/channel/dc:subject
/rss/channel/itunes:category
/rss/channel/itunes:keywords
"""
term = models.TextField(
blank=True,
help_text="The category term (keyword).",
)
scheme = models.CharField(
blank=True,
max_length=255,
help_text="The category scheme (domain).",
)
label = models.TextField(
blank=True,
help_text="A human-readable label for the category.",
)
class Meta:
"""Tags meta."""
verbose_name: typing.ClassVar[str] = "Feed tag"
verbose_name_plural: typing.ClassVar[str] = "Feed tags"
db_table_comment: typing.ClassVar[str] = "A list of tags associated with the feed."
def __str__(self: Tags) -> str:
"""Tag term."""
return f"{self.term} {self.scheme} {self.label}"
class TextInput(models.Model):
"""A text input form. No one actually uses this. Why are you?
Comes from the following elements:
/rdf:RDF/rdf:textinput
/rss/channel/textInput
/rss/channel/textinput
Example:
This is a text input in a feed:
```xml
<textInput>
<title>Go!</title>
<link>http://example.org/search</link>
<name>keyword</name>
<description>Search this site:</description>
</textInput>
```
This is how it could be rendered in HTML:
```html
<form method="get" action="http://example.org/search">
<label for="keyword">Search this site:</label>
<input type="text" id="keyword" name="keyword" value="">
<input type="submit" value="Go!">
</form>
```
"""
title = models.TextField(
blank=True,
help_text="The title of the text input form, which would go in the value attribute of the form's submit button.", # noqa: E501
)
link = models.URLField(
blank=True,
help_text="The link of the script which processes the text input form, which would go in the action attribute of the form.", # noqa: E501
)
name = models.TextField(
blank=True,
help_text="The name of the text input box in the form, which would go in the name attribute of the form's input box.", # noqa: E501
)
description = models.TextField(
blank=True,
help_text="A short description of the text input form, which would go in the label element of the form.",
)
class Meta:
"""TextInput meta."""
verbose_name: typing.ClassVar[str] = "Feed text input"
verbose_name_plural: typing.ClassVar[str] = "Feed text inputs"
db_table_comment: typing.ClassVar[str] = "A text input form. No one actually uses this. Why are you?"
def __str__(self: TextInput) -> str:
"""TextInput title."""
return f"{self.title} {self.link} {self.name} {self.description}"
class Title(models.Model):
"""Details about the title of a feed.
Comes from the following elements:
/atom03:feed/atom03:title
/atom10:feed/atom10:title
/rdf:RDF/rdf:channel/dc:title
/rdf:RDF/rdf:channel/rdf:title
/rss/channel/dc:title
/rss/channel/title
"""
value = models.TextField(
blank=True,
help_text="The title of the feed.",
)
title_type = models.CharField(
max_length=255,
blank=True,
help_text="The content type of the feed title. Most likely text/plain, text/html, or application/xhtml+xml.",
)
language = models.CharField(
max_length=255,
blank=True,
help_text="The language of the feed title.",
)
base = models.URLField(
blank=True,
help_text="The original base URI for links within the feed title.",
)
class Meta:
"""Title meta."""
verbose_name: typing.ClassVar[str] = "Feed title"
verbose_name_plural: typing.ClassVar[str] = "Feed titles"
db_table_comment: typing.ClassVar[str] = "Details about the title of a feed."
def __str__(self: Title) -> str:
"""Title value."""
return f"{self.value} {self.title_type} {self.language} {self.base}"
class Feed(models.Model):
"""A feed."""
url = models.URLField(
unique=True,
help_text="The feed URL.",
)
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
created_by = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True)
# Error tracking
error = models.BooleanField(default=False)
error_message = models.TextField(default="")
error_at = models.DateTimeField(null=True)
# bozo may not be present. Some platforms, such as Mac OS X 10.2 and some versions of FreeBSD, do not include an XML
# parser in their Python distributions. Universal Feed Parser will still work on these platforms, but it will not be
# able to detect whether a feed is well-formed. However, it can detect whether a feed's character encoding is
# incorrectly declared. (This is done in Python, not by the XML parser.)
# See: https://feedparser.readthedocs.io/en/latest/bozo.html#advanced-bozo
bozo = models.BooleanField(default=False, help_text="Is the feed well-formed XML?")
# bozo_exception will only be present if bozo is True.
bozo_exception = models.TextField(
default="",
help_text="The exception raised when attempting to parse a non-well-formed feed.",
)
# The process by which Universal Feed Parser determines the character encoding of the feed is explained can be found here: # noqa: E501
# https://feedparser.readthedocs.io/en/latest/character-encoding.html#advanced-encoding
# This element always exists, although it may be an empty string if the character encoding cannot be determined.
encoding = models.CharField(
max_length=255,
default="",
help_text="The character encoding that was used to parse the feed.",
)
# The purpose of etag is explained here: https://feedparser.readthedocs.io/en/latest/http-etag.html#http-etag
etag = models.CharField(
max_length=255,
default="",
help_text="The ETag of the feed, as specified in the HTTP headers.",
)
# Headers will only be present if the feed was retrieved from a web server. If the feed was parsed from a local file
# or from a string in memory, headers will not be present.
headers = models.TextField(
default="",
help_text="HTTP headers received from the web server when retrieving the feed.",
)
# href will only be present if the feed was retrieved from a web server. If the feed was parsed from a local file or
# from a string in memory, href will not be present.
href = models.URLField(
default="",
help_text="The final URL of the feed that was parsed. If the feed was redirected from the original requested address, href will contain the final (redirected) address.", # noqa: E501
)
# last_modified will only be present if the feed was retrieved from a web server, and only if the web server
# provided a Last-Modified HTTP header for the feed. If the feed was parsed from a local file or from a string
# in memory, last_modified will not be present.
last_modified = models.DateTimeField(
null=True,
help_text="The last-modified date of the feed, as specified in the HTTP headers.",
)
# The prefixes listed in the namespaces dictionary may not match the prefixes defined in the original feed.
# See: https://feedparser.readthedocs.io/en/latest/namespace-handling.html#advanced-namespaces
# This element always exists, although it may be an empty dictionary if the feed does not define any namespaces (such as an RSS 2.0 feed with no extensions). # noqa: E501
namespaces = models.TextField(
default="",
help_text="A dictionary of all XML namespaces defined in the feed, as {prefix: namespaceURI}.",
)
# If the feed was redirected from its original URL, status will contain the redirect status code, not the
# final status code.
# If status is 301, the feed was permanently redirected to a new URL. Clients should update
# their address book to request the new URL from now on.
# If status is 410, the feed is gone. Clients should stop polling the feed.
# status will only be present if the feed was retrieved from a web server. If the feed was parsed from a local file
# or from a string in memory, status will not be present.
# TODO(TheLovinator): #1 We should change feed URL if we get a HTTP 301.
# https://github.com/TheLovinator1/FeedVault/issues/1
# TODO(TheLovinator): #2 We should stop polling a feed if we get a HTTP 410.
# https://github.com/TheLovinator1/FeedVault/issues/2
http_status_code = models.IntegerField(
default=0,
help_text="The HTTP status code that was returned by the web server when the feed was fetched.",
)
# The format and version of the feed. the feed type is completely unknown, version will be an empty string.
version = models.CharField(
max_length=255,
choices=FEED_VERSION_CHOICES,
default="",
help_text="The version of the feed, as determined by Universal Feed Parser.",
)
# /atom03:feed, /atom10:feed, /rdf:RDF/rdf:channel, /rss/channel
feed_data = models.TextField(
default="",
help_text="A dictionary of data about the feed.",
)
author = models.ForeignKey(
Author,
on_delete=models.CASCADE,
null=True,
help_text="The author of the feed.",
)
cloud = models.ForeignKey(
Cloud,
on_delete=models.CASCADE,
null=True,
help_text="Cloud enables realtime push notifications or distributed publish/subscribe communication for feeds.",
)
contributors = models.ManyToManyField(
Contributor,
help_text="A list of contributors (secondary authors) to this feed.",
)
# This element is rare. The reasoning was that in 25 years, someone will stumble on
# an RSS feed and not know what it is, so we should waste everyone's bandwidth with
# useless links until then. Most publishers skip it, and all clients ignore it. If
# this is a relative URI, it is resolved according to a set of rules.
# Comes from /rss/channel/docs
docs = models.URLField(
blank=True,
help_text="A URL pointing to the specification which this feed conforms to.",
)
# Comes from /rdf:RDF/admin:errorReportsTo/@rdf:resource
error_reports_to = models.EmailField(
blank=True,
help_text="An email address for reporting errors in the feed itself.",
)
# Comes from /atom10:feed/atom10:icon
generator = models.ForeignKey(
Generator,
on_delete=models.CASCADE,
null=True,
help_text="Details about the software used to generate the feed.",
)
# If this is a relative URI, it is resolved according to a set of rules.
# Comes from/atom03:feed/atom03:id or /atom10:feed/atom10:id
feed_id = models.CharField(
max_length=255,
blank=True,
help_text="A globally unique identifier for this feed.",
)
image = models.ForeignKey(
Image,
on_delete=models.CASCADE,
null=True,
help_text="A feed image can be a logo, banner, or a picture of the author.",
)
info = models.ForeignKey(
Info,
on_delete=models.CASCADE,
null=True,
help_text="Details about the feed.",
)
# Comes from:
# /atom03:feed/@xml:lang
# /atom10:feed/@xml:lang
# /rdf:RDF/rdf:channel/dc:language
# /rss/channel/dc:language
# /rss/channel/language
language = models.CharField(
max_length=255,
blank=True,
help_text="The primary language of the feed.",
)
# Comes from /atom10:feed/atom10:link[@rel=”license”]/@href, /rdf:RDF/cc:license/@rdf:resource or /rss/channel/creativeCommons:license # noqa: E501
license = models.URLField(
blank=True,
help_text="A URL pointing to the license of the feed.",
)
link = models.ForeignKey(
Link,
on_delete=models.CASCADE,
null=True,
help_text="A list of dictionaries with details on the links associated with the feed.",
)
# Comes from /atom10:feed/atom10:logo
logo = models.URLField(
blank=True,
help_text="A URL to a graphic representing a logo for the feed.",
)
# Comes from /rss/channel/pubDate
# See feed.published_parsed for a more useful version of this value.
published = models.CharField(
max_length=255,
blank=True,
help_text="The date the feed was published, as a string in the same format as it was published in the original feed.", # noqa: E501
)
# Parsed version of feed.published. Feedparser gives us a standard Python 9-tuple.
published_parsed = models.DateTimeField(
null=True,
help_text="The date the feed was published.",
)
publisher = models.ForeignKey(
Publisher,
on_delete=models.CASCADE,
null=True,
help_text="The publisher of the feed.",
)
rights = models.ForeignKey(
Rights,
on_delete=models.CASCADE,
null=True,
help_text="Details about the rights of a feed.",
)
subtitle = models.ForeignKey(
Subtitle,
on_delete=models.CASCADE,
null=True,
help_text="A subtitle, tagline, slogan, or other short description of the feed.",
)
tags = models.ManyToManyField(
Tags,
help_text="A list of tags associated with the feed.",
)
text_input = models.ForeignKey(
TextInput,
on_delete=models.CASCADE,
null=True,
help_text="A text input form. No one actually uses this. Why are you?",
)
title = models.ForeignKey(
Title,
on_delete=models.CASCADE,
null=True,
help_text="Details about the title of a feed.",
)
# No one is quite sure what this means, and no one publishes feeds via file-sharing networks.
# Some clients have interpreted this element to be some sort of inline caching mechanism, albeit one
# that completely ignores the underlying HTTP protocol, its robust caching mechanisms, and the huge
# amount of HTTP-savvy network infrastructure that understands them. Given the vague documentation,
# it is impossible to say that this interpretation is any more ridiculous than the element itself.
# Comes from /rss/channel/ttl
ttl = models.CharField(
max_length=255,
blank=True,
help_text='According to the RSS specification, “None"',
)
# if this key doesn't exist but feed.published does, the value of feed.published will be returned.
updated = models.CharField(
max_length=255,
blank=True,
help_text="The date the feed was last updated, as a string in the same format as it was published in the original feed.", # noqa: E501
)
updated_parsed = models.DateTimeField(
null=True,
help_text="The date the feed was last updated.",
)
class Meta:
"""Feed meta."""
verbose_name: typing.ClassVar[str] = "Feed"
verbose_name_plural: typing.ClassVar[str] = "Feeds"
db_table_comment: typing.ClassVar[str] = "A feed. This is the main model of the app."
def __str__(self: Feed) -> str:
"""Feed URL."""
return f"{self.url}"
class Blocklist(models.Model):
"""A list of URLs to block."""
url = models.CharField(max_length=2000, unique=True, help_text="The URL to block.")
active = models.BooleanField(default=True, help_text="Is this URL still blocked?")
class Meta:
"""Blocklist meta."""
verbose_name: typing.ClassVar[str] = "Blocklist"
verbose_name_plural: typing.ClassVar[str] = "Blocklists"
db_table_comment: typing.ClassVar[str] = "A list of URLs to block."
def __str__(self: Blocklist) -> str:
"""Blocklist URL."""
return f"{self.url}"

51
feeds/tests.py
View file

@ -1,51 +0,0 @@
"""https://docs.djangoproject.com/en/5.0/topics/testing/."""
from __future__ import annotations
import random
from typing import TYPE_CHECKING
from django.test import Client, TestCase
from feeds.validator import is_ip, validate_scheme
if TYPE_CHECKING:
from django.http import HttpResponse
class TestHomePage(TestCase):
"""Test case for the home page view."""
def setUp(self: TestHomePage) -> None:
"""Set up the test client for the test case."""
self.client = Client()
def test_home_page(self: TestHomePage) -> None:
"""Test that a GET request to the home page returns a 200 status code."""
response: HttpResponse = self.client.get("/")
assert response.status_code == 200
class TestValidator(TestCase):
"""Test case for the validator."""
def setUp(self: TestValidator) -> None:
"""Set up the test client for the test case."""
self.client = Client()
def test_is_ip(self: TestValidator) -> None:
"""Test that is_ip() returns True for a valid IP address."""
# Test random IP address
random_ip: str = ".".join(str(random.randint(0, 255)) for _ in range(4)) # noqa: S311
assert is_ip(feed_url=random_ip)
# Test domain name
assert not is_ip(feed_url="https://example.com")
def test_validate_scheme(self: TestValidator) -> None:
"""Test that validate_scheme() returns True for a valid scheme."""
assert validate_scheme(feed_url="https://example.com")
assert validate_scheme(feed_url="http://example.com")
assert not validate_scheme(feed_url="ftp://example.com")
assert not validate_scheme(feed_url="example.com")
assert not validate_scheme(feed_url="127.0.0.1")

22
feeds/urls.py
View file

@ -1,22 +0,0 @@
"""URLs for the feeds app."""
from django.urls import path
from feeds.views import APIView, DonateView, FeedsView, IndexView, add_feeds, upload_opml
app_name = "feeds"
urlpatterns = [
# /
path("", IndexView.as_view(), name="index"),
# /feeds
path("feeds", FeedsView.as_view(), name="feeds"),
# /add
path("add", add_feeds, name="add"),
# /api
path("api", APIView.as_view(), name="api"),
# /donate
path("donate", DonateView.as_view(), name="donate"),
# /upload_opml
path("upload_opml", upload_opml, name="upload_opml"),
]

124
feeds/validator.py
View file

@ -1,124 +0,0 @@
"""Validate feeds before adding them to the database."""
from __future__ import annotations
import ipaddress
import logging
import socket
from urllib.parse import urlparse
import requests
from django.core.exceptions import ValidationError
from django.core.validators import URLValidator
from feeds.models import Blocklist
BLOCKLISTS: list[str] = [
"https://malware-filter.gitlab.io/malware-filter/urlhaus-filter-dnscrypt-blocked-names.txt",
"https://malware-filter.gitlab.io/malware-filter/phishing-filter-dnscrypt-blocked-names.txt",
]
logger: logging.Logger = logging.getLogger(__name__)
def validate_scheme(feed_url: str) -> bool:
"""Validate the scheme of a URL. Only allow http and https.
Args:
feed_url: The URL to validate.
Returns:
True if the URL is valid, False otherwise.
"""
validator = URLValidator(schemes=["http", "https"])
# TODO(TheLovinator): Should we allow other schemes? # noqa: TD003
try:
validator(feed_url)
except ValidationError:
return False
else:
return True
def is_ip(feed_url: str) -> bool:
"""Check if feed is an IP address."""
try:
ipaddress.ip_address(feed_url)
except ValueError:
logger.info(f"{feed_url} passed isn't either a v4 or a v6 address") # noqa: G004
return False
else:
logger.info(f"{feed_url} is an IP address") # noqa: G004
return True
def update_blocklist() -> str:
"""Download the blocklist and add to database."""
# URLs found in the blocklist
found_urls = set()
for _blocklist in BLOCKLISTS:
with requests.get(url=_blocklist, timeout=10) as r:
r.raise_for_status()
logger.debug(f"Downloaded {_blocklist}") # noqa: G004
# Split the blocklist into a list of URLs
blocked_urls = set(r.text.splitlines())
# Remove comments and whitespace
blocked_urls = {url for url in blocked_urls if not url.startswith("#")}
blocked_urls = {url.strip() for url in blocked_urls}
logger.debug(f"Found {len(blocked_urls)} URLs in {_blocklist}") # noqa: G004
# Add URLs to the found URLs set
found_urls.update(blocked_urls)
logger.debug(f"Found {len(found_urls)} URLs in total") # noqa: G004
# Mark all URLs as inactive
Blocklist.objects.all().update(active=False)
logger.debug("Marked all URLs as inactive")
# Bulk create the blocklist
Blocklist.objects.bulk_create(
[Blocklist(url=url, active=True) for url in found_urls],
update_conflicts=True,
unique_fields=["url"],
update_fields=["active"],
batch_size=1000,
)
logger.debug(f"Added {len(found_urls)} URLs to the blocklist") # noqa: G004
return f"Added {len(found_urls)} URLs to the blocklist"
def is_local(feed_url: str) -> bool:
"""Check if feed is a local address."""
network_location: str = urlparse(url=feed_url).netloc
# Check if network location is an IP address
if is_ip(feed_url=network_location):
try:
ip: ipaddress.IPv4Address | ipaddress.IPv6Address = ipaddress.ip_address(address=network_location)
except ValueError:
return False
else:
return ip.is_private
try:
ip_address: str = socket.gethostbyname(network_location)
is_private: bool = ipaddress.ip_address(address=ip_address).is_private
except socket.gaierror as e:
logger.info(f"{feed_url} failed to resolve: {e}") # noqa: G004
return True
except ValueError as e:
logger.info(f"{feed_url} failed to resolve: {e}") # noqa: G004
return True
msg: str = f"{feed_url} is a local URL" if is_private else f"{feed_url} is not a local URL"
logger.info(msg)
return is_private

283
feeds/views.py
View file

@ -1,283 +0,0 @@
"""Views for the feeds app.
IndexView - /
FeedsView - /feeds
"""
from __future__ import annotations
import logging
import typing
from urllib import parse
import listparser
from django.contrib import messages
from django.core.exceptions import ValidationError
from django.db import connection
from django.shortcuts import redirect
from django.views.generic.base import TemplateView
from django.views.generic.list import ListView
from feeds.forms import UploadOPMLForm
from feeds.models import Blocklist, Feed
from feeds.validator import is_ip, is_local, validate_scheme
if typing.TYPE_CHECKING:
from django.http import HttpRequest, HttpResponse
from listparser.common import SuperDict
logger: logging.Logger = logging.getLogger(__name__)
def get_database_size() -> int:
"""Get the size of a database.
Returns:
The size of the database in megabytes.
"""
with connection.cursor() as cursor:
# Query to get the size of the database
query = "SELECT pg_database_size('feedvault')"
cursor.execute(sql=query)
if not cursor:
return 0
size_in_bytes = cursor.fetchone()[0] # type: ignore # noqa: PGH003
if not size_in_bytes:
return 0
return int(size_in_bytes / (1024 * 1024))
class IndexView(TemplateView):
"""Index page."""
template_name = "index.html"
def get_context_data(self: IndexView, **kwargs: dict) -> dict:
"""Add feed count and database size to context data."""
context: dict = super().get_context_data(**kwargs)
context["feed_count"] = Feed.objects.count()
context["database_size"] = get_database_size()
logger.info(f"Found {context['feed_count']} feeds in the database") # noqa: G004
logger.info(f"Database size is {context['database_size']} MB") # noqa: G004
return context
class FeedsView(ListView):
"""Feeds page."""
model = Feed
template_name = "feeds.html"
context_object_name = "feeds"
paginate_by = 100
ordering: typing.ClassVar[list[str]] = ["-created_at"]
def get_context_data(self: FeedsView, **kwargs: dict) -> dict:
"""Add feed count and database size to context data."""
context: dict = super().get_context_data(**kwargs)
context["feed_count"] = Feed.objects.count()
context["database_size"] = get_database_size()
return context
def add_feeds(request: HttpRequest) -> HttpResponse:
"""Add feeds to the database.
Args:
request: The request object.
Returns:
A redirect to the index page if there are errors, otherwise a redirect to the feeds page.
"""
if request.method == "POST":
urls: str | None = request.POST.get("urls")
if not urls:
messages.error(request, "No URLs provided")
return redirect("feeds:index")
if urls == "Test":
messages.error(request, "Test test hello")
return redirect("feeds:index")
for url in urls.splitlines():
check_feeds(feed_urls=[url], request=request)
return redirect("feeds:index")
msg: str = f"You must use a POST request. You used a {request.method} request. You can find out how to use this endpoint here: <a href=''>http://127.0.0.1:8000/</a>. If you think this is a mistake, please contact the administrator." # noqa: E501
messages.error(request, msg)
return redirect("feeds:index")
def handle_opml(opml_url: str, request: HttpRequest) -> None:
"""Add feeds from an OPML file.
Args:
opml_url: The URL of the OPML file.
request: The request object.
Returns:
Errors
"""
if not opml_url:
msg: str = "No URL provided when parsing OPML file."
messages.error(request, msg)
logger.error(msg)
return
url_html: str = f"<a href='{opml_url}'>{opml_url}</a>"
result: SuperDict = listparser.parse(opml_url)
if result.bozo:
msg: str = f"Error when parsing {url_html}: '{result.bozo_exception}'"
messages.error(request, msg)
logger.error(msg)
return
for feed in result.feeds:
logger.debug(f"Found {feed.url} in OPML file '{opml_url}' for '{feed.title}'") # noqa: G004
check_feeds(feed_urls=feed.url, request=request)
def validate_and_add(url: str, request: HttpRequest) -> None:
"""Check if a feed is valid.
Args:
url: The URL of the feed.
request: The request object.
"""
# TODO(TheLovinator): #4 Rewrite this so we check the content instead of the URL
# https://github.com/TheLovinator1/FeedVault/issues/4
list_of_opml_urls: list[str] = [".opml", ".ttl", ".trig", ".rdf"]
if url.endswith(tuple(list_of_opml_urls)):
handle_opml(opml_url=url, request=request)
return
url_html: str = f"<a href='{url}'>{url}</a>"
if Feed.objects.filter(url=url).exists():
msg: str = f"{url_html} is already in the database."
messages.error(request, msg)
return
# Only allow HTTP and HTTPS URLs
if not validate_scheme(feed_url=url):
msg = f"{url_html} is not a HTTP or HTTPS URL."
messages.error(request, msg)
return
# Don't allow IP addresses
if is_ip(feed_url=url):
msg = f"{url_html} is an IP address. IP addresses are not allowed."
messages.error(request, msg)
return
# Check if in blocklist
domain: str = parse.urlparse(url).netloc
if Blocklist.objects.filter(url=domain).exists():
msg = f"{url_html} is in the blocklist."
messages.error(request, msg)
return
# Check if local URL
if is_local(feed_url=url):
msg = f"{url_html} is not accessible from the internet."
messages.error(request, msg)
return
# Create feed
try:
Feed.objects.create(url=url)
msg = f"{url_html} was added to the database."
messages.success(request, msg)
except ValidationError:
msg = f"{url_html} is not a valid URL."
messages.error(request, msg)
def check_feeds(feed_urls: list[str], request: HttpRequest) -> HttpResponse:
"""Check feeds before adding them to the database.
Args:
feed_urls: The feed URLs to check.
request: The request object.
Returns:
A redirect to the index page if there are errors, otherwise a redirect to the feeds page.
"""
for url in feed_urls:
validate_and_add(url=url, request=request)
# Return to feeds page if no errors
# TODO(TheLovinator): Return to search page with our new feeds # noqa: TD003
logger.info(f"Added {len(feed_urls)} feeds to the database") # noqa: G004
return redirect("feeds:feeds")
class APIView(TemplateView):
"""API page."""
template_name = "api.html"
def get_context_data(self: APIView, **kwargs: dict) -> dict:
"""Add feed count and database size to context data."""
context: dict = super().get_context_data(**kwargs)
context["feed_count"] = Feed.objects.count()
context["database_size"] = get_database_size()
return context
class DonateView(TemplateView):
"""Donate page."""
template_name = "donate.html"
def get_context_data(self: DonateView, **kwargs: dict) -> dict:
"""Add feed count and database size to context data."""
context: dict = super().get_context_data(**kwargs)
context["feed_count"] = Feed.objects.count()
context["database_size"] = get_database_size()
return context
def upload_opml(request: HttpRequest) -> HttpResponse:
"""Upload an OPML file.
Args:
request: The request object.
Returns:
A redirect to the index page if there are errors, otherwise a redirect to the feeds page.
"""
if request.method == "POST":
form = UploadOPMLForm(request.POST, request.FILES)
if form.is_valid():
opml_file = request.FILES["file"]
# Read file
with opml_file.open() as file:
opml_file = file.read().decode("utf-8")
result: SuperDict = listparser.parse(opml_file)
if result.bozo:
msg: str = f"Error when parsing OPML file: '{result.bozo_exception}'"
messages.error(request, msg)
logger.error(msg)
return redirect("feeds:index")
for feed in result.feeds:
logger.debug(f"Found {feed.url} in OPML file for '{feed.title}'") # noqa: G004
validate_and_add(url=feed.url, request=request)
for _list in result.lists:
logger.debug(f"Found {_list.url} in OPML file for '{_list.title}'") # noqa: G004
validate_and_add(url=_list.url, request=request)
return redirect("feeds:index")
msg: str = "Invalid form"
messages.error(request, msg)
logger.error(msg)
return redirect("feeds:index")