TheLovinator/feedvault.se
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add feed and item extensions

Browse source
This commit is contained in:
Joakim Hellsén 2024-02-17 21:17:58 +01:00
commit 3113b844bc
9 changed files with 448 additions and 54 deletions
Download patch file Download diff file Expand all files Collapse all files

12
.vscode/launch.json vendored
View file

@ -1,13 +1,15 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "Python Debugger: Django",
"type": "debugpy",
"name": "Launch Package",
"type": "go",
"request": "launch",
"program": "${workspaceFolder}\\manage.py",
"args": ["runserver"],
"django": true
"mode": "auto",
"program": "${workspaceFolder}"
}
]
}

236
db/feeds.sql.go
View file

@ -85,7 +85,9 @@ VALUES
$18,
$19,
$20
) RETURNING id, url, created_at, updated_at, deleted_at, title, description, link, feed_link, links, updated, updated_parsed, published, published_parsed, language, copyright, generator, categories, custom, feed_type, feed_version
)
RETURNING
id, url, created_at, updated_at, deleted_at, title, description, link, feed_link, links, updated, updated_parsed, published, published_parsed, language, copyright, generator, categories, custom, feed_type, feed_version
`
type CreateFeedParams struct {
@ -161,6 +163,61 @@ func (q *Queries) CreateFeed(ctx context.Context, arg CreateFeedParams) (Feed, e
return i, err
}
const createFeedExtension = `-- name: CreateFeedExtension :one
INSERT INTO
feed_extensions (
created_at,
updated_at,
deleted_at,
"name",
"value",
attrs,
children,
feed_id
)
VALUES
($1, $2, $3, $4, $5, $6, $7, $8)
RETURNING
id, created_at, updated_at, deleted_at, name, value, attrs, children, feed_id
`
type CreateFeedExtensionParams struct {
CreatedAt pgtype.Timestamptz `json:"created_at"`
UpdatedAt pgtype.Timestamptz `json:"updated_at"`
DeletedAt pgtype.Timestamptz `json:"deleted_at"`
Name pgtype.Text `json:"name"`
Value pgtype.Text `json:"value"`
Attrs []byte `json:"attrs"`
Children []byte `json:"children"`
FeedID int64 `json:"feed_id"`
}
func (q *Queries) CreateFeedExtension(ctx context.Context, arg CreateFeedExtensionParams) (FeedExtension, error) {
row := q.db.QueryRow(ctx, createFeedExtension,
arg.CreatedAt,
arg.UpdatedAt,
arg.DeletedAt,
arg.Name,
arg.Value,
arg.Attrs,
arg.Children,
arg.FeedID,
)
var i FeedExtension
err := row.Scan(
&i.ID,
&i.CreatedAt,
&i.UpdatedAt,
&i.DeletedAt,
&i.Name,
&i.Value,
&i.Attrs,
&i.Children,
&i.FeedID,
)
return i, err
}
const createItem = `-- name: CreateItem :one
INSERT INTO
items (
@ -199,7 +256,9 @@ VALUES
$14,
$15,
$16
) RETURNING id, created_at, updated_at, deleted_at, title, description, content, link, links, updated, updated_parsed, published, published_parsed, guid, categories, custom, feed_id
)
RETURNING
id, created_at, updated_at, deleted_at, title, description, content, link, links, updated, updated_parsed, published, published_parsed, guid, categories, custom, feed_id
`
type CreateItemParams struct {
@ -212,9 +271,9 @@ type CreateItemParams struct {
Link pgtype.Text `json:"link"`
Links []string `json:"links"`
Updated pgtype.Text `json:"updated"`
UpdatedParsed pgtype.Timestamp `json:"updated_parsed"`
UpdatedParsed pgtype.Timestamptz `json:"updated_parsed"`
Published pgtype.Text `json:"published"`
PublishedParsed pgtype.Timestamp `json:"published_parsed"`
PublishedParsed pgtype.Timestamptz `json:"published_parsed"`
Guid pgtype.Text `json:"guid"`
Categories []string `json:"categories"`
Custom []byte `json:"custom"`
@ -263,6 +322,61 @@ func (q *Queries) CreateItem(ctx context.Context, arg CreateItemParams) (Item, e
return i, err
}
const createItemExtension = `-- name: CreateItemExtension :one
INSERT INTO
item_extensions (
created_at,
updated_at,
deleted_at,
"name",
"value",
attrs,
children,
item_id
)
VALUES
($1, $2, $3, $4, $5, $6, $7, $8)
RETURNING
id, created_at, updated_at, deleted_at, name, value, attrs, children, item_id
`
type CreateItemExtensionParams struct {
CreatedAt pgtype.Timestamptz `json:"created_at"`
UpdatedAt pgtype.Timestamptz `json:"updated_at"`
DeletedAt pgtype.Timestamptz `json:"deleted_at"`
Name pgtype.Text `json:"name"`
Value pgtype.Text `json:"value"`
Attrs []byte `json:"attrs"`
Children []byte `json:"children"`
ItemID int64 `json:"item_id"`
}
func (q *Queries) CreateItemExtension(ctx context.Context, arg CreateItemExtensionParams) (ItemExtension, error) {
row := q.db.QueryRow(ctx, createItemExtension,
arg.CreatedAt,
arg.UpdatedAt,
arg.DeletedAt,
arg.Name,
arg.Value,
arg.Attrs,
arg.Children,
arg.ItemID,
)
var i ItemExtension
err := row.Scan(
&i.ID,
&i.CreatedAt,
&i.UpdatedAt,
&i.DeletedAt,
&i.Name,
&i.Value,
&i.Attrs,
&i.Children,
&i.ItemID,
)
return i, err
}
const getFeed = `-- name: GetFeed :one
SELECT
id, url, created_at, updated_at, deleted_at, title, description, link, feed_link, links, updated, updated_parsed, published, published_parsed, language, copyright, generator, categories, custom, feed_type, feed_version
@ -301,6 +415,57 @@ func (q *Queries) GetFeed(ctx context.Context, id int64) (Feed, error) {
return i, err
}
const getFeedExtensions = `-- name: GetFeedExtensions :many
SELECT
id, created_at, updated_at, deleted_at, name, value, attrs, children, feed_id
FROM
feed_extensions
WHERE
feed_id = $1
ORDER BY
created_at DESC
LIMIT
$2
OFFSET
$3
`
type GetFeedExtensionsParams struct {
FeedID int64 `json:"feed_id"`
Limit int32 `json:"limit"`
Offset int32 `json:"offset"`
}
func (q *Queries) GetFeedExtensions(ctx context.Context, arg GetFeedExtensionsParams) ([]FeedExtension, error) {
rows, err := q.db.Query(ctx, getFeedExtensions, arg.FeedID, arg.Limit, arg.Offset)
if err != nil {
return nil, err
}
defer rows.Close()
items := []FeedExtension{}
for rows.Next() {
var i FeedExtension
if err := rows.Scan(
&i.ID,
&i.CreatedAt,
&i.UpdatedAt,
&i.DeletedAt,
&i.Name,
&i.Value,
&i.Attrs,
&i.Children,
&i.FeedID,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const getFeeds = `-- name: GetFeeds :many
SELECT
id, url, created_at, updated_at, deleted_at, title, description, link, feed_link, links, updated, updated_parsed, published, published_parsed, language, copyright, generator, categories, custom, feed_type, feed_version
@ -308,8 +473,10 @@ FROM
feeds
ORDER BY
created_at DESC
LIMIT $1
OFFSET $2
LIMIT
$1
OFFSET
$2
`
type GetFeedsParams struct {
@ -393,6 +560,57 @@ func (q *Queries) GetItem(ctx context.Context, id int64) (Item, error) {
return i, err
}
const getItemExtensions = `-- name: GetItemExtensions :many
SELECT
id, created_at, updated_at, deleted_at, name, value, attrs, children, item_id
FROM
item_extensions
WHERE
item_id = $1
ORDER BY
created_at DESC
LIMIT
$2
OFFSET
$3
`
type GetItemExtensionsParams struct {
ItemID int64 `json:"item_id"`
Limit int32 `json:"limit"`
Offset int32 `json:"offset"`
}
func (q *Queries) GetItemExtensions(ctx context.Context, arg GetItemExtensionsParams) ([]ItemExtension, error) {
rows, err := q.db.Query(ctx, getItemExtensions, arg.ItemID, arg.Limit, arg.Offset)
if err != nil {
return nil, err
}
defer rows.Close()
items := []ItemExtension{}
for rows.Next() {
var i ItemExtension
if err := rows.Scan(
&i.ID,
&i.CreatedAt,
&i.UpdatedAt,
&i.DeletedAt,
&i.Name,
&i.Value,
&i.Attrs,
&i.Children,
&i.ItemID,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const getItems = `-- name: GetItems :many
SELECT
id, created_at, updated_at, deleted_at, title, description, content, link, links, updated, updated_parsed, published, published_parsed, guid, categories, custom, feed_id
@ -402,8 +620,10 @@ WHERE
feed_id = $1
ORDER BY
created_at DESC
LIMIT $2
OFFSET $3
LIMIT
$2
OFFSET
$3
`
type GetItemsParams struct {

32
db/models.go
View file

@ -129,9 +129,9 @@ type Item struct {
Link pgtype.Text `json:"link"`
Links []string `json:"links"`
Updated pgtype.Text `json:"updated"`
UpdatedParsed pgtype.Timestamp `json:"updated_parsed"`
UpdatedParsed pgtype.Timestamptz `json:"updated_parsed"`
Published pgtype.Text `json:"published"`
PublishedParsed pgtype.Timestamp `json:"published_parsed"`
PublishedParsed pgtype.Timestamptz `json:"published_parsed"`
Guid pgtype.Text `json:"guid"`
Categories []string `json:"categories"`
Custom []byte `json:"custom"`
@ -217,21 +217,21 @@ type ItemItune struct {
}
type ItunesCategory struct {
ID int64 `json:"id"`
CreatedAt pgtype.Timestamp `json:"created_at"`
UpdatedAt pgtype.Timestamp `json:"updated_at"`
DeletedAt pgtype.Timestamp `json:"deleted_at"`
Text pgtype.Text `json:"text"`
Subcategory pgtype.Text `json:"subcategory"`
ItunesID int64 `json:"itunes_id"`
ID int64 `json:"id"`
CreatedAt pgtype.Timestamptz `json:"created_at"`
UpdatedAt pgtype.Timestamptz `json:"updated_at"`
DeletedAt pgtype.Timestamptz `json:"deleted_at"`
Text pgtype.Text `json:"text"`
Subcategory pgtype.Text `json:"subcategory"`
ItunesID int64 `json:"itunes_id"`
}
type ItunesOwner struct {
ID int64 `json:"id"`
CreatedAt pgtype.Timestamp `json:"created_at"`
UpdatedAt pgtype.Timestamp `json:"updated_at"`
DeletedAt pgtype.Timestamp `json:"deleted_at"`
Email pgtype.Text `json:"email"`
Name pgtype.Text `json:"name"`
ItunesID int64 `json:"itunes_id"`
ID int64 `json:"id"`
CreatedAt pgtype.Timestamptz `json:"created_at"`
UpdatedAt pgtype.Timestamptz `json:"updated_at"`
DeletedAt pgtype.Timestamptz `json:"deleted_at"`
Email pgtype.Text `json:"email"`
Name pgtype.Text `json:"name"`
ItunesID int64 `json:"itunes_id"`
}

108
feeds.go
View file

@ -57,6 +57,15 @@ func makeCreateFeedParams(feedURL string, feed *gofeed.Feed) db.CreateFeedParams
}
func makeCreateItemParams(item *gofeed.Item, feedID int64) db.CreateItemParams {
var updatedTime time.Time
if item.UpdatedParsed != nil {
updatedTime = *item.UpdatedParsed
}
var publishedTime time.Time
if item.PublishedParsed != nil {
publishedTime = *item.PublishedParsed
}
itemCustom := []byte("{}")
if item.Custom != nil {
var err error
@ -77,9 +86,9 @@ func makeCreateItemParams(item *gofeed.Item, feedID int64) db.CreateItemParams {
Link: pgtype.Text{String: item.Link, Valid: item.Link != ""},
Links: item.Links,
Updated: pgtype.Text{String: item.Updated, Valid: item.Updated != ""},
UpdatedParsed: pgtype.Timestamp{Time: *item.UpdatedParsed, Valid: item.UpdatedParsed != nil},
UpdatedParsed: pgtype.Timestamptz{Time: updatedTime, Valid: !updatedTime.IsZero()},
Published: pgtype.Text{String: item.Published, Valid: item.Published != ""},
PublishedParsed: pgtype.Timestamp{Time: *item.PublishedParsed, Valid: item.PublishedParsed != nil},
PublishedParsed: pgtype.Timestamptz{Time: publishedTime, Valid: !publishedTime.IsZero()},
Guid: pgtype.Text{String: item.GUID, Valid: item.GUID != ""},
Categories: item.Categories,
Custom: itemCustom,
@ -110,14 +119,105 @@ func AddFeedToDB(feedURL string) error {
if err != nil {
return fmt.Errorf("Error adding feed to database: %s", err)
}
log.Printf("Added feed to database: %+v", newFeed)
log.Printf("Feed added to database")
// Add the items to the database
for _, item := range feed.Items {
_, err := DB.CreateItem(ctx, makeCreateItemParams(item, newFeed.ID))
newItem, err := DB.CreateItem(ctx, makeCreateItemParams(item, newFeed.ID))
if err != nil {
log.Printf("Error adding item to database: %s", err)
}
log.Printf("Item added to database")
// Add extensions to the database
for _, ext := range item.Extensions {
for _, exts := range ext {
for _, e := range exts {
attrsCustom := []byte("{}")
if e.Attrs != nil {
var err error
attrsCustom, err = json.Marshal(e.Attrs)
if err != nil {
fmt.Println("Error marshalling extension attributes:", err)
attrsCustom = []byte("{}")
}
log.Printf("Extension attributes: %s", attrsCustom)
}
childrenCustom := []byte("{}")
if e.Children != nil {
var err error
childrenCustom, err = json.Marshal(e.Children)
if err != nil {
fmt.Println("Error marshalling extension children:", err)
childrenCustom = []byte("{}")
}
log.Printf("Extension children: %s", childrenCustom)
}
_, err := DB.CreateItemExtension(ctx, db.CreateItemExtensionParams{
CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
DeletedAt: pgtype.Timestamptz{Valid: false},
Name: pgtype.Text{String: e.Name, Valid: e.Name != ""},
Value: pgtype.Text{String: e.Value, Valid: e.Value != ""},
Attrs: attrsCustom,
Children: childrenCustom,
ItemID: newItem.ID,
})
if err != nil {
log.Printf("Error adding extension to database: %s", err)
}
log.Printf("Extension added to database")
}
}
}
}
// Add extensions to the database
// TODO: Check if this is correct and works
for _, ext := range feed.Extensions {
for _, exts := range ext {
for _, e := range exts {
attrsCustom := []byte("{}")
if e.Attrs != nil {
var err error
attrsCustom, err = json.Marshal(e.Attrs)
if err != nil {
fmt.Println("Error marshalling extension attributes:", err)
attrsCustom = []byte("{}")
}
}
childrenCustom := []byte("{}")
if e.Children != nil {
var err error
childrenCustom, err = json.Marshal(e.Children)
if err != nil {
fmt.Println("Error marshalling extension children:", err)
childrenCustom = []byte("{}")
}
}
_, err := DB.CreateFeedExtension(ctx, db.CreateFeedExtensionParams{
CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
DeletedAt: pgtype.Timestamptz{Valid: false},
Name: pgtype.Text{String: e.Name, Valid: e.Name != ""},
Value: pgtype.Text{String: e.Value, Valid: e.Value != ""},
Attrs: attrsCustom,
Children: childrenCustom,
FeedID: newFeed.ID,
})
if err != nil {
log.Printf("Error adding extension to database: %s", err)
}
}
}
}
fmt.Println(feed.Title)

9
html.go
View file

@ -8,6 +8,14 @@ import (
"strings"
)
// Used for success/error message at the top of the page after adding a feed.
type ParseResult struct {
FeedURL string
Msg string
IsError bool
}
// HTMLData is the data passed to the HTML template.
type HTMLData struct {
Title string
Description string
@ -18,6 +26,7 @@ type HTMLData struct {
ParseResult []ParseResult
}
// Our CSS that is included in the HTML.
var style = `
html {
max-width: 70ch;

7
models.go
View file

@ -1,7 +0,0 @@
package main
type ParseResult struct {
FeedURL string
Msg string
IsError bool
}

82
sql/queries/feeds.sql
View file

@ -44,7 +44,9 @@ VALUES
$18,
$19,
$20
) RETURNING *;
)
RETURNING
*;
-- name: CountFeeds :one
SELECT
@ -90,7 +92,9 @@ VALUES
$14,
$15,
$16
) RETURNING *;
)
RETURNING
*;
-- name: CountItems :one
SELECT
@ -113,8 +117,10 @@ FROM
feeds
ORDER BY
created_at DESC
LIMIT $1
OFFSET $2;
LIMIT
$1
OFFSET
$2;
-- name: GetItem :one
SELECT
@ -133,5 +139,69 @@ WHERE
feed_id = $1
ORDER BY
created_at DESC
LIMIT $2
OFFSET $3;
LIMIT
$2
OFFSET
$3;
-- name: CreateFeedExtension :one
INSERT INTO
feed_extensions (
created_at,
updated_at,
deleted_at,
"name",
"value",
attrs,
children,
feed_id
)
VALUES
($1, $2, $3, $4, $5, $6, $7, $8)
RETURNING
*;
-- name: CreateItemExtension :one
INSERT INTO
item_extensions (
created_at,
updated_at,
deleted_at,
"name",
"value",
attrs,
children,
item_id
)
VALUES
($1, $2, $3, $4, $5, $6, $7, $8)
RETURNING
*;
-- name: GetFeedExtensions :many
SELECT
*
FROM
feed_extensions
WHERE
feed_id = $1
ORDER BY
created_at DESC
LIMIT
$2
OFFSET
$3;
-- name: GetItemExtensions :many
SELECT
*
FROM
item_extensions
WHERE
item_id = $1
ORDER BY
created_at DESC
LIMIT
$2
OFFSET
$3;

4
sql/schema/20240214043229_feeds.sql
View file

@ -46,9 +46,9 @@ CREATE TABLE IF NOT EXISTS items (
link TEXT,
links TEXT [],
updated TEXT,
updated_parsed TIMESTAMP,
updated_parsed TIMESTAMPTZ,
published TEXT,
published_parsed TIMESTAMP,
published_parsed TIMESTAMPTZ,
-- Authors - See item_authors
"guid" TEXT,
-- Image - See item_images

12
sql/schema/20240215232318_itunes.sql
View file

@ -54,9 +54,9 @@ CREATE TABLE IF NOT EXISTS item_itunes (
-- https://github.com/mmcdole/gofeed/blob/master/extensions/itunes.go#L39
CREATE TABLE IF NOT EXISTS itunes_categories (
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
deleted_at TIMESTAMP DEFAULT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
deleted_at TIMESTAMPTZ DEFAULT NULL,
-- From gofeed:
"text" TEXT,
subcategory TEXT,
@ -69,9 +69,9 @@ CREATE TABLE IF NOT EXISTS itunes_categories (
-- https://github.com/mmcdole/gofeed/blob/master/extensions/itunes.go#L45
CREATE TABLE IF NOT EXISTS itunes_owners (
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
deleted_at TIMESTAMP DEFAULT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
deleted_at TIMESTAMPTZ DEFAULT NULL,
-- From gofeed:
email TEXT,
"name" TEXT,