Add feed and item extensions

.vscode/launch.json

@@ -1,13 +1,15 @@
 {
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
   "version": "0.2.0",
   "configurations": [
     {
-      "name": "Python Debugger: Django",
+      "name": "Launch Package",
-      "type": "debugpy",
+      "type": "go",
       "request": "launch",
-      "program": "${workspaceFolder}\\manage.py",
+      "mode": "auto",
-      "args": ["runserver"],
+      "program": "${workspaceFolder}"
-      "django": true
     }
   ]
 }

db/feeds.sql.go

@@ -85,7 +85,9 @@ VALUES
         $18,
         $19,
         $20
-    ) RETURNING id, url, created_at, updated_at, deleted_at, title, description, link, feed_link, links, updated, updated_parsed, published, published_parsed, language, copyright, generator, categories, custom, feed_type, feed_version
+    )
+RETURNING
+    id, url, created_at, updated_at, deleted_at, title, description, link, feed_link, links, updated, updated_parsed, published, published_parsed, language, copyright, generator, categories, custom, feed_type, feed_version
 `
 
 type CreateFeedParams struct {
@@ -161,6 +163,61 @@ func (q *Queries) CreateFeed(ctx context.Context, arg CreateFeedParams) (Feed, e
 	return i, err
 }
 
+const createFeedExtension = `-- name: CreateFeedExtension :one
+INSERT INTO
+    feed_extensions (
+        created_at,
+        updated_at,
+        deleted_at,
+        "name",
+        "value",
+        attrs,
+        children,
+        feed_id
+    )
+VALUES
+    ($1, $2, $3, $4, $5, $6, $7, $8)
+RETURNING
+    id, created_at, updated_at, deleted_at, name, value, attrs, children, feed_id
+`
+
+type CreateFeedExtensionParams struct {
+	CreatedAt pgtype.Timestamptz `json:"created_at"`
+	UpdatedAt pgtype.Timestamptz `json:"updated_at"`
+	DeletedAt pgtype.Timestamptz `json:"deleted_at"`
+	Name      pgtype.Text        `json:"name"`
+	Value     pgtype.Text        `json:"value"`
+	Attrs     []byte             `json:"attrs"`
+	Children  []byte             `json:"children"`
+	FeedID    int64              `json:"feed_id"`
+}
+
+func (q *Queries) CreateFeedExtension(ctx context.Context, arg CreateFeedExtensionParams) (FeedExtension, error) {
+	row := q.db.QueryRow(ctx, createFeedExtension,
+		arg.CreatedAt,
+		arg.UpdatedAt,
+		arg.DeletedAt,
+		arg.Name,
+		arg.Value,
+		arg.Attrs,
+		arg.Children,
+		arg.FeedID,
+	)
+	var i FeedExtension
+	err := row.Scan(
+		&i.ID,
+		&i.CreatedAt,
+		&i.UpdatedAt,
+		&i.DeletedAt,
+		&i.Name,
+		&i.Value,
+		&i.Attrs,
+		&i.Children,
+		&i.FeedID,
+	)
+	return i, err
+}
+
 const createItem = `-- name: CreateItem :one
 INSERT INTO
     items (
@@ -199,7 +256,9 @@ VALUES
         $14,
         $15,
         $16
-    ) RETURNING id, created_at, updated_at, deleted_at, title, description, content, link, links, updated, updated_parsed, published, published_parsed, guid, categories, custom, feed_id
+    )
+RETURNING
+    id, created_at, updated_at, deleted_at, title, description, content, link, links, updated, updated_parsed, published, published_parsed, guid, categories, custom, feed_id
 `
 
 type CreateItemParams struct {
@@ -212,9 +271,9 @@ type CreateItemParams struct {
 	Link            pgtype.Text        `json:"link"`
 	Links           []string           `json:"links"`
 	Updated         pgtype.Text        `json:"updated"`
-	UpdatedParsed   pgtype.Timestamp   `json:"updated_parsed"`
+	UpdatedParsed   pgtype.Timestamptz `json:"updated_parsed"`
 	Published       pgtype.Text        `json:"published"`
-	PublishedParsed pgtype.Timestamp   `json:"published_parsed"`
+	PublishedParsed pgtype.Timestamptz `json:"published_parsed"`
 	Guid            pgtype.Text        `json:"guid"`
 	Categories      []string           `json:"categories"`
 	Custom          []byte             `json:"custom"`
@@ -263,6 +322,61 @@ func (q *Queries) CreateItem(ctx context.Context, arg CreateItemParams) (Item, e
 	return i, err
 }
 
+const createItemExtension = `-- name: CreateItemExtension :one
+INSERT INTO
+    item_extensions (
+        created_at,
+        updated_at,
+        deleted_at,
+        "name",
+        "value",
+        attrs,
+        children,
+        item_id
+    )
+VALUES
+    ($1, $2, $3, $4, $5, $6, $7, $8)
+RETURNING
+    id, created_at, updated_at, deleted_at, name, value, attrs, children, item_id
+`
+
+type CreateItemExtensionParams struct {
+	CreatedAt pgtype.Timestamptz `json:"created_at"`
+	UpdatedAt pgtype.Timestamptz `json:"updated_at"`
+	DeletedAt pgtype.Timestamptz `json:"deleted_at"`
+	Name      pgtype.Text        `json:"name"`
+	Value     pgtype.Text        `json:"value"`
+	Attrs     []byte             `json:"attrs"`
+	Children  []byte             `json:"children"`
+	ItemID    int64              `json:"item_id"`
+}
+
+func (q *Queries) CreateItemExtension(ctx context.Context, arg CreateItemExtensionParams) (ItemExtension, error) {
+	row := q.db.QueryRow(ctx, createItemExtension,
+		arg.CreatedAt,
+		arg.UpdatedAt,
+		arg.DeletedAt,
+		arg.Name,
+		arg.Value,
+		arg.Attrs,
+		arg.Children,
+		arg.ItemID,
+	)
+	var i ItemExtension
+	err := row.Scan(
+		&i.ID,
+		&i.CreatedAt,
+		&i.UpdatedAt,
+		&i.DeletedAt,
+		&i.Name,
+		&i.Value,
+		&i.Attrs,
+		&i.Children,
+		&i.ItemID,
+	)
+	return i, err
+}
+
 const getFeed = `-- name: GetFeed :one
 SELECT
     id, url, created_at, updated_at, deleted_at, title, description, link, feed_link, links, updated, updated_parsed, published, published_parsed, language, copyright, generator, categories, custom, feed_type, feed_version
@@ -301,6 +415,57 @@ func (q *Queries) GetFeed(ctx context.Context, id int64) (Feed, error) {
 	return i, err
 }
 
+const getFeedExtensions = `-- name: GetFeedExtensions :many
+SELECT
+    id, created_at, updated_at, deleted_at, name, value, attrs, children, feed_id
+FROM
+    feed_extensions
+WHERE
+    feed_id = $1
+ORDER BY
+    created_at DESC
+LIMIT
+    $2
+OFFSET
+    $3
+`
+
+type GetFeedExtensionsParams struct {
+	FeedID int64 `json:"feed_id"`
+	Limit  int32 `json:"limit"`
+	Offset int32 `json:"offset"`
+}
+
+func (q *Queries) GetFeedExtensions(ctx context.Context, arg GetFeedExtensionsParams) ([]FeedExtension, error) {
+	rows, err := q.db.Query(ctx, getFeedExtensions, arg.FeedID, arg.Limit, arg.Offset)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	items := []FeedExtension{}
+	for rows.Next() {
+		var i FeedExtension
+		if err := rows.Scan(
+			&i.ID,
+			&i.CreatedAt,
+			&i.UpdatedAt,
+			&i.DeletedAt,
+			&i.Name,
+			&i.Value,
+			&i.Attrs,
+			&i.Children,
+			&i.FeedID,
+		); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
 const getFeeds = `-- name: GetFeeds :many
 SELECT
     id, url, created_at, updated_at, deleted_at, title, description, link, feed_link, links, updated, updated_parsed, published, published_parsed, language, copyright, generator, categories, custom, feed_type, feed_version
@@ -308,8 +473,10 @@ FROM
     feeds
 ORDER BY
     created_at DESC
-LIMIT $1
+LIMIT
-OFFSET $2
+    $1
+OFFSET
+    $2
 `
 
 type GetFeedsParams struct {
@@ -393,6 +560,57 @@ func (q *Queries) GetItem(ctx context.Context, id int64) (Item, error) {
 	return i, err
 }
 
+const getItemExtensions = `-- name: GetItemExtensions :many
+SELECT
+    id, created_at, updated_at, deleted_at, name, value, attrs, children, item_id
+FROM
+    item_extensions
+WHERE
+    item_id = $1
+ORDER BY
+    created_at DESC
+LIMIT
+    $2
+OFFSET
+    $3
+`
+
+type GetItemExtensionsParams struct {
+	ItemID int64 `json:"item_id"`
+	Limit  int32 `json:"limit"`
+	Offset int32 `json:"offset"`
+}
+
+func (q *Queries) GetItemExtensions(ctx context.Context, arg GetItemExtensionsParams) ([]ItemExtension, error) {
+	rows, err := q.db.Query(ctx, getItemExtensions, arg.ItemID, arg.Limit, arg.Offset)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	items := []ItemExtension{}
+	for rows.Next() {
+		var i ItemExtension
+		if err := rows.Scan(
+			&i.ID,
+			&i.CreatedAt,
+			&i.UpdatedAt,
+			&i.DeletedAt,
+			&i.Name,
+			&i.Value,
+			&i.Attrs,
+			&i.Children,
+			&i.ItemID,
+		); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
 const getItems = `-- name: GetItems :many
 SELECT
     id, created_at, updated_at, deleted_at, title, description, content, link, links, updated, updated_parsed, published, published_parsed, guid, categories, custom, feed_id
@@ -402,8 +620,10 @@ WHERE
     feed_id = $1
 ORDER BY
     created_at DESC
-LIMIT $2
+LIMIT
-OFFSET $3
+    $2
+OFFSET
+    $3
 `
 
 type GetItemsParams struct {

db/models.go

@@ -129,9 +129,9 @@ type Item struct {
 	Link            pgtype.Text        `json:"link"`
 	Links           []string           `json:"links"`
 	Updated         pgtype.Text        `json:"updated"`
-	UpdatedParsed   pgtype.Timestamp   `json:"updated_parsed"`
+	UpdatedParsed   pgtype.Timestamptz `json:"updated_parsed"`
 	Published       pgtype.Text        `json:"published"`
-	PublishedParsed pgtype.Timestamp   `json:"published_parsed"`
+	PublishedParsed pgtype.Timestamptz `json:"published_parsed"`
 	Guid            pgtype.Text        `json:"guid"`
 	Categories      []string           `json:"categories"`
 	Custom          []byte             `json:"custom"`
@@ -217,21 +217,21 @@ type ItemItune struct {
 }
 
 type ItunesCategory struct {
-	ID          int64            `json:"id"`
+	ID          int64              `json:"id"`
-	CreatedAt   pgtype.Timestamp `json:"created_at"`
+	CreatedAt   pgtype.Timestamptz `json:"created_at"`
-	UpdatedAt   pgtype.Timestamp `json:"updated_at"`
+	UpdatedAt   pgtype.Timestamptz `json:"updated_at"`
-	DeletedAt   pgtype.Timestamp `json:"deleted_at"`
+	DeletedAt   pgtype.Timestamptz `json:"deleted_at"`
-	Text        pgtype.Text      `json:"text"`
+	Text        pgtype.Text        `json:"text"`
-	Subcategory pgtype.Text      `json:"subcategory"`
+	Subcategory pgtype.Text        `json:"subcategory"`
-	ItunesID    int64            `json:"itunes_id"`
+	ItunesID    int64              `json:"itunes_id"`
 }
 
 type ItunesOwner struct {
-	ID        int64            `json:"id"`
+	ID        int64              `json:"id"`
-	CreatedAt pgtype.Timestamp `json:"created_at"`
+	CreatedAt pgtype.Timestamptz `json:"created_at"`
-	UpdatedAt pgtype.Timestamp `json:"updated_at"`
+	UpdatedAt pgtype.Timestamptz `json:"updated_at"`
-	DeletedAt pgtype.Timestamp `json:"deleted_at"`
+	DeletedAt pgtype.Timestamptz `json:"deleted_at"`
-	Email     pgtype.Text      `json:"email"`
+	Email     pgtype.Text        `json:"email"`
-	Name      pgtype.Text      `json:"name"`
+	Name      pgtype.Text        `json:"name"`
-	ItunesID  int64            `json:"itunes_id"`
+	ItunesID  int64              `json:"itunes_id"`
 }

feeds.go

@@ -57,6 +57,15 @@ func makeCreateFeedParams(feedURL string, feed *gofeed.Feed) db.CreateFeedParams
 }
 
 func makeCreateItemParams(item *gofeed.Item, feedID int64) db.CreateItemParams {
+	var updatedTime time.Time
+	if item.UpdatedParsed != nil {
+		updatedTime = *item.UpdatedParsed
+	}
+	var publishedTime time.Time
+	if item.PublishedParsed != nil {
+		publishedTime = *item.PublishedParsed
+	}
+
 	itemCustom := []byte("{}")
 	if item.Custom != nil {
 		var err error
@@ -77,9 +86,9 @@ func makeCreateItemParams(item *gofeed.Item, feedID int64) db.CreateItemParams {
 		Link:            pgtype.Text{String: item.Link, Valid: item.Link != ""},
 		Links:           item.Links,
 		Updated:         pgtype.Text{String: item.Updated, Valid: item.Updated != ""},
-		UpdatedParsed:   pgtype.Timestamp{Time: *item.UpdatedParsed, Valid: item.UpdatedParsed != nil},
+		UpdatedParsed:   pgtype.Timestamptz{Time: updatedTime, Valid: !updatedTime.IsZero()},
 		Published:       pgtype.Text{String: item.Published, Valid: item.Published != ""},
-		PublishedParsed: pgtype.Timestamp{Time: *item.PublishedParsed, Valid: item.PublishedParsed != nil},
+		PublishedParsed: pgtype.Timestamptz{Time: publishedTime, Valid: !publishedTime.IsZero()},
 		Guid:            pgtype.Text{String: item.GUID, Valid: item.GUID != ""},
 		Categories:      item.Categories,
 		Custom:          itemCustom,
@@ -110,14 +119,105 @@ func AddFeedToDB(feedURL string) error {
 	if err != nil {
 		return fmt.Errorf("Error adding feed to database: %s", err)
 	}
-	log.Printf("Added feed to database: %+v", newFeed)
+	log.Printf("Feed added to database")
 
 	// Add the items to the database
 	for _, item := range feed.Items {
-		_, err := DB.CreateItem(ctx, makeCreateItemParams(item, newFeed.ID))
+		newItem, err := DB.CreateItem(ctx, makeCreateItemParams(item, newFeed.ID))
 		if err != nil {
 			log.Printf("Error adding item to database: %s", err)
 		}
+		log.Printf("Item added to database")
+
+		// Add extensions to the database
+		for _, ext := range item.Extensions {
+			for _, exts := range ext {
+				for _, e := range exts {
+					attrsCustom := []byte("{}")
+					if e.Attrs != nil {
+						var err error
+						attrsCustom, err = json.Marshal(e.Attrs)
+						if err != nil {
+							fmt.Println("Error marshalling extension attributes:", err)
+							attrsCustom = []byte("{}")
+						}
+						log.Printf("Extension attributes: %s", attrsCustom)
+					}
+
+					childrenCustom := []byte("{}")
+					if e.Children != nil {
+						var err error
+						childrenCustom, err = json.Marshal(e.Children)
+						if err != nil {
+							fmt.Println("Error marshalling extension children:", err)
+							childrenCustom = []byte("{}")
+						}
+						log.Printf("Extension children: %s", childrenCustom)
+					}
+
+					_, err := DB.CreateItemExtension(ctx, db.CreateItemExtensionParams{
+						CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
+						UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
+						DeletedAt: pgtype.Timestamptz{Valid: false},
+						Name:      pgtype.Text{String: e.Name, Valid: e.Name != ""},
+						Value:     pgtype.Text{String: e.Value, Valid: e.Value != ""},
+						Attrs:     attrsCustom,
+						Children:  childrenCustom,
+						ItemID:    newItem.ID,
+					})
+
+					if err != nil {
+						log.Printf("Error adding extension to database: %s", err)
+					}
+
+					log.Printf("Extension added to database")
+				}
+			}
+		}
+
+	}
+
+	// Add extensions to the database
+	// TODO: Check if this is correct and works
+	for _, ext := range feed.Extensions {
+		for _, exts := range ext {
+			for _, e := range exts {
+				attrsCustom := []byte("{}")
+				if e.Attrs != nil {
+					var err error
+					attrsCustom, err = json.Marshal(e.Attrs)
+					if err != nil {
+						fmt.Println("Error marshalling extension attributes:", err)
+						attrsCustom = []byte("{}")
+					}
+				}
+
+				childrenCustom := []byte("{}")
+				if e.Children != nil {
+					var err error
+					childrenCustom, err = json.Marshal(e.Children)
+					if err != nil {
+						fmt.Println("Error marshalling extension children:", err)
+						childrenCustom = []byte("{}")
+					}
+				}
+
+				_, err := DB.CreateFeedExtension(ctx, db.CreateFeedExtensionParams{
+					CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
+					UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
+					DeletedAt: pgtype.Timestamptz{Valid: false},
+					Name:      pgtype.Text{String: e.Name, Valid: e.Name != ""},
+					Value:     pgtype.Text{String: e.Value, Valid: e.Value != ""},
+					Attrs:     attrsCustom,
+					Children:  childrenCustom,
+					FeedID:    newFeed.ID,
+				})
+
+				if err != nil {
+					log.Printf("Error adding extension to database: %s", err)
+				}
+			}
+		}
 	}
 
 	fmt.Println(feed.Title)

html.go

@@ -8,6 +8,14 @@ import (
 	"strings"
 )
 
+// Used for success/error message at the top of the page after adding a feed.
+type ParseResult struct {
+	FeedURL string
+	Msg     string
+	IsError bool
+}
+
+// HTMLData is the data passed to the HTML template.
 type HTMLData struct {
 	Title        string
 	Description  string
@@ -18,6 +26,7 @@ type HTMLData struct {
 	ParseResult  []ParseResult
 }
 
+// Our CSS that is included in the HTML.
 var style = `
 html {
 	max-width: 70ch;

models.go

@@ -1,7 +0,0 @@
-package main
-
-type ParseResult struct {
-	FeedURL string
-	Msg     string
-	IsError bool
-}

sql/queries/feeds.sql

@@ -44,7 +44,9 @@ VALUES
         $18,
         $19,
         $20
-    ) RETURNING *;
+    )
+RETURNING
+    *;
 
 -- name: CountFeeds :one
 SELECT
@@ -90,7 +92,9 @@ VALUES
         $14,
         $15,
         $16
-    ) RETURNING *;
+    )
+RETURNING
+    *;
 
 -- name: CountItems :one
 SELECT
@@ -113,8 +117,10 @@ FROM
     feeds
 ORDER BY
     created_at DESC
-LIMIT $1
+LIMIT
-OFFSET $2;
+    $1
+OFFSET
+    $2;
 
 -- name: GetItem :one
 SELECT
@@ -133,5 +139,69 @@ WHERE
     feed_id = $1
 ORDER BY
     created_at DESC
-LIMIT $2
+LIMIT
-OFFSET $3;
+    $2
+OFFSET
+    $3;
+
+-- name: CreateFeedExtension :one
+INSERT INTO
+    feed_extensions (
+        created_at,
+        updated_at,
+        deleted_at,
+        "name",
+        "value",
+        attrs,
+        children,
+        feed_id
+    )
+VALUES
+    ($1, $2, $3, $4, $5, $6, $7, $8)
+RETURNING
+    *;
+
+-- name: CreateItemExtension :one
+INSERT INTO
+    item_extensions (
+        created_at,
+        updated_at,
+        deleted_at,
+        "name",
+        "value",
+        attrs,
+        children,
+        item_id
+    )
+VALUES
+    ($1, $2, $3, $4, $5, $6, $7, $8)
+RETURNING
+    *;
+
+-- name: GetFeedExtensions :many
+SELECT
+    *
+FROM
+    feed_extensions
+WHERE
+    feed_id = $1
+ORDER BY
+    created_at DESC
+LIMIT
+    $2
+OFFSET
+    $3;
+
+-- name: GetItemExtensions :many
+SELECT
+    *
+FROM
+    item_extensions
+WHERE
+    item_id = $1
+ORDER BY
+    created_at DESC
+LIMIT
+    $2
+OFFSET
+    $3;

sql/schema/20240214043229_feeds.sql

@@ -46,9 +46,9 @@ CREATE TABLE IF NOT EXISTS items (
     link TEXT,
     links TEXT [],
     updated TEXT,
-    updated_parsed TIMESTAMP,
+    updated_parsed TIMESTAMPTZ,
     published TEXT,
-    published_parsed TIMESTAMP,
+    published_parsed TIMESTAMPTZ,
     -- Authors - See item_authors
     "guid" TEXT,
     -- Image - See item_images

sql/schema/20240215232318_itunes.sql

@@ -54,9 +54,9 @@ CREATE TABLE IF NOT EXISTS item_itunes (
 -- https://github.com/mmcdole/gofeed/blob/master/extensions/itunes.go#L39
 CREATE TABLE IF NOT EXISTS itunes_categories (
     id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
-    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    deleted_at TIMESTAMP DEFAULT NULL,
+    deleted_at TIMESTAMPTZ DEFAULT NULL,
     -- From gofeed:
     "text" TEXT,
     subcategory TEXT,
@@ -69,9 +69,9 @@ CREATE TABLE IF NOT EXISTS itunes_categories (
 -- https://github.com/mmcdole/gofeed/blob/master/extensions/itunes.go#L45
 CREATE TABLE IF NOT EXISTS itunes_owners (
     id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
-    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    deleted_at TIMESTAMP DEFAULT NULL,
+    deleted_at TIMESTAMPTZ DEFAULT NULL,
     -- From gofeed:
     email TEXT,
     "name" TEXT,