diff --git a/.vscode/launch.json b/.vscode/launch.json index bf72045..afb1b51 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,13 +1,15 @@ { + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ { - "name": "Python Debugger: Django", - "type": "debugpy", + "name": "Launch Package", + "type": "go", "request": "launch", - "program": "${workspaceFolder}\\manage.py", - "args": ["runserver"], - "django": true + "mode": "auto", + "program": "${workspaceFolder}" } ] } diff --git a/db/feeds.sql.go b/db/feeds.sql.go index ccb60dd..9b9469f 100644 --- a/db/feeds.sql.go +++ b/db/feeds.sql.go @@ -85,7 +85,9 @@ VALUES $18, $19, $20 - ) RETURNING id, url, created_at, updated_at, deleted_at, title, description, link, feed_link, links, updated, updated_parsed, published, published_parsed, language, copyright, generator, categories, custom, feed_type, feed_version + ) +RETURNING + id, url, created_at, updated_at, deleted_at, title, description, link, feed_link, links, updated, updated_parsed, published, published_parsed, language, copyright, generator, categories, custom, feed_type, feed_version ` type CreateFeedParams struct { @@ -161,6 +163,61 @@ func (q *Queries) CreateFeed(ctx context.Context, arg CreateFeedParams) (Feed, e return i, err } +const createFeedExtension = `-- name: CreateFeedExtension :one +INSERT INTO + feed_extensions ( + created_at, + updated_at, + deleted_at, + "name", + "value", + attrs, + children, + feed_id + ) +VALUES + ($1, $2, $3, $4, $5, $6, $7, $8) +RETURNING + id, created_at, updated_at, deleted_at, name, value, attrs, children, feed_id +` + +type CreateFeedExtensionParams struct { + CreatedAt pgtype.Timestamptz `json:"created_at"` + UpdatedAt pgtype.Timestamptz `json:"updated_at"` + DeletedAt pgtype.Timestamptz `json:"deleted_at"` + Name pgtype.Text `json:"name"` + Value pgtype.Text `json:"value"` + Attrs []byte `json:"attrs"` + Children []byte `json:"children"` + FeedID int64 `json:"feed_id"` +} + +func (q *Queries) CreateFeedExtension(ctx context.Context, arg CreateFeedExtensionParams) (FeedExtension, error) { + row := q.db.QueryRow(ctx, createFeedExtension, + arg.CreatedAt, + arg.UpdatedAt, + arg.DeletedAt, + arg.Name, + arg.Value, + arg.Attrs, + arg.Children, + arg.FeedID, + ) + var i FeedExtension + err := row.Scan( + &i.ID, + &i.CreatedAt, + &i.UpdatedAt, + &i.DeletedAt, + &i.Name, + &i.Value, + &i.Attrs, + &i.Children, + &i.FeedID, + ) + return i, err +} + const createItem = `-- name: CreateItem :one INSERT INTO items ( @@ -199,7 +256,9 @@ VALUES $14, $15, $16 - ) RETURNING id, created_at, updated_at, deleted_at, title, description, content, link, links, updated, updated_parsed, published, published_parsed, guid, categories, custom, feed_id + ) +RETURNING + id, created_at, updated_at, deleted_at, title, description, content, link, links, updated, updated_parsed, published, published_parsed, guid, categories, custom, feed_id ` type CreateItemParams struct { @@ -212,9 +271,9 @@ type CreateItemParams struct { Link pgtype.Text `json:"link"` Links []string `json:"links"` Updated pgtype.Text `json:"updated"` - UpdatedParsed pgtype.Timestamp `json:"updated_parsed"` + UpdatedParsed pgtype.Timestamptz `json:"updated_parsed"` Published pgtype.Text `json:"published"` - PublishedParsed pgtype.Timestamp `json:"published_parsed"` + PublishedParsed pgtype.Timestamptz `json:"published_parsed"` Guid pgtype.Text `json:"guid"` Categories []string `json:"categories"` Custom []byte `json:"custom"` @@ -263,6 +322,61 @@ func (q *Queries) CreateItem(ctx context.Context, arg CreateItemParams) (Item, e return i, err } +const createItemExtension = `-- name: CreateItemExtension :one +INSERT INTO + item_extensions ( + created_at, + updated_at, + deleted_at, + "name", + "value", + attrs, + children, + item_id + ) +VALUES + ($1, $2, $3, $4, $5, $6, $7, $8) +RETURNING + id, created_at, updated_at, deleted_at, name, value, attrs, children, item_id +` + +type CreateItemExtensionParams struct { + CreatedAt pgtype.Timestamptz `json:"created_at"` + UpdatedAt pgtype.Timestamptz `json:"updated_at"` + DeletedAt pgtype.Timestamptz `json:"deleted_at"` + Name pgtype.Text `json:"name"` + Value pgtype.Text `json:"value"` + Attrs []byte `json:"attrs"` + Children []byte `json:"children"` + ItemID int64 `json:"item_id"` +} + +func (q *Queries) CreateItemExtension(ctx context.Context, arg CreateItemExtensionParams) (ItemExtension, error) { + row := q.db.QueryRow(ctx, createItemExtension, + arg.CreatedAt, + arg.UpdatedAt, + arg.DeletedAt, + arg.Name, + arg.Value, + arg.Attrs, + arg.Children, + arg.ItemID, + ) + var i ItemExtension + err := row.Scan( + &i.ID, + &i.CreatedAt, + &i.UpdatedAt, + &i.DeletedAt, + &i.Name, + &i.Value, + &i.Attrs, + &i.Children, + &i.ItemID, + ) + return i, err +} + const getFeed = `-- name: GetFeed :one SELECT id, url, created_at, updated_at, deleted_at, title, description, link, feed_link, links, updated, updated_parsed, published, published_parsed, language, copyright, generator, categories, custom, feed_type, feed_version @@ -301,6 +415,57 @@ func (q *Queries) GetFeed(ctx context.Context, id int64) (Feed, error) { return i, err } +const getFeedExtensions = `-- name: GetFeedExtensions :many +SELECT + id, created_at, updated_at, deleted_at, name, value, attrs, children, feed_id +FROM + feed_extensions +WHERE + feed_id = $1 +ORDER BY + created_at DESC +LIMIT + $2 +OFFSET + $3 +` + +type GetFeedExtensionsParams struct { + FeedID int64 `json:"feed_id"` + Limit int32 `json:"limit"` + Offset int32 `json:"offset"` +} + +func (q *Queries) GetFeedExtensions(ctx context.Context, arg GetFeedExtensionsParams) ([]FeedExtension, error) { + rows, err := q.db.Query(ctx, getFeedExtensions, arg.FeedID, arg.Limit, arg.Offset) + if err != nil { + return nil, err + } + defer rows.Close() + items := []FeedExtension{} + for rows.Next() { + var i FeedExtension + if err := rows.Scan( + &i.ID, + &i.CreatedAt, + &i.UpdatedAt, + &i.DeletedAt, + &i.Name, + &i.Value, + &i.Attrs, + &i.Children, + &i.FeedID, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const getFeeds = `-- name: GetFeeds :many SELECT id, url, created_at, updated_at, deleted_at, title, description, link, feed_link, links, updated, updated_parsed, published, published_parsed, language, copyright, generator, categories, custom, feed_type, feed_version @@ -308,8 +473,10 @@ FROM feeds ORDER BY created_at DESC -LIMIT $1 -OFFSET $2 +LIMIT + $1 +OFFSET + $2 ` type GetFeedsParams struct { @@ -393,6 +560,57 @@ func (q *Queries) GetItem(ctx context.Context, id int64) (Item, error) { return i, err } +const getItemExtensions = `-- name: GetItemExtensions :many +SELECT + id, created_at, updated_at, deleted_at, name, value, attrs, children, item_id +FROM + item_extensions +WHERE + item_id = $1 +ORDER BY + created_at DESC +LIMIT + $2 +OFFSET + $3 +` + +type GetItemExtensionsParams struct { + ItemID int64 `json:"item_id"` + Limit int32 `json:"limit"` + Offset int32 `json:"offset"` +} + +func (q *Queries) GetItemExtensions(ctx context.Context, arg GetItemExtensionsParams) ([]ItemExtension, error) { + rows, err := q.db.Query(ctx, getItemExtensions, arg.ItemID, arg.Limit, arg.Offset) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ItemExtension{} + for rows.Next() { + var i ItemExtension + if err := rows.Scan( + &i.ID, + &i.CreatedAt, + &i.UpdatedAt, + &i.DeletedAt, + &i.Name, + &i.Value, + &i.Attrs, + &i.Children, + &i.ItemID, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const getItems = `-- name: GetItems :many SELECT id, created_at, updated_at, deleted_at, title, description, content, link, links, updated, updated_parsed, published, published_parsed, guid, categories, custom, feed_id @@ -402,8 +620,10 @@ WHERE feed_id = $1 ORDER BY created_at DESC -LIMIT $2 -OFFSET $3 +LIMIT + $2 +OFFSET + $3 ` type GetItemsParams struct { diff --git a/db/models.go b/db/models.go index b3a81b2..2f29fb1 100644 --- a/db/models.go +++ b/db/models.go @@ -129,9 +129,9 @@ type Item struct { Link pgtype.Text `json:"link"` Links []string `json:"links"` Updated pgtype.Text `json:"updated"` - UpdatedParsed pgtype.Timestamp `json:"updated_parsed"` + UpdatedParsed pgtype.Timestamptz `json:"updated_parsed"` Published pgtype.Text `json:"published"` - PublishedParsed pgtype.Timestamp `json:"published_parsed"` + PublishedParsed pgtype.Timestamptz `json:"published_parsed"` Guid pgtype.Text `json:"guid"` Categories []string `json:"categories"` Custom []byte `json:"custom"` @@ -217,21 +217,21 @@ type ItemItune struct { } type ItunesCategory struct { - ID int64 `json:"id"` - CreatedAt pgtype.Timestamp `json:"created_at"` - UpdatedAt pgtype.Timestamp `json:"updated_at"` - DeletedAt pgtype.Timestamp `json:"deleted_at"` - Text pgtype.Text `json:"text"` - Subcategory pgtype.Text `json:"subcategory"` - ItunesID int64 `json:"itunes_id"` + ID int64 `json:"id"` + CreatedAt pgtype.Timestamptz `json:"created_at"` + UpdatedAt pgtype.Timestamptz `json:"updated_at"` + DeletedAt pgtype.Timestamptz `json:"deleted_at"` + Text pgtype.Text `json:"text"` + Subcategory pgtype.Text `json:"subcategory"` + ItunesID int64 `json:"itunes_id"` } type ItunesOwner struct { - ID int64 `json:"id"` - CreatedAt pgtype.Timestamp `json:"created_at"` - UpdatedAt pgtype.Timestamp `json:"updated_at"` - DeletedAt pgtype.Timestamp `json:"deleted_at"` - Email pgtype.Text `json:"email"` - Name pgtype.Text `json:"name"` - ItunesID int64 `json:"itunes_id"` + ID int64 `json:"id"` + CreatedAt pgtype.Timestamptz `json:"created_at"` + UpdatedAt pgtype.Timestamptz `json:"updated_at"` + DeletedAt pgtype.Timestamptz `json:"deleted_at"` + Email pgtype.Text `json:"email"` + Name pgtype.Text `json:"name"` + ItunesID int64 `json:"itunes_id"` } diff --git a/feeds.go b/feeds.go index 3c47508..a2fd298 100644 --- a/feeds.go +++ b/feeds.go @@ -57,6 +57,15 @@ func makeCreateFeedParams(feedURL string, feed *gofeed.Feed) db.CreateFeedParams } func makeCreateItemParams(item *gofeed.Item, feedID int64) db.CreateItemParams { + var updatedTime time.Time + if item.UpdatedParsed != nil { + updatedTime = *item.UpdatedParsed + } + var publishedTime time.Time + if item.PublishedParsed != nil { + publishedTime = *item.PublishedParsed + } + itemCustom := []byte("{}") if item.Custom != nil { var err error @@ -77,9 +86,9 @@ func makeCreateItemParams(item *gofeed.Item, feedID int64) db.CreateItemParams { Link: pgtype.Text{String: item.Link, Valid: item.Link != ""}, Links: item.Links, Updated: pgtype.Text{String: item.Updated, Valid: item.Updated != ""}, - UpdatedParsed: pgtype.Timestamp{Time: *item.UpdatedParsed, Valid: item.UpdatedParsed != nil}, + UpdatedParsed: pgtype.Timestamptz{Time: updatedTime, Valid: !updatedTime.IsZero()}, Published: pgtype.Text{String: item.Published, Valid: item.Published != ""}, - PublishedParsed: pgtype.Timestamp{Time: *item.PublishedParsed, Valid: item.PublishedParsed != nil}, + PublishedParsed: pgtype.Timestamptz{Time: publishedTime, Valid: !publishedTime.IsZero()}, Guid: pgtype.Text{String: item.GUID, Valid: item.GUID != ""}, Categories: item.Categories, Custom: itemCustom, @@ -110,14 +119,105 @@ func AddFeedToDB(feedURL string) error { if err != nil { return fmt.Errorf("Error adding feed to database: %s", err) } - log.Printf("Added feed to database: %+v", newFeed) + log.Printf("Feed added to database") // Add the items to the database for _, item := range feed.Items { - _, err := DB.CreateItem(ctx, makeCreateItemParams(item, newFeed.ID)) + newItem, err := DB.CreateItem(ctx, makeCreateItemParams(item, newFeed.ID)) if err != nil { log.Printf("Error adding item to database: %s", err) } + log.Printf("Item added to database") + + // Add extensions to the database + for _, ext := range item.Extensions { + for _, exts := range ext { + for _, e := range exts { + attrsCustom := []byte("{}") + if e.Attrs != nil { + var err error + attrsCustom, err = json.Marshal(e.Attrs) + if err != nil { + fmt.Println("Error marshalling extension attributes:", err) + attrsCustom = []byte("{}") + } + log.Printf("Extension attributes: %s", attrsCustom) + } + + childrenCustom := []byte("{}") + if e.Children != nil { + var err error + childrenCustom, err = json.Marshal(e.Children) + if err != nil { + fmt.Println("Error marshalling extension children:", err) + childrenCustom = []byte("{}") + } + log.Printf("Extension children: %s", childrenCustom) + } + + _, err := DB.CreateItemExtension(ctx, db.CreateItemExtensionParams{ + CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, + UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, + DeletedAt: pgtype.Timestamptz{Valid: false}, + Name: pgtype.Text{String: e.Name, Valid: e.Name != ""}, + Value: pgtype.Text{String: e.Value, Valid: e.Value != ""}, + Attrs: attrsCustom, + Children: childrenCustom, + ItemID: newItem.ID, + }) + + if err != nil { + log.Printf("Error adding extension to database: %s", err) + } + + log.Printf("Extension added to database") + } + } + } + + } + + // Add extensions to the database + // TODO: Check if this is correct and works + for _, ext := range feed.Extensions { + for _, exts := range ext { + for _, e := range exts { + attrsCustom := []byte("{}") + if e.Attrs != nil { + var err error + attrsCustom, err = json.Marshal(e.Attrs) + if err != nil { + fmt.Println("Error marshalling extension attributes:", err) + attrsCustom = []byte("{}") + } + } + + childrenCustom := []byte("{}") + if e.Children != nil { + var err error + childrenCustom, err = json.Marshal(e.Children) + if err != nil { + fmt.Println("Error marshalling extension children:", err) + childrenCustom = []byte("{}") + } + } + + _, err := DB.CreateFeedExtension(ctx, db.CreateFeedExtensionParams{ + CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, + UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, + DeletedAt: pgtype.Timestamptz{Valid: false}, + Name: pgtype.Text{String: e.Name, Valid: e.Name != ""}, + Value: pgtype.Text{String: e.Value, Valid: e.Value != ""}, + Attrs: attrsCustom, + Children: childrenCustom, + FeedID: newFeed.ID, + }) + + if err != nil { + log.Printf("Error adding extension to database: %s", err) + } + } + } } fmt.Println(feed.Title) diff --git a/html.go b/html.go index 7bf077f..048bef0 100644 --- a/html.go +++ b/html.go @@ -8,6 +8,14 @@ import ( "strings" ) +// Used for success/error message at the top of the page after adding a feed. +type ParseResult struct { + FeedURL string + Msg string + IsError bool +} + +// HTMLData is the data passed to the HTML template. type HTMLData struct { Title string Description string @@ -18,6 +26,7 @@ type HTMLData struct { ParseResult []ParseResult } +// Our CSS that is included in the HTML. var style = ` html { max-width: 70ch; diff --git a/models.go b/models.go deleted file mode 100644 index 9a5539b..0000000 --- a/models.go +++ /dev/null @@ -1,7 +0,0 @@ -package main - -type ParseResult struct { - FeedURL string - Msg string - IsError bool -} diff --git a/sql/queries/feeds.sql b/sql/queries/feeds.sql index 90731b1..21b29b5 100644 --- a/sql/queries/feeds.sql +++ b/sql/queries/feeds.sql @@ -44,7 +44,9 @@ VALUES $18, $19, $20 - ) RETURNING *; + ) +RETURNING + *; -- name: CountFeeds :one SELECT @@ -90,7 +92,9 @@ VALUES $14, $15, $16 - ) RETURNING *; + ) +RETURNING + *; -- name: CountItems :one SELECT @@ -113,8 +117,10 @@ FROM feeds ORDER BY created_at DESC -LIMIT $1 -OFFSET $2; +LIMIT + $1 +OFFSET + $2; -- name: GetItem :one SELECT @@ -133,5 +139,69 @@ WHERE feed_id = $1 ORDER BY created_at DESC -LIMIT $2 -OFFSET $3; +LIMIT + $2 +OFFSET + $3; + +-- name: CreateFeedExtension :one +INSERT INTO + feed_extensions ( + created_at, + updated_at, + deleted_at, + "name", + "value", + attrs, + children, + feed_id + ) +VALUES + ($1, $2, $3, $4, $5, $6, $7, $8) +RETURNING + *; + +-- name: CreateItemExtension :one +INSERT INTO + item_extensions ( + created_at, + updated_at, + deleted_at, + "name", + "value", + attrs, + children, + item_id + ) +VALUES + ($1, $2, $3, $4, $5, $6, $7, $8) +RETURNING + *; + +-- name: GetFeedExtensions :many +SELECT + * +FROM + feed_extensions +WHERE + feed_id = $1 +ORDER BY + created_at DESC +LIMIT + $2 +OFFSET + $3; + +-- name: GetItemExtensions :many +SELECT + * +FROM + item_extensions +WHERE + item_id = $1 +ORDER BY + created_at DESC +LIMIT + $2 +OFFSET + $3; diff --git a/sql/schema/20240214043229_feeds.sql b/sql/schema/20240214043229_feeds.sql index 87911c3..6fb80c6 100644 --- a/sql/schema/20240214043229_feeds.sql +++ b/sql/schema/20240214043229_feeds.sql @@ -46,9 +46,9 @@ CREATE TABLE IF NOT EXISTS items ( link TEXT, links TEXT [], updated TEXT, - updated_parsed TIMESTAMP, + updated_parsed TIMESTAMPTZ, published TEXT, - published_parsed TIMESTAMP, + published_parsed TIMESTAMPTZ, -- Authors - See item_authors "guid" TEXT, -- Image - See item_images diff --git a/sql/schema/20240215232318_itunes.sql b/sql/schema/20240215232318_itunes.sql index 6c356ac..6bf75eb 100644 --- a/sql/schema/20240215232318_itunes.sql +++ b/sql/schema/20240215232318_itunes.sql @@ -54,9 +54,9 @@ CREATE TABLE IF NOT EXISTS item_itunes ( -- https://github.com/mmcdole/gofeed/blob/master/extensions/itunes.go#L39 CREATE TABLE IF NOT EXISTS itunes_categories ( id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - deleted_at TIMESTAMP DEFAULT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + deleted_at TIMESTAMPTZ DEFAULT NULL, -- From gofeed: "text" TEXT, subcategory TEXT, @@ -69,9 +69,9 @@ CREATE TABLE IF NOT EXISTS itunes_categories ( -- https://github.com/mmcdole/gofeed/blob/master/extensions/itunes.go#L45 CREATE TABLE IF NOT EXISTS itunes_owners ( id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - deleted_at TIMESTAMP DEFAULT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + deleted_at TIMESTAMPTZ DEFAULT NULL, -- From gofeed: email TEXT, "name" TEXT,