From 8f7690d8f453bb39a8111d43016aa5ccbc409e01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Hells=C3=A9n?= Date: Sun, 18 Feb 2024 01:05:30 +0100 Subject: [PATCH] Add support for Dublin Core --- add_feed.go | 91 ---------- add_item.go | 85 +--------- authors.go | 53 ++++++ db/feeds.sql.go | 374 ++++++++++++++++++++++++++++++++++++++++++ dublincore.go | 84 ++++++++++ extensions.go | 105 ++++++++++++ feeds.go | 4 + images.go | 49 ++++++ sql/queries/feeds.sql | 128 +++++++++++++++ 9 files changed, 800 insertions(+), 173 deletions(-) create mode 100644 authors.go create mode 100644 dublincore.go create mode 100644 extensions.go create mode 100644 images.go diff --git a/add_feed.go b/add_feed.go index eaa37c0..6244281 100644 --- a/add_feed.go +++ b/add_feed.go @@ -1,7 +1,6 @@ package main import ( - "context" "encoding/json" "fmt" "log" @@ -55,93 +54,3 @@ func makeCreateFeedParams(feedURL string, feed *gofeed.Feed) db.CreateFeedParams return params } - -func addFeedExtensionToDB(ctx context.Context, feed *gofeed.Feed, newFeed db.Feed) { - // Add extensions to the database - // TODO: Check if this is correct and works - for _, ext := range feed.Extensions { - for _, exts := range ext { - for _, e := range exts { - attrsCustom := []byte("{}") - if e.Attrs != nil { - var err error - attrsCustom, err = json.Marshal(e.Attrs) - if err != nil { - fmt.Println("Error marshalling extension attributes:", err) - attrsCustom = []byte("{}") - } - } - - childrenCustom := []byte("{}") - if e.Children != nil { - var err error - childrenCustom, err = json.Marshal(e.Children) - if err != nil { - fmt.Println("Error marshalling extension children:", err) - childrenCustom = []byte("{}") - } - } - - _, err := DB.CreateFeedExtension(ctx, db.CreateFeedExtensionParams{ - CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, - UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, - DeletedAt: pgtype.Timestamptz{Valid: false}, - Name: pgtype.Text{String: e.Name, Valid: e.Name != ""}, - Value: pgtype.Text{String: e.Value, Valid: e.Value != ""}, - Attrs: attrsCustom, - Children: childrenCustom, - FeedID: newFeed.ID, - }) - - if err != nil { - log.Printf("Error adding extension to database: %s", err) - } - } - } - } -} - -func addFeedAuthors(ctx context.Context, feed *gofeed.Feed, newFeed db.Feed) { - if feed.Authors == nil { - log.Printf("No authors to add to database") - return - } - - // Add authors to the database - for _, author := range feed.Authors { - _, err := DB.CreateFeedAuthor(ctx, db.CreateFeedAuthorParams{ - CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, - UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, - DeletedAt: pgtype.Timestamptz{Valid: false}, - Name: pgtype.Text{String: author.Name, Valid: author.Name != ""}, - FeedID: newFeed.ID, - }) - if err != nil { - log.Printf("Error adding author %s (%s) to database: %s", author.Name, author.Email, err) - continue - } - log.Printf("Author %s (%s) added to database", author.Name, author.Email) - } -} - -func addFeedImages(ctx context.Context, feed *gofeed.Feed, newFeed db.Feed) { - if feed.Image == nil { - log.Printf("No image to add to database") - return - } - - // TODO: Download the image and store it on the server - _, err := DB.CreateFeedImage(ctx, db.CreateFeedImageParams{ - CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, - UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, - DeletedAt: pgtype.Timestamptz{Valid: false}, - Url: pgtype.Text{String: feed.Image.URL, Valid: feed.Image.URL != ""}, - Title: pgtype.Text{String: feed.Image.Title, Valid: feed.Image.Title != ""}, - FeedID: newFeed.ID, - }) - if err != nil { - log.Printf("Error adding image to database: %s", err) - return - } - log.Printf("Image added to database: %s", feed.Image.URL) -} diff --git a/add_item.go b/add_item.go index a296770..c1f02b4 100644 --- a/add_item.go +++ b/add_item.go @@ -29,6 +29,9 @@ func addItemToDB(item *gofeed.Item, ctx context.Context, newFeed db.Feed) { addItemImages(ctx, item, newItem) } + // Add Dublin Core to the database + createItemDublinCore(ctx, item, newItem) + log.Printf("Item added to database") } @@ -75,85 +78,3 @@ func makeCreateItemParams(item *gofeed.Item, feedID int64) db.CreateItemParams { return params } - -func addItemExtensionToDB(ctx context.Context, item *gofeed.Item, newItem db.Item) { - // Add extensions to the database - for _, ext := range item.Extensions { - for _, exts := range ext { - for _, e := range exts { - attrsCustom := []byte("{}") - if e.Attrs != nil { - var err error - attrsCustom, err = json.Marshal(e.Attrs) - if err != nil { - fmt.Println("Error marshalling extension attributes:", err) - attrsCustom = []byte("{}") - } - } - - childrenCustom := []byte("{}") - if e.Children != nil { - var err error - childrenCustom, err = json.Marshal(e.Children) - if err != nil { - fmt.Println("Error marshalling extension children:", err) - childrenCustom = []byte("{}") - } - } - - _, err := DB.CreateItemExtension(ctx, db.CreateItemExtensionParams{ - CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, - UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, - DeletedAt: pgtype.Timestamptz{Valid: false}, - Name: pgtype.Text{String: e.Name, Valid: e.Name != ""}, - Value: pgtype.Text{String: e.Value, Valid: e.Value != ""}, - Attrs: attrsCustom, - Children: childrenCustom, - ItemID: newItem.ID, - }) - - if err != nil { - log.Printf("Error adding extension to database: %s", err) - continue - } - - log.Printf("Extension added to database") - } - } - } -} - -func addItemAuthors(ctx context.Context, item *gofeed.Item, newItem db.Item) { - for _, author := range item.Authors { - _, err := DB.CreateItemAuthor(ctx, db.CreateItemAuthorParams{ - CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, - UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, - DeletedAt: pgtype.Timestamptz{Valid: false}, - Name: pgtype.Text{String: author.Name, Valid: author.Name != ""}, - Email: pgtype.Text{String: author.Email, Valid: author.Email != ""}, - ItemID: newItem.ID, - }) - - if err != nil { - log.Printf("Error adding author %s (%s) to database: %s", author.Name, author.Email, err) - continue - } - log.Printf("Author %s (%s) added to database", author.Name, author.Email) - } -} - -func addItemImages(ctx context.Context, item *gofeed.Item, newItem db.Item) { - _, err := DB.CreateItemImage(ctx, db.CreateItemImageParams{ - CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, - UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, - DeletedAt: pgtype.Timestamptz{Valid: false}, - Url: pgtype.Text{String: item.Image.URL, Valid: item.Image.URL != ""}, - Title: pgtype.Text{String: item.Image.Title, Valid: item.Image.Title != ""}, - ItemID: newItem.ID, - }) - if err != nil { - log.Printf("Error adding image to database: %s", err) - return - } - log.Printf("Image added to database: %s", item.Image.URL) -} diff --git a/authors.go b/authors.go new file mode 100644 index 0000000..709f40f --- /dev/null +++ b/authors.go @@ -0,0 +1,53 @@ +package main + +import ( + "context" + "log" + "time" + + "github.com/TheLovinator1/FeedVault/db" + "github.com/jackc/pgx/v5/pgtype" + "github.com/mmcdole/gofeed" +) + +func addFeedAuthors(ctx context.Context, feed *gofeed.Feed, newFeed db.Feed) { + if feed.Authors == nil { + log.Printf("No authors to add to database") + return + } + + // Add authors to the database + for _, author := range feed.Authors { + _, err := DB.CreateFeedAuthor(ctx, db.CreateFeedAuthorParams{ + CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, + UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, + DeletedAt: pgtype.Timestamptz{Valid: false}, + Name: pgtype.Text{String: author.Name, Valid: author.Name != ""}, + FeedID: newFeed.ID, + }) + if err != nil { + log.Printf("Error adding author %s (%s) to database: %s", author.Name, author.Email, err) + continue + } + log.Printf("Author %s (%s) added to database", author.Name, author.Email) + } +} + +func addItemAuthors(ctx context.Context, item *gofeed.Item, newItem db.Item) { + for _, author := range item.Authors { + _, err := DB.CreateItemAuthor(ctx, db.CreateItemAuthorParams{ + CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, + UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, + DeletedAt: pgtype.Timestamptz{Valid: false}, + Name: pgtype.Text{String: author.Name, Valid: author.Name != ""}, + Email: pgtype.Text{String: author.Email, Valid: author.Email != ""}, + ItemID: newItem.ID, + }) + + if err != nil { + log.Printf("Error adding author %s (%s) to database: %s", author.Name, author.Email, err) + continue + } + log.Printf("Author %s (%s) added to database", author.Name, author.Email) + } +} diff --git a/db/feeds.sql.go b/db/feeds.sql.go index 94aea86..ac771b3 100644 --- a/db/feeds.sql.go +++ b/db/feeds.sql.go @@ -210,6 +210,130 @@ func (q *Queries) CreateFeedAuthor(ctx context.Context, arg CreateFeedAuthorPara return i, err } +const createFeedDublinCore = `-- name: CreateFeedDublinCore :one +INSERT INTO + feed_dublin_cores ( + created_at, + updated_at, + deleted_at, + title, + creator, + author, + "subject", + "description", + publisher, + contributor, + "date", + "type", + format, + identifier, + source, + "language", + relation, + coverage, + rights, + feed_id + ) +VALUES + ( + $1, + $2, + $3, + $4, + $5, + $6, + $7, + $8, + $9, + $10, + $11, + $12, + $13, + $14, + $15, + $16, + $17, + $18, + $19, + $20 + ) +RETURNING + id, created_at, updated_at, deleted_at, title, creator, author, subject, description, publisher, contributor, date, type, format, identifier, source, language, relation, coverage, rights, feed_id +` + +type CreateFeedDublinCoreParams struct { + CreatedAt pgtype.Timestamptz `json:"created_at"` + UpdatedAt pgtype.Timestamptz `json:"updated_at"` + DeletedAt pgtype.Timestamptz `json:"deleted_at"` + Title []string `json:"title"` + Creator []string `json:"creator"` + Author []string `json:"author"` + Subject []string `json:"subject"` + Description []string `json:"description"` + Publisher []string `json:"publisher"` + Contributor []string `json:"contributor"` + Date []string `json:"date"` + Type []string `json:"type"` + Format []string `json:"format"` + Identifier []string `json:"identifier"` + Source []string `json:"source"` + Language []string `json:"language"` + Relation []string `json:"relation"` + Coverage []string `json:"coverage"` + Rights []string `json:"rights"` + FeedID int64 `json:"feed_id"` +} + +func (q *Queries) CreateFeedDublinCore(ctx context.Context, arg CreateFeedDublinCoreParams) (FeedDublinCore, error) { + row := q.db.QueryRow(ctx, createFeedDublinCore, + arg.CreatedAt, + arg.UpdatedAt, + arg.DeletedAt, + arg.Title, + arg.Creator, + arg.Author, + arg.Subject, + arg.Description, + arg.Publisher, + arg.Contributor, + arg.Date, + arg.Type, + arg.Format, + arg.Identifier, + arg.Source, + arg.Language, + arg.Relation, + arg.Coverage, + arg.Rights, + arg.FeedID, + ) + var i FeedDublinCore + err := row.Scan( + &i.ID, + &i.CreatedAt, + &i.UpdatedAt, + &i.DeletedAt, + &i.Title, + &i.Creator, + &i.Author, + &i.Subject, + &i.Description, + &i.Publisher, + &i.Contributor, + &i.Date, + &i.Type, + &i.Format, + &i.Identifier, + &i.Source, + &i.Language, + &i.Relation, + &i.Coverage, + &i.Rights, + &i.FeedID, + ) + return i, err +} + const createFeedExtension = `-- name: CreateFeedExtension :one INSERT INTO feed_extensions ( @@ -463,6 +587,130 @@ func (q *Queries) CreateItemAuthor(ctx context.Context, arg CreateItemAuthorPara return i, err } +const createItemDublinCore = `-- name: CreateItemDublinCore :one +INSERT INTO + item_dublin_cores ( + created_at, + updated_at, + deleted_at, + title, + creator, + author, + "subject", + "description", + publisher, + contributor, + "date", + "type", + format, + identifier, + source, + "language", + relation, + coverage, + rights, + item_id + ) +VALUES + ( + $1, + $2, + $3, + $4, + $5, + $6, + $7, + $8, + $9, + $10, + $11, + $12, + $13, + $14, + $15, + $16, + $17, + $18, + $19, + $20 + ) +RETURNING + id, created_at, updated_at, deleted_at, title, creator, author, subject, description, publisher, contributor, date, type, format, identifier, source, language, relation, coverage, rights, item_id +` + +type CreateItemDublinCoreParams struct { + CreatedAt pgtype.Timestamptz `json:"created_at"` + UpdatedAt pgtype.Timestamptz `json:"updated_at"` + DeletedAt pgtype.Timestamptz `json:"deleted_at"` + Title []string `json:"title"` + Creator []string `json:"creator"` + Author []string `json:"author"` + Subject []string `json:"subject"` + Description []string `json:"description"` + Publisher []string `json:"publisher"` + Contributor []string `json:"contributor"` + Date []string `json:"date"` + Type []string `json:"type"` + Format []string `json:"format"` + Identifier []string `json:"identifier"` + Source []string `json:"source"` + Language []string `json:"language"` + Relation []string `json:"relation"` + Coverage []string `json:"coverage"` + Rights []string `json:"rights"` + ItemID int64 `json:"item_id"` +} + +func (q *Queries) CreateItemDublinCore(ctx context.Context, arg CreateItemDublinCoreParams) (ItemDublinCore, error) { + row := q.db.QueryRow(ctx, createItemDublinCore, + arg.CreatedAt, + arg.UpdatedAt, + arg.DeletedAt, + arg.Title, + arg.Creator, + arg.Author, + arg.Subject, + arg.Description, + arg.Publisher, + arg.Contributor, + arg.Date, + arg.Type, + arg.Format, + arg.Identifier, + arg.Source, + arg.Language, + arg.Relation, + arg.Coverage, + arg.Rights, + arg.ItemID, + ) + var i ItemDublinCore + err := row.Scan( + &i.ID, + &i.CreatedAt, + &i.UpdatedAt, + &i.DeletedAt, + &i.Title, + &i.Creator, + &i.Author, + &i.Subject, + &i.Description, + &i.Publisher, + &i.Contributor, + &i.Date, + &i.Type, + &i.Format, + &i.Identifier, + &i.Source, + &i.Language, + &i.Relation, + &i.Coverage, + &i.Rights, + &i.ItemID, + ) + return i, err +} + const createItemExtension = `-- name: CreateItemExtension :one INSERT INTO item_extensions ( @@ -652,6 +900,69 @@ func (q *Queries) GetFeedAuthors(ctx context.Context, arg GetFeedAuthorsParams) return items, nil } +const getFeedDublinCores = `-- name: GetFeedDublinCores :many +SELECT + id, created_at, updated_at, deleted_at, title, creator, author, subject, description, publisher, contributor, date, type, format, identifier, source, language, relation, coverage, rights, feed_id +FROM + feed_dublin_cores +WHERE + feed_id = $1 +ORDER BY + created_at DESC +LIMIT + $2 +OFFSET + $3 +` + +type GetFeedDublinCoresParams struct { + FeedID int64 `json:"feed_id"` + Limit int32 `json:"limit"` + Offset int32 `json:"offset"` +} + +func (q *Queries) GetFeedDublinCores(ctx context.Context, arg GetFeedDublinCoresParams) ([]FeedDublinCore, error) { + rows, err := q.db.Query(ctx, getFeedDublinCores, arg.FeedID, arg.Limit, arg.Offset) + if err != nil { + return nil, err + } + defer rows.Close() + items := []FeedDublinCore{} + for rows.Next() { + var i FeedDublinCore + if err := rows.Scan( + &i.ID, + &i.CreatedAt, + &i.UpdatedAt, + &i.DeletedAt, + &i.Title, + &i.Creator, + &i.Author, + &i.Subject, + &i.Description, + &i.Publisher, + &i.Contributor, + &i.Date, + &i.Type, + &i.Format, + &i.Identifier, + &i.Source, + &i.Language, + &i.Relation, + &i.Coverage, + &i.Rights, + &i.FeedID, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const getFeedExtensions = `-- name: GetFeedExtensions :many SELECT id, created_at, updated_at, deleted_at, name, value, attrs, children, feed_id @@ -895,6 +1206,69 @@ func (q *Queries) GetItemAuthors(ctx context.Context, arg GetItemAuthorsParams) return items, nil } +const getItemDublinCores = `-- name: GetItemDublinCores :many +SELECT + id, created_at, updated_at, deleted_at, title, creator, author, subject, description, publisher, contributor, date, type, format, identifier, source, language, relation, coverage, rights, item_id +FROM + item_dublin_cores +WHERE + item_id = $1 +ORDER BY + created_at DESC +LIMIT + $2 +OFFSET + $3 +` + +type GetItemDublinCoresParams struct { + ItemID int64 `json:"item_id"` + Limit int32 `json:"limit"` + Offset int32 `json:"offset"` +} + +func (q *Queries) GetItemDublinCores(ctx context.Context, arg GetItemDublinCoresParams) ([]ItemDublinCore, error) { + rows, err := q.db.Query(ctx, getItemDublinCores, arg.ItemID, arg.Limit, arg.Offset) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ItemDublinCore{} + for rows.Next() { + var i ItemDublinCore + if err := rows.Scan( + &i.ID, + &i.CreatedAt, + &i.UpdatedAt, + &i.DeletedAt, + &i.Title, + &i.Creator, + &i.Author, + &i.Subject, + &i.Description, + &i.Publisher, + &i.Contributor, + &i.Date, + &i.Type, + &i.Format, + &i.Identifier, + &i.Source, + &i.Language, + &i.Relation, + &i.Coverage, + &i.Rights, + &i.ItemID, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const getItemExtensions = `-- name: GetItemExtensions :many SELECT id, created_at, updated_at, deleted_at, name, value, attrs, children, item_id diff --git a/dublincore.go b/dublincore.go new file mode 100644 index 0000000..8b9fde4 --- /dev/null +++ b/dublincore.go @@ -0,0 +1,84 @@ +package main + +import ( + "context" + "log" + "time" + + "github.com/TheLovinator1/FeedVault/db" + "github.com/jackc/pgx/v5/pgtype" + "github.com/mmcdole/gofeed" +) + +func createFeedDublinCore(ctx context.Context, feed *gofeed.Feed, newFeed db.Feed) { + // TODO: Check if this is correct and works. I can't find a feed that has Dublin Core to test with :-) + if feed.DublinCoreExt == nil { + log.Printf("No Dublin Core to add to database") + return + } + + // Add Dublin Core to the database + _, err := DB.CreateFeedDublinCore(ctx, db.CreateFeedDublinCoreParams{ + CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, + UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, + DeletedAt: pgtype.Timestamptz{Valid: false}, + Title: feed.DublinCoreExt.Title, + Creator: feed.DublinCoreExt.Creator, + Subject: feed.DublinCoreExt.Subject, + Source: feed.DublinCoreExt.Source, + Publisher: feed.DublinCoreExt.Publisher, + Contributor: feed.DublinCoreExt.Contributor, + Description: feed.DublinCoreExt.Description, + Date: feed.DublinCoreExt.Date, + Type: feed.DublinCoreExt.Type, + Format: feed.DublinCoreExt.Format, + Identifier: feed.DublinCoreExt.Identifier, + Language: feed.DublinCoreExt.Language, + Relation: feed.DublinCoreExt.Relation, + Coverage: feed.DublinCoreExt.Coverage, + Rights: feed.DublinCoreExt.Rights, + FeedID: newFeed.ID, + }) + + if err != nil { + log.Printf("Error adding Dublin Core to database: %s", err) + return + } + log.Printf("Dublin Core added to database") +} + +func createItemDublinCore(ctx context.Context, item *gofeed.Item, newItem db.Item) { + if item.DublinCoreExt == nil { + log.Printf("No Dublin Core to add to database") + return + } + + // Add Dublin Core to the database + _, err := DB.CreateItemDublinCore(ctx, db.CreateItemDublinCoreParams{ + CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, + UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, + DeletedAt: pgtype.Timestamptz{Valid: false}, + Title: item.DublinCoreExt.Title, + Creator: item.DublinCoreExt.Creator, + Subject: item.DublinCoreExt.Subject, + Source: item.DublinCoreExt.Source, + Publisher: item.DublinCoreExt.Publisher, + Contributor: item.DublinCoreExt.Contributor, + Description: item.DublinCoreExt.Description, + Date: item.DublinCoreExt.Date, + Type: item.DublinCoreExt.Type, + Format: item.DublinCoreExt.Format, + Identifier: item.DublinCoreExt.Identifier, + Language: item.DublinCoreExt.Language, + Relation: item.DublinCoreExt.Relation, + Coverage: item.DublinCoreExt.Coverage, + Rights: item.DublinCoreExt.Rights, + ItemID: newItem.ID, + }) + + if err != nil { + log.Printf("Error adding Dublin Core to database: %s", err) + return + } + log.Printf("Dublin Core added to database") +} diff --git a/extensions.go b/extensions.go new file mode 100644 index 0000000..c45b5c5 --- /dev/null +++ b/extensions.go @@ -0,0 +1,105 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "log" + "time" + + "github.com/TheLovinator1/FeedVault/db" + "github.com/jackc/pgx/v5/pgtype" + "github.com/mmcdole/gofeed" +) + +func addFeedExtensionToDB(ctx context.Context, feed *gofeed.Feed, newFeed db.Feed) { + // Add extensions to the database + // TODO: Check if this is correct and works + for _, ext := range feed.Extensions { + for _, exts := range ext { + for _, e := range exts { + attrsCustom := []byte("{}") + if e.Attrs != nil { + var err error + attrsCustom, err = json.Marshal(e.Attrs) + if err != nil { + fmt.Println("Error marshalling extension attributes:", err) + attrsCustom = []byte("{}") + } + } + + childrenCustom := []byte("{}") + if e.Children != nil { + var err error + childrenCustom, err = json.Marshal(e.Children) + if err != nil { + fmt.Println("Error marshalling extension children:", err) + childrenCustom = []byte("{}") + } + } + + _, err := DB.CreateFeedExtension(ctx, db.CreateFeedExtensionParams{ + CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, + UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, + DeletedAt: pgtype.Timestamptz{Valid: false}, + Name: pgtype.Text{String: e.Name, Valid: e.Name != ""}, + Value: pgtype.Text{String: e.Value, Valid: e.Value != ""}, + Attrs: attrsCustom, + Children: childrenCustom, + FeedID: newFeed.ID, + }) + + if err != nil { + log.Printf("Error adding extension to database: %s", err) + } + } + } + } +} + +func addItemExtensionToDB(ctx context.Context, item *gofeed.Item, newItem db.Item) { + // Add extensions to the database + for _, ext := range item.Extensions { + for _, exts := range ext { + for _, e := range exts { + attrsCustom := []byte("{}") + if e.Attrs != nil { + var err error + attrsCustom, err = json.Marshal(e.Attrs) + if err != nil { + fmt.Println("Error marshalling extension attributes:", err) + attrsCustom = []byte("{}") + } + } + + childrenCustom := []byte("{}") + if e.Children != nil { + var err error + childrenCustom, err = json.Marshal(e.Children) + if err != nil { + fmt.Println("Error marshalling extension children:", err) + childrenCustom = []byte("{}") + } + } + + _, err := DB.CreateItemExtension(ctx, db.CreateItemExtensionParams{ + CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, + UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, + DeletedAt: pgtype.Timestamptz{Valid: false}, + Name: pgtype.Text{String: e.Name, Valid: e.Name != ""}, + Value: pgtype.Text{String: e.Value, Valid: e.Value != ""}, + Attrs: attrsCustom, + Children: childrenCustom, + ItemID: newItem.ID, + }) + + if err != nil { + log.Printf("Error adding extension to database: %s", err) + continue + } + + log.Printf("Extension added to database") + } + } + } +} diff --git a/feeds.go b/feeds.go index bb14492..1d1192b 100644 --- a/feeds.go +++ b/feeds.go @@ -49,6 +49,10 @@ func AddFeedToDB(feedURL string) error { log.Printf("Adding images to the database") addFeedImages(ctx, feed, newFeed) + // Add Dublin Core to the database + log.Printf("Adding Dublin Core to the database") + createFeedDublinCore(ctx, feed, newFeed) + log.Printf("Feed added to database") return nil } diff --git a/images.go b/images.go new file mode 100644 index 0000000..af83e92 --- /dev/null +++ b/images.go @@ -0,0 +1,49 @@ +package main + +import ( + "context" + "log" + "time" + + "github.com/TheLovinator1/FeedVault/db" + "github.com/jackc/pgx/v5/pgtype" + "github.com/mmcdole/gofeed" +) + +func addFeedImages(ctx context.Context, feed *gofeed.Feed, newFeed db.Feed) { + if feed.Image == nil { + log.Printf("No image to add to database") + return + } + + // TODO: Download the image and store it on the server + _, err := DB.CreateFeedImage(ctx, db.CreateFeedImageParams{ + CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, + UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, + DeletedAt: pgtype.Timestamptz{Valid: false}, + Url: pgtype.Text{String: feed.Image.URL, Valid: feed.Image.URL != ""}, + Title: pgtype.Text{String: feed.Image.Title, Valid: feed.Image.Title != ""}, + FeedID: newFeed.ID, + }) + if err != nil { + log.Printf("Error adding image to database: %s", err) + return + } + log.Printf("Image added to database: %s", feed.Image.URL) +} + +func addItemImages(ctx context.Context, item *gofeed.Item, newItem db.Item) { + _, err := DB.CreateItemImage(ctx, db.CreateItemImageParams{ + CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, + UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, + DeletedAt: pgtype.Timestamptz{Valid: false}, + Url: pgtype.Text{String: item.Image.URL, Valid: item.Image.URL != ""}, + Title: pgtype.Text{String: item.Image.Title, Valid: item.Image.Title != ""}, + ItemID: newItem.ID, + }) + if err != nil { + log.Printf("Error adding image to database: %s", err) + return + } + log.Printf("Image added to database: %s", item.Image.URL) +} diff --git a/sql/queries/feeds.sql b/sql/queries/feeds.sql index becedfc..187c15a 100644 --- a/sql/queries/feeds.sql +++ b/sql/queries/feeds.sql @@ -321,3 +321,131 @@ LIMIT $2 OFFSET $3; + +-- name: CreateFeedDublinCore :one +INSERT INTO + feed_dublin_cores ( + created_at, + updated_at, + deleted_at, + title, + creator, + author, + "subject", + "description", + publisher, + contributor, + "date", + "type", + format, + identifier, + source, + "language", + relation, + coverage, + rights, + feed_id + ) +VALUES + ( + $1, + $2, + $3, + $4, + $5, + $6, + $7, + $8, + $9, + $10, + $11, + $12, + $13, + $14, + $15, + $16, + $17, + $18, + $19, + $20 + ) +RETURNING + *; + +-- name: CreateItemDublinCore :one +INSERT INTO + item_dublin_cores ( + created_at, + updated_at, + deleted_at, + title, + creator, + author, + "subject", + "description", + publisher, + contributor, + "date", + "type", + format, + identifier, + source, + "language", + relation, + coverage, + rights, + item_id + ) +VALUES + ( + $1, + $2, + $3, + $4, + $5, + $6, + $7, + $8, + $9, + $10, + $11, + $12, + $13, + $14, + $15, + $16, + $17, + $18, + $19, + $20 + ) +RETURNING + *; + +-- name: GetFeedDublinCores :many +SELECT + * +FROM + feed_dublin_cores +WHERE + feed_id = $1 +ORDER BY + created_at DESC +LIMIT + $2 +OFFSET + $3; + +-- name: GetItemDublinCores :many +SELECT + * +FROM + item_dublin_cores +WHERE + item_id = $1 +ORDER BY + created_at DESC +LIMIT + $2 +OFFSET + $3;