You can now get authors
This commit is contained in:
parent
3113b844bc
commit
5016f576b9
9 changed files with 597 additions and 189 deletions
119
add_feed.go
Normal file
119
add_feed.go
Normal file
|
|
@ -0,0 +1,119 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/TheLovinator1/FeedVault/db"
|
||||||
|
"github.com/jackc/pgx/v5/pgtype"
|
||||||
|
"github.com/mmcdole/gofeed"
|
||||||
|
)
|
||||||
|
|
||||||
|
func makeCreateFeedParams(feedURL string, feed *gofeed.Feed) db.CreateFeedParams {
|
||||||
|
var updatedTime time.Time
|
||||||
|
if feed.UpdatedParsed != nil {
|
||||||
|
updatedTime = *feed.UpdatedParsed
|
||||||
|
}
|
||||||
|
var publishedTime time.Time
|
||||||
|
if feed.PublishedParsed != nil {
|
||||||
|
publishedTime = *feed.PublishedParsed
|
||||||
|
}
|
||||||
|
|
||||||
|
feedCustom, err := json.Marshal(feed.Custom)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Println("Error marshalling feed custom data:", err)
|
||||||
|
feedCustom = []byte("{}")
|
||||||
|
}
|
||||||
|
|
||||||
|
params := db.CreateFeedParams{
|
||||||
|
Url: feedURL,
|
||||||
|
CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||||
|
UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||||
|
DeletedAt: pgtype.Timestamptz{Valid: false},
|
||||||
|
Title: pgtype.Text{String: feed.Title, Valid: feed.Title != ""},
|
||||||
|
Description: pgtype.Text{String: feed.Description, Valid: feed.Description != ""},
|
||||||
|
Link: pgtype.Text{String: feed.Link, Valid: feed.Link != ""},
|
||||||
|
FeedLink: pgtype.Text{String: feed.FeedLink, Valid: feed.FeedLink != ""},
|
||||||
|
Links: feed.Links,
|
||||||
|
Updated: pgtype.Text{String: feed.Updated, Valid: feed.Updated != ""},
|
||||||
|
UpdatedParsed: pgtype.Timestamptz{Time: updatedTime, Valid: !updatedTime.IsZero()},
|
||||||
|
Published: pgtype.Text{String: feed.Published, Valid: feed.Published != ""},
|
||||||
|
PublishedParsed: pgtype.Timestamptz{Time: publishedTime, Valid: !publishedTime.IsZero()},
|
||||||
|
Language: pgtype.Text{String: feed.Language, Valid: feed.Language != ""},
|
||||||
|
Copyright: pgtype.Text{String: feed.Copyright, Valid: feed.Copyright != ""},
|
||||||
|
Generator: pgtype.Text{String: feed.Generator, Valid: feed.Generator != ""},
|
||||||
|
Categories: feed.Categories,
|
||||||
|
Custom: feedCustom,
|
||||||
|
FeedType: pgtype.Text{String: feed.FeedType, Valid: feed.FeedType != ""},
|
||||||
|
FeedVersion: pgtype.Text{String: feed.FeedVersion, Valid: feed.FeedVersion != ""},
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("Created feed params: %+v", params)
|
||||||
|
|
||||||
|
return params
|
||||||
|
}
|
||||||
|
|
||||||
|
func addFeedExtensionToDB(ctx context.Context, feed *gofeed.Feed, newFeed db.Feed) {
|
||||||
|
// Add extensions to the database
|
||||||
|
// TODO: Check if this is correct and works
|
||||||
|
for _, ext := range feed.Extensions {
|
||||||
|
for _, exts := range ext {
|
||||||
|
for _, e := range exts {
|
||||||
|
attrsCustom := []byte("{}")
|
||||||
|
if e.Attrs != nil {
|
||||||
|
var err error
|
||||||
|
attrsCustom, err = json.Marshal(e.Attrs)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Println("Error marshalling extension attributes:", err)
|
||||||
|
attrsCustom = []byte("{}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
childrenCustom := []byte("{}")
|
||||||
|
if e.Children != nil {
|
||||||
|
var err error
|
||||||
|
childrenCustom, err = json.Marshal(e.Children)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Println("Error marshalling extension children:", err)
|
||||||
|
childrenCustom = []byte("{}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := DB.CreateFeedExtension(ctx, db.CreateFeedExtensionParams{
|
||||||
|
CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||||
|
UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||||
|
DeletedAt: pgtype.Timestamptz{Valid: false},
|
||||||
|
Name: pgtype.Text{String: e.Name, Valid: e.Name != ""},
|
||||||
|
Value: pgtype.Text{String: e.Value, Valid: e.Value != ""},
|
||||||
|
Attrs: attrsCustom,
|
||||||
|
Children: childrenCustom,
|
||||||
|
FeedID: newFeed.ID,
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("Error adding extension to database: %s", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func addFeedAuthors(ctx context.Context, feed *gofeed.Feed, newFeed db.Feed) {
|
||||||
|
// Add authors to the database
|
||||||
|
for _, author := range feed.Authors {
|
||||||
|
_, err := DB.CreateFeedAuthor(ctx, db.CreateFeedAuthorParams{
|
||||||
|
CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||||
|
UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||||
|
DeletedAt: pgtype.Timestamptz{Valid: false},
|
||||||
|
Name: pgtype.Text{String: author.Name, Valid: author.Name != ""},
|
||||||
|
FeedID: newFeed.ID,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("Error adding author %s (%s) to database: %s", author.Name, author.Email, err)
|
||||||
|
}
|
||||||
|
log.Printf("Author %s (%s) added to database", author.Name, author.Email)
|
||||||
|
}
|
||||||
|
}
|
||||||
138
add_item.go
Normal file
138
add_item.go
Normal file
|
|
@ -0,0 +1,138 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/TheLovinator1/FeedVault/db"
|
||||||
|
"github.com/jackc/pgx/v5/pgtype"
|
||||||
|
"github.com/mmcdole/gofeed"
|
||||||
|
)
|
||||||
|
|
||||||
|
func addItemToDB(item *gofeed.Item, ctx context.Context, newFeed db.Feed) {
|
||||||
|
newItem, err := DB.CreateItem(ctx, makeCreateItemParams(item, newFeed.ID))
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("Error adding item to database: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add extensions to the database
|
||||||
|
addItemExtensionToDB(ctx, item, newItem)
|
||||||
|
|
||||||
|
// Add authors to the database
|
||||||
|
addItemAuthors(ctx, item, newItem)
|
||||||
|
|
||||||
|
log.Printf("Item added to database")
|
||||||
|
}
|
||||||
|
|
||||||
|
func makeCreateItemParams(item *gofeed.Item, feedID int64) db.CreateItemParams {
|
||||||
|
var updatedTime time.Time
|
||||||
|
if item.UpdatedParsed != nil {
|
||||||
|
updatedTime = *item.UpdatedParsed
|
||||||
|
}
|
||||||
|
var publishedTime time.Time
|
||||||
|
if item.PublishedParsed != nil {
|
||||||
|
publishedTime = *item.PublishedParsed
|
||||||
|
}
|
||||||
|
|
||||||
|
itemCustom := []byte("{}")
|
||||||
|
if item.Custom != nil {
|
||||||
|
var err error
|
||||||
|
itemCustom, err = json.Marshal(item.Custom)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Println("Error marshalling item custom data:", err)
|
||||||
|
itemCustom = []byte("{}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
params := db.CreateItemParams{
|
||||||
|
CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||||
|
UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||||
|
DeletedAt: pgtype.Timestamptz{Valid: false},
|
||||||
|
Title: pgtype.Text{String: item.Title, Valid: item.Title != ""},
|
||||||
|
Description: pgtype.Text{String: item.Description, Valid: item.Description != ""},
|
||||||
|
Content: pgtype.Text{String: item.Content, Valid: item.Content != ""},
|
||||||
|
Link: pgtype.Text{String: item.Link, Valid: item.Link != ""},
|
||||||
|
Links: item.Links,
|
||||||
|
Updated: pgtype.Text{String: item.Updated, Valid: item.Updated != ""},
|
||||||
|
UpdatedParsed: pgtype.Timestamptz{Time: updatedTime, Valid: !updatedTime.IsZero()},
|
||||||
|
Published: pgtype.Text{String: item.Published, Valid: item.Published != ""},
|
||||||
|
PublishedParsed: pgtype.Timestamptz{Time: publishedTime, Valid: !publishedTime.IsZero()},
|
||||||
|
Guid: pgtype.Text{String: item.GUID, Valid: item.GUID != ""},
|
||||||
|
Categories: item.Categories,
|
||||||
|
Custom: itemCustom,
|
||||||
|
FeedID: feedID,
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("Created item params: %+v", params)
|
||||||
|
|
||||||
|
return params
|
||||||
|
}
|
||||||
|
|
||||||
|
func addItemExtensionToDB(ctx context.Context, item *gofeed.Item, newItem db.Item) {
|
||||||
|
// Add extensions to the database
|
||||||
|
for _, ext := range item.Extensions {
|
||||||
|
for _, exts := range ext {
|
||||||
|
for _, e := range exts {
|
||||||
|
attrsCustom := []byte("{}")
|
||||||
|
if e.Attrs != nil {
|
||||||
|
var err error
|
||||||
|
attrsCustom, err = json.Marshal(e.Attrs)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Println("Error marshalling extension attributes:", err)
|
||||||
|
attrsCustom = []byte("{}")
|
||||||
|
}
|
||||||
|
log.Printf("Extension attributes: %s", attrsCustom)
|
||||||
|
}
|
||||||
|
|
||||||
|
childrenCustom := []byte("{}")
|
||||||
|
if e.Children != nil {
|
||||||
|
var err error
|
||||||
|
childrenCustom, err = json.Marshal(e.Children)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Println("Error marshalling extension children:", err)
|
||||||
|
childrenCustom = []byte("{}")
|
||||||
|
}
|
||||||
|
log.Printf("Extension children: %s", childrenCustom)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := DB.CreateItemExtension(ctx, db.CreateItemExtensionParams{
|
||||||
|
CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||||
|
UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||||
|
DeletedAt: pgtype.Timestamptz{Valid: false},
|
||||||
|
Name: pgtype.Text{String: e.Name, Valid: e.Name != ""},
|
||||||
|
Value: pgtype.Text{String: e.Value, Valid: e.Value != ""},
|
||||||
|
Attrs: attrsCustom,
|
||||||
|
Children: childrenCustom,
|
||||||
|
ItemID: newItem.ID,
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("Error adding extension to database: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("Extension added to database")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func addItemAuthors(ctx context.Context, item *gofeed.Item, newItem db.Item) {
|
||||||
|
for _, author := range item.Authors {
|
||||||
|
_, err := DB.CreateItemAuthor(ctx, db.CreateItemAuthorParams{
|
||||||
|
CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||||
|
UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||||
|
DeletedAt: pgtype.Timestamptz{Valid: false},
|
||||||
|
Name: pgtype.Text{String: author.Name, Valid: author.Name != ""},
|
||||||
|
Email: pgtype.Text{String: author.Email, Valid: author.Email != ""},
|
||||||
|
ItemID: newItem.ID,
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("Error adding author %s (%s) to database: %s", author.Name, author.Email, err)
|
||||||
|
}
|
||||||
|
log.Printf("Author %s (%s) added to database", author.Name, author.Email)
|
||||||
|
}
|
||||||
|
}
|
||||||
192
db/feeds.sql.go
192
db/feeds.sql.go
|
|
@ -163,6 +163,53 @@ func (q *Queries) CreateFeed(ctx context.Context, arg CreateFeedParams) (Feed, e
|
||||||
return i, err
|
return i, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const createFeedAuthor = `-- name: CreateFeedAuthor :one
|
||||||
|
INSERT INTO
|
||||||
|
feed_authors (
|
||||||
|
created_at,
|
||||||
|
updated_at,
|
||||||
|
deleted_at,
|
||||||
|
"name",
|
||||||
|
email,
|
||||||
|
feed_id
|
||||||
|
)
|
||||||
|
VALUES
|
||||||
|
($1, $2, $3, $4, $5, $6)
|
||||||
|
RETURNING
|
||||||
|
id, created_at, updated_at, deleted_at, name, email, feed_id
|
||||||
|
`
|
||||||
|
|
||||||
|
type CreateFeedAuthorParams struct {
|
||||||
|
CreatedAt pgtype.Timestamptz `json:"created_at"`
|
||||||
|
UpdatedAt pgtype.Timestamptz `json:"updated_at"`
|
||||||
|
DeletedAt pgtype.Timestamptz `json:"deleted_at"`
|
||||||
|
Name pgtype.Text `json:"name"`
|
||||||
|
Email pgtype.Text `json:"email"`
|
||||||
|
FeedID int64 `json:"feed_id"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *Queries) CreateFeedAuthor(ctx context.Context, arg CreateFeedAuthorParams) (FeedAuthor, error) {
|
||||||
|
row := q.db.QueryRow(ctx, createFeedAuthor,
|
||||||
|
arg.CreatedAt,
|
||||||
|
arg.UpdatedAt,
|
||||||
|
arg.DeletedAt,
|
||||||
|
arg.Name,
|
||||||
|
arg.Email,
|
||||||
|
arg.FeedID,
|
||||||
|
)
|
||||||
|
var i FeedAuthor
|
||||||
|
err := row.Scan(
|
||||||
|
&i.ID,
|
||||||
|
&i.CreatedAt,
|
||||||
|
&i.UpdatedAt,
|
||||||
|
&i.DeletedAt,
|
||||||
|
&i.Name,
|
||||||
|
&i.Email,
|
||||||
|
&i.FeedID,
|
||||||
|
)
|
||||||
|
return i, err
|
||||||
|
}
|
||||||
|
|
||||||
const createFeedExtension = `-- name: CreateFeedExtension :one
|
const createFeedExtension = `-- name: CreateFeedExtension :one
|
||||||
INSERT INTO
|
INSERT INTO
|
||||||
feed_extensions (
|
feed_extensions (
|
||||||
|
|
@ -322,6 +369,53 @@ func (q *Queries) CreateItem(ctx context.Context, arg CreateItemParams) (Item, e
|
||||||
return i, err
|
return i, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const createItemAuthor = `-- name: CreateItemAuthor :one
|
||||||
|
INSERT INTO
|
||||||
|
item_authors (
|
||||||
|
created_at,
|
||||||
|
updated_at,
|
||||||
|
deleted_at,
|
||||||
|
"name",
|
||||||
|
email,
|
||||||
|
item_id
|
||||||
|
)
|
||||||
|
VALUES
|
||||||
|
($1, $2, $3, $4, $5, $6)
|
||||||
|
RETURNING
|
||||||
|
id, created_at, updated_at, deleted_at, name, email, item_id
|
||||||
|
`
|
||||||
|
|
||||||
|
type CreateItemAuthorParams struct {
|
||||||
|
CreatedAt pgtype.Timestamptz `json:"created_at"`
|
||||||
|
UpdatedAt pgtype.Timestamptz `json:"updated_at"`
|
||||||
|
DeletedAt pgtype.Timestamptz `json:"deleted_at"`
|
||||||
|
Name pgtype.Text `json:"name"`
|
||||||
|
Email pgtype.Text `json:"email"`
|
||||||
|
ItemID int64 `json:"item_id"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *Queries) CreateItemAuthor(ctx context.Context, arg CreateItemAuthorParams) (ItemAuthor, error) {
|
||||||
|
row := q.db.QueryRow(ctx, createItemAuthor,
|
||||||
|
arg.CreatedAt,
|
||||||
|
arg.UpdatedAt,
|
||||||
|
arg.DeletedAt,
|
||||||
|
arg.Name,
|
||||||
|
arg.Email,
|
||||||
|
arg.ItemID,
|
||||||
|
)
|
||||||
|
var i ItemAuthor
|
||||||
|
err := row.Scan(
|
||||||
|
&i.ID,
|
||||||
|
&i.CreatedAt,
|
||||||
|
&i.UpdatedAt,
|
||||||
|
&i.DeletedAt,
|
||||||
|
&i.Name,
|
||||||
|
&i.Email,
|
||||||
|
&i.ItemID,
|
||||||
|
)
|
||||||
|
return i, err
|
||||||
|
}
|
||||||
|
|
||||||
const createItemExtension = `-- name: CreateItemExtension :one
|
const createItemExtension = `-- name: CreateItemExtension :one
|
||||||
INSERT INTO
|
INSERT INTO
|
||||||
item_extensions (
|
item_extensions (
|
||||||
|
|
@ -415,6 +509,55 @@ func (q *Queries) GetFeed(ctx context.Context, id int64) (Feed, error) {
|
||||||
return i, err
|
return i, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const getFeedAuthors = `-- name: GetFeedAuthors :many
|
||||||
|
SELECT
|
||||||
|
id, created_at, updated_at, deleted_at, name, email, feed_id
|
||||||
|
FROM
|
||||||
|
feed_authors
|
||||||
|
WHERE
|
||||||
|
feed_id = $1
|
||||||
|
ORDER BY
|
||||||
|
created_at DESC
|
||||||
|
LIMIT
|
||||||
|
$2
|
||||||
|
OFFSET
|
||||||
|
$3
|
||||||
|
`
|
||||||
|
|
||||||
|
type GetFeedAuthorsParams struct {
|
||||||
|
FeedID int64 `json:"feed_id"`
|
||||||
|
Limit int32 `json:"limit"`
|
||||||
|
Offset int32 `json:"offset"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *Queries) GetFeedAuthors(ctx context.Context, arg GetFeedAuthorsParams) ([]FeedAuthor, error) {
|
||||||
|
rows, err := q.db.Query(ctx, getFeedAuthors, arg.FeedID, arg.Limit, arg.Offset)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
items := []FeedAuthor{}
|
||||||
|
for rows.Next() {
|
||||||
|
var i FeedAuthor
|
||||||
|
if err := rows.Scan(
|
||||||
|
&i.ID,
|
||||||
|
&i.CreatedAt,
|
||||||
|
&i.UpdatedAt,
|
||||||
|
&i.DeletedAt,
|
||||||
|
&i.Name,
|
||||||
|
&i.Email,
|
||||||
|
&i.FeedID,
|
||||||
|
); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
items = append(items, i)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
const getFeedExtensions = `-- name: GetFeedExtensions :many
|
const getFeedExtensions = `-- name: GetFeedExtensions :many
|
||||||
SELECT
|
SELECT
|
||||||
id, created_at, updated_at, deleted_at, name, value, attrs, children, feed_id
|
id, created_at, updated_at, deleted_at, name, value, attrs, children, feed_id
|
||||||
|
|
@ -560,6 +703,55 @@ func (q *Queries) GetItem(ctx context.Context, id int64) (Item, error) {
|
||||||
return i, err
|
return i, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const getItemAuthors = `-- name: GetItemAuthors :many
|
||||||
|
SELECT
|
||||||
|
id, created_at, updated_at, deleted_at, name, email, item_id
|
||||||
|
FROM
|
||||||
|
item_authors
|
||||||
|
WHERE
|
||||||
|
item_id = $1
|
||||||
|
ORDER BY
|
||||||
|
created_at DESC
|
||||||
|
LIMIT
|
||||||
|
$2
|
||||||
|
OFFSET
|
||||||
|
$3
|
||||||
|
`
|
||||||
|
|
||||||
|
type GetItemAuthorsParams struct {
|
||||||
|
ItemID int64 `json:"item_id"`
|
||||||
|
Limit int32 `json:"limit"`
|
||||||
|
Offset int32 `json:"offset"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *Queries) GetItemAuthors(ctx context.Context, arg GetItemAuthorsParams) ([]ItemAuthor, error) {
|
||||||
|
rows, err := q.db.Query(ctx, getItemAuthors, arg.ItemID, arg.Limit, arg.Offset)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
items := []ItemAuthor{}
|
||||||
|
for rows.Next() {
|
||||||
|
var i ItemAuthor
|
||||||
|
if err := rows.Scan(
|
||||||
|
&i.ID,
|
||||||
|
&i.CreatedAt,
|
||||||
|
&i.UpdatedAt,
|
||||||
|
&i.DeletedAt,
|
||||||
|
&i.Name,
|
||||||
|
&i.Email,
|
||||||
|
&i.ItemID,
|
||||||
|
); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
items = append(items, i)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
const getItemExtensions = `-- name: GetItemExtensions :many
|
const getItemExtensions = `-- name: GetItemExtensions :many
|
||||||
SELECT
|
SELECT
|
||||||
id, created_at, updated_at, deleted_at, name, value, attrs, children, item_id
|
id, created_at, updated_at, deleted_at, name, value, attrs, children, item_id
|
||||||
|
|
|
||||||
|
|
@ -50,7 +50,6 @@ type FeedAuthor struct {
|
||||||
DeletedAt pgtype.Timestamptz `json:"deleted_at"`
|
DeletedAt pgtype.Timestamptz `json:"deleted_at"`
|
||||||
Name pgtype.Text `json:"name"`
|
Name pgtype.Text `json:"name"`
|
||||||
Email pgtype.Text `json:"email"`
|
Email pgtype.Text `json:"email"`
|
||||||
Uri pgtype.Text `json:"uri"`
|
|
||||||
FeedID int64 `json:"feed_id"`
|
FeedID int64 `json:"feed_id"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -145,7 +144,6 @@ type ItemAuthor struct {
|
||||||
DeletedAt pgtype.Timestamptz `json:"deleted_at"`
|
DeletedAt pgtype.Timestamptz `json:"deleted_at"`
|
||||||
Name pgtype.Text `json:"name"`
|
Name pgtype.Text `json:"name"`
|
||||||
Email pgtype.Text `json:"email"`
|
Email pgtype.Text `json:"email"`
|
||||||
Uri pgtype.Text `json:"uri"`
|
|
||||||
ItemID int64 `json:"item_id"`
|
ItemID int64 `json:"item_id"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
187
feeds.go
187
feeds.go
|
|
@ -2,104 +2,13 @@ package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/TheLovinator1/FeedVault/db"
|
|
||||||
"github.com/jackc/pgx/v5/pgtype"
|
|
||||||
"github.com/mmcdole/gofeed"
|
"github.com/mmcdole/gofeed"
|
||||||
)
|
)
|
||||||
|
|
||||||
func makeCreateFeedParams(feedURL string, feed *gofeed.Feed) db.CreateFeedParams {
|
|
||||||
var updatedTime time.Time
|
|
||||||
if feed.UpdatedParsed != nil {
|
|
||||||
updatedTime = *feed.UpdatedParsed
|
|
||||||
}
|
|
||||||
var publishedTime time.Time
|
|
||||||
if feed.PublishedParsed != nil {
|
|
||||||
publishedTime = *feed.PublishedParsed
|
|
||||||
}
|
|
||||||
|
|
||||||
feedCustom, err := json.Marshal(feed.Custom)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Println("Error marshalling feed custom data:", err)
|
|
||||||
feedCustom = []byte("{}")
|
|
||||||
}
|
|
||||||
|
|
||||||
params := db.CreateFeedParams{
|
|
||||||
Url: feedURL,
|
|
||||||
CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
|
||||||
UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
|
||||||
DeletedAt: pgtype.Timestamptz{Valid: false},
|
|
||||||
Title: pgtype.Text{String: feed.Title, Valid: feed.Title != ""},
|
|
||||||
Description: pgtype.Text{String: feed.Description, Valid: feed.Description != ""},
|
|
||||||
Link: pgtype.Text{String: feed.Link, Valid: feed.Link != ""},
|
|
||||||
FeedLink: pgtype.Text{String: feed.FeedLink, Valid: feed.FeedLink != ""},
|
|
||||||
Links: feed.Links,
|
|
||||||
Updated: pgtype.Text{String: feed.Updated, Valid: feed.Updated != ""},
|
|
||||||
UpdatedParsed: pgtype.Timestamptz{Time: updatedTime, Valid: !updatedTime.IsZero()},
|
|
||||||
Published: pgtype.Text{String: feed.Published, Valid: feed.Published != ""},
|
|
||||||
PublishedParsed: pgtype.Timestamptz{Time: publishedTime, Valid: !publishedTime.IsZero()},
|
|
||||||
Language: pgtype.Text{String: feed.Language, Valid: feed.Language != ""},
|
|
||||||
Copyright: pgtype.Text{String: feed.Copyright, Valid: feed.Copyright != ""},
|
|
||||||
Generator: pgtype.Text{String: feed.Generator, Valid: feed.Generator != ""},
|
|
||||||
Categories: feed.Categories,
|
|
||||||
Custom: feedCustom,
|
|
||||||
FeedType: pgtype.Text{String: feed.FeedType, Valid: feed.FeedType != ""},
|
|
||||||
FeedVersion: pgtype.Text{String: feed.FeedVersion, Valid: feed.FeedVersion != ""},
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Printf("Created feed params: %+v", params)
|
|
||||||
|
|
||||||
return params
|
|
||||||
}
|
|
||||||
|
|
||||||
func makeCreateItemParams(item *gofeed.Item, feedID int64) db.CreateItemParams {
|
|
||||||
var updatedTime time.Time
|
|
||||||
if item.UpdatedParsed != nil {
|
|
||||||
updatedTime = *item.UpdatedParsed
|
|
||||||
}
|
|
||||||
var publishedTime time.Time
|
|
||||||
if item.PublishedParsed != nil {
|
|
||||||
publishedTime = *item.PublishedParsed
|
|
||||||
}
|
|
||||||
|
|
||||||
itemCustom := []byte("{}")
|
|
||||||
if item.Custom != nil {
|
|
||||||
var err error
|
|
||||||
itemCustom, err = json.Marshal(item.Custom)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Println("Error marshalling item custom data:", err)
|
|
||||||
itemCustom = []byte("{}")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
params := db.CreateItemParams{
|
|
||||||
CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
|
||||||
UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
|
||||||
DeletedAt: pgtype.Timestamptz{Valid: false},
|
|
||||||
Title: pgtype.Text{String: item.Title, Valid: item.Title != ""},
|
|
||||||
Description: pgtype.Text{String: item.Description, Valid: item.Description != ""},
|
|
||||||
Content: pgtype.Text{String: item.Content, Valid: item.Content != ""},
|
|
||||||
Link: pgtype.Text{String: item.Link, Valid: item.Link != ""},
|
|
||||||
Links: item.Links,
|
|
||||||
Updated: pgtype.Text{String: item.Updated, Valid: item.Updated != ""},
|
|
||||||
UpdatedParsed: pgtype.Timestamptz{Time: updatedTime, Valid: !updatedTime.IsZero()},
|
|
||||||
Published: pgtype.Text{String: item.Published, Valid: item.Published != ""},
|
|
||||||
PublishedParsed: pgtype.Timestamptz{Time: publishedTime, Valid: !publishedTime.IsZero()},
|
|
||||||
Guid: pgtype.Text{String: item.GUID, Valid: item.GUID != ""},
|
|
||||||
Categories: item.Categories,
|
|
||||||
Custom: itemCustom,
|
|
||||||
FeedID: feedID,
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Printf("Created item params: %+v", params)
|
|
||||||
|
|
||||||
return params
|
|
||||||
}
|
|
||||||
|
|
||||||
func AddFeedToDB(feedURL string) error {
|
func AddFeedToDB(feedURL string) error {
|
||||||
// Cancel the request after 60 seconds if it hasn't finished
|
// Cancel the request after 60 seconds if it hasn't finished
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||||
|
|
@ -123,102 +32,14 @@ func AddFeedToDB(feedURL string) error {
|
||||||
|
|
||||||
// Add the items to the database
|
// Add the items to the database
|
||||||
for _, item := range feed.Items {
|
for _, item := range feed.Items {
|
||||||
newItem, err := DB.CreateItem(ctx, makeCreateItemParams(item, newFeed.ID))
|
addItemToDB(item, ctx, newFeed)
|
||||||
if err != nil {
|
|
||||||
log.Printf("Error adding item to database: %s", err)
|
|
||||||
}
|
|
||||||
log.Printf("Item added to database")
|
|
||||||
|
|
||||||
// Add extensions to the database
|
|
||||||
for _, ext := range item.Extensions {
|
|
||||||
for _, exts := range ext {
|
|
||||||
for _, e := range exts {
|
|
||||||
attrsCustom := []byte("{}")
|
|
||||||
if e.Attrs != nil {
|
|
||||||
var err error
|
|
||||||
attrsCustom, err = json.Marshal(e.Attrs)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Println("Error marshalling extension attributes:", err)
|
|
||||||
attrsCustom = []byte("{}")
|
|
||||||
}
|
|
||||||
log.Printf("Extension attributes: %s", attrsCustom)
|
|
||||||
}
|
|
||||||
|
|
||||||
childrenCustom := []byte("{}")
|
|
||||||
if e.Children != nil {
|
|
||||||
var err error
|
|
||||||
childrenCustom, err = json.Marshal(e.Children)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Println("Error marshalling extension children:", err)
|
|
||||||
childrenCustom = []byte("{}")
|
|
||||||
}
|
|
||||||
log.Printf("Extension children: %s", childrenCustom)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := DB.CreateItemExtension(ctx, db.CreateItemExtensionParams{
|
|
||||||
CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
|
||||||
UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
|
||||||
DeletedAt: pgtype.Timestamptz{Valid: false},
|
|
||||||
Name: pgtype.Text{String: e.Name, Valid: e.Name != ""},
|
|
||||||
Value: pgtype.Text{String: e.Value, Valid: e.Value != ""},
|
|
||||||
Attrs: attrsCustom,
|
|
||||||
Children: childrenCustom,
|
|
||||||
ItemID: newItem.ID,
|
|
||||||
})
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("Error adding extension to database: %s", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Printf("Extension added to database")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add extensions to the database
|
// Add extensions to the database
|
||||||
// TODO: Check if this is correct and works
|
addFeedExtensionToDB(ctx, feed, newFeed)
|
||||||
for _, ext := range feed.Extensions {
|
|
||||||
for _, exts := range ext {
|
|
||||||
for _, e := range exts {
|
|
||||||
attrsCustom := []byte("{}")
|
|
||||||
if e.Attrs != nil {
|
|
||||||
var err error
|
|
||||||
attrsCustom, err = json.Marshal(e.Attrs)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Println("Error marshalling extension attributes:", err)
|
|
||||||
attrsCustom = []byte("{}")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
childrenCustom := []byte("{}")
|
// Add authors to the database
|
||||||
if e.Children != nil {
|
addFeedAuthors(ctx, feed, newFeed)
|
||||||
var err error
|
|
||||||
childrenCustom, err = json.Marshal(e.Children)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Println("Error marshalling extension children:", err)
|
|
||||||
childrenCustom = []byte("{}")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := DB.CreateFeedExtension(ctx, db.CreateFeedExtensionParams{
|
|
||||||
CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
|
||||||
UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
|
||||||
DeletedAt: pgtype.Timestamptz{Valid: false},
|
|
||||||
Name: pgtype.Text{String: e.Name, Valid: e.Name != ""},
|
|
||||||
Value: pgtype.Text{String: e.Value, Valid: e.Value != ""},
|
|
||||||
Attrs: attrsCustom,
|
|
||||||
Children: childrenCustom,
|
|
||||||
FeedID: newFeed.ID,
|
|
||||||
})
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("Error adding extension to database: %s", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Println(feed.Title)
|
fmt.Println(feed.Title)
|
||||||
return nil
|
return nil
|
||||||
|
|
|
||||||
87
handlers.go
87
handlers.go
|
|
@ -39,8 +39,52 @@ func FeedsHandler(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
|
||||||
fb := strings.Builder{}
|
fb := strings.Builder{}
|
||||||
for _, feed := range feeds {
|
for _, feed := range feeds {
|
||||||
|
authors, err := DB.GetFeedAuthors(context.Background(), db.GetFeedAuthorsParams{
|
||||||
|
FeedID: feed.ID,
|
||||||
|
Limit: 100,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, "Error getting authors", http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
extensions, err := DB.GetFeedExtensions(context.Background(), db.GetFeedExtensionsParams{
|
||||||
|
FeedID: feed.ID,
|
||||||
|
Limit: 100,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, "Error getting extensions", http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
fb.WriteString("<li>")
|
fb.WriteString("<li>")
|
||||||
fb.WriteString("<a href=\"/feed/" + strconv.FormatInt(feed.ID, 10) + "\">" + feed.Title.String + "</a> - <a href=\"" + feed.Link.String + "\">" + feed.Link.String + "</a>")
|
fb.WriteString(feed.Title.String)
|
||||||
|
fb.WriteString("<ul>")
|
||||||
|
for _, author := range authors {
|
||||||
|
if author.Name.Valid {
|
||||||
|
fb.WriteString("<li>Author: " + author.Name.String + "</li>")
|
||||||
|
}
|
||||||
|
if author.Email.Valid {
|
||||||
|
fb.WriteString("<li>Email: " + author.Email.String + "</li>")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, ext := range extensions {
|
||||||
|
if ext.Attrs != nil {
|
||||||
|
fb.WriteString("<li>Attrs: " + string(ext.Attrs) + "</li>")
|
||||||
|
}
|
||||||
|
if ext.Children != nil {
|
||||||
|
fb.WriteString("<li>Children: " + string(ext.Children) + "</li>")
|
||||||
|
}
|
||||||
|
if ext.Name.Valid {
|
||||||
|
fb.WriteString("<li>Name: " + ext.Name.String + "</li>")
|
||||||
|
}
|
||||||
|
if ext.Value.Valid {
|
||||||
|
fb.WriteString("<li>Value: " + ext.Value.String + "</li>")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fb.WriteString("</ul>")
|
||||||
|
fb.WriteString("<a href=\"/feed/" + strconv.FormatInt(feed.ID, 10) + "\">" + feed.Url + "</a>")
|
||||||
fb.WriteString("</li>")
|
fb.WriteString("</li>")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -279,9 +323,50 @@ func FeedHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
// Build the HTML
|
// Build the HTML
|
||||||
fb := strings.Builder{}
|
fb := strings.Builder{}
|
||||||
for _, item := range items {
|
for _, item := range items {
|
||||||
|
// Get authors for the item
|
||||||
|
authors, err := DB.GetItemAuthors(context.Background(), db.GetItemAuthorsParams{
|
||||||
|
ItemID: item.ID,
|
||||||
|
Limit: 100,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, "Error getting authors", http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get extensions for the item
|
||||||
|
extensions, err := DB.GetItemExtensions(context.Background(), db.GetItemExtensionsParams{
|
||||||
|
ItemID: item.ID,
|
||||||
|
Limit: 100,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, "Error getting extensions", http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
fb.WriteString("<li>")
|
fb.WriteString("<li>")
|
||||||
fb.WriteString("<a href=\"" + item.Link.String + "\">" + item.Title.String + "</a>")
|
fb.WriteString("<a href=\"" + item.Link.String + "\">" + item.Title.String + "</a>")
|
||||||
fb.WriteString("<ul>")
|
fb.WriteString("<ul>")
|
||||||
|
for _, author := range authors {
|
||||||
|
fb.WriteString("<li>Author: " + author.Name.String + "</li>")
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, ext := range extensions {
|
||||||
|
fb.WriteString("<ul>")
|
||||||
|
fb.WriteString("<li>Extension: " + ext.Name.String + "</li>")
|
||||||
|
if ext.Value.Valid {
|
||||||
|
fb.WriteString("<li>Name: " + ext.Value.String + "</li>")
|
||||||
|
}
|
||||||
|
if ext.Attrs != nil {
|
||||||
|
fb.WriteString("<li>Attrs: " + string(ext.Attrs) + "</li>")
|
||||||
|
}
|
||||||
|
if ext.Children != nil {
|
||||||
|
fb.WriteString("<li>Children: " + string(ext.Children) + "</li>")
|
||||||
|
}
|
||||||
|
fb.WriteString("</ul>")
|
||||||
|
|
||||||
|
}
|
||||||
|
fb.WriteString("</ul>")
|
||||||
|
fb.WriteString("<ul>")
|
||||||
if item.Published.Valid {
|
if item.Published.Valid {
|
||||||
fb.WriteString("<li>Published: " + item.Published.String + "</li>")
|
fb.WriteString("<li>Published: " + item.Published.String + "</li>")
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,5 @@
|
||||||
package main
|
package main
|
||||||
|
|
||||||
|
|
||||||
// "Fun" messages that will be displayed in the footer
|
// "Fun" messages that will be displayed in the footer
|
||||||
var FunMsg = []string{
|
var FunMsg = []string{
|
||||||
"Web scraping is not a crime.",
|
"Web scraping is not a crime.",
|
||||||
|
|
|
||||||
|
|
@ -205,3 +205,61 @@ LIMIT
|
||||||
$2
|
$2
|
||||||
OFFSET
|
OFFSET
|
||||||
$3;
|
$3;
|
||||||
|
|
||||||
|
-- name: CreateFeedAuthor :one
|
||||||
|
INSERT INTO
|
||||||
|
feed_authors (
|
||||||
|
created_at,
|
||||||
|
updated_at,
|
||||||
|
deleted_at,
|
||||||
|
"name",
|
||||||
|
email,
|
||||||
|
feed_id
|
||||||
|
)
|
||||||
|
VALUES
|
||||||
|
($1, $2, $3, $4, $5, $6)
|
||||||
|
RETURNING
|
||||||
|
*;
|
||||||
|
|
||||||
|
-- name: CreateItemAuthor :one
|
||||||
|
INSERT INTO
|
||||||
|
item_authors (
|
||||||
|
created_at,
|
||||||
|
updated_at,
|
||||||
|
deleted_at,
|
||||||
|
"name",
|
||||||
|
email,
|
||||||
|
item_id
|
||||||
|
)
|
||||||
|
VALUES
|
||||||
|
($1, $2, $3, $4, $5, $6)
|
||||||
|
RETURNING
|
||||||
|
*;
|
||||||
|
|
||||||
|
-- name: GetFeedAuthors :many
|
||||||
|
SELECT
|
||||||
|
*
|
||||||
|
FROM
|
||||||
|
feed_authors
|
||||||
|
WHERE
|
||||||
|
feed_id = $1
|
||||||
|
ORDER BY
|
||||||
|
created_at DESC
|
||||||
|
LIMIT
|
||||||
|
$2
|
||||||
|
OFFSET
|
||||||
|
$3;
|
||||||
|
|
||||||
|
-- name: GetItemAuthors :many
|
||||||
|
SELECT
|
||||||
|
*
|
||||||
|
FROM
|
||||||
|
item_authors
|
||||||
|
WHERE
|
||||||
|
item_id = $1
|
||||||
|
ORDER BY
|
||||||
|
created_at DESC
|
||||||
|
LIMIT
|
||||||
|
$2
|
||||||
|
OFFSET
|
||||||
|
$3;
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,6 @@ CREATE TABLE IF NOT EXISTS feed_authors (
|
||||||
-- From gofeed:
|
-- From gofeed:
|
||||||
"name" TEXT,
|
"name" TEXT,
|
||||||
email TEXT,
|
email TEXT,
|
||||||
uri TEXT,
|
|
||||||
-- Link to feed
|
-- Link to feed
|
||||||
feed_id BIGINT NOT NULL,
|
feed_id BIGINT NOT NULL,
|
||||||
CONSTRAINT fk_feed_id FOREIGN KEY (feed_id) REFERENCES feeds (id) ON DELETE CASCADE
|
CONSTRAINT fk_feed_id FOREIGN KEY (feed_id) REFERENCES feeds (id) ON DELETE CASCADE
|
||||||
|
|
@ -26,7 +25,6 @@ CREATE TABLE IF NOT EXISTS item_authors (
|
||||||
-- From gofeed:
|
-- From gofeed:
|
||||||
"name" TEXT,
|
"name" TEXT,
|
||||||
email TEXT,
|
email TEXT,
|
||||||
uri TEXT,
|
|
||||||
-- Link to feed item (Also called feed entry)
|
-- Link to feed item (Also called feed entry)
|
||||||
item_id BIGINT NOT NULL,
|
item_id BIGINT NOT NULL,
|
||||||
CONSTRAINT fk_item_id FOREIGN KEY (item_id) REFERENCES items (id) ON DELETE CASCADE
|
CONSTRAINT fk_item_id FOREIGN KEY (item_id) REFERENCES items (id) ON DELETE CASCADE
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue