You can now get authors
This commit is contained in:
parent
3113b844bc
commit
5016f576b9
9 changed files with 597 additions and 189 deletions
119
add_feed.go
Normal file
119
add_feed.go
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/TheLovinator1/FeedVault/db"
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
"github.com/mmcdole/gofeed"
|
||||
)
|
||||
|
||||
func makeCreateFeedParams(feedURL string, feed *gofeed.Feed) db.CreateFeedParams {
|
||||
var updatedTime time.Time
|
||||
if feed.UpdatedParsed != nil {
|
||||
updatedTime = *feed.UpdatedParsed
|
||||
}
|
||||
var publishedTime time.Time
|
||||
if feed.PublishedParsed != nil {
|
||||
publishedTime = *feed.PublishedParsed
|
||||
}
|
||||
|
||||
feedCustom, err := json.Marshal(feed.Custom)
|
||||
if err != nil {
|
||||
fmt.Println("Error marshalling feed custom data:", err)
|
||||
feedCustom = []byte("{}")
|
||||
}
|
||||
|
||||
params := db.CreateFeedParams{
|
||||
Url: feedURL,
|
||||
CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||
UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||
DeletedAt: pgtype.Timestamptz{Valid: false},
|
||||
Title: pgtype.Text{String: feed.Title, Valid: feed.Title != ""},
|
||||
Description: pgtype.Text{String: feed.Description, Valid: feed.Description != ""},
|
||||
Link: pgtype.Text{String: feed.Link, Valid: feed.Link != ""},
|
||||
FeedLink: pgtype.Text{String: feed.FeedLink, Valid: feed.FeedLink != ""},
|
||||
Links: feed.Links,
|
||||
Updated: pgtype.Text{String: feed.Updated, Valid: feed.Updated != ""},
|
||||
UpdatedParsed: pgtype.Timestamptz{Time: updatedTime, Valid: !updatedTime.IsZero()},
|
||||
Published: pgtype.Text{String: feed.Published, Valid: feed.Published != ""},
|
||||
PublishedParsed: pgtype.Timestamptz{Time: publishedTime, Valid: !publishedTime.IsZero()},
|
||||
Language: pgtype.Text{String: feed.Language, Valid: feed.Language != ""},
|
||||
Copyright: pgtype.Text{String: feed.Copyright, Valid: feed.Copyright != ""},
|
||||
Generator: pgtype.Text{String: feed.Generator, Valid: feed.Generator != ""},
|
||||
Categories: feed.Categories,
|
||||
Custom: feedCustom,
|
||||
FeedType: pgtype.Text{String: feed.FeedType, Valid: feed.FeedType != ""},
|
||||
FeedVersion: pgtype.Text{String: feed.FeedVersion, Valid: feed.FeedVersion != ""},
|
||||
}
|
||||
|
||||
log.Printf("Created feed params: %+v", params)
|
||||
|
||||
return params
|
||||
}
|
||||
|
||||
func addFeedExtensionToDB(ctx context.Context, feed *gofeed.Feed, newFeed db.Feed) {
|
||||
// Add extensions to the database
|
||||
// TODO: Check if this is correct and works
|
||||
for _, ext := range feed.Extensions {
|
||||
for _, exts := range ext {
|
||||
for _, e := range exts {
|
||||
attrsCustom := []byte("{}")
|
||||
if e.Attrs != nil {
|
||||
var err error
|
||||
attrsCustom, err = json.Marshal(e.Attrs)
|
||||
if err != nil {
|
||||
fmt.Println("Error marshalling extension attributes:", err)
|
||||
attrsCustom = []byte("{}")
|
||||
}
|
||||
}
|
||||
|
||||
childrenCustom := []byte("{}")
|
||||
if e.Children != nil {
|
||||
var err error
|
||||
childrenCustom, err = json.Marshal(e.Children)
|
||||
if err != nil {
|
||||
fmt.Println("Error marshalling extension children:", err)
|
||||
childrenCustom = []byte("{}")
|
||||
}
|
||||
}
|
||||
|
||||
_, err := DB.CreateFeedExtension(ctx, db.CreateFeedExtensionParams{
|
||||
CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||
UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||
DeletedAt: pgtype.Timestamptz{Valid: false},
|
||||
Name: pgtype.Text{String: e.Name, Valid: e.Name != ""},
|
||||
Value: pgtype.Text{String: e.Value, Valid: e.Value != ""},
|
||||
Attrs: attrsCustom,
|
||||
Children: childrenCustom,
|
||||
FeedID: newFeed.ID,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
log.Printf("Error adding extension to database: %s", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func addFeedAuthors(ctx context.Context, feed *gofeed.Feed, newFeed db.Feed) {
|
||||
// Add authors to the database
|
||||
for _, author := range feed.Authors {
|
||||
_, err := DB.CreateFeedAuthor(ctx, db.CreateFeedAuthorParams{
|
||||
CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||
UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||
DeletedAt: pgtype.Timestamptz{Valid: false},
|
||||
Name: pgtype.Text{String: author.Name, Valid: author.Name != ""},
|
||||
FeedID: newFeed.ID,
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("Error adding author %s (%s) to database: %s", author.Name, author.Email, err)
|
||||
}
|
||||
log.Printf("Author %s (%s) added to database", author.Name, author.Email)
|
||||
}
|
||||
}
|
||||
138
add_item.go
Normal file
138
add_item.go
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/TheLovinator1/FeedVault/db"
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
"github.com/mmcdole/gofeed"
|
||||
)
|
||||
|
||||
func addItemToDB(item *gofeed.Item, ctx context.Context, newFeed db.Feed) {
|
||||
newItem, err := DB.CreateItem(ctx, makeCreateItemParams(item, newFeed.ID))
|
||||
if err != nil {
|
||||
log.Printf("Error adding item to database: %s", err)
|
||||
}
|
||||
|
||||
// Add extensions to the database
|
||||
addItemExtensionToDB(ctx, item, newItem)
|
||||
|
||||
// Add authors to the database
|
||||
addItemAuthors(ctx, item, newItem)
|
||||
|
||||
log.Printf("Item added to database")
|
||||
}
|
||||
|
||||
func makeCreateItemParams(item *gofeed.Item, feedID int64) db.CreateItemParams {
|
||||
var updatedTime time.Time
|
||||
if item.UpdatedParsed != nil {
|
||||
updatedTime = *item.UpdatedParsed
|
||||
}
|
||||
var publishedTime time.Time
|
||||
if item.PublishedParsed != nil {
|
||||
publishedTime = *item.PublishedParsed
|
||||
}
|
||||
|
||||
itemCustom := []byte("{}")
|
||||
if item.Custom != nil {
|
||||
var err error
|
||||
itemCustom, err = json.Marshal(item.Custom)
|
||||
if err != nil {
|
||||
fmt.Println("Error marshalling item custom data:", err)
|
||||
itemCustom = []byte("{}")
|
||||
}
|
||||
}
|
||||
|
||||
params := db.CreateItemParams{
|
||||
CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||
UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||
DeletedAt: pgtype.Timestamptz{Valid: false},
|
||||
Title: pgtype.Text{String: item.Title, Valid: item.Title != ""},
|
||||
Description: pgtype.Text{String: item.Description, Valid: item.Description != ""},
|
||||
Content: pgtype.Text{String: item.Content, Valid: item.Content != ""},
|
||||
Link: pgtype.Text{String: item.Link, Valid: item.Link != ""},
|
||||
Links: item.Links,
|
||||
Updated: pgtype.Text{String: item.Updated, Valid: item.Updated != ""},
|
||||
UpdatedParsed: pgtype.Timestamptz{Time: updatedTime, Valid: !updatedTime.IsZero()},
|
||||
Published: pgtype.Text{String: item.Published, Valid: item.Published != ""},
|
||||
PublishedParsed: pgtype.Timestamptz{Time: publishedTime, Valid: !publishedTime.IsZero()},
|
||||
Guid: pgtype.Text{String: item.GUID, Valid: item.GUID != ""},
|
||||
Categories: item.Categories,
|
||||
Custom: itemCustom,
|
||||
FeedID: feedID,
|
||||
}
|
||||
|
||||
log.Printf("Created item params: %+v", params)
|
||||
|
||||
return params
|
||||
}
|
||||
|
||||
func addItemExtensionToDB(ctx context.Context, item *gofeed.Item, newItem db.Item) {
|
||||
// Add extensions to the database
|
||||
for _, ext := range item.Extensions {
|
||||
for _, exts := range ext {
|
||||
for _, e := range exts {
|
||||
attrsCustom := []byte("{}")
|
||||
if e.Attrs != nil {
|
||||
var err error
|
||||
attrsCustom, err = json.Marshal(e.Attrs)
|
||||
if err != nil {
|
||||
fmt.Println("Error marshalling extension attributes:", err)
|
||||
attrsCustom = []byte("{}")
|
||||
}
|
||||
log.Printf("Extension attributes: %s", attrsCustom)
|
||||
}
|
||||
|
||||
childrenCustom := []byte("{}")
|
||||
if e.Children != nil {
|
||||
var err error
|
||||
childrenCustom, err = json.Marshal(e.Children)
|
||||
if err != nil {
|
||||
fmt.Println("Error marshalling extension children:", err)
|
||||
childrenCustom = []byte("{}")
|
||||
}
|
||||
log.Printf("Extension children: %s", childrenCustom)
|
||||
}
|
||||
|
||||
_, err := DB.CreateItemExtension(ctx, db.CreateItemExtensionParams{
|
||||
CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||
UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||
DeletedAt: pgtype.Timestamptz{Valid: false},
|
||||
Name: pgtype.Text{String: e.Name, Valid: e.Name != ""},
|
||||
Value: pgtype.Text{String: e.Value, Valid: e.Value != ""},
|
||||
Attrs: attrsCustom,
|
||||
Children: childrenCustom,
|
||||
ItemID: newItem.ID,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
log.Printf("Error adding extension to database: %s", err)
|
||||
}
|
||||
|
||||
log.Printf("Extension added to database")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func addItemAuthors(ctx context.Context, item *gofeed.Item, newItem db.Item) {
|
||||
for _, author := range item.Authors {
|
||||
_, err := DB.CreateItemAuthor(ctx, db.CreateItemAuthorParams{
|
||||
CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||
UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||
DeletedAt: pgtype.Timestamptz{Valid: false},
|
||||
Name: pgtype.Text{String: author.Name, Valid: author.Name != ""},
|
||||
Email: pgtype.Text{String: author.Email, Valid: author.Email != ""},
|
||||
ItemID: newItem.ID,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
log.Printf("Error adding author %s (%s) to database: %s", author.Name, author.Email, err)
|
||||
}
|
||||
log.Printf("Author %s (%s) added to database", author.Name, author.Email)
|
||||
}
|
||||
}
|
||||
192
db/feeds.sql.go
192
db/feeds.sql.go
|
|
@ -163,6 +163,53 @@ func (q *Queries) CreateFeed(ctx context.Context, arg CreateFeedParams) (Feed, e
|
|||
return i, err
|
||||
}
|
||||
|
||||
const createFeedAuthor = `-- name: CreateFeedAuthor :one
|
||||
INSERT INTO
|
||||
feed_authors (
|
||||
created_at,
|
||||
updated_at,
|
||||
deleted_at,
|
||||
"name",
|
||||
email,
|
||||
feed_id
|
||||
)
|
||||
VALUES
|
||||
($1, $2, $3, $4, $5, $6)
|
||||
RETURNING
|
||||
id, created_at, updated_at, deleted_at, name, email, feed_id
|
||||
`
|
||||
|
||||
type CreateFeedAuthorParams struct {
|
||||
CreatedAt pgtype.Timestamptz `json:"created_at"`
|
||||
UpdatedAt pgtype.Timestamptz `json:"updated_at"`
|
||||
DeletedAt pgtype.Timestamptz `json:"deleted_at"`
|
||||
Name pgtype.Text `json:"name"`
|
||||
Email pgtype.Text `json:"email"`
|
||||
FeedID int64 `json:"feed_id"`
|
||||
}
|
||||
|
||||
func (q *Queries) CreateFeedAuthor(ctx context.Context, arg CreateFeedAuthorParams) (FeedAuthor, error) {
|
||||
row := q.db.QueryRow(ctx, createFeedAuthor,
|
||||
arg.CreatedAt,
|
||||
arg.UpdatedAt,
|
||||
arg.DeletedAt,
|
||||
arg.Name,
|
||||
arg.Email,
|
||||
arg.FeedID,
|
||||
)
|
||||
var i FeedAuthor
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.CreatedAt,
|
||||
&i.UpdatedAt,
|
||||
&i.DeletedAt,
|
||||
&i.Name,
|
||||
&i.Email,
|
||||
&i.FeedID,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const createFeedExtension = `-- name: CreateFeedExtension :one
|
||||
INSERT INTO
|
||||
feed_extensions (
|
||||
|
|
@ -322,6 +369,53 @@ func (q *Queries) CreateItem(ctx context.Context, arg CreateItemParams) (Item, e
|
|||
return i, err
|
||||
}
|
||||
|
||||
const createItemAuthor = `-- name: CreateItemAuthor :one
|
||||
INSERT INTO
|
||||
item_authors (
|
||||
created_at,
|
||||
updated_at,
|
||||
deleted_at,
|
||||
"name",
|
||||
email,
|
||||
item_id
|
||||
)
|
||||
VALUES
|
||||
($1, $2, $3, $4, $5, $6)
|
||||
RETURNING
|
||||
id, created_at, updated_at, deleted_at, name, email, item_id
|
||||
`
|
||||
|
||||
type CreateItemAuthorParams struct {
|
||||
CreatedAt pgtype.Timestamptz `json:"created_at"`
|
||||
UpdatedAt pgtype.Timestamptz `json:"updated_at"`
|
||||
DeletedAt pgtype.Timestamptz `json:"deleted_at"`
|
||||
Name pgtype.Text `json:"name"`
|
||||
Email pgtype.Text `json:"email"`
|
||||
ItemID int64 `json:"item_id"`
|
||||
}
|
||||
|
||||
func (q *Queries) CreateItemAuthor(ctx context.Context, arg CreateItemAuthorParams) (ItemAuthor, error) {
|
||||
row := q.db.QueryRow(ctx, createItemAuthor,
|
||||
arg.CreatedAt,
|
||||
arg.UpdatedAt,
|
||||
arg.DeletedAt,
|
||||
arg.Name,
|
||||
arg.Email,
|
||||
arg.ItemID,
|
||||
)
|
||||
var i ItemAuthor
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.CreatedAt,
|
||||
&i.UpdatedAt,
|
||||
&i.DeletedAt,
|
||||
&i.Name,
|
||||
&i.Email,
|
||||
&i.ItemID,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const createItemExtension = `-- name: CreateItemExtension :one
|
||||
INSERT INTO
|
||||
item_extensions (
|
||||
|
|
@ -415,6 +509,55 @@ func (q *Queries) GetFeed(ctx context.Context, id int64) (Feed, error) {
|
|||
return i, err
|
||||
}
|
||||
|
||||
const getFeedAuthors = `-- name: GetFeedAuthors :many
|
||||
SELECT
|
||||
id, created_at, updated_at, deleted_at, name, email, feed_id
|
||||
FROM
|
||||
feed_authors
|
||||
WHERE
|
||||
feed_id = $1
|
||||
ORDER BY
|
||||
created_at DESC
|
||||
LIMIT
|
||||
$2
|
||||
OFFSET
|
||||
$3
|
||||
`
|
||||
|
||||
type GetFeedAuthorsParams struct {
|
||||
FeedID int64 `json:"feed_id"`
|
||||
Limit int32 `json:"limit"`
|
||||
Offset int32 `json:"offset"`
|
||||
}
|
||||
|
||||
func (q *Queries) GetFeedAuthors(ctx context.Context, arg GetFeedAuthorsParams) ([]FeedAuthor, error) {
|
||||
rows, err := q.db.Query(ctx, getFeedAuthors, arg.FeedID, arg.Limit, arg.Offset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := []FeedAuthor{}
|
||||
for rows.Next() {
|
||||
var i FeedAuthor
|
||||
if err := rows.Scan(
|
||||
&i.ID,
|
||||
&i.CreatedAt,
|
||||
&i.UpdatedAt,
|
||||
&i.DeletedAt,
|
||||
&i.Name,
|
||||
&i.Email,
|
||||
&i.FeedID,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const getFeedExtensions = `-- name: GetFeedExtensions :many
|
||||
SELECT
|
||||
id, created_at, updated_at, deleted_at, name, value, attrs, children, feed_id
|
||||
|
|
@ -560,6 +703,55 @@ func (q *Queries) GetItem(ctx context.Context, id int64) (Item, error) {
|
|||
return i, err
|
||||
}
|
||||
|
||||
const getItemAuthors = `-- name: GetItemAuthors :many
|
||||
SELECT
|
||||
id, created_at, updated_at, deleted_at, name, email, item_id
|
||||
FROM
|
||||
item_authors
|
||||
WHERE
|
||||
item_id = $1
|
||||
ORDER BY
|
||||
created_at DESC
|
||||
LIMIT
|
||||
$2
|
||||
OFFSET
|
||||
$3
|
||||
`
|
||||
|
||||
type GetItemAuthorsParams struct {
|
||||
ItemID int64 `json:"item_id"`
|
||||
Limit int32 `json:"limit"`
|
||||
Offset int32 `json:"offset"`
|
||||
}
|
||||
|
||||
func (q *Queries) GetItemAuthors(ctx context.Context, arg GetItemAuthorsParams) ([]ItemAuthor, error) {
|
||||
rows, err := q.db.Query(ctx, getItemAuthors, arg.ItemID, arg.Limit, arg.Offset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := []ItemAuthor{}
|
||||
for rows.Next() {
|
||||
var i ItemAuthor
|
||||
if err := rows.Scan(
|
||||
&i.ID,
|
||||
&i.CreatedAt,
|
||||
&i.UpdatedAt,
|
||||
&i.DeletedAt,
|
||||
&i.Name,
|
||||
&i.Email,
|
||||
&i.ItemID,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const getItemExtensions = `-- name: GetItemExtensions :many
|
||||
SELECT
|
||||
id, created_at, updated_at, deleted_at, name, value, attrs, children, item_id
|
||||
|
|
|
|||
|
|
@ -50,7 +50,6 @@ type FeedAuthor struct {
|
|||
DeletedAt pgtype.Timestamptz `json:"deleted_at"`
|
||||
Name pgtype.Text `json:"name"`
|
||||
Email pgtype.Text `json:"email"`
|
||||
Uri pgtype.Text `json:"uri"`
|
||||
FeedID int64 `json:"feed_id"`
|
||||
}
|
||||
|
||||
|
|
@ -145,7 +144,6 @@ type ItemAuthor struct {
|
|||
DeletedAt pgtype.Timestamptz `json:"deleted_at"`
|
||||
Name pgtype.Text `json:"name"`
|
||||
Email pgtype.Text `json:"email"`
|
||||
Uri pgtype.Text `json:"uri"`
|
||||
ItemID int64 `json:"item_id"`
|
||||
}
|
||||
|
||||
|
|
|
|||
187
feeds.go
187
feeds.go
|
|
@ -2,104 +2,13 @@ package main
|
|||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/TheLovinator1/FeedVault/db"
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
"github.com/mmcdole/gofeed"
|
||||
)
|
||||
|
||||
func makeCreateFeedParams(feedURL string, feed *gofeed.Feed) db.CreateFeedParams {
|
||||
var updatedTime time.Time
|
||||
if feed.UpdatedParsed != nil {
|
||||
updatedTime = *feed.UpdatedParsed
|
||||
}
|
||||
var publishedTime time.Time
|
||||
if feed.PublishedParsed != nil {
|
||||
publishedTime = *feed.PublishedParsed
|
||||
}
|
||||
|
||||
feedCustom, err := json.Marshal(feed.Custom)
|
||||
if err != nil {
|
||||
fmt.Println("Error marshalling feed custom data:", err)
|
||||
feedCustom = []byte("{}")
|
||||
}
|
||||
|
||||
params := db.CreateFeedParams{
|
||||
Url: feedURL,
|
||||
CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||
UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||
DeletedAt: pgtype.Timestamptz{Valid: false},
|
||||
Title: pgtype.Text{String: feed.Title, Valid: feed.Title != ""},
|
||||
Description: pgtype.Text{String: feed.Description, Valid: feed.Description != ""},
|
||||
Link: pgtype.Text{String: feed.Link, Valid: feed.Link != ""},
|
||||
FeedLink: pgtype.Text{String: feed.FeedLink, Valid: feed.FeedLink != ""},
|
||||
Links: feed.Links,
|
||||
Updated: pgtype.Text{String: feed.Updated, Valid: feed.Updated != ""},
|
||||
UpdatedParsed: pgtype.Timestamptz{Time: updatedTime, Valid: !updatedTime.IsZero()},
|
||||
Published: pgtype.Text{String: feed.Published, Valid: feed.Published != ""},
|
||||
PublishedParsed: pgtype.Timestamptz{Time: publishedTime, Valid: !publishedTime.IsZero()},
|
||||
Language: pgtype.Text{String: feed.Language, Valid: feed.Language != ""},
|
||||
Copyright: pgtype.Text{String: feed.Copyright, Valid: feed.Copyright != ""},
|
||||
Generator: pgtype.Text{String: feed.Generator, Valid: feed.Generator != ""},
|
||||
Categories: feed.Categories,
|
||||
Custom: feedCustom,
|
||||
FeedType: pgtype.Text{String: feed.FeedType, Valid: feed.FeedType != ""},
|
||||
FeedVersion: pgtype.Text{String: feed.FeedVersion, Valid: feed.FeedVersion != ""},
|
||||
}
|
||||
|
||||
log.Printf("Created feed params: %+v", params)
|
||||
|
||||
return params
|
||||
}
|
||||
|
||||
func makeCreateItemParams(item *gofeed.Item, feedID int64) db.CreateItemParams {
|
||||
var updatedTime time.Time
|
||||
if item.UpdatedParsed != nil {
|
||||
updatedTime = *item.UpdatedParsed
|
||||
}
|
||||
var publishedTime time.Time
|
||||
if item.PublishedParsed != nil {
|
||||
publishedTime = *item.PublishedParsed
|
||||
}
|
||||
|
||||
itemCustom := []byte("{}")
|
||||
if item.Custom != nil {
|
||||
var err error
|
||||
itemCustom, err = json.Marshal(item.Custom)
|
||||
if err != nil {
|
||||
fmt.Println("Error marshalling item custom data:", err)
|
||||
itemCustom = []byte("{}")
|
||||
}
|
||||
}
|
||||
|
||||
params := db.CreateItemParams{
|
||||
CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||
UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||
DeletedAt: pgtype.Timestamptz{Valid: false},
|
||||
Title: pgtype.Text{String: item.Title, Valid: item.Title != ""},
|
||||
Description: pgtype.Text{String: item.Description, Valid: item.Description != ""},
|
||||
Content: pgtype.Text{String: item.Content, Valid: item.Content != ""},
|
||||
Link: pgtype.Text{String: item.Link, Valid: item.Link != ""},
|
||||
Links: item.Links,
|
||||
Updated: pgtype.Text{String: item.Updated, Valid: item.Updated != ""},
|
||||
UpdatedParsed: pgtype.Timestamptz{Time: updatedTime, Valid: !updatedTime.IsZero()},
|
||||
Published: pgtype.Text{String: item.Published, Valid: item.Published != ""},
|
||||
PublishedParsed: pgtype.Timestamptz{Time: publishedTime, Valid: !publishedTime.IsZero()},
|
||||
Guid: pgtype.Text{String: item.GUID, Valid: item.GUID != ""},
|
||||
Categories: item.Categories,
|
||||
Custom: itemCustom,
|
||||
FeedID: feedID,
|
||||
}
|
||||
|
||||
log.Printf("Created item params: %+v", params)
|
||||
|
||||
return params
|
||||
}
|
||||
|
||||
func AddFeedToDB(feedURL string) error {
|
||||
// Cancel the request after 60 seconds if it hasn't finished
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
|
|
@ -123,102 +32,14 @@ func AddFeedToDB(feedURL string) error {
|
|||
|
||||
// Add the items to the database
|
||||
for _, item := range feed.Items {
|
||||
newItem, err := DB.CreateItem(ctx, makeCreateItemParams(item, newFeed.ID))
|
||||
if err != nil {
|
||||
log.Printf("Error adding item to database: %s", err)
|
||||
}
|
||||
log.Printf("Item added to database")
|
||||
|
||||
// Add extensions to the database
|
||||
for _, ext := range item.Extensions {
|
||||
for _, exts := range ext {
|
||||
for _, e := range exts {
|
||||
attrsCustom := []byte("{}")
|
||||
if e.Attrs != nil {
|
||||
var err error
|
||||
attrsCustom, err = json.Marshal(e.Attrs)
|
||||
if err != nil {
|
||||
fmt.Println("Error marshalling extension attributes:", err)
|
||||
attrsCustom = []byte("{}")
|
||||
}
|
||||
log.Printf("Extension attributes: %s", attrsCustom)
|
||||
}
|
||||
|
||||
childrenCustom := []byte("{}")
|
||||
if e.Children != nil {
|
||||
var err error
|
||||
childrenCustom, err = json.Marshal(e.Children)
|
||||
if err != nil {
|
||||
fmt.Println("Error marshalling extension children:", err)
|
||||
childrenCustom = []byte("{}")
|
||||
}
|
||||
log.Printf("Extension children: %s", childrenCustom)
|
||||
}
|
||||
|
||||
_, err := DB.CreateItemExtension(ctx, db.CreateItemExtensionParams{
|
||||
CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||
UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||
DeletedAt: pgtype.Timestamptz{Valid: false},
|
||||
Name: pgtype.Text{String: e.Name, Valid: e.Name != ""},
|
||||
Value: pgtype.Text{String: e.Value, Valid: e.Value != ""},
|
||||
Attrs: attrsCustom,
|
||||
Children: childrenCustom,
|
||||
ItemID: newItem.ID,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
log.Printf("Error adding extension to database: %s", err)
|
||||
}
|
||||
|
||||
log.Printf("Extension added to database")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
addItemToDB(item, ctx, newFeed)
|
||||
}
|
||||
|
||||
// Add extensions to the database
|
||||
// TODO: Check if this is correct and works
|
||||
for _, ext := range feed.Extensions {
|
||||
for _, exts := range ext {
|
||||
for _, e := range exts {
|
||||
attrsCustom := []byte("{}")
|
||||
if e.Attrs != nil {
|
||||
var err error
|
||||
attrsCustom, err = json.Marshal(e.Attrs)
|
||||
if err != nil {
|
||||
fmt.Println("Error marshalling extension attributes:", err)
|
||||
attrsCustom = []byte("{}")
|
||||
}
|
||||
}
|
||||
addFeedExtensionToDB(ctx, feed, newFeed)
|
||||
|
||||
childrenCustom := []byte("{}")
|
||||
if e.Children != nil {
|
||||
var err error
|
||||
childrenCustom, err = json.Marshal(e.Children)
|
||||
if err != nil {
|
||||
fmt.Println("Error marshalling extension children:", err)
|
||||
childrenCustom = []byte("{}")
|
||||
}
|
||||
}
|
||||
|
||||
_, err := DB.CreateFeedExtension(ctx, db.CreateFeedExtensionParams{
|
||||
CreatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||
UpdatedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
||||
DeletedAt: pgtype.Timestamptz{Valid: false},
|
||||
Name: pgtype.Text{String: e.Name, Valid: e.Name != ""},
|
||||
Value: pgtype.Text{String: e.Value, Valid: e.Value != ""},
|
||||
Attrs: attrsCustom,
|
||||
Children: childrenCustom,
|
||||
FeedID: newFeed.ID,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
log.Printf("Error adding extension to database: %s", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Add authors to the database
|
||||
addFeedAuthors(ctx, feed, newFeed)
|
||||
|
||||
fmt.Println(feed.Title)
|
||||
return nil
|
||||
|
|
|
|||
87
handlers.go
87
handlers.go
|
|
@ -39,8 +39,52 @@ func FeedsHandler(w http.ResponseWriter, _ *http.Request) {
|
|||
|
||||
fb := strings.Builder{}
|
||||
for _, feed := range feeds {
|
||||
authors, err := DB.GetFeedAuthors(context.Background(), db.GetFeedAuthorsParams{
|
||||
FeedID: feed.ID,
|
||||
Limit: 100,
|
||||
})
|
||||
if err != nil {
|
||||
http.Error(w, "Error getting authors", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
extensions, err := DB.GetFeedExtensions(context.Background(), db.GetFeedExtensionsParams{
|
||||
FeedID: feed.ID,
|
||||
Limit: 100,
|
||||
})
|
||||
if err != nil {
|
||||
http.Error(w, "Error getting extensions", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
fb.WriteString("<li>")
|
||||
fb.WriteString("<a href=\"/feed/" + strconv.FormatInt(feed.ID, 10) + "\">" + feed.Title.String + "</a> - <a href=\"" + feed.Link.String + "\">" + feed.Link.String + "</a>")
|
||||
fb.WriteString(feed.Title.String)
|
||||
fb.WriteString("<ul>")
|
||||
for _, author := range authors {
|
||||
if author.Name.Valid {
|
||||
fb.WriteString("<li>Author: " + author.Name.String + "</li>")
|
||||
}
|
||||
if author.Email.Valid {
|
||||
fb.WriteString("<li>Email: " + author.Email.String + "</li>")
|
||||
}
|
||||
}
|
||||
|
||||
for _, ext := range extensions {
|
||||
if ext.Attrs != nil {
|
||||
fb.WriteString("<li>Attrs: " + string(ext.Attrs) + "</li>")
|
||||
}
|
||||
if ext.Children != nil {
|
||||
fb.WriteString("<li>Children: " + string(ext.Children) + "</li>")
|
||||
}
|
||||
if ext.Name.Valid {
|
||||
fb.WriteString("<li>Name: " + ext.Name.String + "</li>")
|
||||
}
|
||||
if ext.Value.Valid {
|
||||
fb.WriteString("<li>Value: " + ext.Value.String + "</li>")
|
||||
}
|
||||
}
|
||||
fb.WriteString("</ul>")
|
||||
fb.WriteString("<a href=\"/feed/" + strconv.FormatInt(feed.ID, 10) + "\">" + feed.Url + "</a>")
|
||||
fb.WriteString("</li>")
|
||||
}
|
||||
|
||||
|
|
@ -279,9 +323,50 @@ func FeedHandler(w http.ResponseWriter, r *http.Request) {
|
|||
// Build the HTML
|
||||
fb := strings.Builder{}
|
||||
for _, item := range items {
|
||||
// Get authors for the item
|
||||
authors, err := DB.GetItemAuthors(context.Background(), db.GetItemAuthorsParams{
|
||||
ItemID: item.ID,
|
||||
Limit: 100,
|
||||
})
|
||||
if err != nil {
|
||||
http.Error(w, "Error getting authors", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// Get extensions for the item
|
||||
extensions, err := DB.GetItemExtensions(context.Background(), db.GetItemExtensionsParams{
|
||||
ItemID: item.ID,
|
||||
Limit: 100,
|
||||
})
|
||||
if err != nil {
|
||||
http.Error(w, "Error getting extensions", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
fb.WriteString("<li>")
|
||||
fb.WriteString("<a href=\"" + item.Link.String + "\">" + item.Title.String + "</a>")
|
||||
fb.WriteString("<ul>")
|
||||
for _, author := range authors {
|
||||
fb.WriteString("<li>Author: " + author.Name.String + "</li>")
|
||||
}
|
||||
|
||||
for _, ext := range extensions {
|
||||
fb.WriteString("<ul>")
|
||||
fb.WriteString("<li>Extension: " + ext.Name.String + "</li>")
|
||||
if ext.Value.Valid {
|
||||
fb.WriteString("<li>Name: " + ext.Value.String + "</li>")
|
||||
}
|
||||
if ext.Attrs != nil {
|
||||
fb.WriteString("<li>Attrs: " + string(ext.Attrs) + "</li>")
|
||||
}
|
||||
if ext.Children != nil {
|
||||
fb.WriteString("<li>Children: " + string(ext.Children) + "</li>")
|
||||
}
|
||||
fb.WriteString("</ul>")
|
||||
|
||||
}
|
||||
fb.WriteString("</ul>")
|
||||
fb.WriteString("<ul>")
|
||||
if item.Published.Valid {
|
||||
fb.WriteString("<li>Published: " + item.Published.String + "</li>")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
package main
|
||||
|
||||
|
||||
// "Fun" messages that will be displayed in the footer
|
||||
var FunMsg = []string{
|
||||
"Web scraping is not a crime.",
|
||||
|
|
|
|||
|
|
@ -205,3 +205,61 @@ LIMIT
|
|||
$2
|
||||
OFFSET
|
||||
$3;
|
||||
|
||||
-- name: CreateFeedAuthor :one
|
||||
INSERT INTO
|
||||
feed_authors (
|
||||
created_at,
|
||||
updated_at,
|
||||
deleted_at,
|
||||
"name",
|
||||
email,
|
||||
feed_id
|
||||
)
|
||||
VALUES
|
||||
($1, $2, $3, $4, $5, $6)
|
||||
RETURNING
|
||||
*;
|
||||
|
||||
-- name: CreateItemAuthor :one
|
||||
INSERT INTO
|
||||
item_authors (
|
||||
created_at,
|
||||
updated_at,
|
||||
deleted_at,
|
||||
"name",
|
||||
email,
|
||||
item_id
|
||||
)
|
||||
VALUES
|
||||
($1, $2, $3, $4, $5, $6)
|
||||
RETURNING
|
||||
*;
|
||||
|
||||
-- name: GetFeedAuthors :many
|
||||
SELECT
|
||||
*
|
||||
FROM
|
||||
feed_authors
|
||||
WHERE
|
||||
feed_id = $1
|
||||
ORDER BY
|
||||
created_at DESC
|
||||
LIMIT
|
||||
$2
|
||||
OFFSET
|
||||
$3;
|
||||
|
||||
-- name: GetItemAuthors :many
|
||||
SELECT
|
||||
*
|
||||
FROM
|
||||
item_authors
|
||||
WHERE
|
||||
item_id = $1
|
||||
ORDER BY
|
||||
created_at DESC
|
||||
LIMIT
|
||||
$2
|
||||
OFFSET
|
||||
$3;
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ CREATE TABLE IF NOT EXISTS feed_authors (
|
|||
-- From gofeed:
|
||||
"name" TEXT,
|
||||
email TEXT,
|
||||
uri TEXT,
|
||||
-- Link to feed
|
||||
feed_id BIGINT NOT NULL,
|
||||
CONSTRAINT fk_feed_id FOREIGN KEY (feed_id) REFERENCES feeds (id) ON DELETE CASCADE
|
||||
|
|
@ -26,7 +25,6 @@ CREATE TABLE IF NOT EXISTS item_authors (
|
|||
-- From gofeed:
|
||||
"name" TEXT,
|
||||
email TEXT,
|
||||
uri TEXT,
|
||||
-- Link to feed item (Also called feed entry)
|
||||
item_id BIGINT NOT NULL,
|
||||
CONSTRAINT fk_item_id FOREIGN KEY (item_id) REFERENCES items (id) ON DELETE CASCADE
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue