TheLovinator/feedvault.se
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

You can now add feeds to the database

Browse source
This commit is contained in:
Joakim Hellsén 2024-02-16 04:56:44 +01:00
commit a5b6352a4c
24 changed files with 1049 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.air.toml
View file

@ -16,6 +16,7 @@ exclude_dir = [
"tests",
".git",
".vscode",
"data",
]
exclude_file = []
exclude_regex = ["_test.go"]

3
.gitignore vendored
View file

@ -22,5 +22,4 @@ tmp/
.env
# Database
*.db
*.sqlite3
data/

20
.vscode/settings.json vendored
View file

@ -31,6 +31,7 @@
"homerouter",
"hotspot",
"huaweimobilewifi",
"jackc",
"ldflags",
"leftright",
"levelname",
@ -49,6 +50,7 @@
"pacman",
"PGHOST",
"PGPORT",
"pgtype",
"PGUSER",
"Prés",
"psql",
@ -67,6 +69,7 @@
"stylesheet",
"sunt",
"tdewolff",
"Timestamptz",
"tmpl",
"tplinkap",
"tplinkeap",
@ -79,5 +82,20 @@
"webmail",
"XOXO",
"zerolog"
]
],
"terminal.integrated.env.windows": {
"GOOSE_DRIVER": "postgres",
"GOOSE_DBSTRING": "user=feedvault password=feedvault dbname=feedvault sslmode=disable",
"GOOSE_MIGRATION_DIR": "${workspaceFolder}/sql/schema"
},
"terminal.integrated.env.linux": {
"GOOSE_DRIVER": "postgres",
"GOOSE_DBSTRING": "user=feedvault password=feedvault dbname=feedvault sslmode=disable",
"GOOSE_MIGRATION_DIR": "${workspaceFolder}/sql/schema"
},
"terminal.integrated.env.osx": {
"GOOSE_DRIVER": "postgres",
"GOOSE_DBSTRING": "user=feedvault password=feedvault dbname=feedvault sslmode=disable",
"GOOSE_MIGRATION_DIR": "${workspaceFolder}/sql/schema"
}
}

17
README.md
View file

@ -22,18 +22,25 @@ _Note: Some features are currently in development._
- Add your favorite feeds to start archiving content.
- Explore, manage, and enjoy your centralized feed archive.
## Docker
Please see [Docker.md](Docker.md).
## Contributing
All contributions are welcome regardless of skill level or experience.
Please read the [CONTRIBUTING.md](CONTRIBUTING.md) file for more information on how to contribute to FeedVault.
Please create a new issue before submitting a big pull request. I am probably okay with anything, but I don't want you to waste your time on something that I won't accept.
Try to minimize the number of dependencies you add to the project. If you need to add a new dependency, please create an issue first.
## Contact
For any inquiries or support, please create an issue on GitHub.
## Development
- I use [goose](https://github.com/pressly/goose) and [sqlc](https://github.com/sqlc-dev/sqlc) for database migrations and queries.
- To create a new migration, run `goose create <migration_name> sql`. Then, edit the file in `sql/schema/<date>_<migration_name>.sql` and run `goose up` to apply the migration.
- You will have to install `goose` first. See the [Goose documentation](https://pressly.github.io/goose/installation/).
- You will also have to install `sqlc`. See the [sqlc documentation](https://docs.sqlc.dev/en/latest/overview/install.html).
- You have to set some environment variables for this. See [.vscode/settings.json](.vscode/settings.json) for local development.
- To generate new queries, run `sqlc generate`.
Thank you for using FeedVault! Happy archiving!

32
db/db.go Normal file
View file

@ -0,0 +1,32 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.25.0
package db
import (
"context"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
)
type DBTX interface {
Exec(context.Context, string, ...interface{}) (pgconn.CommandTag, error)
Query(context.Context, string, ...interface{}) (pgx.Rows, error)
QueryRow(context.Context, string, ...interface{}) pgx.Row
}
func New(db DBTX) *Queries {
return &Queries{db: db}
}
type Queries struct {
db DBTX
}
func (q *Queries) WithTx(tx pgx.Tx) *Queries {
return &Queries{
db: tx,
}
}

148
db/feeds.sql.go Normal file
View file

@ -0,0 +1,148 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.25.0
// source: feeds.sql
package db
import (
"context"
"github.com/jackc/pgx/v5/pgtype"
)
const CountFeeds = `-- name: CountFeeds :one
SELECT
COUNT(*)
FROM
feeds
`
func (q *Queries) CountFeeds(ctx context.Context) (int64, error) {
row := q.db.QueryRow(ctx, CountFeeds)
var count int64
err := row.Scan(&count)
return count, err
}
const CreateFeed = `-- name: CreateFeed :one
INSERT INTO
feeds (
"url",
created_at,
updated_at,
deleted_at,
title,
"description",
link,
feed_link,
links,
updated,
updated_parsed,
published,
published_parsed,
"language",
copyright,
generator,
categories,
custom,
feed_type,
feed_version
)
VALUES
(
$1,
$2,
$3,
$4,
$5,
$6,
$7,
$8,
$9,
$10,
$11,
$12,
$13,
$14,
$15,
$16,
$17,
$18,
$19,
$20
) RETURNING id, url, created_at, updated_at, deleted_at, title, description, link, feed_link, links, updated, updated_parsed, published, published_parsed, language, copyright, generator, categories, custom, feed_type, feed_version
`
type CreateFeedParams struct {
Url string `json:"url"`
CreatedAt pgtype.Timestamp `json:"created_at"`
UpdatedAt pgtype.Timestamp `json:"updated_at"`
DeletedAt pgtype.Timestamp `json:"deleted_at"`
Title pgtype.Text `json:"title"`
Description pgtype.Text `json:"description"`
Link pgtype.Text `json:"link"`
FeedLink pgtype.Text `json:"feed_link"`
Links []string `json:"links"`
Updated pgtype.Text `json:"updated"`
UpdatedParsed pgtype.Timestamp `json:"updated_parsed"`
Published pgtype.Text `json:"published"`
PublishedParsed pgtype.Timestamp `json:"published_parsed"`
Language pgtype.Text `json:"language"`
Copyright pgtype.Text `json:"copyright"`
Generator pgtype.Text `json:"generator"`
Categories []string `json:"categories"`
Custom []byte `json:"custom"`
FeedType pgtype.Text `json:"feed_type"`
FeedVersion pgtype.Text `json:"feed_version"`
}
func (q *Queries) CreateFeed(ctx context.Context, arg CreateFeedParams) (Feed, error) {
row := q.db.QueryRow(ctx, CreateFeed,
arg.Url,
arg.CreatedAt,
arg.UpdatedAt,
arg.DeletedAt,
arg.Title,
arg.Description,
arg.Link,
arg.FeedLink,
arg.Links,
arg.Updated,
arg.UpdatedParsed,
arg.Published,
arg.PublishedParsed,
arg.Language,
arg.Copyright,
arg.Generator,
arg.Categories,
arg.Custom,
arg.FeedType,
arg.FeedVersion,
)
var i Feed
err := row.Scan(
&i.ID,
&i.Url,
&i.CreatedAt,
&i.UpdatedAt,
&i.DeletedAt,
&i.Title,
&i.Description,
&i.Link,
&i.FeedLink,
&i.Links,
&i.Updated,
&i.UpdatedParsed,
&i.Published,
&i.PublishedParsed,
&i.Language,
&i.Copyright,
&i.Generator,
&i.Categories,
&i.Custom,
&i.FeedType,
&i.FeedVersion,
)
return i, err
}

237
db/models.go Normal file
View file

@ -0,0 +1,237 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.25.0
package db
import (
"github.com/jackc/pgx/v5/pgtype"
)
type Enclosure struct {
ID int32 `json:"id"`
CreatedAt pgtype.Timestamp `json:"created_at"`
UpdatedAt pgtype.Timestamp `json:"updated_at"`
DeletedAt pgtype.Timestamp `json:"deleted_at"`
Url pgtype.Text `json:"url"`
Length pgtype.Text `json:"length"`
Type pgtype.Text `json:"type"`
ItemID int32 `json:"item_id"`
}
type Feed struct {
ID int32 `json:"id"`
Url string `json:"url"`
CreatedAt pgtype.Timestamp `json:"created_at"`
UpdatedAt pgtype.Timestamp `json:"updated_at"`
DeletedAt pgtype.Timestamp `json:"deleted_at"`
Title pgtype.Text `json:"title"`
Description pgtype.Text `json:"description"`
Link pgtype.Text `json:"link"`
FeedLink pgtype.Text `json:"feed_link"`
Links []string `json:"links"`
Updated pgtype.Text `json:"updated"`
UpdatedParsed pgtype.Timestamp `json:"updated_parsed"`
Published pgtype.Text `json:"published"`
PublishedParsed pgtype.Timestamp `json:"published_parsed"`
Language pgtype.Text `json:"language"`
Copyright pgtype.Text `json:"copyright"`
Generator pgtype.Text `json:"generator"`
Categories []string `json:"categories"`
Custom []byte `json:"custom"`
FeedType pgtype.Text `json:"feed_type"`
FeedVersion pgtype.Text `json:"feed_version"`
}
type FeedAuthor struct {
ID int32 `json:"id"`
CreatedAt pgtype.Timestamp `json:"created_at"`
UpdatedAt pgtype.Timestamp `json:"updated_at"`
DeletedAt pgtype.Timestamp `json:"deleted_at"`
Name pgtype.Text `json:"name"`
Email pgtype.Text `json:"email"`
Uri pgtype.Text `json:"uri"`
FeedID int32 `json:"feed_id"`
}
type FeedDublinCore struct {
ID int32 `json:"id"`
CreatedAt pgtype.Timestamp `json:"created_at"`
UpdatedAt pgtype.Timestamp `json:"updated_at"`
DeletedAt pgtype.Timestamp `json:"deleted_at"`
Title []string `json:"title"`
Creator []string `json:"creator"`
Author []string `json:"author"`
Subject []string `json:"subject"`
Description []string `json:"description"`
Publisher []string `json:"publisher"`
Contributor []string `json:"contributor"`
Date []string `json:"date"`
Type []string `json:"type"`
Format []string `json:"format"`
Identifier []string `json:"identifier"`
Source []string `json:"source"`
Language []string `json:"language"`
Relation []string `json:"relation"`
Coverage []string `json:"coverage"`
Rights []string `json:"rights"`
FeedID int32 `json:"feed_id"`
}
type FeedExtension struct {
ID int32 `json:"id"`
CreatedAt pgtype.Timestamp `json:"created_at"`
UpdatedAt pgtype.Timestamp `json:"updated_at"`
DeletedAt pgtype.Timestamp `json:"deleted_at"`
Name pgtype.Text `json:"name"`
Value pgtype.Text `json:"value"`
Attrs []byte `json:"attrs"`
Children []byte `json:"children"`
FeedID int32 `json:"feed_id"`
}
type FeedImage struct {
ID int32 `json:"id"`
CreatedAt pgtype.Timestamp `json:"created_at"`
UpdatedAt pgtype.Timestamp `json:"updated_at"`
DeletedAt pgtype.Timestamp `json:"deleted_at"`
Url pgtype.Text `json:"url"`
Title pgtype.Text `json:"title"`
FeedID int32 `json:"feed_id"`
}
type FeedItune struct {
ID int32 `json:"id"`
CreatedAt pgtype.Timestamp `json:"created_at"`
UpdatedAt pgtype.Timestamp `json:"updated_at"`
DeletedAt pgtype.Timestamp `json:"deleted_at"`
Author pgtype.Text `json:"author"`
Block pgtype.Text `json:"block"`
Explicit pgtype.Text `json:"explicit"`
Keywords pgtype.Text `json:"keywords"`
Subtitle pgtype.Text `json:"subtitle"`
Summary pgtype.Text `json:"summary"`
Image pgtype.Text `json:"image"`
Complete pgtype.Text `json:"complete"`
NewFeedUrl pgtype.Text `json:"new_feed_url"`
Type pgtype.Text `json:"type"`
FeedID int32 `json:"feed_id"`
}
type Item struct {
ID int32 `json:"id"`
CreatedAt pgtype.Timestamp `json:"created_at"`
UpdatedAt pgtype.Timestamp `json:"updated_at"`
DeletedAt pgtype.Timestamp `json:"deleted_at"`
Title pgtype.Text `json:"title"`
Description pgtype.Text `json:"description"`
Content pgtype.Text `json:"content"`
Link pgtype.Text `json:"link"`
Links []string `json:"links"`
Updated pgtype.Text `json:"updated"`
UpdatedParsed pgtype.Timestamp `json:"updated_parsed"`
Published pgtype.Text `json:"published"`
PublishedParsed pgtype.Timestamp `json:"published_parsed"`
Guid pgtype.Text `json:"guid"`
Categories []string `json:"categories"`
Custom []byte `json:"custom"`
FeedID int32 `json:"feed_id"`
}
type ItemAuthor struct {
ID int32 `json:"id"`
CreatedAt pgtype.Timestamp `json:"created_at"`
UpdatedAt pgtype.Timestamp `json:"updated_at"`
DeletedAt pgtype.Timestamp `json:"deleted_at"`
Name pgtype.Text `json:"name"`
Email pgtype.Text `json:"email"`
Uri pgtype.Text `json:"uri"`
ItemID int32 `json:"item_id"`
}
type ItemDublinCore struct {
ID int32 `json:"id"`
CreatedAt pgtype.Timestamp `json:"created_at"`
UpdatedAt pgtype.Timestamp `json:"updated_at"`
DeletedAt pgtype.Timestamp `json:"deleted_at"`
Title []string `json:"title"`
Creator []string `json:"creator"`
Author []string `json:"author"`
Subject []string `json:"subject"`
Description []string `json:"description"`
Publisher []string `json:"publisher"`
Contributor []string `json:"contributor"`
Date []string `json:"date"`
Type []string `json:"type"`
Format []string `json:"format"`
Identifier []string `json:"identifier"`
Source []string `json:"source"`
Language []string `json:"language"`
Relation []string `json:"relation"`
Coverage []string `json:"coverage"`
Rights []string `json:"rights"`
ItemID int32 `json:"item_id"`
}
type ItemExtension struct {
ID int32 `json:"id"`
CreatedAt pgtype.Timestamp `json:"created_at"`
UpdatedAt pgtype.Timestamp `json:"updated_at"`
DeletedAt pgtype.Timestamp `json:"deleted_at"`
Name pgtype.Text `json:"name"`
Value pgtype.Text `json:"value"`
Attrs []byte `json:"attrs"`
Children []byte `json:"children"`
ItemID int32 `json:"item_id"`
}
type ItemImage struct {
ID int32 `json:"id"`
CreatedAt pgtype.Timestamp `json:"created_at"`
UpdatedAt pgtype.Timestamp `json:"updated_at"`
DeletedAt pgtype.Timestamp `json:"deleted_at"`
Url pgtype.Text `json:"url"`
Title pgtype.Text `json:"title"`
ItemID int32 `json:"item_id"`
}
type ItemItune struct {
ID int32 `json:"id"`
CreatedAt pgtype.Timestamp `json:"created_at"`
UpdatedAt pgtype.Timestamp `json:"updated_at"`
DeletedAt pgtype.Timestamp `json:"deleted_at"`
Author pgtype.Text `json:"author"`
Block pgtype.Text `json:"block"`
Duration pgtype.Text `json:"duration"`
Explicit pgtype.Text `json:"explicit"`
Keywords pgtype.Text `json:"keywords"`
Subtitle pgtype.Text `json:"subtitle"`
Summary pgtype.Text `json:"summary"`
Image pgtype.Text `json:"image"`
IsClosedCaptioned pgtype.Text `json:"is_closed_captioned"`
Episode pgtype.Text `json:"episode"`
Season pgtype.Text `json:"season"`
Order pgtype.Text `json:"order"`
EpisodeType pgtype.Text `json:"episode_type"`
ItemID int32 `json:"item_id"`
}
type ItunesCategory struct {
ID int32 `json:"id"`
CreatedAt pgtype.Timestamp `json:"created_at"`
UpdatedAt pgtype.Timestamp `json:"updated_at"`
DeletedAt pgtype.Timestamp `json:"deleted_at"`
Text pgtype.Text `json:"text"`
Subcategory pgtype.Text `json:"subcategory"`
ItunesID int32 `json:"itunes_id"`
}
type ItunesOwner struct {
ID int32 `json:"id"`
CreatedAt pgtype.Timestamp `json:"created_at"`
UpdatedAt pgtype.Timestamp `json:"updated_at"`
DeletedAt pgtype.Timestamp `json:"deleted_at"`
Email pgtype.Text `json:"email"`
Name pgtype.Text `json:"name"`
ItunesID int32 `json:"itunes_id"`
}

16
db/querier.go Normal file
View file

@ -0,0 +1,16 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.25.0
package db
import (
"context"
)
type Querier interface {
CountFeeds(ctx context.Context) (int64, error)
CreateFeed(ctx context.Context, arg CreateFeedParams) (Feed, error)
}
var _ Querier = (*Queries)(nil)

54
feeds.go
View file

@ -2,12 +2,60 @@ package main
import (
"context"
"encoding/json"
"fmt"
"time"
"github.com/TheLovinator1/FeedVault/db"
"github.com/jackc/pgx/v5/pgtype"
"github.com/mmcdole/gofeed"
)
func makeCreateFeedParams(feedURL string, feed *gofeed.Feed) db.CreateFeedParams {
links := make([]string, len(feed.Items))
for i, item := range feed.Items {
links[i] = item.Link
}
var updatedTime time.Time
if feed.UpdatedParsed != nil {
updatedTime = *feed.UpdatedParsed
}
var publishedTime time.Time
if feed.PublishedParsed != nil {
publishedTime = *feed.PublishedParsed
}
feedCustom, err := json.Marshal(feed.Custom)
if err != nil {
fmt.Println("Error marshalling feed custom data:", err)
feedCustom = []byte("{}")
}
return db.CreateFeedParams{
Url: feedURL,
CreatedAt: pgtype.Timestamp{Time: time.Now(), Valid: true},
UpdatedAt: pgtype.Timestamp{Time: time.Now(), Valid: true},
DeletedAt: pgtype.Timestamp{Valid: false},
Title: pgtype.Text{String: feed.Title, Valid: feed.Title != ""},
Description: pgtype.Text{String: feed.Description, Valid: feed.Description != ""},
Link: pgtype.Text{String: feed.Link, Valid: feed.Link != ""},
FeedLink: pgtype.Text{String: feed.FeedLink, Valid: feed.FeedLink != ""},
Links: links,
Updated: pgtype.Text{String: feed.Updated, Valid: feed.Updated != ""},
UpdatedParsed: pgtype.Timestamp{Time: updatedTime, Valid: !updatedTime.IsZero()},
Published: pgtype.Text{String: feed.Published, Valid: feed.Published != ""},
PublishedParsed: pgtype.Timestamp{Time: publishedTime, Valid: !publishedTime.IsZero()},
Language: pgtype.Text{String: feed.Language, Valid: feed.Language != ""},
Copyright: pgtype.Text{String: feed.Copyright, Valid: feed.Copyright != ""},
Generator: pgtype.Text{String: feed.Generator, Valid: feed.Generator != ""},
Categories: feed.Categories,
Custom: feedCustom,
FeedType: pgtype.Text{String: feed.FeedType, Valid: feed.FeedType != ""},
FeedVersion: pgtype.Text{String: feed.FeedVersion, Valid: feed.FeedVersion != ""},
}
}
func AddFeedToDB(feedURL string) error {
// Cancel the request after 60 seconds if it hasn't finished
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
@ -22,6 +70,12 @@ func AddFeedToDB(feedURL string) error {
return fmt.Errorf("Error parsing feed: %s", err)
}
// Add the feed to the database
_, err = DB.CreateFeed(ctx, makeCreateFeedParams(feedURL, feed))
if err != nil {
return fmt.Errorf("Error adding feed to database: %s", err)
}
fmt.Println(feed.Title)
return nil
}

BIN
feedvault.db Normal file
View file

Binary file not shown.

12
go.mod
View file

@ -2,15 +2,21 @@ module github.com/TheLovinator1/FeedVault
go 1.22.0
require github.com/mmcdole/gofeed v1.2.1
require (
github.com/jackc/pgx/v5 v5.5.3
github.com/mmcdole/gofeed v1.2.1
)
require (
github.com/PuerkitoBio/goquery v1.8.0 // indirect
github.com/andybalholm/cascadia v1.3.1 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/mmcdole/goxpp v1.1.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
golang.org/x/net v0.4.0 // indirect
golang.org/x/text v0.5.0 // indirect
golang.org/x/crypto v0.17.0 // indirect
golang.org/x/net v0.10.0 // indirect
golang.org/x/text v0.14.0 // indirect
)

23
go.sum
View file

@ -6,6 +6,14 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.5.3 h1:Ces6/M3wbDXYpM8JyyPD57ivTtJACFZJd885pdIaV2s=
github.com/jackc/pgx/v5 v5.5.3/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A=
github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/mmcdole/gofeed v1.2.1 h1:tPbFN+mfOLcM1kDF1x2c/N68ChbdBatkppdzf/vDe1s=
@ -21,17 +29,24 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU=
golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM=
golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

22
handlers.go
View file

@ -18,7 +18,10 @@ func ApiHandler(w http.ResponseWriter, _ *http.Request) {
Content: "<p>Here be dragons.</p>",
}
html := FullHTML(htmlData)
w.Write([]byte(html))
_, err := w.Write([]byte(html))
if err != nil {
log.Println("Error writing response:", err)
}
}
func FeedsHandler(w http.ResponseWriter, _ *http.Request) {
@ -31,7 +34,10 @@ func FeedsHandler(w http.ResponseWriter, _ *http.Request) {
Content: "<p>Here be </p>",
}
html := FullHTML(htmlData)
w.Write([]byte(html))
_, err := w.Write([]byte(html))
if err != nil {
log.Println("Error writing response:", err)
}
}
func AddFeedHandler(w http.ResponseWriter, r *http.Request) {
@ -81,7 +87,11 @@ func AddFeedHandler(w http.ResponseWriter, r *http.Request) {
}
html := FullHTML(htmlData)
w.Write([]byte(html))
_, err = w.Write([]byte(html))
if err != nil {
log.Println("Error writing response:", err)
}
}
func IndexHandler(w http.ResponseWriter, _ *http.Request) {
@ -158,7 +168,11 @@ func IndexHandler(w http.ResponseWriter, _ *http.Request) {
func UploadOpmlHandler(w http.ResponseWriter, r *http.Request) {
// Parse the form and get the file
r.ParseMultipartForm(10 << 20) // 10 MB
err := r.ParseMultipartForm(10 << 20) // 10 MB
if err != nil {
http.Error(w, "Error parsing form", http.StatusInternalServerError)
return
}
file, _, err := r.FormFile("file")
if err != nil {
http.Error(w, "No file provided", http.StatusBadRequest)

9
html.go
View file

@ -1,6 +1,7 @@
package main
import (
"context"
"fmt"
"log"
"math/rand"
@ -165,7 +166,13 @@ func buildErrorList(parseResults []ParseResult) string {
func FullHTML(h HTMLData) string {
statusMsg := buildErrorList(h.ParseResult)
feedCount := 0
feedCount, err := DB.CountFeeds(context.Background())
if err != nil {
log.Fatalf("DB.CountFeeds(): %v", err)
feedCount = 0
}
databaseSize, err := GetDBSize()
if err != nil {
databaseSize = "0 KiB"

23
main.go
View file

@ -1,14 +1,37 @@
package main
import (
"context"
"log"
"net/http"
"github.com/TheLovinator1/FeedVault/db"
"github.com/jackc/pgx/v5"
)
var (
conn *pgx.Conn
DB *db.Queries
)
func init() { log.SetFlags(log.LstdFlags | log.Lshortfile) }
func init() {
ctx := context.Background()
// Open a database connection
conn, err := pgx.Connect(ctx, "postgresql://localhost/feedvault?user=feedvault&password=feedvault")
if err != nil {
log.Fatalf("pgx.Connect(): %v", err)
}
DB = db.New(conn)
}
func main() {
defer conn.Close(context.Background())
log.Print("Starting server")
// Create a new ServeMux

53
sql/queries/feeds.sql Normal file
View file

@ -0,0 +1,53 @@
-- name: CreateFeed :one
INSERT INTO
feeds (
"url",
created_at,
updated_at,
deleted_at,
title,
"description",
link,
feed_link,
links,
updated,
updated_parsed,
published,
published_parsed,
"language",
copyright,
generator,
categories,
custom,
feed_type,
feed_version
)
VALUES
(
$1,
$2,
$3,
$4,
$5,
$6,
$7,
$8,
$9,
$10,
$11,
$12,
$13,
$14,
$15,
$16,
$17,
$18,
$19,
$20
) RETURNING *;
-- name: CountFeeds :one
SELECT
COUNT(*)
FROM
feeds;

73
sql/schema/20240214043229_feeds.sql Normal file
View file

@ -0,0 +1,73 @@
-- +goose Up
-- +goose StatementBegin
-- Create table feeds if not exists
CREATE TABLE IF NOT EXISTS feeds (
id SERIAL PRIMARY KEY,
"url" TEXT NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
deleted_at TIMESTAMP DEFAULT NULL,
-- From gofeed: https://github.com/mmcdole/gofeed/blob/master/feed.go
title TEXT,
"description" TEXT,
link TEXT,
feed_link TEXT,
links TEXT [],
updated TEXT,
updated_parsed TIMESTAMP,
published TEXT,
published_parsed TIMESTAMP,
-- Authors - See feed_authors
"language" TEXT,
-- Image - See feed_images
copyright TEXT,
generator TEXT,
categories TEXT [],
-- Dublin Core - See feed_dublin_cores
-- Itunes - See feed_itunes
-- Extensions - See feed_extensions
custom JSONB,
-- Items - See items
feed_type TEXT,
feed_version TEXT
);
-- Feed item
-- https://github.com/mmcdole/gofeed/blob/master/feed.go#L49
CREATE TABLE IF NOT EXISTS items (
id SERIAL PRIMARY KEY,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
deleted_at TIMESTAMP DEFAULT NULL,
-- From gofeed:
title TEXT,
"description" TEXT,
content TEXT,
link TEXT,
links TEXT [],
updated TEXT,
updated_parsed TIMESTAMP,
published TEXT,
published_parsed TIMESTAMP,
-- Authors - See item_authors
"guid" TEXT,
-- Image - See item_images
categories TEXT [],
-- Enclosures - See enclosures
-- Dublin Core - See item_dublin_cores
-- Itunes - See item_itunes
-- Extensions - See item_extensions
custom JSONB,
-- Link to feed
feed_id INTEGER NOT NULL,
CONSTRAINT fk_feed_id FOREIGN KEY (feed_id) REFERENCES feeds (id) ON DELETE CASCADE
);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
DROP TABLE IF EXISTS feeds;
DROP TABLE IF EXISTS items;
-- +goose StatementEnd

44
sql/schema/20240215232236_extensions.sql Normal file
View file

@ -0,0 +1,44 @@
-- +goose Up
-- +goose StatementBegin
-- Extensions for feeds
-- https://github.com/mmcdole/gofeed/blob/master/extensions/extensions.go#L3
CREATE TABLE IF NOT EXISTS feed_extensions (
id SERIAL PRIMARY KEY,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
deleted_at TIMESTAMP DEFAULT NULL,
-- From gofeed:
"name" TEXT,
"value" TEXT,
attrs JSONB,
children JSONB,
-- Link to feed
feed_id INTEGER NOT NULL,
CONSTRAINT fk_feed_id FOREIGN KEY (feed_id) REFERENCES feeds (id) ON DELETE CASCADE
);
-- Extensions for items
-- https://github.com/mmcdole/gofeed/blob/master/extensions/extensions.go#L3
CREATE TABLE IF NOT EXISTS item_extensions (
id SERIAL PRIMARY KEY,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
deleted_at TIMESTAMP DEFAULT NULL,
-- From gofeed:
"name" TEXT,
"value" TEXT,
attrs JSONB,
children JSONB,
-- Link to feed item (Also called feed entry)
item_id INTEGER NOT NULL,
CONSTRAINT fk_item_id FOREIGN KEY (item_id) REFERENCES items (id) ON DELETE CASCADE
);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
DROP TABLE IF EXISTS feed_extensions;
DROP TABLE IF EXISTS item_extensions;
-- +goose StatementEnd

42
sql/schema/20240215232245_authors.sql Normal file
View file

@ -0,0 +1,42 @@
-- +goose Up
-- +goose StatementBegin
-- Person for feeds
-- https://github.com/mmcdole/gofeed/blob/master/feed.go#L73
CREATE TABLE IF NOT EXISTS feed_authors (
id SERIAL PRIMARY KEY,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
deleted_at TIMESTAMP DEFAULT NULL,
-- From gofeed:
"name" TEXT,
email TEXT,
uri TEXT,
-- Link to feed
feed_id INTEGER NOT NULL,
CONSTRAINT fk_feed_id FOREIGN KEY (feed_id) REFERENCES feeds (id) ON DELETE CASCADE
);
-- Person for items
-- https://github.com/mmcdole/gofeed/blob/master/feed.go#L73
CREATE TABLE IF NOT EXISTS item_authors (
id SERIAL PRIMARY KEY,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
deleted_at TIMESTAMP DEFAULT NULL,
-- From gofeed:
"name" TEXT,
email TEXT,
uri TEXT,
-- Link to feed item (Also called feed entry)
item_id INTEGER NOT NULL,
CONSTRAINT fk_item_id FOREIGN KEY (item_id) REFERENCES items (id) ON DELETE CASCADE
);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
DROP TABLE IF EXISTS feed_authors;
DROP TABLE IF EXISTS item_authors;
-- +goose StatementEnd

40
sql/schema/20240215232251_images.sql Normal file
View file

@ -0,0 +1,40 @@
-- +goose Up
-- +goose StatementBegin
-- Image for feeds
-- https://github.com/mmcdole/gofeed/blob/master/feed.go#L80
CREATE TABLE IF NOT EXISTS feed_images (
id SERIAL PRIMARY KEY,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
deleted_at TIMESTAMP DEFAULT NULL,
-- From gofeed:
"url" TEXT,
title TEXT,
-- Link to feed
feed_id INTEGER NOT NULL,
CONSTRAINT fk_feed_id FOREIGN KEY (feed_id) REFERENCES feeds (id) ON DELETE CASCADE
);
-- Image for items
-- https://github.com/mmcdole/gofeed/blob/master/feed.go#L80
CREATE TABLE IF NOT EXISTS item_images (
id SERIAL PRIMARY KEY,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
deleted_at TIMESTAMP DEFAULT NULL,
-- From gofeed:
"url" TEXT,
title TEXT,
-- Link to feed item (Also called feed entry)
item_id INTEGER NOT NULL,
CONSTRAINT fk_item_id FOREIGN KEY (item_id) REFERENCES items (id) ON DELETE CASCADE
);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
DROP TABLE IF EXISTS feed_images;
DROP TABLE IF EXISTS item_images;
-- +goose StatementEnd

68
sql/schema/20240215232259_dublin_cores.sql Normal file
View file

@ -0,0 +1,68 @@
-- +goose Up
-- +goose StatementBegin
-- Dublin Core for feeds
-- https://github.com/mmcdole/gofeed/blob/master/extensions/dublincore.go#L5
CREATE TABLE IF NOT EXISTS feed_dublin_cores (
id SERIAL PRIMARY KEY,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
deleted_at TIMESTAMP DEFAULT NULL,
-- From gofeed:
title TEXT [],
creator TEXT [],
author TEXT [],
"subject" TEXT [],
"description" TEXT [],
publisher TEXT [],
contributor TEXT [],
"date" TEXT [],
"type" TEXT [],
format TEXT [],
identifier TEXT [],
source TEXT [],
"language" TEXT [],
relation TEXT [],
coverage TEXT [],
rights TEXT [],
-- Link to feed
feed_id INTEGER NOT NULL,
CONSTRAINT fk_feed_id FOREIGN KEY (feed_id) REFERENCES feeds (id) ON DELETE CASCADE
);
-- Dublin Core for items
-- https://github.com/mmcdole/gofeed/blob/master/extensions/dublincore.go#L5
CREATE TABLE IF NOT EXISTS item_dublin_cores (
id SERIAL PRIMARY KEY,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
deleted_at TIMESTAMP DEFAULT NULL,
-- From gofeed:
title TEXT [],
creator TEXT [],
author TEXT [],
"subject" TEXT [],
"description" TEXT [],
publisher TEXT [],
contributor TEXT [],
"date" TEXT [],
"type" TEXT [],
format TEXT [],
identifier TEXT [],
source TEXT [],
"language" TEXT [],
relation TEXT [],
coverage TEXT [],
rights TEXT [],
-- Link to feed item (Also called feed entry)
item_id INTEGER NOT NULL,
CONSTRAINT fk_item_id FOREIGN KEY (item_id) REFERENCES items (id) ON DELETE CASCADE
);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
DROP TABLE IF EXISTS feed_dublin_cores;
DROP TABLE IF EXISTS item_dublin_cores;
-- +goose StatementEnd

94
sql/schema/20240215232318_itunes.sql Normal file
View file

@ -0,0 +1,94 @@
-- +goose Up
-- +goose StatementBegin
-- Itunes for feeds
-- https://github.com/mmcdole/gofeed/blob/master/extensions/itunes.go#L5
CREATE TABLE IF NOT EXISTS feed_itunes (
id SERIAL PRIMARY KEY,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
deleted_at TIMESTAMP DEFAULT NULL,
-- From gofeed:
author TEXT,
"block" TEXT,
"explicit" TEXT,
keywords TEXT,
-- Owner
subtitle TEXT,
summary TEXT,
"image" TEXT,
complete TEXT,
new_feed_url TEXT,
"type" TEXT,
-- Link to feed
feed_id INTEGER NOT NULL,
CONSTRAINT fk_feed_id FOREIGN KEY (feed_id) REFERENCES feeds (id) ON DELETE CASCADE
);
-- Itunes for items
-- https://github.com/mmcdole/gofeed/blob/master/extensions/itunes.go#L22
CREATE TABLE IF NOT EXISTS item_itunes (
id SERIAL PRIMARY KEY,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
deleted_at TIMESTAMP DEFAULT NULL,
-- From gofeed:
author TEXT,
"block" TEXT,
duration TEXT,
"explicit" TEXT,
keywords TEXT,
subtitle TEXT,
summary TEXT,
"image" TEXT,
is_closed_captioned TEXT,
episode TEXT,
season TEXT,
"order" TEXT,
episode_type TEXT,
-- Link to feed item (Also called feed entry)
item_id INTEGER NOT NULL,
CONSTRAINT fk_item_id FOREIGN KEY (item_id) REFERENCES items (id) ON DELETE CASCADE
);
-- Itunes categories
-- https://github.com/mmcdole/gofeed/blob/master/extensions/itunes.go#L39
CREATE TABLE IF NOT EXISTS itunes_categories (
id SERIAL PRIMARY KEY,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
deleted_at TIMESTAMP DEFAULT NULL,
-- From gofeed:
"text" TEXT,
subcategory TEXT,
-- Link to itunes
itunes_id INTEGER NOT NULL,
CONSTRAINT fk_itunes_id FOREIGN KEY (itunes_id) REFERENCES feed_itunes (id) ON DELETE CASCADE
);
-- Itunes owners
-- https://github.com/mmcdole/gofeed/blob/master/extensions/itunes.go#L45
CREATE TABLE IF NOT EXISTS itunes_owners (
id SERIAL PRIMARY KEY,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
deleted_at TIMESTAMP DEFAULT NULL,
-- From gofeed:
email TEXT,
"name" TEXT,
-- Link to itunes
itunes_id INTEGER NOT NULL,
CONSTRAINT fk_itunes_id FOREIGN KEY (itunes_id) REFERENCES feed_itunes (id) ON DELETE CASCADE
);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
DROP TABLE IF EXISTS feed_itunes;
DROP TABLE IF EXISTS item_itunes;
DROP TABLE IF EXISTS itunes_categories;
DROP TABLE IF EXISTS itunes_owners;
-- +goose StatementEnd

24
sql/schema/20240215232334_enclousures.sql Normal file
View file

@ -0,0 +1,24 @@
-- +goose Up
-- +goose StatementBegin
-- Enclosures
-- https://github.com/mmcdole/gofeed/blob/master/feed.go#L86
CREATE TABLE IF NOT EXISTS enclosures (
id SERIAL PRIMARY KEY,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
deleted_at TIMESTAMP DEFAULT NULL,
-- From gofeed:
"url" TEXT,
"length" TEXT,
"type" TEXT,
-- Link to feed item (Also called feed entry)
item_id INTEGER NOT NULL,
CONSTRAINT fk_item_id FOREIGN KEY (item_id) REFERENCES items (id) ON DELETE CASCADE
);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
DROP TABLE IF EXISTS enclosures;
-- +goose StatementEnd

14
sqlc.yaml Normal file
View file

@ -0,0 +1,14 @@
version: "2"
sql:
- engine: "postgresql"
queries: "sql/queries"
schema: "sql/schema"
gen:
go:
out: "db"
sql_package: "pgx/v5"
emit_prepared_queries: true
emit_interface: true
emit_empty_slices: true
emit_exported_queries: true
emit_json_tags: true