diff --git a/README.md b/README.md index febac91..86c74bc 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ FeedVault is an open-source web application written in Golang that allows users _Note: Some features are currently in development._ -- **Unified Feed Archiving**: Archive RSS (0.90 to 2.0), Atom (0.3, 1.0), and JSON (1.0, 1.1) feeds seamlessly in one centralized location. +- **Unified Feed Archiving**: Archive RSS (0.90 to 2.0), Atom (0.3, 1.0), JSON (1.0, 1.1), Dublin Core, and ITunes feeds seamlessly in one centralized location. - **Content Search**: Easily search your archive for specific content. - **Export Options**: Export your archive to various formats, including JSON, CSV, HTML, ODS, RST, TSV, XLS, XLSX, or YAML. - **API**: Access your archive programmatically through a API. diff --git a/go.mod b/go.mod index 73d5a4b..98788e6 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,16 @@ module github.com/TheLovinator1/FeedVault go 1.22.0 + +require github.com/mmcdole/gofeed v1.2.1 + +require ( + github.com/PuerkitoBio/goquery v1.8.0 // indirect + github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mmcdole/goxpp v1.1.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + golang.org/x/net v0.4.0 // indirect + golang.org/x/text v0.5.0 // indirect +) diff --git a/go.sum b/go.sum index e69de29..2ce6a9b 100644 --- a/go.sum +++ b/go.sum @@ -0,0 +1,37 @@ +github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= +github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= +github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/mmcdole/gofeed v1.2.1 h1:tPbFN+mfOLcM1kDF1x2c/N68ChbdBatkppdzf/vDe1s= +github.com/mmcdole/gofeed v1.2.1/go.mod h1:2wVInNpgmC85q16QTTuwbuKxtKkHLCDDtf0dCmnrNr4= +github.com/mmcdole/goxpp v1.1.0 h1:WwslZNF7KNAXTFuzRtn/OKZxFLJAAyOA9w82mDz2ZGI= +github.com/mmcdole/goxpp v1.1.0/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= +golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/feeds/feeds.go b/pkg/feeds/feeds.go new file mode 100644 index 0000000..95b05e1 --- /dev/null +++ b/pkg/feeds/feeds.go @@ -0,0 +1,27 @@ +package feeds + +import ( + "context" + "fmt" + "time" + + "github.com/mmcdole/gofeed" +) + +func AddFeedToDB(feedURL string) error { + // Cancel the request after 60 seconds if it hasn't finished + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + // Parse the feed + fp := gofeed.NewParser() + fp.UserAgent = "FeedVault/1.0 (RSS feed archive; https://feedvault.se; bot@feedvault.se; TheLovinator#9276)" + + feed, err := fp.ParseURLWithContext(feedURL, ctx) + if err != nil { + return fmt.Errorf("Error parsing feed: %s", err) + } + + fmt.Println(feed.Title) + return nil +} diff --git a/pkg/handlers/feeds.go b/pkg/handlers/feeds.go index c5ba21f..6d8677e 100644 --- a/pkg/handlers/feeds.go +++ b/pkg/handlers/feeds.go @@ -1,10 +1,10 @@ package handlers import ( - "log" "net/http" "strings" + "github.com/TheLovinator1/FeedVault/pkg/feeds" "github.com/TheLovinator1/FeedVault/pkg/html" "github.com/TheLovinator1/FeedVault/pkg/models" "github.com/TheLovinator1/FeedVault/pkg/validate" @@ -27,7 +27,12 @@ func AddFeedHandler(w http.ResponseWriter, r *http.Request) { var parseErrors []models.ParseResult // Parse the form and get the URLs - r.ParseForm() + err := r.ParseForm() + if err != nil { + http.Error(w, "Error parsing form", http.StatusInternalServerError) + return + } + urls := r.Form.Get("urls") if urls == "" { http.Error(w, "No URLs provided", http.StatusBadRequest) @@ -44,8 +49,13 @@ func AddFeedHandler(w http.ResponseWriter, r *http.Request) { continue } - // "Add" the feed to the database - log.Println("Adding feed:", feed_url) + err = feeds.AddFeedToDB(feed_url) + if err != nil { + parseErrors = append(parseErrors, models.ParseResult{FeedURL: feed_url, Msg: err.Error(), IsError: true}) + continue + } + + // Feed was added successfully parseErrors = append(parseErrors, models.ParseResult{FeedURL: feed_url, Msg: "Added", IsError: false}) }