TheLovinator/feedvault.se
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Refactor folder structure

Browse source
This commit is contained in:
Joakim Hellsén 2024-02-13 21:21:08 +01:00
commit c7cca02ca7
30 changed files with 610 additions and 933 deletions
Download patch file Download diff file Expand all files Collapse all files

20
pkg/handlers/api.go Normal file
View file

@ -0,0 +1,20 @@
package handlers
import (
"net/http"
"github.com/TheLovinator1/FeedVault/pkg/html"
)
func ApiHandler(w http.ResponseWriter, _ *http.Request) {
htmlData := html.HTMLData{
Title: "FeedVault API",
Description: "FeedVault API - A feed archive",
Keywords: "RSS, Atom, Feed, Archive, API",
Author: "TheLovinator",
CanonicalURL: "http://localhost:8000/api",
Content: "<p>Here be dragons.</p>",
}
html := html.FullHTML(htmlData)
w.Write([]byte(html))
}

64
pkg/handlers/feeds.go Normal file
View file

@ -0,0 +1,64 @@
package handlers
import (
"log"
"net/http"
"strings"
"github.com/TheLovinator1/FeedVault/pkg/html"
"github.com/TheLovinator1/FeedVault/pkg/models"
"github.com/TheLovinator1/FeedVault/pkg/validate"
)
func FeedsHandler(w http.ResponseWriter, _ *http.Request) {
htmlData := html.HTMLData{
Title: "FeedVault Feeds",
Description: "FeedVault Feeds - A feed archive",
Keywords: "RSS, Atom, Feed, Archive",
Author: "TheLovinator",
CanonicalURL: "http://localhost:8000/feeds",
Content: "<p>Here be feeds.</p>",
}
html := html.FullHTML(htmlData)
w.Write([]byte(html))
}
func AddFeedHandler(w http.ResponseWriter, r *http.Request) {
var parseErrors []models.ParseResult
// Parse the form and get the URLs
r.ParseForm()
urls := r.Form.Get("urls")
if urls == "" {
http.Error(w, "No URLs provided", http.StatusBadRequest)
return
}
for _, feed_url := range strings.Split(urls, "\n") {
// TODO: Try to upgrade to https if http is provided
// Validate the URL
err := validate.ValidateFeedURL(feed_url)
if err != nil {
parseErrors = append(parseErrors, models.ParseResult{FeedURL: feed_url, Msg: err.Error(), IsError: true})
continue
}
// "Add" the feed to the database
log.Println("Adding feed:", feed_url)
parseErrors = append(parseErrors, models.ParseResult{FeedURL: feed_url, Msg: "Added", IsError: false})
}
htmlData := html.HTMLData{
Title: "FeedVault",
Description: "FeedVault - A feed archive",
Keywords: "RSS, Atom, Feed, Archive",
Author: "TheLovinator",
CanonicalURL: "http://localhost:8000/",
Content: "<p>Feeds added.</p>",
ParseResult: parseErrors,
}
html := html.FullHTML(htmlData)
w.Write([]byte(html))
}

79
pkg/handlers/index.go Normal file
View file

@ -0,0 +1,79 @@
package handlers
import (
"net/http"
"github.com/TheLovinator1/FeedVault/pkg/html"
)
func IndexHandler(w http.ResponseWriter, _ *http.Request) {
content := `<h2>Feeds to archive</h2>
<p>
Input the URLs of the feeds you wish to archive below. You can add as many as needed, and access them through the website or API. Alternatively, include links to .opml files, and the feeds within will be archived.
</p>
<form action="/add" method="post">
<textarea id="urls" name="urls" rows="5" cols="50" required></textarea>
<button type="submit">Add feeds</button>
</form>
<br>
<p>You can also upload .opml files containing the feeds you wish to archive:</p>
<form enctype="multipart/form-data" method="post" action="/upload_opml">
<input type="file" name="file" id="file" accept=".opml" required>
<button type="submit">Upload OPML</button>
</form>
`
FAQ := `
<h2>FAQ</h2>
<details>
<summary>What are web feeds?</summary>
<p>
Web feeds are a way to distribute content on the web. They allow users to access updates from websites without having to visit them directly. Feeds are typically used for news websites, blogs, and other sites that frequently update content.
<br>
You can read more about web feeds on <a href="https://en.wikipedia.org/wiki/Web_feed">Wikipedia</a>.
</p>
<hr>
</details>
<details>
<summary>What is FeedVault?</summary>
<p>
FeedVault is a service that archives web feeds. It allows users to access and search for historical content from various websites. The service is designed to preserve the history of the web and provide a reliable source for accessing content that may no longer be available on the original websites.
</p>
<hr>
</details>
<details>
<summary>Why archive feeds?</summary>
<p>
Web feeds are a valuable source of information, and archiving them ensures that the content is preserved for future reference. By archiving feeds, we can ensure that historical content is available for research, analysis, and other purposes. Additionally, archiving feeds can help prevent the loss of valuable information due to website changes, outages, or other issues.
</p>
<hr>
</details>
<details>
<summary>How does it work?</summary>
<p>
FeedVault is written in Go and uses the <a href="https://github.com/mmcdole/gofeed">gofeed</a> library to parse feeds. The service periodically checks for new content in the feeds and stores it in a database. Users can access the archived feeds through the website or API.
<hr>
</details>
<details>
<summary>How can I access the archived feeds?</summary>
<p>
You can access the archived feeds through the website or API. The website provides a user interface for searching and browsing the feeds, while the API allows you to access the feeds programmatically. You can also download the feeds in various formats, such as JSON, XML, or RSS.
</p>
</details>
`
content += FAQ
htmlData := html.HTMLData{
Title: "FeedVault",
Description: "FeedVault - A feed archive",
Keywords: "RSS, Atom, Feed, Archive",
Author: "TheLovinator",
CanonicalURL: "http://localhost:8000/",
Content: content,
}
html := html.FullHTML(htmlData)
w.Write([]byte(html))
}

63
pkg/handlers/opml.go Normal file
View file

@ -0,0 +1,63 @@
package handlers
import (
"io"
"log"
"net/http"
"github.com/TheLovinator1/FeedVault/pkg/html"
"github.com/TheLovinator1/FeedVault/pkg/models"
"github.com/TheLovinator1/FeedVault/pkg/opml"
"github.com/TheLovinator1/FeedVault/pkg/validate"
)
func UploadOpmlHandler(w http.ResponseWriter, r *http.Request) {
// Parse the form and get the file
r.ParseMultipartForm(10 << 20) // 10 MB
file, _, err := r.FormFile("file")
if err != nil {
http.Error(w, "No file provided", http.StatusBadRequest)
return
}
defer file.Close()
// Read the file
all, err := io.ReadAll(file)
if err != nil {
http.Error(w, "Failed to read file", http.StatusInternalServerError)
return
}
// Parse the OPML file
parseResult := []models.ParseResult{}
links, err := opml.ParseOpml(string(all))
if err != nil {
parseResult = append(parseResult, models.ParseResult{FeedURL: "/upload_opml", Msg: err.Error(), IsError: true})
} else {
// Add the feeds to the database
for _, feed_url := range links.XMLLinks {
log.Println("Adding feed:", feed_url)
// Validate the URL
err := validate.ValidateFeedURL(feed_url)
if err != nil {
parseResult = append(parseResult, models.ParseResult{FeedURL: feed_url, Msg: err.Error(), IsError: true})
continue
}
parseResult = append(parseResult, models.ParseResult{FeedURL: feed_url, Msg: "Added", IsError: false})
}
}
// Return the results
htmlData := html.HTMLData{
Title: "FeedVault",
Description: "FeedVault - A feed archive",
Keywords: "RSS, Atom, Feed, Archive",
Author: "TheLovinator",
CanonicalURL: "http://localhost:8000/",
Content: "<p>Feeds added.</p>",
ParseResult: parseResult,
}
html := html.FullHTML(htmlData)
w.Write([]byte(html))
}

191
pkg/html/html.go Normal file
View file

@ -0,0 +1,191 @@
package html
import (
"fmt"
"math/rand"
"strings"
"github.com/TheLovinator1/FeedVault/pkg/models"
"github.com/TheLovinator1/FeedVault/pkg/quotes"
"github.com/TheLovinator1/FeedVault/pkg/stats"
)
type HTMLData struct {
Title string
Description string
Keywords string
Author string
CanonicalURL string
Content string
ParseResult []models.ParseResult
}
var style = `
html {
max-width: 70ch;
padding: calc(1vmin + 0.5rem);
margin-inline: auto;
font-size: clamp(1em, 0.909em + 0.45vmin, 1.25em);
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
color-scheme: light dark;
}
h1 {
font-size: 2.5rem;
font-weight: 600;
margin: 0;
}
.title {
text-align: center;
}
.search {
display: flex;
justify-content: center;
margin-top: 1rem;
margin-inline: auto;
}
.leftright {
display: flex;
justify-content: center;
}
.left {
margin-right: auto;
}
.right {
margin-left: auto;
}
textarea {
width: 100%;
height: 10rem;
resize: vertical;
}
.messages {
list-style-type: none;
}
.error {
color: red;
}
.success {
color: green;
}
`
func FullHTML(h HTMLData) string {
var sb strings.Builder
var errorBuilder strings.Builder
FeedCount := 0
DatabaseSize := stats.GetDBSize()
// This is the error message that will be displayed if there are any errors
if len(h.ParseResult) > 0 {
errorBuilder.WriteString("<ul>")
for _, result := range h.ParseResult {
var listItemClass, statusMsg string
if result.IsError {
listItemClass = "error"
statusMsg = result.Msg
} else {
listItemClass = "success"
statusMsg = result.Msg
}
errorBuilder.WriteString(fmt.Sprintf(`<li class="%s"><a href="%s">%s</a> - %s</li>`, listItemClass, result.FeedURL, result.FeedURL, statusMsg))
}
errorBuilder.WriteString("</ul>")
}
StatusMsg := errorBuilder.String()
sb.WriteString(`
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
`)
if h.Description != "" {
sb.WriteString(`<meta name="description" content="` + h.Description + `">`)
}
if h.Keywords != "" {
sb.WriteString(`<meta name="keywords" content="` + h.Keywords + `">`)
}
if h.Author != "" {
sb.WriteString(`<meta name="author" content="` + h.Author + `">`)
}
if h.CanonicalURL != "" {
sb.WriteString(`<link rel="canonical" href="` + h.CanonicalURL + `">`)
}
sb.WriteString(`
<title>` + h.Title + `</title>
<style>` + style + `</style>
</head>
<body>
` + StatusMsg + `
<span class="title"><h1><a href="/">FeedVault</a></h1></span>
<div class="leftright">
<div class="left">
<small>Archive of <a href="https://en.wikipedia.org/wiki/Web_feed">web feeds</a>. ` + fmt.Sprintf("%d", FeedCount) + ` feeds. ~` + DatabaseSize + `.</small>
</div>
<div class="right">
<!-- Search -->
<form action="#" method="get">
<input type="text" name="q" placeholder="Search">
<button type="submit">Search</button>
</form>
</div>
</div>
<nav>
<small>
<div class="leftright">
<div class="left">
<a href="/">Home</a> | <a href="/feeds">Feeds</a> | <a href="/api">API</a>
</div>
<div class="right">
<a href="https://github.com/TheLovinator1/FeedVault">GitHub</a> | <a href="https://github.com/sponsors/TheLovinator1">Donate</a>
</div>
</div>
</small>
</nav>
<hr>
<main>
` + h.Content + `
</main>
<hr>
<footer>
<small>
<div class="leftright">
<div class="left">
Made by <a href="">Joakim Hellsén</a>.
</div>
<div class="right">No rights reserved.</div>
</div>
<div class="leftright">
<div class="left">
<a href="mailto:hello@feedvault.se">hello@feedvault.se</a>
</div>
<div class="right">
` + quotes.FunMsg[rand.Intn(len(quotes.FunMsg))] + `
</div>
</div>
</small>
</footer>
</body>
</html>`)
return sb.String()
}

32
pkg/models/models.go Normal file
View file

@ -0,0 +1,32 @@
package models
import (
"net/http"
"github.com/TheLovinator1/FeedVault/pkg/stats"
)
type TemplateData struct {
Title string
Description string
Keywords string
Author string
CanonicalURL string
FeedCount int
DatabaseSize string
Request *http.Request
ParseErrors []ParseResult
}
type ParseResult struct {
FeedURL string
Msg string
IsError bool
}
func (d *TemplateData) GetDatabaseSizeAndFeedCount() {
// TODO: Get the feed count from the database
// TODO: Add amount of entries
// TODO: Add amount of users
d.DatabaseSize = stats.GetDBSize()
}

94
pkg/opml/opml.go Normal file
View file

@ -0,0 +1,94 @@
package opml
import "encoding/xml"
type opml struct {
Head struct {
Title string `xml:"title"`
} `xml:"head"`
Body Body `xml:"body"`
}
type Body struct {
Outlines []Outline `xml:"outline"`
}
type Outline struct {
Outlines []Outline `xml:"outline"`
XmlUrl string `xml:"xmlUrl,attr,omitempty"`
HtmlUrl string `xml:"htmlUrl,attr,omitempty"`
}
func (o *opml) ParseString(s string) error {
return xml.Unmarshal([]byte(s), o)
}
func (o *opml) String() (string, error) {
b, err := xml.Marshal(o)
if err != nil {
return "", err
}
return xml.Header + string(b), nil
}
type linksFromOpml struct {
XMLLinks []string `json:"xmlLinks"`
HTMLLinks []string `json:"htmlLinks"`
}
func RemoveDuplicates(s []string) []string {
seen := make(map[string]struct{}, len(s))
j := 0
for _, v := range s {
if _, ok := seen[v]; ok {
continue
}
seen[v] = struct{}{}
s[j] = v
j++
}
return s[:j]
}
func ParseOpml(s string) (linksFromOpml, error) {
// Get all the feeds from the OPML and return them as linksFromOpml
opml := &opml{}
err := opml.ParseString(s)
if err != nil {
return linksFromOpml{}, err
}
links := linksFromOpml{}
for _, outline := range opml.Body.Outlines {
links.XMLLinks = append(links.XMLLinks, outline.XmlUrl)
links.HTMLLinks = append(links.HTMLLinks, outline.HtmlUrl)
}
// Also check outlines for nested outlines
for _, outline := range opml.Body.Outlines {
for _, nestedOutline := range outline.Outlines {
links.XMLLinks = append(links.XMLLinks, nestedOutline.XmlUrl)
links.HTMLLinks = append(links.HTMLLinks, nestedOutline.HtmlUrl)
}
}
// Remove any empty strings
for i := 0; i < len(links.XMLLinks); i++ {
if links.XMLLinks[i] == "" {
links.XMLLinks = append(links.XMLLinks[:i], links.XMLLinks[i+1:]...)
i--
}
}
for i := 0; i < len(links.HTMLLinks); i++ {
if links.HTMLLinks[i] == "" {
links.HTMLLinks = append(links.HTMLLinks[:i], links.HTMLLinks[i+1:]...)
i--
}
}
// Remove any duplicates
links.XMLLinks = RemoveDuplicates(links.XMLLinks)
links.HTMLLinks = RemoveDuplicates(links.HTMLLinks)
return links, nil
}

57
pkg/quotes/quotes.go Normal file
View file

@ -0,0 +1,57 @@
package quotes
// "Fun" messages that will be displayed in the footer
var FunMsg = []string{
"Web scraping is not a crime.",
"Made in Sweden.",
"🙃",
"Hello.",
"<3",
"So many feeds, so little time.",
"A feed in the hand is worth two in the bush.",
"Death begets death begets death.",
"I am Eo's dream.",
"Freedom in an unjust system is no freedom at all.",
"Omnis vir lupus.",
"Shit escalates.",
"Break the chains, my love.",
"Sharpened by hate. Strengthened by love.",
"Hic sunt leones.",
"Keyboard not found. Press F1 to continue.",
"The stars shine brighter when shared among comrades.",
"Zzz... 🛌",
"Generated in 0 ms.",
"Open source, open heart.",
"RSS is the new black.",
"Unsubscribe.",
"ChatGPT made 99% of this website :-)",
"👁️👄👁️",
"From each planet, to each star—equality in the cosmos.",
"In the vastness of space, no one should own more than they can share.",
"Workers of the universe, unite! The stars are our common heritage.",
"Space is for all, not just the privileged few.",
"From the red planet to the black hole, solidarity knows no borders.",
"Astronauts of the world, unite for a cosmic revolution!",
"Space is for everyone, not just the 1%.",
"No class struggle in zero gravity.",
"Only solidarity among the cosmic proletariat.",
"The red glow of the stars reflects the spirit of collective effort.",
"The final frontier is a shared frontier, where no one is left behind.",
"Vote for a space utopia!",
"From the Milky Way to Andromeda, the stars belong to the people.",
"Space is for the workers, not the bosses.",
"Let the fruits of progress be the common heritage of all.",
"From the moon to the asteroid belt, the cosmos is for the common good.",
"The stars do not discriminate; neither should we.",
"In the vacuum of space, let equality fill the void.",
"From Big Bang to the heat death of the universe, solidarity is eternal.",
"In dark times, should the stars also go out?",
"One day I will return to your side.",
"Un Jour Je Serai de Retour Prés de Toi",
"You should build Space Communism — precisely *because* it's impossible.",
"She thinks you are an idiot, sire.",
"The song of death is sweet and endless.",
"Child-murdering billionaires still rule the world with a shit-eating grin.",
"Eight billion people - and you failed every single one of them.",
"You are the first crack. From you it will spread.",
}

58
pkg/stats/stats.go Normal file
View file

@ -0,0 +1,58 @@
package stats
import (
"fmt"
"log"
"os"
"time"
)
type Cache struct {
timestamp time.Time
data string
}
var cache Cache
// Get the size of the database and return as nearest human readable size.
//
// e.g. 1.23 KiB, 4.56 MiB, 7.89 GiB, 0.12 TiB
// The size is cached for 10 minutes
func GetDBSize() string {
// If cache is less than 10 minutes old, return cached data
if time.Since(cache.timestamp).Minutes() < 10 {
return cache.data
}
fileInfo, err := os.Stat("feedvault.db")
if err != nil {
log.Println("Error getting file info:", err)
return "0 B"
}
// Get the file size in bytes
fileSize := fileInfo.Size()
// Convert to human readable size and append the unit (KiB, MiB, GiB, TiB)
var size float64
if fileSize < 1024*1024 {
size = float64(fileSize) / 1024
cache.data = fmt.Sprintf("%.2f KiB", size)
} else if fileSize < 1024*1024*1024 {
size = float64(fileSize) / (1024 * 1024)
cache.data = fmt.Sprintf("%.2f MiB", size)
} else if fileSize < 1024*1024*1024*1024 {
size = float64(fileSize) / (1024 * 1024 * 1024)
cache.data = fmt.Sprintf("%.2f GiB", size)
} else {
size = float64(fileSize) / (1024 * 1024 * 1024 * 1024)
cache.data = fmt.Sprintf("%.2f TiB", size)
}
// Update cache timestamp
cache.timestamp = time.Now()
log.Println("Returning database size, it is", cache.data)
return cache.data
}

133
pkg/validate/validate.go Normal file
View file

@ -0,0 +1,133 @@
package validate
import (
"errors"
"net"
"net/http"
"net/url"
"strings"
)
// Run some simple validation on the URL
func ValidateFeedURL(feed_url string) error {
// Check if URL starts with http or https
if !strings.HasPrefix(feed_url, "http://") && !strings.HasPrefix(feed_url, "https://") {
return errors.New("URL must start with http:// or https://")
}
// Parse a url into a URL structure
u, err := url.Parse(feed_url)
if err != nil {
return errors.New("failed to parse URL")
}
// Get the domain from the URL
domain := u.Hostname()
domain = strings.TrimSpace(domain)
if domain == "" {
return errors.New("URL does not contain a domain")
}
// Don't allow IP address URLs
ip := net.ParseIP(domain)
if ip != nil {
return errors.New("IP address URLs are not allowed")
}
// Don't allow local URLs (e.g. router URLs)
// Taken from https://github.com/uBlockOrigin/uAssets/blob/master/filters/lan-block.txt
// https://github.com/gwarser/filter-lists
localURLs := []string{
"[::]",
"[::1]",
"airbox.home",
"airport",
"arcor.easybox",
"aterm.me",
"bthomehub.home",
"bthub.home",
"congstar.box",
"connect.box",
"console.gl-inet.com",
"easy.box",
"etxr",
"fire.walla",
"fritz.box",
"fritz.nas",
"fritz.repeater",
"giga.cube",
"hi.link",
"hitronhub.home",
"home.arpa",
"homerouter.cpe",
"host.docker.internal",
"huaweimobilewifi.com",
"localbattle.net",
"localhost",
"mobile.hotspot",
"myfritz.box",
"ntt.setup",
"pi.hole",
"plex.direct",
"repeater.asus.com",
"router.asus.com",
"routerlogin.com",
"routerlogin.net",
"samsung.router",
"speedport.ip",
"steamloopback.host",
"tplinkap.net",
"tplinkeap.net",
"tplinkmodem.net",
"tplinkplclogin.net",
"tplinkrepeater.net",
"tplinkwifi.net",
"web.setup.home",
"web.setup",
}
for _, localURL := range localURLs {
if strings.Contains(domain, localURL) {
return errors.New("local URLs are not allowed")
}
}
// Don't allow URLs that end with .local
if strings.HasSuffix(domain, ".local") {
return errors.New("URLs ending with .local are not allowed")
}
// Don't allow URLs that end with .onion
if strings.HasSuffix(domain, ".onion") {
return errors.New("URLs ending with .onion are not allowed")
}
// Don't allow URLs that end with .home.arpa
if strings.HasSuffix(domain, ".home.arpa") {
return errors.New("URLs ending with .home.arpa are not allowed")
}
// Don't allow URLs that end with .internal
// Docker uses host.docker.internal
if strings.HasSuffix(domain, ".internal") {
return errors.New("URLs ending with .internal are not allowed")
}
// Don't allow URLs that end with .localdomain
if strings.HasSuffix(domain, ".localdomain") {
return errors.New("URLs ending with .localdomain are not allowed")
}
// Check if the domain is resolvable
_, err = net.LookupIP(domain)
if err != nil {
return errors.New("failed to resolve domain")
}
// Check if the URL is reachable
_, err = http.Get(feed_url)
if err != nil {
return errors.New("failed to reach URL")
}
return nil
}