Add tests

.vscode/settings.json

@@ -1,26 +1,54 @@
 {
   "cSpell.words": [
+    "airbox",
+    "arcor",
     "arpa",
+    "asus",
+    "aterm",
     "blocklist",
     "blocklists",
+    "bthomehub",
+    "bthub",
     "chartboost",
+    "congstar",
+    "easybox",
+    "etxr",
     "feedburner",
     "feedparser",
     "feedvault",
     "gaierror",
+    "giga",
     "gofeed",
     "gorm",
+    "hitronhub",
+    "homerouter",
+    "hotspot",
+    "huaweimobilewifi",
     "leftright",
     "levelname",
     "listparser",
+    "localbattle",
+    "localdomain",
+    "malformedurl",
     "mmcdole",
     "Monero",
+    "myfritz",
     "PGHOST",
     "PGPORT",
     "PGUSER",
     "regexes",
+    "routerlogin",
+    "speedport",
+    "steamloopback",
+    "stretchr",
     "stylesheet",
     "tmpl",
+    "tplinkap",
+    "tplinkeap",
+    "tplinkmodem",
+    "tplinkplclogin",
+    "tplinkrepeater",
+    "tplinkwifi",
     "webmail"
   ]
 }

go.mod

@@ -4,13 +4,16 @@ go 1.21.6
 
 require (
 	github.com/go-chi/chi/v5 v5.0.11
-	github.com/mmcdole/gofeed v1.2.1
+	github.com/stretchr/testify v1.8.1
 	gorm.io/driver/sqlite v1.5.4
 	gorm.io/gorm v1.25.6
 )
 
 require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/mattn/go-sqlite3 v1.14.22 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,7 +1,4 @@
-github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
-github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
-github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/go-chi/chi/v5 v5.0.11 h1:BnpYbFZ3T3S1WMpD79r7R5ThWX40TaFB7L31Y8xqSwA=
@@ -10,26 +7,20 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
 github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
-github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
-github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
 github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
-github.com/mmcdole/gofeed v1.2.1 h1:tPbFN+mfOLcM1kDF1x2c/N68ChbdBatkppdzf/vDe1s=
-github.com/mmcdole/gofeed v1.2.1/go.mod h1:2wVInNpgmC85q16QTTuwbuKxtKkHLCDDtf0dCmnrNr4=
-github.com/mmcdole/goxpp v1.1.0 h1:WwslZNF7KNAXTFuzRtn/OKZxFLJAAyOA9w82mDz2ZGI=
-github.com/mmcdole/goxpp v1.1.0/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
-github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
-golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gorm.io/driver/sqlite v1.5.4 h1:IqXwXi8M/ZlPzH/947tn5uik3aYQslP9BVveoax0nV0=

main.go

@@ -125,6 +125,7 @@ func FeedsHandler(w http.ResponseWriter, _ *http.Request) {
 	renderPage(w, "Feeds", "Feeds Page", "feeds, page", "TheLovinator", "http://localhost:8000/feeds", "feeds")
 }
 
+// Run some simple validation on the URL
 func validateURL(feed_url string) error {
 	// Check if URL starts with http or https
 	if !strings.HasPrefix(feed_url, "http://") && !strings.HasPrefix(feed_url, "https://") {
@@ -150,9 +151,61 @@ func validateURL(feed_url string) error {
 		return errors.New("IP address URLs are not allowed")
 	}
 
-	// Don't allow localhost URLs
+	// Don't allow local URLs (e.g. router URLs)
-	if strings.Contains(domain, "localhost") {
+	// Taken from https://github.com/uBlockOrigin/uAssets/blob/master/filters/lan-block.txt
-		return errors.New("localhost are not allowed")
+	// https://github.com/gwarser/filter-lists
+	localURLs := []string{
+		"[::]",
+		"[::1]",
+		"airbox.home",
+		"airport",
+		"arcor.easybox",
+		"aterm.me",
+		"bthomehub.home",
+		"bthub.home",
+		"congstar.box",
+		"connect.box",
+		"console.gl-inet.com",
+		"easy.box",
+		"etxr",
+		"fire.walla",
+		"fritz.box",
+		"fritz.nas",
+		"fritz.repeater",
+		"giga.cube",
+		"hi.link",
+		"hitronhub.home",
+		"home.arpa",
+		"homerouter.cpe",
+		"host.docker.internal",
+		"huaweimobilewifi.com",
+		"localbattle.net",
+		"localhost",
+		"mobile.hotspot",
+		"myfritz.box",
+		"ntt.setup",
+		"pi.hole",
+		"plex.direct",
+		"repeater.asus.com",
+		"router.asus.com",
+		"routerlogin.com",
+		"routerlogin.net",
+		"samsung.router",
+		"speedport.ip",
+		"steamloopback.host",
+		"tplinkap.net",
+		"tplinkeap.net",
+		"tplinkmodem.net",
+		"tplinkplclogin.net",
+		"tplinkrepeater.net",
+		"tplinkwifi.net",
+		"web.setup.home",
+		"web.setup",
+	}
+	for _, localURL := range localURLs {
+		if strings.Contains(domain, localURL) {
+			return errors.New("local URLs are not allowed")
+		}
 	}
 
 	// Don't allow URLs that end with .local
@@ -160,6 +213,27 @@ func validateURL(feed_url string) error {
 		return errors.New("URLs ending with .local are not allowed")
 	}
 
+	// Don't allow URLs that end with .onion
+	if strings.HasSuffix(domain, ".onion") {
+		return errors.New("URLs ending with .onion are not allowed")
+	}
+
+	// Don't allow URLs that end with .home.arpa
+	if strings.HasSuffix(domain, ".home.arpa") {
+		return errors.New("URLs ending with .home.arpa are not allowed")
+	}
+
+	// Don't allow URLs that end with .internal
+	// Docker uses host.docker.internal
+	if strings.HasSuffix(domain, ".internal") {
+		return errors.New("URLs ending with .internal are not allowed")
+	}
+
+	// Don't allow URLs that end with .localdomain
+	if strings.HasSuffix(domain, ".localdomain") {
+		return errors.New("URLs ending with .localdomain are not allowed")
+	}
+
 	// Check if the domain is resolvable
 	_, err = net.LookupIP(domain)
 	if err != nil {
@@ -176,15 +250,16 @@ func validateURL(feed_url string) error {
 }
 
 func AddFeedHandler(w http.ResponseWriter, r *http.Request) {
+	var parseErrors []ParseResult
+
+	// Parse the form and get the URLs
 	r.ParseForm()
 	urls := r.Form.Get("urls")
 	if urls == "" {
-		http.Error(w, "No feed URLs provided", http.StatusBadRequest)
+		http.Error(w, "No URLs provided", http.StatusBadRequest)
 		return
 	}
 
-	var parseErrors []ParseResult
-
 	for _, feed_url := range strings.Split(urls, "\n") {
 		// TODO: Try to upgrade to https if http is provided
 

main_test.go

@@ -0,0 +1,231 @@
+package main
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// URL starts with http://
+func TestURLStartsWithHTTP(t *testing.T) {
+	url := "http://example.com"
+	err := validateURL(url)
+	assert.Nil(t, err)
+}
+
+// URL starts with https://
+func TestURLStartsWithHTTPS(t *testing.T) {
+	url := "https://example.com"
+	err := validateURL(url)
+	assert.Nil(t, err)
+}
+
+// URL contains a valid domain
+func TestURLContainsValidDomain(t *testing.T) {
+	url := "http://example.com"
+	err := validateURL(url)
+	assert.Nil(t, err)
+}
+
+// URL is empty
+func TestURLEmpty(t *testing.T) {
+	url := ""
+	err := validateURL(url)
+	assert.NotNil(t, err)
+	assert.Equal(t, "URL must start with http:// or https://", err.Error())
+}
+
+// URL does not contain a domain
+func TestURLNotNumbers(t *testing.T) {
+	url := "12345"
+	err := validateURL(url)
+	assert.NotNil(t, err)
+	assert.Equal(t, "URL must start with http:// or https://", err.Error())
+}
+
+// URL is not a valid URL
+func TestURLNotValidURL(t *testing.T) {
+	url := "example.com"
+	err := validateURL(url)
+	assert.NotNil(t, err)
+	assert.Equal(t, "URL must start with http:// or https://", err.Error())
+}
+
+// Domain is resolvable
+func TestDomainIsResolvable(t *testing.T) {
+	url := "http://example.com"
+	err := validateURL(url)
+	assert.Nil(t, err)
+}
+
+// Domain does not end with .local
+func TestDomainDoesNotEndWithLocal(t *testing.T) {
+	url := "http://example.com"
+	err := validateURL(url)
+	assert.Nil(t, err)
+}
+
+// Domain is not localhost
+func TestDomainIsNotLocalhost(t *testing.T) {
+	url := "http://example.com"
+	err := validateURL(url)
+	assert.Nil(t, err)
+}
+
+// Domain is not an IP address
+func TestDomainIsNotIPAddress(t *testing.T) {
+	url := "http://example.com"
+	err := validateURL(url)
+	assert.Nil(t, err)
+}
+
+// URL is a file path
+func TestURLIsFilePath(t *testing.T) {
+	url := "/path/to/file"
+	err := validateURL(url)
+	assert.NotNil(t, err)
+	assert.Equal(t, "URL must start with http:// or https://", err.Error())
+}
+
+// URL is a relative path
+func TestURLIsRelativePath(t *testing.T) {
+	url := "/path/to/resource"
+	err := validateURL(url)
+	assert.NotNil(t, err)
+	assert.Equal(t, "URL must start with http:// or https://", err.Error())
+}
+
+// URL is a non-existent domain
+func TestNonExistentDomainURL(t *testing.T) {
+	url := "http://jfsalksajlkfsajklfsajklfllfjffffkfsklslsksassflfskjlfjlfsjkalfsaf.com"
+	err := validateURL(url)
+	assert.NotNil(t, err)
+	assert.Equal(t, "failed to resolve domain", err.Error())
+}
+
+// URL is a malformed URL
+func TestMalformedURL(t *testing.T) {
+	url := "malformedurl"
+	err := validateURL(url)
+	assert.NotNil(t, err)
+	assert.Equal(t, "URL must start with http:// or https://", err.Error())
+}
+
+// URL is a domain that does not support HTTP/HTTPS
+func TestURLDomainNotSupportHTTP(t *testing.T) {
+	url := "ftp://example.com"
+	err := validateURL(url)
+	assert.NotNil(t, err)
+	assert.Equal(t, "URL must start with http:// or https://", err.Error())
+}
+
+// URL is an unreachable domain
+func TestUnreachableDomain(t *testing.T) {
+	url := "http://fafsffsfsfsfsafsasafassfs.com"
+	err := validateURL(url)
+	assert.NotNil(t, err)
+	assert.Equal(t, "failed to resolve domain", err.Error())
+}
+
+// URL is an IP address
+func TestURLIsIPAddress(t *testing.T) {
+	url := "http://84.55.107.42"
+	err := validateURL(url)
+	assert.NotNil(t, err)
+	assert.Equal(t, "IP address URLs are not allowed", err.Error())
+}
+
+// URL ends with .local
+func TestURLEndsWithLocal(t *testing.T) {
+	url := "http://example.local"
+	err := validateURL(url)
+	assert.NotNil(t, err)
+	assert.Equal(t, "URLs ending with .local are not allowed", err.Error())
+}
+
+func TestLocalURLs(t *testing.T) {
+	localURLs := []string{
+		"https://localhost",
+		"https://home.arpa",
+		"https://airbox.home",
+		"https://airport",
+		"https://arcor.easybox",
+		"https://aterm.me",
+		"https://bthub.home",
+		"https://bthomehub.home",
+		"https://congstar.box",
+		"https://connect.box",
+		"https://console.gl-inet.com",
+		"https://easy.box",
+		"https://etxr",
+		"https://fire.walla",
+		"https://fritz.box",
+		"https://fritz.nas",
+		"https://fritz.repeater",
+		"https://giga.cube",
+		"https://hi.link",
+		"https://hitronhub.home",
+		"https://homerouter.cpe",
+		"https://huaweimobilewifi.com",
+		"https://localbattle.net",
+		"https://myfritz.box",
+		"https://mobile.hotspot",
+		"https://ntt.setup",
+		"https://pi.hole",
+		"https://plex.direct",
+		"https://repeater.asus.com",
+		"https://router.asus.com",
+		"https://routerlogin.com",
+		"https://routerlogin.net",
+		"https://samsung.router",
+		"https://speedport.ip",
+		"https://steamloopback.host",
+		"https://tplinkap.net",
+		"https://tplinkeap.net",
+		"https://tplinkmodem.net",
+		"https://tplinkplclogin.net",
+		"https://tplinkrepeater.net",
+		"https://tplinkwifi.net",
+		"https://web.setup",
+		"https://web.setup.home",
+	}
+
+	for _, localURL := range localURLs {
+		err := validateURL(localURL)
+		if err == nil {
+			t.Errorf("Expected an error for local URL %s, got nil", localURL)
+		}
+		assert.Equal(t, "local URLs are not allowed", err.Error())
+	}
+}
+
+func TestIndexHandler(t *testing.T) {
+	// Create a request to pass to our handler.
+	req, err := http.NewRequest("GET", "/", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// We create a ResponseRecorder (which satisfies http.ResponseWriter) to record the response.
+	rr := httptest.NewRecorder()
+	handler := http.HandlerFunc(IndexHandler)
+
+	// Our handlers satisfy http.Handler, so we can call their ServeHTTP method
+	// directly and pass in our Request and ResponseRecorder.
+	handler.ServeHTTP(rr, req)
+
+	// Check the status code is what we expect.
+	if status := rr.Code; status != http.StatusOK {
+		t.Errorf("handler returned wrong status code: got %v want %v",
+			status, http.StatusOK)
+	}
+
+	// Check the response contains the expected string.
+	shouldContain := "Input the URLs of the feeds you wish to archive below. You can add as many as needed, and access them through the website or API. Alternatively, include links to .opml files, and the feeds within will be archived."
+	if rr.Body.String() != shouldContain {
+		t.Errorf("handler returned unexpected body: got %v want %v",
+			rr.Body.String(), shouldContain)
+	}
+}