diff --git a/.vscode/settings.json b/.vscode/settings.json index d13f184..0d84518 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,26 +1,54 @@ { "cSpell.words": [ + "airbox", + "arcor", "arpa", + "asus", + "aterm", "blocklist", "blocklists", + "bthomehub", + "bthub", "chartboost", + "congstar", + "easybox", + "etxr", "feedburner", "feedparser", "feedvault", "gaierror", + "giga", "gofeed", "gorm", + "hitronhub", + "homerouter", + "hotspot", + "huaweimobilewifi", "leftright", "levelname", "listparser", + "localbattle", + "localdomain", + "malformedurl", "mmcdole", "Monero", + "myfritz", "PGHOST", "PGPORT", "PGUSER", "regexes", + "routerlogin", + "speedport", + "steamloopback", + "stretchr", "stylesheet", "tmpl", + "tplinkap", + "tplinkeap", + "tplinkmodem", + "tplinkplclogin", + "tplinkrepeater", + "tplinkwifi", "webmail" ] } diff --git a/go.mod b/go.mod index d84e269..a309772 100644 --- a/go.mod +++ b/go.mod @@ -4,13 +4,16 @@ go 1.21.6 require ( github.com/go-chi/chi/v5 v5.0.11 - github.com/mmcdole/gofeed v1.2.1 + github.com/stretchr/testify v1.8.1 gorm.io/driver/sqlite v1.5.4 gorm.io/gorm v1.25.6 ) require ( + github.com/davecgh/go-spew v1.1.1 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/mattn/go-sqlite3 v1.14.22 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 9545d77..4cd49f8 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,4 @@ -github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= -github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= -github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= -github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-chi/chi/v5 v5.0.11 h1:BnpYbFZ3T3S1WMpD79r7R5ThWX40TaFB7L31Y8xqSwA= @@ -10,26 +7,20 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/mmcdole/gofeed v1.2.1 h1:tPbFN+mfOLcM1kDF1x2c/N68ChbdBatkppdzf/vDe1s= -github.com/mmcdole/gofeed v1.2.1/go.mod h1:2wVInNpgmC85q16QTTuwbuKxtKkHLCDDtf0dCmnrNr4= -github.com/mmcdole/goxpp v1.1.0 h1:WwslZNF7KNAXTFuzRtn/OKZxFLJAAyOA9w82mDz2ZGI= -github.com/mmcdole/goxpp v1.1.0/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= -golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= -golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= -golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/driver/sqlite v1.5.4 h1:IqXwXi8M/ZlPzH/947tn5uik3aYQslP9BVveoax0nV0= diff --git a/main.go b/main.go index 3d80a07..c55a87e 100644 --- a/main.go +++ b/main.go @@ -125,6 +125,7 @@ func FeedsHandler(w http.ResponseWriter, _ *http.Request) { renderPage(w, "Feeds", "Feeds Page", "feeds, page", "TheLovinator", "http://localhost:8000/feeds", "feeds") } +// Run some simple validation on the URL func validateURL(feed_url string) error { // Check if URL starts with http or https if !strings.HasPrefix(feed_url, "http://") && !strings.HasPrefix(feed_url, "https://") { @@ -150,9 +151,61 @@ func validateURL(feed_url string) error { return errors.New("IP address URLs are not allowed") } - // Don't allow localhost URLs - if strings.Contains(domain, "localhost") { - return errors.New("localhost are not allowed") + // Don't allow local URLs (e.g. router URLs) + // Taken from https://github.com/uBlockOrigin/uAssets/blob/master/filters/lan-block.txt + // https://github.com/gwarser/filter-lists + localURLs := []string{ + "[::]", + "[::1]", + "airbox.home", + "airport", + "arcor.easybox", + "aterm.me", + "bthomehub.home", + "bthub.home", + "congstar.box", + "connect.box", + "console.gl-inet.com", + "easy.box", + "etxr", + "fire.walla", + "fritz.box", + "fritz.nas", + "fritz.repeater", + "giga.cube", + "hi.link", + "hitronhub.home", + "home.arpa", + "homerouter.cpe", + "host.docker.internal", + "huaweimobilewifi.com", + "localbattle.net", + "localhost", + "mobile.hotspot", + "myfritz.box", + "ntt.setup", + "pi.hole", + "plex.direct", + "repeater.asus.com", + "router.asus.com", + "routerlogin.com", + "routerlogin.net", + "samsung.router", + "speedport.ip", + "steamloopback.host", + "tplinkap.net", + "tplinkeap.net", + "tplinkmodem.net", + "tplinkplclogin.net", + "tplinkrepeater.net", + "tplinkwifi.net", + "web.setup.home", + "web.setup", + } + for _, localURL := range localURLs { + if strings.Contains(domain, localURL) { + return errors.New("local URLs are not allowed") + } } // Don't allow URLs that end with .local @@ -160,6 +213,27 @@ func validateURL(feed_url string) error { return errors.New("URLs ending with .local are not allowed") } + // Don't allow URLs that end with .onion + if strings.HasSuffix(domain, ".onion") { + return errors.New("URLs ending with .onion are not allowed") + } + + // Don't allow URLs that end with .home.arpa + if strings.HasSuffix(domain, ".home.arpa") { + return errors.New("URLs ending with .home.arpa are not allowed") + } + + // Don't allow URLs that end with .internal + // Docker uses host.docker.internal + if strings.HasSuffix(domain, ".internal") { + return errors.New("URLs ending with .internal are not allowed") + } + + // Don't allow URLs that end with .localdomain + if strings.HasSuffix(domain, ".localdomain") { + return errors.New("URLs ending with .localdomain are not allowed") + } + // Check if the domain is resolvable _, err = net.LookupIP(domain) if err != nil { @@ -176,15 +250,16 @@ func validateURL(feed_url string) error { } func AddFeedHandler(w http.ResponseWriter, r *http.Request) { + var parseErrors []ParseResult + + // Parse the form and get the URLs r.ParseForm() urls := r.Form.Get("urls") if urls == "" { - http.Error(w, "No feed URLs provided", http.StatusBadRequest) + http.Error(w, "No URLs provided", http.StatusBadRequest) return } - var parseErrors []ParseResult - for _, feed_url := range strings.Split(urls, "\n") { // TODO: Try to upgrade to https if http is provided diff --git a/main_test.go b/main_test.go new file mode 100644 index 0000000..ba94003 --- /dev/null +++ b/main_test.go @@ -0,0 +1,231 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" +) + +// URL starts with http:// +func TestURLStartsWithHTTP(t *testing.T) { + url := "http://example.com" + err := validateURL(url) + assert.Nil(t, err) +} + +// URL starts with https:// +func TestURLStartsWithHTTPS(t *testing.T) { + url := "https://example.com" + err := validateURL(url) + assert.Nil(t, err) +} + +// URL contains a valid domain +func TestURLContainsValidDomain(t *testing.T) { + url := "http://example.com" + err := validateURL(url) + assert.Nil(t, err) +} + +// URL is empty +func TestURLEmpty(t *testing.T) { + url := "" + err := validateURL(url) + assert.NotNil(t, err) + assert.Equal(t, "URL must start with http:// or https://", err.Error()) +} + +// URL does not contain a domain +func TestURLNotNumbers(t *testing.T) { + url := "12345" + err := validateURL(url) + assert.NotNil(t, err) + assert.Equal(t, "URL must start with http:// or https://", err.Error()) +} + +// URL is not a valid URL +func TestURLNotValidURL(t *testing.T) { + url := "example.com" + err := validateURL(url) + assert.NotNil(t, err) + assert.Equal(t, "URL must start with http:// or https://", err.Error()) +} + +// Domain is resolvable +func TestDomainIsResolvable(t *testing.T) { + url := "http://example.com" + err := validateURL(url) + assert.Nil(t, err) +} + +// Domain does not end with .local +func TestDomainDoesNotEndWithLocal(t *testing.T) { + url := "http://example.com" + err := validateURL(url) + assert.Nil(t, err) +} + +// Domain is not localhost +func TestDomainIsNotLocalhost(t *testing.T) { + url := "http://example.com" + err := validateURL(url) + assert.Nil(t, err) +} + +// Domain is not an IP address +func TestDomainIsNotIPAddress(t *testing.T) { + url := "http://example.com" + err := validateURL(url) + assert.Nil(t, err) +} + +// URL is a file path +func TestURLIsFilePath(t *testing.T) { + url := "/path/to/file" + err := validateURL(url) + assert.NotNil(t, err) + assert.Equal(t, "URL must start with http:// or https://", err.Error()) +} + +// URL is a relative path +func TestURLIsRelativePath(t *testing.T) { + url := "/path/to/resource" + err := validateURL(url) + assert.NotNil(t, err) + assert.Equal(t, "URL must start with http:// or https://", err.Error()) +} + +// URL is a non-existent domain +func TestNonExistentDomainURL(t *testing.T) { + url := "http://jfsalksajlkfsajklfsajklfllfjffffkfsklslsksassflfskjlfjlfsjkalfsaf.com" + err := validateURL(url) + assert.NotNil(t, err) + assert.Equal(t, "failed to resolve domain", err.Error()) +} + +// URL is a malformed URL +func TestMalformedURL(t *testing.T) { + url := "malformedurl" + err := validateURL(url) + assert.NotNil(t, err) + assert.Equal(t, "URL must start with http:// or https://", err.Error()) +} + +// URL is a domain that does not support HTTP/HTTPS +func TestURLDomainNotSupportHTTP(t *testing.T) { + url := "ftp://example.com" + err := validateURL(url) + assert.NotNil(t, err) + assert.Equal(t, "URL must start with http:// or https://", err.Error()) +} + +// URL is an unreachable domain +func TestUnreachableDomain(t *testing.T) { + url := "http://fafsffsfsfsfsafsasafassfs.com" + err := validateURL(url) + assert.NotNil(t, err) + assert.Equal(t, "failed to resolve domain", err.Error()) +} + +// URL is an IP address +func TestURLIsIPAddress(t *testing.T) { + url := "http://84.55.107.42" + err := validateURL(url) + assert.NotNil(t, err) + assert.Equal(t, "IP address URLs are not allowed", err.Error()) +} + +// URL ends with .local +func TestURLEndsWithLocal(t *testing.T) { + url := "http://example.local" + err := validateURL(url) + assert.NotNil(t, err) + assert.Equal(t, "URLs ending with .local are not allowed", err.Error()) +} + +func TestLocalURLs(t *testing.T) { + localURLs := []string{ + "https://localhost", + "https://home.arpa", + "https://airbox.home", + "https://airport", + "https://arcor.easybox", + "https://aterm.me", + "https://bthub.home", + "https://bthomehub.home", + "https://congstar.box", + "https://connect.box", + "https://console.gl-inet.com", + "https://easy.box", + "https://etxr", + "https://fire.walla", + "https://fritz.box", + "https://fritz.nas", + "https://fritz.repeater", + "https://giga.cube", + "https://hi.link", + "https://hitronhub.home", + "https://homerouter.cpe", + "https://huaweimobilewifi.com", + "https://localbattle.net", + "https://myfritz.box", + "https://mobile.hotspot", + "https://ntt.setup", + "https://pi.hole", + "https://plex.direct", + "https://repeater.asus.com", + "https://router.asus.com", + "https://routerlogin.com", + "https://routerlogin.net", + "https://samsung.router", + "https://speedport.ip", + "https://steamloopback.host", + "https://tplinkap.net", + "https://tplinkeap.net", + "https://tplinkmodem.net", + "https://tplinkplclogin.net", + "https://tplinkrepeater.net", + "https://tplinkwifi.net", + "https://web.setup", + "https://web.setup.home", + } + + for _, localURL := range localURLs { + err := validateURL(localURL) + if err == nil { + t.Errorf("Expected an error for local URL %s, got nil", localURL) + } + assert.Equal(t, "local URLs are not allowed", err.Error()) + } +} + +func TestIndexHandler(t *testing.T) { + // Create a request to pass to our handler. + req, err := http.NewRequest("GET", "/", nil) + if err != nil { + t.Fatal(err) + } + + // We create a ResponseRecorder (which satisfies http.ResponseWriter) to record the response. + rr := httptest.NewRecorder() + handler := http.HandlerFunc(IndexHandler) + + // Our handlers satisfy http.Handler, so we can call their ServeHTTP method + // directly and pass in our Request and ResponseRecorder. + handler.ServeHTTP(rr, req) + + // Check the status code is what we expect. + if status := rr.Code; status != http.StatusOK { + t.Errorf("handler returned wrong status code: got %v want %v", + status, http.StatusOK) + } + + // Check the response contains the expected string. + shouldContain := "Input the URLs of the feeds you wish to archive below. You can add as many as needed, and access them through the website or API. Alternatively, include links to .opml files, and the feeds within will be archived." + if rr.Body.String() != shouldContain { + t.Errorf("handler returned unexpected body: got %v want %v", + rr.Body.String(), shouldContain) + } +}