Check if the domain is on a blocklist

main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bufio"
 	"errors"
 	"fmt"
 	"html/template"
@@ -9,6 +10,7 @@ import (
 	"net/http"
 	"net/url"
 	"strings"
+	"time"
 
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/chi/v5/middleware"
@@ -16,24 +18,32 @@ import (
 	"gorm.io/gorm"
 )
 
-func main() {
-	log.Println("Starting FeedVault...")
-	db, err := gorm.Open(sqlite.Open("feedvault.db"), &gorm.Config{})
+var db *gorm.DB
+
+// Initialize the database
+func init() {
+	var err error
+	db, err = gorm.Open(sqlite.Open("feedvault.db"), &gorm.Config{})
 	if err != nil {
 		panic("Failed to connect to database")
 	}
-
-	sqlDB, err := db.DB()
-	if err != nil {
-		panic("Failed to get database connection")
+	if db == nil {
+		panic("db nil")
 	}
-	defer sqlDB.Close()
+	log.Println("Connected to database")
 
 	// Migrate the schema
-	err = db.AutoMigrate(&Feed{}, &Item{}, &Person{}, &Image{}, &Enclosure{}, &DublinCoreExtension{}, &ITunesFeedExtension{}, &ITunesItemExtension{}, &ITunesCategory{}, &ITunesOwner{}, &Extension{})
+	err = db.AutoMigrate(&BadURLsMeta{}, &BadURLs{}, &Feed{}, &Item{}, &Person{}, &Image{}, &Enclosure{}, &DublinCoreExtension{}, &ITunesFeedExtension{}, &ITunesItemExtension{}, &ITunesCategory{}, &ITunesOwner{}, &Extension{})
 	if err != nil {
 		panic("Failed to migrate the database")
 	}
+}
+
+func main() {
+	log.Println("Starting FeedVault...")
+
+	// Scrape the bad URLs in the background
+	scrapeBadURLs()
 
 	// Create a new router
 	r := chi.NewRouter()
@@ -59,6 +69,85 @@ func main() {
 	http.ListenAndServe("127.0.0.1:8000", r)
 }
 
+func scrapeBadURLs() {
+	// TODO: We should only scrape the bad URLs if the file has been updated
+	filterListURLs := []string{
+		"https://malware-filter.gitlab.io/malware-filter/phishing-filter-dnscrypt-blocked-names.txt",
+		"https://malware-filter.gitlab.io/malware-filter/urlhaus-filter-dnscrypt-blocked-names-online.txt",
+	}
+
+	// Scrape the bad URLs
+	badURLs := []BadURLs{}
+	for _, url := range filterListURLs {
+		// Check if we have scraped the bad URLs in the last 24 hours
+		var meta BadURLsMeta
+		db.Where("url = ?", url).First(&meta)
+		if time.Since(meta.LastScraped).Hours() < 24 {
+			log.Printf("%s was last scraped %.1f hours ago\n", url, time.Since(meta.LastScraped).Hours())
+			continue
+		}
+
+		// Create the meta if it doesn't exist
+		if meta.ID == 0 {
+			meta = BadURLsMeta{URL: url}
+			db.Create(&meta)
+		}
+
+		// Update the last scraped time
+		db.Model(&meta).Update("last_scraped", time.Now())
+
+		// Get the filter list
+		resp, err := http.Get(url)
+		if err != nil {
+			log.Println("Failed to get filter list:", err)
+			continue
+		}
+		defer resp.Body.Close()
+
+		scanner := bufio.NewScanner(resp.Body)
+		for scanner.Scan() {
+			line := scanner.Text()
+			if strings.HasPrefix(line, "#") {
+				log.Println("Comment:", line)
+				continue
+			}
+
+			// Skip the URL if it already exists in the database
+			var count int64
+			db.Model(&BadURLs{}).Where("url = ?", line).Count(&count)
+			if count > 0 {
+				log.Println("URL already exists:", line)
+				continue
+			}
+
+			// Add the bad URL to the list
+			badURLs = append(badURLs, BadURLs{URL: line, Active: true})
+		}
+
+		if err := scanner.Err(); err != nil {
+			log.Println("Failed to scan filter list:", err)
+		}
+	}
+
+	if len(badURLs) == 0 {
+		log.Println("No new URLs found in", len(filterListURLs), "filter lists")
+		return
+	}
+
+	// Log how many bad URLs we found
+	log.Println("Found", len(badURLs), "bad URLs")
+
+	// Mark all the bad URLs as inactive if we have any in the database
+	var count int64
+	db.Model(&BadURLs{}).Count(&count)
+	if count > 0 {
+		db.Model(&BadURLs{}).Update("active", false)
+	}
+
+	// Save the bad URLs to the database
+	db.Create(&badURLs)
+}
+
 func renderPage(w http.ResponseWriter, title, description, keywords, author, url, templateName string) {
 	data := TemplateData{
 		Title:        title,
@@ -208,6 +297,13 @@ func validateURL(feed_url string) error {
 		}
 	}
 
+	// Check if the domain is in BadURLs
+	var count int64
+	db.Model(&BadURLs{}).Where("url = ?", domain).Count(&count)
+	if count > 0 {
+		return errors.New("URL is in the bad URLs list")
+	}
+
 	// Don't allow URLs that end with .local
 	if strings.HasSuffix(domain, ".local") {
 		return errors.New("URLs ending with .local are not allowed")