Add GOBLOG_DATA_DIR: one directory for everything goblog writes

.github/workflows/install-smoke.yml

@@ -8,12 +8,19 @@ on:
 jobs:
 
   install:
-    name: Fresh install (${{ matrix.database }})
+    name: Fresh install (${{ matrix.database }}${{ matrix.legacy && ', no data directory' || '' }})
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         database: [sqlite, mysql, postgres]
+        # A matrix key of its own: an include that only added "legacy" to
+        # sqlite would modify the sqlite job instead of adding a second one.
+        legacy: [""]
+        include:
+          # The layout of installs that predate GOBLOG_DATA_DIR.
+          - database: sqlite
+            legacy: "1"
     steps:
 
       - name: Check out code
@@ -22,4 +29,6 @@ jobs:
       # Builds the Docker image and walks the install wizard against a
       # fresh database (#651).
       - name: Install smoke test
+        env:
+          GOBLOG_SMOKE_LEGACY: ${{ matrix.legacy }}
         run: scripts/install-smoke-test.sh ${{ matrix.database }}

README.md

@@ -66,8 +66,11 @@ Visit http://localhost:7000 and follow the install wizard.
 
 ### Docker
 ```bash
-docker run -p 7000:7000 compscidr/goblog:latest
+docker run -p 7000:7000 -e GOBLOG_DATA_DIR=/data -v goblog-data:/data compscidr/goblog:latest
 ```
+`GOBLOG_DATA_DIR` is where goblog keeps everything it writes: `.env` (the session key, database settings and GitHub credentials the wizard saves), the SQLite database, uploads, and installed plugins and themes. With it on a volume, the site survives the container being replaced, for example when you pull a newer image. Without it those files are written inside the container and are lost with it.
+
+A relative SQLite path such as the wizard's default `goblog.db` is created inside the data directory; an absolute path is used as given. Sites set up before `GOBLOG_DATA_DIR` existed keep working unchanged when it is not set.
 
 ### Database
 SQLite is the default and needs no setup. To use MySQL or PostgreSQL instead, pick it in the install wizard or set the variables in `.env` (see `template.env`):
@@ -195,7 +198,7 @@ The installer writes this sidecar for directory installs; an operator dropping a
 
 #### Installing from the directory
 **Admin → Plugins** lists what is installed and lets you browse and search the [plugin directory](https://www.goblog.live/plugins), install a plugin with one click, update it when the directory has a newer release, or uninstall it. WebAssembly is the only format the directory installs; requirements:
-- `plugins/wasm/` writable by goblog (WASM loading is on by default; set `ENABLE_WASM_PLUGINS=false` to disable it entirely). With Docker, bind-mount that directory (see below) — otherwise installed plugins vanish with the container.
+- `plugins/wasm/` writable by goblog (WASM loading is on by default; set `ENABLE_WASM_PLUGINS=false` to disable it entirely). With Docker, set `GOBLOG_DATA_DIR` (see [Docker](#docker)) or bind-mount that directory (see below) — otherwise installed plugins vanish with the container.
 - The directory URL is the `plugin_directory_url` setting (default `https://www.goblog.live/plugins/index.json`); point it elsewhere to run a private directory. `plugin_directory_url` is a trust decision: whatever it points at can offer code that runs inside goblog once you click Install.
 
 Install downloads the plugin's `.wasm` asset, verifies its sha256 against the directory index, loads it, checks that its name and version match, and only then writes it (plus the `allowed_hosts` sidecar) to `plugins/wasm/` and starts it — no restart. Updates keep the plugin's settings; uninstall removes both files. Install only from sources you trust. A dynamic (`.go`) plugin installed before the directory went wasm-only can still be updated to a wasm release or uninstalled, just not reinstalled as `.go`.

admin/admin.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"goblog/auth"
 	"goblog/blog"
+	"goblog/datadir"
 	gplugin "goblog/plugin"
 	"goblog/plugin/installer"
 	"goblog/plugins/directory"
@@ -209,7 +210,13 @@ func (a *Admin) UploadFile(c *gin.Context) {
 	}
 
 	filename := UploadsFolder + filepath.Base(file.Filename)
-	if err := c.SaveUploadedFile(file, WWWFolder+filename); err != nil {
+	// With a data directory the file goes to <data>/uploads, which main
+	// serves at /uploads; otherwise into www, served as it always was.
+	dst := WWWFolder + filename
+	if dir := datadir.Dir(); dir != "" {
+		dst = filepath.Join(dir, filename)
+	}
+	if err := c.SaveUploadedFile(file, dst); err != nil {
 		log.Println(fmt.Sprintf("Save Upload File Error erorr: %s", err.Error()))
 		c.JSON(http.StatusBadRequest, fmt.Sprintf("upload file err: %s", err.Error()))
 		return

auth/auth.go

@@ -3,6 +3,7 @@ package auth
 import (
 	"encoding/json"
 	"errors"
+	"goblog/datadir"
 	"io/ioutil"
 	"log"
 	"net/http"
@@ -77,7 +78,7 @@ func (a *Auth) requestAccessToken(parsedCode string) (*AccessTokenResponse, erro
 	// .env is the usual home for these, but a missing file is not itself an
 	// error: a deployment may set them in the real environment. What matters
 	// is ending up with both values, which is checked below.
-	if err := godotenv.Load(".env"); err != nil {
+	if err := godotenv.Load(datadir.Path(".env")); err != nil {
 		_ = godotenv.Load("local.env")
 	}
 	clientID := os.Getenv("client_id")

blog/blog.go

@@ -12,6 +12,7 @@ import (
 	"sort"
 
 	"goblog/auth"
+	"goblog/datadir"
 	"html/template"
 	"log"
 	"net"
@@ -1395,7 +1396,7 @@ func (b *Blog) RobotsTxt(c *gin.Context) {
 
 // Login to the blog
 func (b *Blog) Login(c *gin.Context) {
-	err := godotenv.Load(".env")
+	err := godotenv.Load(datadir.Path(".env"))
 	if err != nil {
 		//fall back to local config
 		err = godotenv.Load("local.env")
@@ -1550,7 +1551,7 @@ func GithubAuthorizeURL(origin, clientID, state string) string {
 // The state it mints, and where the visitor was heading, are kept in the
 // session for GithubCallback to check when GitHub returns (#637).
 func (b *Blog) GithubLogin(c *gin.Context) {
-	if err := godotenv.Load(".env"); err != nil {
+	if err := godotenv.Load(datadir.Path(".env")); err != nil {
 		_ = godotenv.Load("local.env")
 	}
 	clientID := os.Getenv("client_id")

datadir/datadir.go

@@ -0,0 +1,32 @@
+// Package datadir says where goblog keeps the files it writes: .env, a
+// SQLite database, uploads, and installed plugins and themes.
+//
+// By default those live in the working directory, next to the templates
+// and static files goblog ships with, which is how every install before
+// this package existed is laid out. Setting GOBLOG_DATA_DIR moves all of
+// them under one directory, so a container needs a single volume to keep
+// its site (#653).
+package datadir
+
+import (
+	"os"
+	"path/filepath"
+)
+
+// EnvVar names the environment variable that sets the data directory.
+const EnvVar = "GOBLOG_DATA_DIR"
+
+// Dir returns the data directory, or "" when none is set.
+func Dir() string { return os.Getenv(EnvVar) }
+
+// Path returns where the file or directory rel lives: under the data
+// directory when one is set, otherwise rel unchanged (relative to the
+// working directory). An absolute rel is returned as it is, so a path the
+// operator spelled out in full is never moved.
+func Path(rel string) string {
+	dir := Dir()
+	if dir == "" || filepath.IsAbs(rel) {
+		return rel
+	}
+	return filepath.Join(dir, rel)
+}

datadir/datadir_test.go

@@ -0,0 +1,21 @@
+package datadir
+
+import "testing"
+
+func TestPath(t *testing.T) {
+	t.Setenv(EnvVar, "")
+	if got := Path(".env"); got != ".env" {
+		t.Errorf("no data dir: Path(.env) = %q, want .env unchanged", got)
+	}
+	t.Setenv(EnvVar, "/data")
+	for rel, want := range map[string]string{
+		".env":             "/data/.env",
+		"goblog.db":        "/data/goblog.db",
+		"plugins/wasm":     "/data/plugins/wasm",
+		"/var/lib/blog.db": "/var/lib/blog.db", // absolute paths are left alone
+	} {
+		if got := Path(rel); got != want {
+			t.Errorf("Path(%q) = %q, want %q", rel, got, want)
+		}
+	}
+}

db.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"goblog/datadir"
 	"strings"
 
 	"github.com/gin-gonic/gin"
@@ -139,7 +140,7 @@ func (cfg dbConfig) dsn() string {
 func openDatabase(cfg dbConfig) (*gorm.DB, error) {
 	switch cfg.Type {
 	case "sqlite":
-		return gorm.Open(sqlite.Open(cfg.SQLiteFile), &gorm.Config{
+		return gorm.Open(sqlite.Open(datadir.Path(cfg.SQLiteFile)), &gorm.Config{
 			DisableForeignKeyConstraintWhenMigrating: true,
 		})
 	case "mysql":

goblog.go

@@ -7,6 +7,7 @@ import (
 	"goblog/admin"
 	"goblog/auth"
 	"goblog/blog"
+	"goblog/datadir"
 	"goblog/mail"
 	gplugin "goblog/plugin"
 	"goblog/plugin/installer"
@@ -48,15 +49,15 @@ type goblog struct {
 }
 
 func envFilePresent() bool {
-	_, err := os.Stat(".env")
+	_, err := os.Stat(datadir.Path(".env"))
 	if err != nil {
 		return false
 	}
 	return true
 }
 
 func isAuthConfigured() bool {
-	envFile, err := godotenv.Read(".env")
+	envFile, err := godotenv.Read(datadir.Path(".env"))
 	if err != nil {
 		log.Println("Couldn't read the .env file: " + err.Error())
 		return false
@@ -68,7 +69,7 @@ func isAuthConfigured() bool {
 }
 
 func attemptConnectDb() *gorm.DB {
-	envFile, err := godotenv.Read(".env")
+	envFile, err := godotenv.Read(datadir.Path(".env"))
 	if err != nil {
 		log.Println("Couldn't read the .env file: " + err.Error())
 		return nil
@@ -98,7 +99,7 @@ func (g *goblog) rootHandler(c *gin.Context) {
 		return
 	} else {
 		log.Println("Root handler:  Found .env file")
-		envFile, err := godotenv.Read(".env")
+		envFile, err := godotenv.Read(datadir.Path(".env"))
 		if err != nil {
 			c.HTML(http.StatusOK, "wizard_db.html", gin.H{
 				"version": Version,
@@ -216,12 +217,19 @@ func main() {
 		os.Exit(runValidatePlugin(os.Args[2:], os.Stdout, os.Stderr))
 	}
 	log.Println("Starting blog version: ", Version)
+	if dir := datadir.Dir(); dir != "" {
+		log.Println("Data directory: " + dir)
+		if err := os.MkdirAll(dir, 0755); err != nil {
+			log.Println("Couldn't create the data directory: " + err.Error())
+			return
+		}
+	}
 	var sessionKey string
 	var db *gorm.DB = nil
 	if !envFilePresent() {
 		log.Println("No .env file found, creating one with a new session key")
 		sessionKey = uuid.New().String()
-		f, err := os.Create(".env")
+		f, err := os.Create(datadir.Path(".env"))
 		if err != nil {
 			log.Println("Couldn't create the .env file: " + err.Error())
 			return
@@ -234,7 +242,7 @@ func main() {
 		}
 	} else {
 		log.Println("Found .env file")
-		envFile, err := godotenv.Read(".env")
+		envFile, err := godotenv.Read(datadir.Path(".env"))
 		if err != nil {
 			log.Println("Couldn't read the .env file: " + err.Error())
 			return
@@ -243,7 +251,7 @@ func main() {
 		if (sessionKey == "") || (len(sessionKey) != 36) {
 			log.Println("No session key found or it's invalid, creating a new one")
 			sessionKey = uuid.New().String()
-			f, err := os.OpenFile(".env", os.O_APPEND|os.O_WRONLY, 0644)
+			f, err := os.OpenFile(datadir.Path(".env"), os.O_APPEND|os.O_WRONLY, 0644)
 			if err != nil {
 				log.Println("Couldn't open the .env file: " + err.Error())
 				return
@@ -279,7 +287,7 @@ func main() {
 
 	// The wizard and admin pin read settings lazily via os.Getenv after a
 	// godotenv.Load; SMTP is needed at startup, so load once here too.
-	if err := godotenv.Load(".env"); err != nil {
+	if err := godotenv.Load(datadir.Path(".env")); err != nil {
 		log.Println("Couldn't load .env into the environment: " + err.Error())
 	}
 
@@ -313,15 +321,15 @@ func main() {
 	registry.Register(docs.New())
 	dynamicEnabled := os.Getenv("ENABLE_DYNAMIC_PLUGINS") == "true"
 	if dynamicEnabled {
-		gplugin.LoadDynamicPlugins(registry, "plugins/dynamic")
+		gplugin.LoadDynamicPlugins(registry, datadir.Path("plugins/dynamic"))
 	}
 	wasmEnabled := os.Getenv("ENABLE_WASM_PLUGINS") != "false"
 	if wasmEnabled {
-		wasm.LoadWasmPlugins(registry, "plugins/wasm", registry.Store())
+		wasm.LoadWasmPlugins(registry, datadir.Path("plugins/wasm"), registry.Store())
 	}
 	pluginInstaller := &installer.Installer{
-		Dir:         "plugins/dynamic",
-		WasmDir:     "plugins/wasm",
+		Dir:         datadir.Path("plugins/dynamic"),
+		WasmDir:     datadir.Path("plugins/wasm"),
 		Registry:    registry,
 		Directory:   installer.NewFetcher(nil),
 		Version:     Version,
@@ -470,6 +478,11 @@ func main() {
 	router.POST("/api/v1/directory/repos/:id/rebuild", goblog._admin.RebuildDirectoryRepo)
 	router.DELETE("/api/v1/directory/repos/:id", goblog._admin.DelistDirectoryRepo)
 	//if we use true here - it will override the home route and just show files
+	if datadir.Dir() != "" {
+		// Uploads are written under the data directory; anything that is
+		// not there (a file the image ships in www/uploads) falls through.
+		router.Use(static.Serve("/uploads", static.LocalFile(datadir.Path("uploads"), false)))
+	}
 	router.Use(static.Serve("/", static.LocalFile("www", false)))
 	if err != nil {
 		log.Println("Couldn't get the hostname")
@@ -650,7 +663,7 @@ func updateDB(c *gin.Context) {
 		fail("Invalid database type")
 		return
 	}
-	if err := os.WriteFile(".env", []byte(cfg.envFile()), 0600); err != nil {
+	if err := os.WriteFile(datadir.Path(".env"), []byte(cfg.envFile()), 0600); err != nil {
 		fail("Couldn't write the .env file: " + err.Error())
 		return
 	}