diff --git a/.github/workflows/install-smoke.yml b/.github/workflows/install-smoke.yml index c7d37725..5610efe2 100644 --- a/.github/workflows/install-smoke.yml +++ b/.github/workflows/install-smoke.yml @@ -8,12 +8,19 @@ on: jobs: install: - name: Fresh install (${{ matrix.database }}) + name: Fresh install (${{ matrix.database }}${{ matrix.legacy && ', no data directory' || '' }}) runs-on: ubuntu-latest strategy: fail-fast: false matrix: database: [sqlite, mysql, postgres] + # A matrix key of its own: an include that only added "legacy" to + # sqlite would modify the sqlite job instead of adding a second one. + legacy: [""] + include: + # The layout of installs that predate GOBLOG_DATA_DIR. + - database: sqlite + legacy: "1" steps: - name: Check out code @@ -22,4 +29,6 @@ jobs: # Builds the Docker image and walks the install wizard against a # fresh database (#651). - name: Install smoke test + env: + GOBLOG_SMOKE_LEGACY: ${{ matrix.legacy }} run: scripts/install-smoke-test.sh ${{ matrix.database }} diff --git a/README.md b/README.md index fcd1ddcd..8b2b7b0d 100644 --- a/README.md +++ b/README.md @@ -66,8 +66,11 @@ Visit http://localhost:7000 and follow the install wizard. ### Docker ```bash -docker run -p 7000:7000 compscidr/goblog:latest +docker run -p 7000:7000 -e GOBLOG_DATA_DIR=/data -v goblog-data:/data compscidr/goblog:latest ``` +`GOBLOG_DATA_DIR` is where goblog keeps everything it writes: `.env` (the session key, database settings and GitHub credentials the wizard saves), the SQLite database, uploads, and installed plugins and themes. With it on a volume, the site survives the container being replaced, for example when you pull a newer image. Without it those files are written inside the container and are lost with it. + +A relative SQLite path such as the wizard's default `goblog.db` is created inside the data directory; an absolute path is used as given. Sites set up before `GOBLOG_DATA_DIR` existed keep working unchanged when it is not set. ### Database SQLite is the default and needs no setup. To use MySQL or PostgreSQL instead, pick it in the install wizard or set the variables in `.env` (see `template.env`): @@ -195,7 +198,7 @@ The installer writes this sidecar for directory installs; an operator dropping a #### Installing from the directory **Admin → Plugins** lists what is installed and lets you browse and search the [plugin directory](https://www.goblog.live/plugins), install a plugin with one click, update it when the directory has a newer release, or uninstall it. WebAssembly is the only format the directory installs; requirements: -- `plugins/wasm/` writable by goblog (WASM loading is on by default; set `ENABLE_WASM_PLUGINS=false` to disable it entirely). With Docker, bind-mount that directory (see below) — otherwise installed plugins vanish with the container. +- `plugins/wasm/` writable by goblog (WASM loading is on by default; set `ENABLE_WASM_PLUGINS=false` to disable it entirely). With Docker, set `GOBLOG_DATA_DIR` (see [Docker](#docker)) or bind-mount that directory (see below) — otherwise installed plugins vanish with the container. - The directory URL is the `plugin_directory_url` setting (default `https://www.goblog.live/plugins/index.json`); point it elsewhere to run a private directory. `plugin_directory_url` is a trust decision: whatever it points at can offer code that runs inside goblog once you click Install. Install downloads the plugin's `.wasm` asset, verifies its sha256 against the directory index, loads it, checks that its name and version match, and only then writes it (plus the `allowed_hosts` sidecar) to `plugins/wasm/` and starts it — no restart. Updates keep the plugin's settings; uninstall removes both files. Install only from sources you trust. A dynamic (`.go`) plugin installed before the directory went wasm-only can still be updated to a wasm release or uninstalled, just not reinstalled as `.go`. diff --git a/admin/admin.go b/admin/admin.go index ebbb6cf6..f578c520 100644 --- a/admin/admin.go +++ b/admin/admin.go @@ -5,6 +5,7 @@ import ( "fmt" "goblog/auth" "goblog/blog" + "goblog/datadir" gplugin "goblog/plugin" "goblog/plugin/installer" "goblog/plugins/directory" @@ -209,7 +210,13 @@ func (a *Admin) UploadFile(c *gin.Context) { } filename := UploadsFolder + filepath.Base(file.Filename) - if err := c.SaveUploadedFile(file, WWWFolder+filename); err != nil { + // With a data directory the file goes to /uploads, which main + // serves at /uploads; otherwise into www, served as it always was. + dst := WWWFolder + filename + if dir := datadir.Dir(); dir != "" { + dst = filepath.Join(dir, filename) + } + if err := c.SaveUploadedFile(file, dst); err != nil { log.Println(fmt.Sprintf("Save Upload File Error erorr: %s", err.Error())) c.JSON(http.StatusBadRequest, fmt.Sprintf("upload file err: %s", err.Error())) return diff --git a/auth/auth.go b/auth/auth.go index 2847d097..1a096205 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -3,6 +3,7 @@ package auth import ( "encoding/json" "errors" + "goblog/datadir" "io/ioutil" "log" "net/http" @@ -77,7 +78,7 @@ func (a *Auth) requestAccessToken(parsedCode string) (*AccessTokenResponse, erro // .env is the usual home for these, but a missing file is not itself an // error: a deployment may set them in the real environment. What matters // is ending up with both values, which is checked below. - if err := godotenv.Load(".env"); err != nil { + if err := godotenv.Load(datadir.Path(".env")); err != nil { _ = godotenv.Load("local.env") } clientID := os.Getenv("client_id") diff --git a/blog/blog.go b/blog/blog.go index 15fb9913..cca0b7f7 100644 --- a/blog/blog.go +++ b/blog/blog.go @@ -12,6 +12,7 @@ import ( "sort" "goblog/auth" + "goblog/datadir" "html/template" "log" "net" @@ -1395,7 +1396,7 @@ func (b *Blog) RobotsTxt(c *gin.Context) { // Login to the blog func (b *Blog) Login(c *gin.Context) { - err := godotenv.Load(".env") + err := godotenv.Load(datadir.Path(".env")) if err != nil { //fall back to local config err = godotenv.Load("local.env") @@ -1550,7 +1551,7 @@ func GithubAuthorizeURL(origin, clientID, state string) string { // The state it mints, and where the visitor was heading, are kept in the // session for GithubCallback to check when GitHub returns (#637). func (b *Blog) GithubLogin(c *gin.Context) { - if err := godotenv.Load(".env"); err != nil { + if err := godotenv.Load(datadir.Path(".env")); err != nil { _ = godotenv.Load("local.env") } clientID := os.Getenv("client_id") diff --git a/datadir/datadir.go b/datadir/datadir.go new file mode 100644 index 00000000..f4353958 --- /dev/null +++ b/datadir/datadir.go @@ -0,0 +1,32 @@ +// Package datadir says where goblog keeps the files it writes: .env, a +// SQLite database, uploads, and installed plugins and themes. +// +// By default those live in the working directory, next to the templates +// and static files goblog ships with, which is how every install before +// this package existed is laid out. Setting GOBLOG_DATA_DIR moves all of +// them under one directory, so a container needs a single volume to keep +// its site (#653). +package datadir + +import ( + "os" + "path/filepath" +) + +// EnvVar names the environment variable that sets the data directory. +const EnvVar = "GOBLOG_DATA_DIR" + +// Dir returns the data directory, or "" when none is set. +func Dir() string { return os.Getenv(EnvVar) } + +// Path returns where the file or directory rel lives: under the data +// directory when one is set, otherwise rel unchanged (relative to the +// working directory). An absolute rel is returned as it is, so a path the +// operator spelled out in full is never moved. +func Path(rel string) string { + dir := Dir() + if dir == "" || filepath.IsAbs(rel) { + return rel + } + return filepath.Join(dir, rel) +} diff --git a/datadir/datadir_test.go b/datadir/datadir_test.go new file mode 100644 index 00000000..91973d75 --- /dev/null +++ b/datadir/datadir_test.go @@ -0,0 +1,21 @@ +package datadir + +import "testing" + +func TestPath(t *testing.T) { + t.Setenv(EnvVar, "") + if got := Path(".env"); got != ".env" { + t.Errorf("no data dir: Path(.env) = %q, want .env unchanged", got) + } + t.Setenv(EnvVar, "/data") + for rel, want := range map[string]string{ + ".env": "/data/.env", + "goblog.db": "/data/goblog.db", + "plugins/wasm": "/data/plugins/wasm", + "/var/lib/blog.db": "/var/lib/blog.db", // absolute paths are left alone + } { + if got := Path(rel); got != want { + t.Errorf("Path(%q) = %q, want %q", rel, got, want) + } + } +} diff --git a/db.go b/db.go index c533120d..70dc1c73 100644 --- a/db.go +++ b/db.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "goblog/datadir" "strings" "github.com/gin-gonic/gin" @@ -139,7 +140,7 @@ func (cfg dbConfig) dsn() string { func openDatabase(cfg dbConfig) (*gorm.DB, error) { switch cfg.Type { case "sqlite": - return gorm.Open(sqlite.Open(cfg.SQLiteFile), &gorm.Config{ + return gorm.Open(sqlite.Open(datadir.Path(cfg.SQLiteFile)), &gorm.Config{ DisableForeignKeyConstraintWhenMigrating: true, }) case "mysql": diff --git a/goblog.go b/goblog.go index c6c7c572..b90341a0 100644 --- a/goblog.go +++ b/goblog.go @@ -7,6 +7,7 @@ import ( "goblog/admin" "goblog/auth" "goblog/blog" + "goblog/datadir" "goblog/mail" gplugin "goblog/plugin" "goblog/plugin/installer" @@ -48,7 +49,7 @@ type goblog struct { } func envFilePresent() bool { - _, err := os.Stat(".env") + _, err := os.Stat(datadir.Path(".env")) if err != nil { return false } @@ -56,7 +57,7 @@ func envFilePresent() bool { } func isAuthConfigured() bool { - envFile, err := godotenv.Read(".env") + envFile, err := godotenv.Read(datadir.Path(".env")) if err != nil { log.Println("Couldn't read the .env file: " + err.Error()) return false @@ -68,7 +69,7 @@ func isAuthConfigured() bool { } func attemptConnectDb() *gorm.DB { - envFile, err := godotenv.Read(".env") + envFile, err := godotenv.Read(datadir.Path(".env")) if err != nil { log.Println("Couldn't read the .env file: " + err.Error()) return nil @@ -98,7 +99,7 @@ func (g *goblog) rootHandler(c *gin.Context) { return } else { log.Println("Root handler: Found .env file") - envFile, err := godotenv.Read(".env") + envFile, err := godotenv.Read(datadir.Path(".env")) if err != nil { c.HTML(http.StatusOK, "wizard_db.html", gin.H{ "version": Version, @@ -216,12 +217,19 @@ func main() { os.Exit(runValidatePlugin(os.Args[2:], os.Stdout, os.Stderr)) } log.Println("Starting blog version: ", Version) + if dir := datadir.Dir(); dir != "" { + log.Println("Data directory: " + dir) + if err := os.MkdirAll(dir, 0755); err != nil { + log.Println("Couldn't create the data directory: " + err.Error()) + return + } + } var sessionKey string var db *gorm.DB = nil if !envFilePresent() { log.Println("No .env file found, creating one with a new session key") sessionKey = uuid.New().String() - f, err := os.Create(".env") + f, err := os.Create(datadir.Path(".env")) if err != nil { log.Println("Couldn't create the .env file: " + err.Error()) return @@ -234,7 +242,7 @@ func main() { } } else { log.Println("Found .env file") - envFile, err := godotenv.Read(".env") + envFile, err := godotenv.Read(datadir.Path(".env")) if err != nil { log.Println("Couldn't read the .env file: " + err.Error()) return @@ -243,7 +251,7 @@ func main() { if (sessionKey == "") || (len(sessionKey) != 36) { log.Println("No session key found or it's invalid, creating a new one") sessionKey = uuid.New().String() - f, err := os.OpenFile(".env", os.O_APPEND|os.O_WRONLY, 0644) + f, err := os.OpenFile(datadir.Path(".env"), os.O_APPEND|os.O_WRONLY, 0644) if err != nil { log.Println("Couldn't open the .env file: " + err.Error()) return @@ -279,7 +287,7 @@ func main() { // The wizard and admin pin read settings lazily via os.Getenv after a // godotenv.Load; SMTP is needed at startup, so load once here too. - if err := godotenv.Load(".env"); err != nil { + if err := godotenv.Load(datadir.Path(".env")); err != nil { log.Println("Couldn't load .env into the environment: " + err.Error()) } @@ -313,15 +321,15 @@ func main() { registry.Register(docs.New()) dynamicEnabled := os.Getenv("ENABLE_DYNAMIC_PLUGINS") == "true" if dynamicEnabled { - gplugin.LoadDynamicPlugins(registry, "plugins/dynamic") + gplugin.LoadDynamicPlugins(registry, datadir.Path("plugins/dynamic")) } wasmEnabled := os.Getenv("ENABLE_WASM_PLUGINS") != "false" if wasmEnabled { - wasm.LoadWasmPlugins(registry, "plugins/wasm", registry.Store()) + wasm.LoadWasmPlugins(registry, datadir.Path("plugins/wasm"), registry.Store()) } pluginInstaller := &installer.Installer{ - Dir: "plugins/dynamic", - WasmDir: "plugins/wasm", + Dir: datadir.Path("plugins/dynamic"), + WasmDir: datadir.Path("plugins/wasm"), Registry: registry, Directory: installer.NewFetcher(nil), Version: Version, @@ -470,6 +478,11 @@ func main() { router.POST("/api/v1/directory/repos/:id/rebuild", goblog._admin.RebuildDirectoryRepo) router.DELETE("/api/v1/directory/repos/:id", goblog._admin.DelistDirectoryRepo) //if we use true here - it will override the home route and just show files + if datadir.Dir() != "" { + // Uploads are written under the data directory; anything that is + // not there (a file the image ships in www/uploads) falls through. + router.Use(static.Serve("/uploads", static.LocalFile(datadir.Path("uploads"), false))) + } router.Use(static.Serve("/", static.LocalFile("www", false))) if err != nil { log.Println("Couldn't get the hostname") @@ -650,7 +663,7 @@ func updateDB(c *gin.Context) { fail("Invalid database type") return } - if err := os.WriteFile(".env", []byte(cfg.envFile()), 0600); err != nil { + if err := os.WriteFile(datadir.Path(".env"), []byte(cfg.envFile()), 0600); err != nil { fail("Couldn't write the .env file: " + err.Error()) return } diff --git a/scripts/install-smoke-test.sh b/scripts/install-smoke-test.sh index 6adf6d16..8cbfab89 100755 --- a/scripts/install-smoke-test.sh +++ b/scripts/install-smoke-test.sh @@ -8,10 +8,16 @@ # GOBLOG_IMAGE names one that already exists. GOBLOG_PORT (default 7007) is # the host port the container is published on. # +# The container runs the way the quick start says to: GOBLOG_DATA_DIR on a +# volume. Half way through, the container is removed and a new one started +# on the same volume, which is what an upgrade does (#653). With +# GOBLOG_SMOKE_LEGACY=1 there is no data directory and the container is +# restarted instead: the layout of installs that predate GOBLOG_DATA_DIR. +# # What it cannot do is the wizard's last step, authorising a GitHub OAuth # app. It writes placeholder client_id/client_secret into the container's -# .env and restarts instead, so GitHub's callback and the first admin login -# are not covered. +# .env instead, so GitHub's callback and the first admin login are not +# covered. set -euo pipefail DB=${1:-} @@ -26,7 +32,15 @@ BASE="http://127.0.0.1:$PORT" NET="goblog-smoke-$DB" APP="goblog-smoke-$DB-app" DBC="goblog-smoke-$DB-db" -APP_DIR=/go/src/github.com/compscidr/goblog +VOL="goblog-smoke-$DB-data" +LEGACY=${GOBLOG_SMOKE_LEGACY:-} +if [ -n "$LEGACY" ]; then + ENV_FILE=/go/src/github.com/compscidr/goblog/.env + run_args=() +else + ENV_FILE=/data/.env + run_args=(-e GOBLOG_DATA_DIR=/data -v "$VOL:/data") +fi TITLE="Smoke Test Blog" TMP=$(mktemp -d) @@ -37,6 +51,7 @@ cleanup() { docker logs "$APP" 2>&1 | tail -60 >&2 || true fi docker rm -f "$APP" "$DBC" >/dev/null 2>&1 || true + docker volume rm "$VOL" >/dev/null 2>&1 || true docker network rm "$NET" >/dev/null 2>&1 || true rm -rf "$TMP" exit "$code" @@ -71,9 +86,15 @@ if ! docker image inspect "$IMAGE" >/dev/null 2>&1; then fi docker rm -f "$APP" "$DBC" >/dev/null 2>&1 || true +docker volume rm "$VOL" >/dev/null 2>&1 || true docker network rm "$NET" >/dev/null 2>&1 || true docker network create "$NET" >/dev/null +start_app() { + docker run -d --name "$APP" --network "$NET" -p "127.0.0.1:$PORT:7000" "${run_args[@]}" "$IMAGE" >/dev/null + wait_for goblog 60 app_up +} + # The wizard's database form, as the browser posts it. case "$DB" in sqlite) @@ -99,9 +120,8 @@ case "$DB" in ;; esac -step "starting goblog ($DB)" -docker run -d --name "$APP" --network "$NET" -p "127.0.0.1:$PORT:7000" "$IMAGE" >/dev/null -wait_for goblog 60 app_up +step "starting goblog ($DB${LEGACY:+, no data directory})" +start_app step "a fresh install shows the database wizard" request "GET /" 200 "$BASE/" @@ -124,18 +144,30 @@ body_has 'id="settings-form"' "GET / after the database step" step "settings step" request "PATCH /api/v1/settings" 202 -X PATCH "$BASE/api/v1/settings" -H 'Content-Type: application/json' \ -d "[{\"key\":\"site_title\",\"value\":\"$TITLE\",\"type\":\"text\"},{\"key\":\"site_subtitle\",\"value\":\"installed by the smoke test\",\"type\":\"text\"}]" +echo "smoke test upload" >"$TMP/smoke-upload.txt" +request "POST /api/v1/upload" 200 -X POST "$BASE/api/v1/upload" -F "file=@$TMP/smoke-upload.txt" +body_has "/uploads/smoke-upload.txt" "POST /api/v1/upload" request "GET /?page=auth" 200 "$BASE/?page=auth" body_has 'name="client_id"' "GET /?page=auth" # The real last step is GitHub redirecting back to /login, whose handler # stores the credentials and registers the site's routes. With placeholders -# there is no callback, so the routes arrive with the restart below instead; -# that also covers a restart of a finished install (.env re-read, database -# reconnected, migrations re-run). -step "auth step (placeholder GitHub credentials written to .env), then restart" -docker exec "$APP" sh -c "printf 'client_id=smoke\nclient_secret=smoke\n' >> $APP_DIR/.env" -docker restart "$APP" >/dev/null -wait_for "goblog after restart" 60 app_up +# there is no callback, so the routes arrive with the next start instead. +step "auth step (placeholder GitHub credentials written to .env)" +docker exec "$APP" sh -c "printf 'client_id=smoke\nclient_secret=smoke\n' >> $ENV_FILE" + +if [ -n "$LEGACY" ]; then + step "restarting the container" + docker restart "$APP" >/dev/null + wait_for "goblog after restart" 60 app_up +else + # Everything the install wrote has to be on the volume: .env, the SQLite + # file, the upload. + step "replacing the container, keeping only the data volume" + docker logs "$APP" >"$TMP/first.log" 2>&1 + docker rm -f "$APP" >/dev/null + start_app +fi check_site() { request "GET /" 200 "$BASE/" @@ -146,6 +178,8 @@ check_site() { done request "GET /sitemap.xml" 200 "$BASE/sitemap.xml" body_has "&1 | grep -v -e "Couldn't connect to the database" -e "failed to initialize database" \ +if { cat "$TMP/first.log" 2>/dev/null; docker logs "$APP" 2>&1; } | grep -v -e "Couldn't connect to the database" -e "failed to initialize database" \ | grep -E "panic|Error [0-9]{4} \(|SQLSTATE|syntax error|SQL logic error|no such (table|column)" >"$TMP/errors"; then head -20 "$TMP/errors" >&2 fail "goblog logged errors" diff --git a/theme/theme.go b/theme/theme.go index 1c90e9af..cef8d4e2 100644 --- a/theme/theme.go +++ b/theme/theme.go @@ -6,6 +6,7 @@ package theme import ( + "goblog/datadir" "os" "path/filepath" "regexp" @@ -24,13 +25,13 @@ var ( // DefaultName is the theme every other theme is layered on. const DefaultName = "default" -// InstalledRoot is where the theme installer writes: $THEMES_INSTALLED_DIR -// or themes/installed. Read on every call so tests can change it. +// InstalledRoot is where the theme installer writes: $THEMES_INSTALLED_DIR, +// or themes/installed (under the data directory when one is set). Read on every call so tests can change it. func InstalledRoot() string { if v := os.Getenv("THEMES_INSTALLED_DIR"); v != "" { return v } - return filepath.Join(BuiltinRoot, "installed") + return datadir.Path(filepath.Join(BuiltinRoot, "installed")) } // namePattern is the on-disk rule, unchanged from the old inline check in diff --git a/wizard/wizard.go b/wizard/wizard.go index 57986b86..ff5b2c32 100644 --- a/wizard/wizard.go +++ b/wizard/wizard.go @@ -7,6 +7,7 @@ import ( "github.com/gin-contrib/sessions" "github.com/gin-gonic/gin" "goblog/auth" + "goblog/datadir" "gorm.io/gorm" "io" "log" @@ -148,7 +149,7 @@ func (w *Wizard) LoginCode(c *gin.Context) error { return errors.New("Error unmarshalling token response: " + err.Error()) } - f, err := os.OpenFile(".env", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600) + f, err := os.OpenFile(datadir.Path(".env"), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600) if err != nil { return errors.New("Error writing the .env file to save settings: " + err.Error()) }