Merge pull request #961 from go-admin-team/fix/sqlite-demo-deploy

Run the demo on sqlite; stop sqlite locking; embed seed SQL

.github/workflows/build.yml

@@ -98,25 +98,20 @@ jobs:
           key: ${{ secrets.DEPLOY_KEY }}
           # 重启的脚本，根据自身情况做相应改动，一般要做的是migrate数据库以及重启服务器
           #
-          # 配置从宿主机挂载，不使用镜像里的那份：演示站连的是托管数据库，
-          # 而 config/settings.demo.yml 会随仓库公开、也会打进镜像，凭据不能写在那里。
-          # 镜像里那份保持 sqlite，供 clone 仓库的人开箱即用。
+          # 演示站用 sqlite，配置与数据库都是镜像里自带的 config/settings.demo.yml 和
+          # go-admin-db.db，不挂载宿主机文件。演示模式不允许用户改数据，所以数据不需要
+          # 留存：每次部署换新容器，就是回到干净状态。
           #
-          # 路径本身走 secret：它不是凭据，但本仓库公开，没有理由把服务器的
-          # 目录结构一并公布。DEMO_CONFIG_PATH 指向宿主机上那份配置。
-          #
-          # 顺序是有意的：迁移先跑，跑不过就保持现有版本不动；
-          # 旧容器改名保留而不是删除，新容器不健康时能原样恢复。
+          # 迁移在新容器里、服务启动之前跑：数据库留在容器内，一次性容器里跑的迁移
+          # 改的是用完即删的副本，对服务没有意义。迁移失败则容器退出、健康检查不过，
+          # 走下面的回滚。旧容器改名保留而不是删除，新容器不健康时能原样恢复。
           # 健康检查两条都要过——HTTP 活着不代表数据库通了。
           script: |
             set -u
-            CFG="${{ secrets.DEMO_CONFIG_PATH }}"
             IMG="${{ env.IMAGE_NAME_TAG }}"
             NAME=go-admin-api
             PREV="$NAME-prev"
 
-            test -f "$CFG" || { echo "宿主机配置缺失，中止部署"; exit 1; }
-
             # Old images of this repository are removed here and nowhere else.
             # Every deployment pulls one tagged with its commit and nothing ever
             # removed the previous one, so they only accumulated: 68 of them
@@ -154,13 +149,6 @@ jobs:
             prune_old_images
             sudo docker pull "$IMG" || { echo "拉取镜像失败，中止部署"; exit 1; }
 
-            # 迁移用新镜像跑。失败时线上仍是旧版本配旧 schema，是自洽的；
-            # 硬切过去才会得到代码与表对不上的服务。
-            if ! sudo docker run --rm -v "$CFG":/config/settings.yml:ro "$IMG" \
-                 /main migrate -c /config/settings.yml; then
-              echo "迁移失败，保持现有版本"; exit 1
-            fi
-
             if sudo docker ps -a --format '{{.Names}}' | grep -qx "$NAME"; then
               sudo docker rm -f "$PREV" >/dev/null 2>&1 || true
               sudo docker rename "$NAME" "$PREV"
@@ -174,9 +162,8 @@ jobs:
               sudo docker stop --timeout 30 "$PREV" >/dev/null
             fi
 
-            sudo docker run -d -p 8000:8000 \
-              -v "$CFG":/config/settings.yml:ro \
-              --name "$NAME" "$IMG"
+            sudo docker run -d -p 8000:8000 --name "$NAME" "$IMG" \
+              sh -c '/main migrate -c /config/settings.yml && exec /main server -c /config/settings.yml'
 
             ok=0
             for i in $(seq 1 20); do

.github/workflows/go.yml

@@ -139,6 +139,12 @@ jobs:
     - name: Test
       run: make test
 
+    # make test builds without the sqlite3 tag, so the tests that open a real
+    # sqlite file are skipped by it. These are the ones that prove concurrent
+    # requests wait for each other instead of failing with "database is locked".
+    - name: Test (sqlite3 driver)
+      run: go test -tags sqlite3 -count=1 ./common/database/
+
     - name: Build
       run: make build
 

app/admin/models/initdb.go

@@ -3,8 +3,8 @@ package models
 import (
 	"fmt"
 	"go-admin/common/global"
+	"go-admin/config"
 	"gorm.io/gorm"
-	"io/ioutil"
 	"log"
 	"strings"
 )
@@ -44,7 +44,7 @@ func ExecSql(db *gorm.DB, filePath string) error {
 }
 
 func Ioutil(filePath string) (string, error) {
-	if contents, err := ioutil.ReadFile(filePath); err == nil {
+	if contents, err := config.ReadSeed(filePath); err == nil {
 		//因为contents是[]byte类型，直接转换成string类型后会多一行空格,需要使用strings.Replace替换换行符
 		result := strings.Replace(string(contents), "\n", "", 1)
 		fmt.Println("Use ioutil.ReadFile to read a file:", result)

cmd/migrate/migration/models/initdb.go

@@ -3,7 +3,7 @@ package models
 import (
 	"fmt"
 	"go-admin/common/global"
-	"io/ioutil"
+	"go-admin/config"
 	"log"
 	"strings"
 
@@ -61,7 +61,7 @@ func ExecSql(db *gorm.DB, filePath string) error {
 }
 
 func Ioutil(filePath string) (string, error) {
-	if contents, err := ioutil.ReadFile(filePath); err == nil {
+	if contents, err := config.ReadSeed(filePath); err == nil {
 		//因为contents是[]byte类型，直接转换成string类型后会多一行空格,需要使用strings.Replace替换换行符
 		result := strings.Replace(string(contents), "\n", "", 1)
 		fmt.Println("Use ioutil.ReadFile to read a file:", result)

common/database/initialize.go

@@ -41,6 +41,7 @@ func setupSimpleDatabase(host string, c *toolsConfig.Database) {
 	if err != nil {
 		log.Fatal(pkg.Red(err.Error()))
 	}
+	open = withSqliteDefaults(c.Driver, open)
 
 	resolverConfig := toolsDB.NewConfigure(c.Source, c.MaxIdleConns, c.MaxOpenConns, c.ConnMaxIdleTime, c.ConnMaxLifeTime, registers)
 	db, err := resolverConfig.Init(&gorm.Config{

common/database/sqlite_dsn.go

@@ -0,0 +1,77 @@
+package database
+
+import (
+	"net/url"
+	"strings"
+
+	"gorm.io/gorm"
+)
+
+// sqlitePragmas are the connection parameters a file-backed sqlite database
+// needs to behave under a server that issues concurrent requests. Each one is
+// applied only when the configured source does not already set it.
+//
+// Without them sqlite runs in rollback-journal mode with no busy timeout: a
+// writer blocks every reader and a second writer fails at once with
+// "database is locked", instead of waiting for the first to finish.
+//
+//	_busy_timeout    wait up to 10s for a lock rather than failing immediately
+//	_journal_mode    WAL lets readers proceed while a write is in progress
+//	_synchronous     NORMAL is the pairing WAL is designed for
+//	_txlock          a transaction takes the write lock when it begins. The
+//	                 default defers it, and a deferred transaction that reads
+//	                 and then writes is refused with SQLITE_BUSY straight away -
+//	                 busy_timeout does not apply to that upgrade.
+var sqlitePragmas = [][2]string{
+	{"_busy_timeout", "10000"},
+	{"_journal_mode", "WAL"},
+	{"_synchronous", "NORMAL"},
+	{"_txlock", "immediate"},
+}
+
+// sqliteDSN returns source with the defaults in sqlitePragmas added.
+// In-memory databases are returned untouched: WAL has no meaning there.
+func sqliteDSN(source string) string {
+	if source == "" {
+		return source
+	}
+
+	path, rawQuery := source, ""
+	if i := strings.IndexByte(source, '?'); i >= 0 {
+		path, rawQuery = source[:i], source[i+1:]
+	}
+	q, err := url.ParseQuery(rawQuery)
+	if err != nil {
+		// Not something to rewrite blind; the driver will report it.
+		return source
+	}
+	if isMemoryDB(path, q) {
+		return source
+	}
+	for _, p := range sqlitePragmas {
+		if _, set := q[p[0]]; !set {
+			q.Set(p[0], p[1])
+		}
+	}
+	return path + "?" + q.Encode()
+}
+
+// isMemoryDB reports whether the source names an in-memory database: the exact
+// filename forms sqlite recognises, or a mode=memory parameter. A file that
+// merely has ":memory:" or "mode=memory" somewhere in its name is a file.
+func isMemoryDB(path string, q url.Values) bool {
+	switch path {
+	case ":memory:", "file::memory:", "file:":
+		return true
+	}
+	return q.Get("mode") == "memory"
+}
+
+// withSqliteDefaults wraps open so every connection string handed to a sqlite3
+// dialector - the primary and any replicas - carries the defaults above.
+func withSqliteDefaults(driver string, open func(string) gorm.Dialector) func(string) gorm.Dialector {
+	if driver != "sqlite3" {
+		return open
+	}
+	return func(dsn string) gorm.Dialector { return open(sqliteDSN(dsn)) }
+}

common/database/sqlite_dsn_test.go

@@ -0,0 +1,54 @@
+package database
+
+import (
+	"net/url"
+	"testing"
+)
+
+func TestSqliteDSN(t *testing.T) {
+	parse := func(s string) (string, url.Values) {
+		u, err := url.Parse(s)
+		if err != nil {
+			t.Fatalf("parse %q: %v", s, err)
+		}
+		return u.Path, u.Query()
+	}
+
+	t.Run("plain path gets every default", func(t *testing.T) {
+		path, q := parse(sqliteDSN("./go-admin-db.db"))
+		if path != "./go-admin-db.db" {
+			t.Errorf("path = %q", path)
+		}
+		for _, p := range sqlitePragmas {
+			if q.Get(p[0]) != p[1] {
+				t.Errorf("%s = %q, want %q", p[0], q.Get(p[0]), p[1])
+			}
+		}
+	})
+
+	t.Run("explicit values win", func(t *testing.T) {
+		_, q := parse(sqliteDSN("a.db?_busy_timeout=500&_journal_mode=DELETE"))
+		if q.Get("_busy_timeout") != "500" || q.Get("_journal_mode") != "DELETE" {
+			t.Errorf("overridden: %v", q)
+		}
+		if q.Get("_txlock") != "immediate" {
+			t.Errorf("missing default alongside overrides: %v", q)
+		}
+	})
+
+	t.Run("files that only look like memory databases get the defaults", func(t *testing.T) {
+		for _, s := range []string{"mode=memory.db", "/tmp/:memory:.db", "data/mode=memory/app.db"} {
+			if got := sqliteDSN(s); got == s {
+				t.Errorf("sqliteDSN(%q) was left untouched", s)
+			}
+		}
+	})
+
+	t.Run("in-memory and empty untouched", func(t *testing.T) {
+		for _, s := range []string{"", ":memory:", "file::memory:?cache=shared", "file:x?mode=memory"} {
+			if got := sqliteDSN(s); got != s {
+				t.Errorf("sqliteDSN(%q) = %q", s, got)
+			}
+		}
+	})
+}

common/database/sqlite_lock_test.go

@@ -0,0 +1,91 @@
+//go:build sqlite3
+
+package database
+
+import (
+	"path/filepath"
+	"sync"
+	"testing"
+
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+)
+
+type lockRow struct {
+	ID int `gorm:"primaryKey"`
+	N  int
+}
+
+// hammer runs writers and readers against one file at the same time and
+// returns how many statements failed.
+func hammer(t *testing.T, dsn string) (failed int, firstErr error) {
+	t.Helper()
+	db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := db.AutoMigrate(&lockRow{}); err != nil {
+		t.Fatal(err)
+	}
+	db.Create(&lockRow{ID: 1})
+
+	var mu sync.Mutex
+	note := func(err error) {
+		if err == nil {
+			return
+		}
+		mu.Lock()
+		failed++
+		if firstErr == nil {
+			firstErr = err
+		}
+		mu.Unlock()
+	}
+
+	var wg sync.WaitGroup
+	for w := 0; w < 8; w++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for i := 0; i < 40; i++ {
+				note(db.Create(&lockRow{N: i}).Error)
+				// read-then-write transaction: the shape a deferred
+				// transaction cannot upgrade without SQLITE_BUSY
+				note(db.Transaction(func(tx *gorm.DB) error {
+					var r lockRow
+					if err := tx.First(&r, 1).Error; err != nil {
+						return err
+					}
+					return tx.Model(&lockRow{}).Where("id = 1").Update("n", r.N+1).Error
+				}))
+			}
+		}()
+	}
+	for r := 0; r < 4; r++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for i := 0; i < 80; i++ {
+				var c int64
+				note(db.Model(&lockRow{}).Count(&c).Error)
+			}
+		}()
+	}
+	wg.Wait()
+	return failed, firstErr
+}
+
+func TestSqliteDefaultsPreventLocking(t *testing.T) {
+	file := filepath.Join(t.TempDir(), "lock.db")
+	if failed, err := hammer(t, sqliteDSN(file)); failed != 0 {
+		t.Fatalf("%d statements failed with the defaults applied, first: %v", failed, err)
+	}
+}
+
+// Documents what the defaults are for: the bare path loses statements.
+func TestSqliteBarePathLocks(t *testing.T) {
+	file := filepath.Join(t.TempDir(), "bare.db")
+	if failed, _ := hammer(t, file); failed == 0 {
+		t.Skip("bare path did not lock on this machine; the comparison proves nothing here")
+	}
+}

config/seed.go

@@ -0,0 +1,31 @@
+package config
+
+import (
+	"embed"
+	"errors"
+	"io/fs"
+	"os"
+	"path"
+)
+
+// The seed scripts ship inside the binary so a database can be created from
+// wherever the binary runs. They used to be read from ./config/ only, which
+// fails for anything not started from the repository root - a Docker image
+// that carries just the binary and its settings file, for one.
+//
+//go:embed db.sql pg.sql db-begin-mysql.sql db-end-mysql.sql
+var seedFS embed.FS
+
+// ReadSeed returns the seed script at name (e.g. "config/db.sql").
+// A file on disk wins, so a deployment that edits its own copy keeps doing
+// so; the embedded copy is the fallback when there is none.
+func ReadSeed(name string) ([]byte, error) {
+	b, err := os.ReadFile(name)
+	if err == nil || !errors.Is(err, fs.ErrNotExist) {
+		return b, err
+	}
+	if emb, embErr := seedFS.ReadFile(path.Base(name)); embErr == nil {
+		return emb, nil
+	}
+	return nil, err
+}

config/seed_test.go

@@ -0,0 +1,33 @@
+package config
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestReadSeedFallsBackToEmbedded(t *testing.T) {
+	// A directory with no config/ in it, as in a container.
+	missing := filepath.Join(t.TempDir(), "config", "db.sql")
+	b, err := ReadSeed(missing)
+	if err != nil || len(b) == 0 {
+		t.Fatalf("embedded fallback: len=%d err=%v", len(b), err)
+	}
+}
+
+func TestReadSeedPrefersDisk(t *testing.T) {
+	p := filepath.Join(t.TempDir(), "db.sql")
+	if err := os.WriteFile(p, []byte("-- mine;"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	b, err := ReadSeed(p)
+	if err != nil || string(b) != "-- mine;" {
+		t.Fatalf("got %q, %v", b, err)
+	}
+}
+
+func TestReadSeedUnknownFileStillErrors(t *testing.T) {
+	if _, err := ReadSeed(filepath.Join(t.TempDir(), "nope.sql")); err == nil {
+		t.Fatal("expected an error for a script that is not embedded")
+	}
+}

test/e2e-apporder/e2e_test.go

@@ -284,9 +284,12 @@ func TestInstallUninstallReinstall(t *testing.T) {
 func TestMigrateExitsNonZeroWhenItFails(t *testing.T) {
 	e := newEnv(t)
 
-	// The framework's first migration reads config/db.sql. Without it the
-	// migration fails, which is the cheapest real failure to arrange.
-	if err := os.Remove(filepath.Join(e.dir, "config", "db.sql")); err != nil {
+	// The framework's first migration runs config/db.sql, and a copy on disk
+	// takes precedence over the embedded one. Statements that do not parse
+	// make that migration fail, which is the cheapest real failure to arrange.
+	// (Deleting the file no longer does: the embedded copy takes over.)
+	bad := "THIS IS NOT SQL;\nNEITHER IS THIS;\n"
+	if err := os.WriteFile(filepath.Join(e.dir, "config", "db.sql"), []byte(bad), 0o600); err != nil {
 		t.Fatal(err)
 	}
 	out, code := e.run("migrate")