From ea679bf00bd3a52c63e7296fc121c7f9b1bacf10 Mon Sep 17 00:00:00 2001 From: zhangwenjian Date: Wed, 7 Oct 2026 17:32:39 +0800 Subject: [PATCH 1/6] =?UTF-8?q?fix=F0=9F=90=9B:=20give=20sqlite=20connecti?= =?UTF-8?q?ons=20a=20busy=20timeout,=20WAL=20and=20immediate=20transaction?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A file-backed sqlite database ran in rollback-journal mode with no busy timeout, so concurrent requests failed with "database is locked" instead of waiting for each other. Add the defaults to the sqlite3 connection string unless the configured source already sets them. --- common/database/initialize.go | 1 + common/database/sqlite_dsn.go | 64 ++++++++++++++++++++ common/database/sqlite_dsn_test.go | 46 +++++++++++++++ common/database/sqlite_lock_test.go | 91 +++++++++++++++++++++++++++++ 4 files changed, 202 insertions(+) create mode 100644 common/database/sqlite_dsn.go create mode 100644 common/database/sqlite_dsn_test.go create mode 100644 common/database/sqlite_lock_test.go diff --git a/common/database/initialize.go b/common/database/initialize.go index 75c2bd68..60d6b6ba 100644 --- a/common/database/initialize.go +++ b/common/database/initialize.go @@ -41,6 +41,7 @@ func setupSimpleDatabase(host string, c *toolsConfig.Database) { if err != nil { log.Fatal(pkg.Red(err.Error())) } + open = withSqliteDefaults(c.Driver, open) resolverConfig := toolsDB.NewConfigure(c.Source, c.MaxIdleConns, c.MaxOpenConns, c.ConnMaxIdleTime, c.ConnMaxLifeTime, registers) db, err := resolverConfig.Init(&gorm.Config{ diff --git a/common/database/sqlite_dsn.go b/common/database/sqlite_dsn.go new file mode 100644 index 00000000..7da673ee --- /dev/null +++ b/common/database/sqlite_dsn.go @@ -0,0 +1,64 @@ +package database + +import ( + "net/url" + "strings" + + "gorm.io/gorm" +) + +// sqlitePragmas are the connection parameters a file-backed sqlite database +// needs to behave under a server that issues concurrent requests. Each one is +// applied only when the configured source does not already set it. +// +// Without them sqlite runs in rollback-journal mode with no busy timeout: a +// writer blocks every reader and a second writer fails at once with +// "database is locked", instead of waiting for the first to finish. +// +// _busy_timeout wait up to 10s for a lock rather than failing immediately +// _journal_mode WAL lets readers proceed while a write is in progress +// _synchronous NORMAL is the pairing WAL is designed for +// _txlock a transaction takes the write lock when it begins. The +// default defers it, and a deferred transaction that reads +// and then writes is refused with SQLITE_BUSY straight away - +// busy_timeout does not apply to that upgrade. +var sqlitePragmas = [][2]string{ + {"_busy_timeout", "10000"}, + {"_journal_mode", "WAL"}, + {"_synchronous", "NORMAL"}, + {"_txlock", "immediate"}, +} + +// sqliteDSN returns source with the defaults in sqlitePragmas added. +// In-memory databases are returned untouched: WAL has no meaning there, and +// every connection to one is its own database anyway. +func sqliteDSN(source string) string { + if source == "" || strings.Contains(source, ":memory:") || strings.Contains(source, "mode=memory") { + return source + } + + path, rawQuery := source, "" + if i := strings.IndexByte(source, '?'); i >= 0 { + path, rawQuery = source[:i], source[i+1:] + } + q, err := url.ParseQuery(rawQuery) + if err != nil { + // Not something to rewrite blind; the driver will report it. + return source + } + for _, p := range sqlitePragmas { + if _, set := q[p[0]]; !set { + q.Set(p[0], p[1]) + } + } + return path + "?" + q.Encode() +} + +// withSqliteDefaults wraps open so every connection string handed to a sqlite3 +// dialector - the primary and any replicas - carries the defaults above. +func withSqliteDefaults(driver string, open func(string) gorm.Dialector) func(string) gorm.Dialector { + if driver != "sqlite3" { + return open + } + return func(dsn string) gorm.Dialector { return open(sqliteDSN(dsn)) } +} diff --git a/common/database/sqlite_dsn_test.go b/common/database/sqlite_dsn_test.go new file mode 100644 index 00000000..e4da5ff4 --- /dev/null +++ b/common/database/sqlite_dsn_test.go @@ -0,0 +1,46 @@ +package database + +import ( + "net/url" + "testing" +) + +func TestSqliteDSN(t *testing.T) { + parse := func(s string) (string, url.Values) { + u, err := url.Parse(s) + if err != nil { + t.Fatalf("parse %q: %v", s, err) + } + return u.Path, u.Query() + } + + t.Run("plain path gets every default", func(t *testing.T) { + path, q := parse(sqliteDSN("./go-admin-db.db")) + if path != "./go-admin-db.db" { + t.Errorf("path = %q", path) + } + for _, p := range sqlitePragmas { + if q.Get(p[0]) != p[1] { + t.Errorf("%s = %q, want %q", p[0], q.Get(p[0]), p[1]) + } + } + }) + + t.Run("explicit values win", func(t *testing.T) { + _, q := parse(sqliteDSN("a.db?_busy_timeout=500&_journal_mode=DELETE")) + if q.Get("_busy_timeout") != "500" || q.Get("_journal_mode") != "DELETE" { + t.Errorf("overridden: %v", q) + } + if q.Get("_txlock") != "immediate" { + t.Errorf("missing default alongside overrides: %v", q) + } + }) + + t.Run("in-memory and empty untouched", func(t *testing.T) { + for _, s := range []string{"", ":memory:", "file::memory:?cache=shared", "file:x?mode=memory"} { + if got := sqliteDSN(s); got != s { + t.Errorf("sqliteDSN(%q) = %q", s, got) + } + } + }) +} diff --git a/common/database/sqlite_lock_test.go b/common/database/sqlite_lock_test.go new file mode 100644 index 00000000..aa5426d8 --- /dev/null +++ b/common/database/sqlite_lock_test.go @@ -0,0 +1,91 @@ +//go:build sqlite3 + +package database + +import ( + "path/filepath" + "sync" + "testing" + + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +type lockRow struct { + ID int `gorm:"primaryKey"` + N int +} + +// hammer runs writers and readers against one file at the same time and +// returns how many statements failed. +func hammer(t *testing.T, dsn string) (failed int, firstErr error) { + t.Helper() + db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err := db.AutoMigrate(&lockRow{}); err != nil { + t.Fatal(err) + } + db.Create(&lockRow{ID: 1}) + + var mu sync.Mutex + note := func(err error) { + if err == nil { + return + } + mu.Lock() + failed++ + if firstErr == nil { + firstErr = err + } + mu.Unlock() + } + + var wg sync.WaitGroup + for w := 0; w < 8; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 40; i++ { + note(db.Create(&lockRow{N: i}).Error) + // read-then-write transaction: the shape a deferred + // transaction cannot upgrade without SQLITE_BUSY + note(db.Transaction(func(tx *gorm.DB) error { + var r lockRow + if err := tx.First(&r, 1).Error; err != nil { + return err + } + return tx.Model(&lockRow{}).Where("id = 1").Update("n", r.N+1).Error + })) + } + }() + } + for r := 0; r < 4; r++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 80; i++ { + var c int64 + note(db.Model(&lockRow{}).Count(&c).Error) + } + }() + } + wg.Wait() + return failed, firstErr +} + +func TestSqliteDefaultsPreventLocking(t *testing.T) { + file := filepath.Join(t.TempDir(), "lock.db") + if failed, err := hammer(t, sqliteDSN(file)); failed != 0 { + t.Fatalf("%d statements failed with the defaults applied, first: %v", failed, err) + } +} + +// Documents what the defaults are for: the bare path loses statements. +func TestSqliteBarePathLocks(t *testing.T) { + file := filepath.Join(t.TempDir(), "bare.db") + if failed, _ := hammer(t, file); failed == 0 { + t.Skip("bare path did not lock on this machine; the comparison proves nothing here") + } +} From 3a96f37391e8f74cccba0757cdc90cd74575ac5b Mon Sep 17 00:00:00 2001 From: zhangwenjian Date: Wed, 7 Oct 2026 17:32:42 +0800 Subject: [PATCH 2/6] =?UTF-8?q?fix=F0=9F=90=9B:=20embed=20the=20seed=20SQL?= =?UTF-8?q?=20so=20migrate=20works=20outside=20the=20repository=20root?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The seed scripts were read from ./config/ only, so migrating an empty database failed anywhere that directory was absent, such as a container holding just the binary and its settings. Embed them and prefer a file on disk when there is one. --- app/admin/models/initdb.go | 4 ++-- cmd/migrate/migration/models/initdb.go | 4 ++-- config/seed.go | 31 ++++++++++++++++++++++++ config/seed_test.go | 33 ++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 4 deletions(-) create mode 100644 config/seed.go create mode 100644 config/seed_test.go diff --git a/app/admin/models/initdb.go b/app/admin/models/initdb.go index cdec260f..ed8fcbfe 100644 --- a/app/admin/models/initdb.go +++ b/app/admin/models/initdb.go @@ -3,8 +3,8 @@ package models import ( "fmt" "go-admin/common/global" + "go-admin/config" "gorm.io/gorm" - "io/ioutil" "log" "strings" ) @@ -44,7 +44,7 @@ func ExecSql(db *gorm.DB, filePath string) error { } func Ioutil(filePath string) (string, error) { - if contents, err := ioutil.ReadFile(filePath); err == nil { + if contents, err := config.ReadSeed(filePath); err == nil { //因为contents是[]byte类型，直接转换成string类型后会多一行空格,需要使用strings.Replace替换换行符 result := strings.Replace(string(contents), "\n", "", 1) fmt.Println("Use ioutil.ReadFile to read a file:", result) diff --git a/cmd/migrate/migration/models/initdb.go b/cmd/migrate/migration/models/initdb.go index ba7f4ca7..61557418 100644 --- a/cmd/migrate/migration/models/initdb.go +++ b/cmd/migrate/migration/models/initdb.go @@ -3,7 +3,7 @@ package models import ( "fmt" "go-admin/common/global" - "io/ioutil" + "go-admin/config" "log" "strings" @@ -61,7 +61,7 @@ func ExecSql(db *gorm.DB, filePath string) error { } func Ioutil(filePath string) (string, error) { - if contents, err := ioutil.ReadFile(filePath); err == nil { + if contents, err := config.ReadSeed(filePath); err == nil { //因为contents是[]byte类型，直接转换成string类型后会多一行空格,需要使用strings.Replace替换换行符 result := strings.Replace(string(contents), "\n", "", 1) fmt.Println("Use ioutil.ReadFile to read a file:", result) diff --git a/config/seed.go b/config/seed.go new file mode 100644 index 00000000..ae2e5f19 --- /dev/null +++ b/config/seed.go @@ -0,0 +1,31 @@ +package config + +import ( + "embed" + "errors" + "io/fs" + "os" + "path" +) + +// The seed scripts ship inside the binary so a database can be created from +// wherever the binary runs. They used to be read from ./config/ only, which +// fails for anything not started from the repository root - a Docker image +// that carries just the binary and its settings file, for one. +// +//go:embed db.sql pg.sql db-begin-mysql.sql db-end-mysql.sql +var seedFS embed.FS + +// ReadSeed returns the seed script at name (e.g. "config/db.sql"). +// A file on disk wins, so a deployment that edits its own copy keeps doing +// so; the embedded copy is the fallback when there is none. +func ReadSeed(name string) ([]byte, error) { + b, err := os.ReadFile(name) + if err == nil || !errors.Is(err, fs.ErrNotExist) { + return b, err + } + if emb, embErr := seedFS.ReadFile(path.Base(name)); embErr == nil { + return emb, nil + } + return nil, err +} diff --git a/config/seed_test.go b/config/seed_test.go new file mode 100644 index 00000000..4040f1f5 --- /dev/null +++ b/config/seed_test.go @@ -0,0 +1,33 @@ +package config + +import ( + "os" + "path/filepath" + "testing" +) + +func TestReadSeedFallsBackToEmbedded(t *testing.T) { + // A directory with no config/ in it, as in a container. + missing := filepath.Join(t.TempDir(), "config", "db.sql") + b, err := ReadSeed(missing) + if err != nil || len(b) == 0 { + t.Fatalf("embedded fallback: len=%d err=%v", len(b), err) + } +} + +func TestReadSeedPrefersDisk(t *testing.T) { + p := filepath.Join(t.TempDir(), "db.sql") + if err := os.WriteFile(p, []byte("-- mine;"), 0o600); err != nil { + t.Fatal(err) + } + b, err := ReadSeed(p) + if err != nil || string(b) != "-- mine;" { + t.Fatalf("got %q, %v", b, err) + } +} + +func TestReadSeedUnknownFileStillErrors(t *testing.T) { + if _, err := ReadSeed(filepath.Join(t.TempDir(), "nope.sql")); err == nil { + t.Fatal("expected an error for a script that is not embedded") + } +} From 8ad42db8b3fcb51cd1f3d14f6fc3141e0417ad26 Mon Sep 17 00:00:00 2001 From: zhangwenjian Date: Wed, 7 Oct 2026 17:32:42 +0800 Subject: [PATCH 3/6] =?UTF-8?q?ci=F0=9F=91=B7:=20run=20the=20demo=20on=20i?= =?UTF-8?q?ts=20bundled=20sqlite=20database?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stop mounting a host configuration and run migrate inside the new container before the server starts, so the migration applies to the database the server actually uses. A failed migration fails the health check and rolls back. --- .github/workflows/build.yml | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6babaa7a..a193b555 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -98,25 +98,20 @@ jobs: key: ${{ secrets.DEPLOY_KEY }} # 重启的脚本，根据自身情况做相应改动，一般要做的是migrate数据库以及重启服务器 # - # 配置从宿主机挂载，不使用镜像里的那份：演示站连的是托管数据库， - # 而 config/settings.demo.yml 会随仓库公开、也会打进镜像，凭据不能写在那里。 - # 镜像里那份保持 sqlite，供 clone 仓库的人开箱即用。 + # 演示站用 sqlite，配置与数据库都是镜像里自带的 config/settings.demo.yml 和 + # go-admin-db.db，不挂载宿主机文件。演示模式不允许用户改数据，所以数据不需要 + # 留存：每次部署换新容器，就是回到干净状态。 # - # 路径本身走 secret：它不是凭据，但本仓库公开，没有理由把服务器的 - # 目录结构一并公布。DEMO_CONFIG_PATH 指向宿主机上那份配置。 - # - # 顺序是有意的：迁移先跑，跑不过就保持现有版本不动； - # 旧容器改名保留而不是删除，新容器不健康时能原样恢复。 + # 迁移在新容器里、服务启动之前跑：数据库留在容器内，一次性容器里跑的迁移 + # 改的是用完即删的副本，对服务没有意义。迁移失败则容器退出、健康检查不过， + # 走下面的回滚。旧容器改名保留而不是删除，新容器不健康时能原样恢复。 # 健康检查两条都要过——HTTP 活着不代表数据库通了。 script: | set -u - CFG="${{ secrets.DEMO_CONFIG_PATH }}" IMG="${{ env.IMAGE_NAME_TAG }}" NAME=go-admin-api PREV="$NAME-prev" - test -f "$CFG" || { echo "宿主机配置缺失，中止部署"; exit 1; } - # Old images of this repository are removed here and nowhere else. # Every deployment pulls one tagged with its commit and nothing ever # removed the previous one, so they only accumulated: 68 of them @@ -154,13 +149,6 @@ jobs: prune_old_images sudo docker pull "$IMG" || { echo "拉取镜像失败，中止部署"; exit 1; } - # 迁移用新镜像跑。失败时线上仍是旧版本配旧 schema，是自洽的； - # 硬切过去才会得到代码与表对不上的服务。 - if ! sudo docker run --rm -v "$CFG":/config/settings.yml:ro "$IMG" \ - /main migrate -c /config/settings.yml; then - echo "迁移失败，保持现有版本"; exit 1 - fi - if sudo docker ps -a --format '{{.Names}}' | grep -qx "$NAME"; then sudo docker rm -f "$PREV" >/dev/null 2>&1 || true sudo docker rename "$NAME" "$PREV" @@ -174,9 +162,8 @@ jobs: sudo docker stop --timeout 30 "$PREV" >/dev/null fi - sudo docker run -d -p 8000:8000 \ - -v "$CFG":/config/settings.yml:ro \ - --name "$NAME" "$IMG" + sudo docker run -d -p 8000:8000 --name "$NAME" "$IMG" \ + sh -c '/main migrate -c /config/settings.yml && exec /main server -c /config/settings.yml' ok=0 for i in $(seq 1 20); do From b082ea5d857771c7c999c06d9928faa237fad810 Mon Sep 17 00:00:00 2001 From: zhangwenjian Date: Wed, 7 Oct 2026 17:40:06 +0800 Subject: [PATCH 4/6] =?UTF-8?q?test=E2=9C=85:=20fail=20the=20migrate=20exi?= =?UTF-8?q?t-code=20test=20with=20bad=20SQL=20instead=20of=20a=20missing?= =?UTF-8?q?=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A missing config/db.sql now falls back to the embedded copy, so it no longer makes the migration fail. Corrupt the on-disk copy, which takes precedence. --- test/e2e-apporder/e2e_test.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/e2e-apporder/e2e_test.go b/test/e2e-apporder/e2e_test.go index a12a1f9d..d310d02e 100644 --- a/test/e2e-apporder/e2e_test.go +++ b/test/e2e-apporder/e2e_test.go @@ -284,9 +284,12 @@ func TestInstallUninstallReinstall(t *testing.T) { func TestMigrateExitsNonZeroWhenItFails(t *testing.T) { e := newEnv(t) - // The framework's first migration reads config/db.sql. Without it the - // migration fails, which is the cheapest real failure to arrange. - if err := os.Remove(filepath.Join(e.dir, "config", "db.sql")); err != nil { + // The framework's first migration runs config/db.sql, and a copy on disk + // takes precedence over the embedded one. Statements that do not parse + // make that migration fail, which is the cheapest real failure to arrange. + // (Deleting the file no longer does: the embedded copy takes over.) + bad := "THIS IS NOT SQL;\nNEITHER IS THIS;\n" + if err := os.WriteFile(filepath.Join(e.dir, "config", "db.sql"), []byte(bad), 0o600); err != nil { t.Fatal(err) } out, code := e.run("migrate") From 8a1218c40401a7343885afbe881de53ec0b46936 Mon Sep 17 00:00:00 2001 From: zhangwenjian Date: Wed, 7 Oct 2026 17:40:49 +0800 Subject: [PATCH 5/6] =?UTF-8?q?fix=F0=9F=90=9B:=20only=20treat=20exact=20i?= =?UTF-8?q?n-memory=20forms=20as=20in-memory=20sqlite=20sources?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A file whose name merely contained :memory: or mode=memory was classified as an in-memory database and skipped the locking defaults. Parse the query and match the exact filename forms or a mode=memory parameter. --- common/database/sqlite_dsn.go | 19 ++++++++++++++++--- common/database/sqlite_dsn_test.go | 8 ++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/common/database/sqlite_dsn.go b/common/database/sqlite_dsn.go index 7da673ee..7f28e5bd 100644 --- a/common/database/sqlite_dsn.go +++ b/common/database/sqlite_dsn.go @@ -30,10 +30,9 @@ var sqlitePragmas = [][2]string{ } // sqliteDSN returns source with the defaults in sqlitePragmas added. -// In-memory databases are returned untouched: WAL has no meaning there, and -// every connection to one is its own database anyway. +// In-memory databases are returned untouched: WAL has no meaning there. func sqliteDSN(source string) string { - if source == "" || strings.Contains(source, ":memory:") || strings.Contains(source, "mode=memory") { + if source == "" { return source } @@ -46,6 +45,9 @@ func sqliteDSN(source string) string { // Not something to rewrite blind; the driver will report it. return source } + if isMemoryDB(path, q) { + return source + } for _, p := range sqlitePragmas { if _, set := q[p[0]]; !set { q.Set(p[0], p[1]) @@ -54,6 +56,17 @@ func sqliteDSN(source string) string { return path + "?" + q.Encode() } +// isMemoryDB reports whether the source names an in-memory database: the exact +// filename forms sqlite recognises, or a mode=memory parameter. A file that +// merely has ":memory:" or "mode=memory" somewhere in its name is a file. +func isMemoryDB(path string, q url.Values) bool { + switch path { + case ":memory:", "file::memory:", "file:": + return true + } + return q.Get("mode") == "memory" +} + // withSqliteDefaults wraps open so every connection string handed to a sqlite3 // dialector - the primary and any replicas - carries the defaults above. func withSqliteDefaults(driver string, open func(string) gorm.Dialector) func(string) gorm.Dialector { diff --git a/common/database/sqlite_dsn_test.go b/common/database/sqlite_dsn_test.go index e4da5ff4..35def066 100644 --- a/common/database/sqlite_dsn_test.go +++ b/common/database/sqlite_dsn_test.go @@ -36,6 +36,14 @@ func TestSqliteDSN(t *testing.T) { } }) + t.Run("files that only look like memory databases get the defaults", func(t *testing.T) { + for _, s := range []string{"mode=memory.db", "/tmp/:memory:.db", "data/mode=memory/app.db"} { + if got := sqliteDSN(s); got == s { + t.Errorf("sqliteDSN(%q) was left untouched", s) + } + } + }) + t.Run("in-memory and empty untouched", func(t *testing.T) { for _, s := range []string{"", ":memory:", "file::memory:?cache=shared", "file:x?mode=memory"} { if got := sqliteDSN(s); got != s { From 9092e1dea7c9c725635786d8d3d3a0d5765c2b14 Mon Sep 17 00:00:00 2001 From: zhangwenjian Date: Wed, 7 Oct 2026 17:40:49 +0800 Subject: [PATCH 6/6] =?UTF-8?q?ci=F0=9F=91=B7:=20run=20the=20sqlite3-tagge?= =?UTF-8?q?d=20database=20tests=20on=20every=20change?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit make test builds without the sqlite3 tag, so the lock regression test never ran. --- .github/workflows/go.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 76768623..676058c6 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -139,6 +139,12 @@ jobs: - name: Test run: make test + # make test builds without the sqlite3 tag, so the tests that open a real + # sqlite file are skipped by it. These are the ones that prove concurrent + # requests wait for each other instead of failing with "database is locked". + - name: Test (sqlite3 driver) + run: go test -tags sqlite3 -count=1 ./common/database/ + - name: Build run: make build