diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6babaa7a..a193b555 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -98,25 +98,20 @@ jobs: key: ${{ secrets.DEPLOY_KEY }} # 重启的脚本，根据自身情况做相应改动，一般要做的是migrate数据库以及重启服务器 # - # 配置从宿主机挂载，不使用镜像里的那份：演示站连的是托管数据库， - # 而 config/settings.demo.yml 会随仓库公开、也会打进镜像，凭据不能写在那里。 - # 镜像里那份保持 sqlite，供 clone 仓库的人开箱即用。 + # 演示站用 sqlite，配置与数据库都是镜像里自带的 config/settings.demo.yml 和 + # go-admin-db.db，不挂载宿主机文件。演示模式不允许用户改数据，所以数据不需要 + # 留存：每次部署换新容器，就是回到干净状态。 # - # 路径本身走 secret：它不是凭据，但本仓库公开，没有理由把服务器的 - # 目录结构一并公布。DEMO_CONFIG_PATH 指向宿主机上那份配置。 - # - # 顺序是有意的：迁移先跑，跑不过就保持现有版本不动； - # 旧容器改名保留而不是删除，新容器不健康时能原样恢复。 + # 迁移在新容器里、服务启动之前跑：数据库留在容器内，一次性容器里跑的迁移 + # 改的是用完即删的副本，对服务没有意义。迁移失败则容器退出、健康检查不过， + # 走下面的回滚。旧容器改名保留而不是删除，新容器不健康时能原样恢复。 # 健康检查两条都要过——HTTP 活着不代表数据库通了。 script: | set -u - CFG="${{ secrets.DEMO_CONFIG_PATH }}" IMG="${{ env.IMAGE_NAME_TAG }}" NAME=go-admin-api PREV="$NAME-prev" - test -f "$CFG" || { echo "宿主机配置缺失，中止部署"; exit 1; } - # Old images of this repository are removed here and nowhere else. # Every deployment pulls one tagged with its commit and nothing ever # removed the previous one, so they only accumulated: 68 of them @@ -154,13 +149,6 @@ jobs: prune_old_images sudo docker pull "$IMG" || { echo "拉取镜像失败，中止部署"; exit 1; } - # 迁移用新镜像跑。失败时线上仍是旧版本配旧 schema，是自洽的； - # 硬切过去才会得到代码与表对不上的服务。 - if ! sudo docker run --rm -v "$CFG":/config/settings.yml:ro "$IMG" \ - /main migrate -c /config/settings.yml; then - echo "迁移失败，保持现有版本"; exit 1 - fi - if sudo docker ps -a --format '{{.Names}}' | grep -qx "$NAME"; then sudo docker rm -f "$PREV" >/dev/null 2>&1 || true sudo docker rename "$NAME" "$PREV" @@ -174,9 +162,8 @@ jobs: sudo docker stop --timeout 30 "$PREV" >/dev/null fi - sudo docker run -d -p 8000:8000 \ - -v "$CFG":/config/settings.yml:ro \ - --name "$NAME" "$IMG" + sudo docker run -d -p 8000:8000 --name "$NAME" "$IMG" \ + sh -c '/main migrate -c /config/settings.yml && exec /main server -c /config/settings.yml' ok=0 for i in $(seq 1 20); do diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 76768623..676058c6 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -139,6 +139,12 @@ jobs: - name: Test run: make test + # make test builds without the sqlite3 tag, so the tests that open a real + # sqlite file are skipped by it. These are the ones that prove concurrent + # requests wait for each other instead of failing with "database is locked". + - name: Test (sqlite3 driver) + run: go test -tags sqlite3 -count=1 ./common/database/ + - name: Build run: make build diff --git a/app/admin/models/initdb.go b/app/admin/models/initdb.go index cdec260f..ed8fcbfe 100644 --- a/app/admin/models/initdb.go +++ b/app/admin/models/initdb.go @@ -3,8 +3,8 @@ package models import ( "fmt" "go-admin/common/global" + "go-admin/config" "gorm.io/gorm" - "io/ioutil" "log" "strings" ) @@ -44,7 +44,7 @@ func ExecSql(db *gorm.DB, filePath string) error { } func Ioutil(filePath string) (string, error) { - if contents, err := ioutil.ReadFile(filePath); err == nil { + if contents, err := config.ReadSeed(filePath); err == nil { //因为contents是[]byte类型，直接转换成string类型后会多一行空格,需要使用strings.Replace替换换行符 result := strings.Replace(string(contents), "\n", "", 1) fmt.Println("Use ioutil.ReadFile to read a file:", result) diff --git a/cmd/migrate/migration/models/initdb.go b/cmd/migrate/migration/models/initdb.go index ba7f4ca7..61557418 100644 --- a/cmd/migrate/migration/models/initdb.go +++ b/cmd/migrate/migration/models/initdb.go @@ -3,7 +3,7 @@ package models import ( "fmt" "go-admin/common/global" - "io/ioutil" + "go-admin/config" "log" "strings" @@ -61,7 +61,7 @@ func ExecSql(db *gorm.DB, filePath string) error { } func Ioutil(filePath string) (string, error) { - if contents, err := ioutil.ReadFile(filePath); err == nil { + if contents, err := config.ReadSeed(filePath); err == nil { //因为contents是[]byte类型，直接转换成string类型后会多一行空格,需要使用strings.Replace替换换行符 result := strings.Replace(string(contents), "\n", "", 1) fmt.Println("Use ioutil.ReadFile to read a file:", result) diff --git a/common/database/initialize.go b/common/database/initialize.go index 75c2bd68..60d6b6ba 100644 --- a/common/database/initialize.go +++ b/common/database/initialize.go @@ -41,6 +41,7 @@ func setupSimpleDatabase(host string, c *toolsConfig.Database) { if err != nil { log.Fatal(pkg.Red(err.Error())) } + open = withSqliteDefaults(c.Driver, open) resolverConfig := toolsDB.NewConfigure(c.Source, c.MaxIdleConns, c.MaxOpenConns, c.ConnMaxIdleTime, c.ConnMaxLifeTime, registers) db, err := resolverConfig.Init(&gorm.Config{ diff --git a/common/database/sqlite_dsn.go b/common/database/sqlite_dsn.go new file mode 100644 index 00000000..7f28e5bd --- /dev/null +++ b/common/database/sqlite_dsn.go @@ -0,0 +1,77 @@ +package database + +import ( + "net/url" + "strings" + + "gorm.io/gorm" +) + +// sqlitePragmas are the connection parameters a file-backed sqlite database +// needs to behave under a server that issues concurrent requests. Each one is +// applied only when the configured source does not already set it. +// +// Without them sqlite runs in rollback-journal mode with no busy timeout: a +// writer blocks every reader and a second writer fails at once with +// "database is locked", instead of waiting for the first to finish. +// +// _busy_timeout wait up to 10s for a lock rather than failing immediately +// _journal_mode WAL lets readers proceed while a write is in progress +// _synchronous NORMAL is the pairing WAL is designed for +// _txlock a transaction takes the write lock when it begins. The +// default defers it, and a deferred transaction that reads +// and then writes is refused with SQLITE_BUSY straight away - +// busy_timeout does not apply to that upgrade. +var sqlitePragmas = [][2]string{ + {"_busy_timeout", "10000"}, + {"_journal_mode", "WAL"}, + {"_synchronous", "NORMAL"}, + {"_txlock", "immediate"}, +} + +// sqliteDSN returns source with the defaults in sqlitePragmas added. +// In-memory databases are returned untouched: WAL has no meaning there. +func sqliteDSN(source string) string { + if source == "" { + return source + } + + path, rawQuery := source, "" + if i := strings.IndexByte(source, '?'); i >= 0 { + path, rawQuery = source[:i], source[i+1:] + } + q, err := url.ParseQuery(rawQuery) + if err != nil { + // Not something to rewrite blind; the driver will report it. + return source + } + if isMemoryDB(path, q) { + return source + } + for _, p := range sqlitePragmas { + if _, set := q[p[0]]; !set { + q.Set(p[0], p[1]) + } + } + return path + "?" + q.Encode() +} + +// isMemoryDB reports whether the source names an in-memory database: the exact +// filename forms sqlite recognises, or a mode=memory parameter. A file that +// merely has ":memory:" or "mode=memory" somewhere in its name is a file. +func isMemoryDB(path string, q url.Values) bool { + switch path { + case ":memory:", "file::memory:", "file:": + return true + } + return q.Get("mode") == "memory" +} + +// withSqliteDefaults wraps open so every connection string handed to a sqlite3 +// dialector - the primary and any replicas - carries the defaults above. +func withSqliteDefaults(driver string, open func(string) gorm.Dialector) func(string) gorm.Dialector { + if driver != "sqlite3" { + return open + } + return func(dsn string) gorm.Dialector { return open(sqliteDSN(dsn)) } +} diff --git a/common/database/sqlite_dsn_test.go b/common/database/sqlite_dsn_test.go new file mode 100644 index 00000000..35def066 --- /dev/null +++ b/common/database/sqlite_dsn_test.go @@ -0,0 +1,54 @@ +package database + +import ( + "net/url" + "testing" +) + +func TestSqliteDSN(t *testing.T) { + parse := func(s string) (string, url.Values) { + u, err := url.Parse(s) + if err != nil { + t.Fatalf("parse %q: %v", s, err) + } + return u.Path, u.Query() + } + + t.Run("plain path gets every default", func(t *testing.T) { + path, q := parse(sqliteDSN("./go-admin-db.db")) + if path != "./go-admin-db.db" { + t.Errorf("path = %q", path) + } + for _, p := range sqlitePragmas { + if q.Get(p[0]) != p[1] { + t.Errorf("%s = %q, want %q", p[0], q.Get(p[0]), p[1]) + } + } + }) + + t.Run("explicit values win", func(t *testing.T) { + _, q := parse(sqliteDSN("a.db?_busy_timeout=500&_journal_mode=DELETE")) + if q.Get("_busy_timeout") != "500" || q.Get("_journal_mode") != "DELETE" { + t.Errorf("overridden: %v", q) + } + if q.Get("_txlock") != "immediate" { + t.Errorf("missing default alongside overrides: %v", q) + } + }) + + t.Run("files that only look like memory databases get the defaults", func(t *testing.T) { + for _, s := range []string{"mode=memory.db", "/tmp/:memory:.db", "data/mode=memory/app.db"} { + if got := sqliteDSN(s); got == s { + t.Errorf("sqliteDSN(%q) was left untouched", s) + } + } + }) + + t.Run("in-memory and empty untouched", func(t *testing.T) { + for _, s := range []string{"", ":memory:", "file::memory:?cache=shared", "file:x?mode=memory"} { + if got := sqliteDSN(s); got != s { + t.Errorf("sqliteDSN(%q) = %q", s, got) + } + } + }) +} diff --git a/common/database/sqlite_lock_test.go b/common/database/sqlite_lock_test.go new file mode 100644 index 00000000..aa5426d8 --- /dev/null +++ b/common/database/sqlite_lock_test.go @@ -0,0 +1,91 @@ +//go:build sqlite3 + +package database + +import ( + "path/filepath" + "sync" + "testing" + + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +type lockRow struct { + ID int `gorm:"primaryKey"` + N int +} + +// hammer runs writers and readers against one file at the same time and +// returns how many statements failed. +func hammer(t *testing.T, dsn string) (failed int, firstErr error) { + t.Helper() + db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err := db.AutoMigrate(&lockRow{}); err != nil { + t.Fatal(err) + } + db.Create(&lockRow{ID: 1}) + + var mu sync.Mutex + note := func(err error) { + if err == nil { + return + } + mu.Lock() + failed++ + if firstErr == nil { + firstErr = err + } + mu.Unlock() + } + + var wg sync.WaitGroup + for w := 0; w < 8; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 40; i++ { + note(db.Create(&lockRow{N: i}).Error) + // read-then-write transaction: the shape a deferred + // transaction cannot upgrade without SQLITE_BUSY + note(db.Transaction(func(tx *gorm.DB) error { + var r lockRow + if err := tx.First(&r, 1).Error; err != nil { + return err + } + return tx.Model(&lockRow{}).Where("id = 1").Update("n", r.N+1).Error + })) + } + }() + } + for r := 0; r < 4; r++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 80; i++ { + var c int64 + note(db.Model(&lockRow{}).Count(&c).Error) + } + }() + } + wg.Wait() + return failed, firstErr +} + +func TestSqliteDefaultsPreventLocking(t *testing.T) { + file := filepath.Join(t.TempDir(), "lock.db") + if failed, err := hammer(t, sqliteDSN(file)); failed != 0 { + t.Fatalf("%d statements failed with the defaults applied, first: %v", failed, err) + } +} + +// Documents what the defaults are for: the bare path loses statements. +func TestSqliteBarePathLocks(t *testing.T) { + file := filepath.Join(t.TempDir(), "bare.db") + if failed, _ := hammer(t, file); failed == 0 { + t.Skip("bare path did not lock on this machine; the comparison proves nothing here") + } +} diff --git a/config/seed.go b/config/seed.go new file mode 100644 index 00000000..ae2e5f19 --- /dev/null +++ b/config/seed.go @@ -0,0 +1,31 @@ +package config + +import ( + "embed" + "errors" + "io/fs" + "os" + "path" +) + +// The seed scripts ship inside the binary so a database can be created from +// wherever the binary runs. They used to be read from ./config/ only, which +// fails for anything not started from the repository root - a Docker image +// that carries just the binary and its settings file, for one. +// +//go:embed db.sql pg.sql db-begin-mysql.sql db-end-mysql.sql +var seedFS embed.FS + +// ReadSeed returns the seed script at name (e.g. "config/db.sql"). +// A file on disk wins, so a deployment that edits its own copy keeps doing +// so; the embedded copy is the fallback when there is none. +func ReadSeed(name string) ([]byte, error) { + b, err := os.ReadFile(name) + if err == nil || !errors.Is(err, fs.ErrNotExist) { + return b, err + } + if emb, embErr := seedFS.ReadFile(path.Base(name)); embErr == nil { + return emb, nil + } + return nil, err +} diff --git a/config/seed_test.go b/config/seed_test.go new file mode 100644 index 00000000..4040f1f5 --- /dev/null +++ b/config/seed_test.go @@ -0,0 +1,33 @@ +package config + +import ( + "os" + "path/filepath" + "testing" +) + +func TestReadSeedFallsBackToEmbedded(t *testing.T) { + // A directory with no config/ in it, as in a container. + missing := filepath.Join(t.TempDir(), "config", "db.sql") + b, err := ReadSeed(missing) + if err != nil || len(b) == 0 { + t.Fatalf("embedded fallback: len=%d err=%v", len(b), err) + } +} + +func TestReadSeedPrefersDisk(t *testing.T) { + p := filepath.Join(t.TempDir(), "db.sql") + if err := os.WriteFile(p, []byte("-- mine;"), 0o600); err != nil { + t.Fatal(err) + } + b, err := ReadSeed(p) + if err != nil || string(b) != "-- mine;" { + t.Fatalf("got %q, %v", b, err) + } +} + +func TestReadSeedUnknownFileStillErrors(t *testing.T) { + if _, err := ReadSeed(filepath.Join(t.TempDir(), "nope.sql")); err == nil { + t.Fatal("expected an error for a script that is not embedded") + } +} diff --git a/test/e2e-apporder/e2e_test.go b/test/e2e-apporder/e2e_test.go index a12a1f9d..d310d02e 100644 --- a/test/e2e-apporder/e2e_test.go +++ b/test/e2e-apporder/e2e_test.go @@ -284,9 +284,12 @@ func TestInstallUninstallReinstall(t *testing.T) { func TestMigrateExitsNonZeroWhenItFails(t *testing.T) { e := newEnv(t) - // The framework's first migration reads config/db.sql. Without it the - // migration fails, which is the cheapest real failure to arrange. - if err := os.Remove(filepath.Join(e.dir, "config", "db.sql")); err != nil { + // The framework's first migration runs config/db.sql, and a copy on disk + // takes precedence over the embedded one. Statements that do not parse + // make that migration fail, which is the cheapest real failure to arrange. + // (Deleting the file no longer does: the embedded copy takes over.) + bad := "THIS IS NOT SQL;\nNEITHER IS THIS;\n" + if err := os.WriteFile(filepath.Join(e.dir, "config", "db.sql"), []byte(bad), 0o600); err != nil { t.Fatal(err) } out, code := e.run("migrate")